2 * btree.c - NILFS B-tree.
4 * Copyright (C) 2005-2008 Nippon Telegraph and Telephone Corporation.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20 * Written by Koji Sato <koji@osrg.net>.
23 #include <linux/slab.h>
24 #include <linux/string.h>
25 #include <linux/errno.h>
26 #include <linux/pagevec.h>
34 * struct nilfs_btree_path - A path on which B-tree operations are executed
35 * @bp_bh: buffer head of node block
36 * @bp_sib_bh: buffer head of sibling node block
37 * @bp_index: index of child node
38 * @bp_oldreq: ptr end request for old ptr
39 * @bp_newreq: ptr alloc request for new ptr
40 * @bp_op: rebalance operation
42 struct nilfs_btree_path {
43 struct buffer_head *bp_bh;
44 struct buffer_head *bp_sib_bh;
46 union nilfs_bmap_ptr_req bp_oldreq;
47 union nilfs_bmap_ptr_req bp_newreq;
48 struct nilfs_btnode_chkey_ctxt bp_ctxt;
49 void (*bp_op)(struct nilfs_btree *, struct nilfs_btree_path *,
50 int, __u64 *, __u64 *);
54 * B-tree path operations
57 static struct kmem_cache *nilfs_btree_path_cache;
59 int __init nilfs_btree_path_cache_init(void)
61 nilfs_btree_path_cache =
62 kmem_cache_create("nilfs2_btree_path_cache",
63 sizeof(struct nilfs_btree_path) *
64 NILFS_BTREE_LEVEL_MAX, 0, 0, NULL);
65 return (nilfs_btree_path_cache != NULL) ? 0 : -ENOMEM;
68 void nilfs_btree_path_cache_destroy(void)
70 kmem_cache_destroy(nilfs_btree_path_cache);
73 static inline struct nilfs_btree_path *
74 nilfs_btree_alloc_path(const struct nilfs_btree *btree)
76 return (struct nilfs_btree_path *)
77 kmem_cache_alloc(nilfs_btree_path_cache, GFP_NOFS);
80 static inline void nilfs_btree_free_path(const struct nilfs_btree *btree,
81 struct nilfs_btree_path *path)
83 kmem_cache_free(nilfs_btree_path_cache, path);
86 static void nilfs_btree_init_path(const struct nilfs_btree *btree,
87 struct nilfs_btree_path *path)
91 for (level = NILFS_BTREE_LEVEL_DATA;
92 level < NILFS_BTREE_LEVEL_MAX;
94 path[level].bp_bh = NULL;
95 path[level].bp_sib_bh = NULL;
96 path[level].bp_index = 0;
97 path[level].bp_oldreq.bpr_ptr = NILFS_BMAP_INVALID_PTR;
98 path[level].bp_newreq.bpr_ptr = NILFS_BMAP_INVALID_PTR;
99 path[level].bp_op = NULL;
103 static void nilfs_btree_clear_path(const struct nilfs_btree *btree,
104 struct nilfs_btree_path *path)
108 for (level = NILFS_BTREE_LEVEL_DATA;
109 level < NILFS_BTREE_LEVEL_MAX;
111 if (path[level].bp_bh != NULL) {
112 brelse(path[level].bp_bh);
113 path[level].bp_bh = NULL;
115 /* sib_bh is released or deleted by prepare or commit
117 path[level].bp_sib_bh = NULL;
118 path[level].bp_index = 0;
119 path[level].bp_oldreq.bpr_ptr = NILFS_BMAP_INVALID_PTR;
120 path[level].bp_newreq.bpr_ptr = NILFS_BMAP_INVALID_PTR;
121 path[level].bp_op = NULL;
127 * B-tree node operations
131 nilfs_btree_node_get_flags(const struct nilfs_btree *btree,
132 const struct nilfs_btree_node *node)
134 return node->bn_flags;
138 nilfs_btree_node_set_flags(struct nilfs_btree *btree,
139 struct nilfs_btree_node *node,
142 node->bn_flags = flags;
145 static inline int nilfs_btree_node_root(const struct nilfs_btree *btree,
146 const struct nilfs_btree_node *node)
148 return nilfs_btree_node_get_flags(btree, node) & NILFS_BTREE_NODE_ROOT;
152 nilfs_btree_node_get_level(const struct nilfs_btree *btree,
153 const struct nilfs_btree_node *node)
155 return node->bn_level;
159 nilfs_btree_node_set_level(struct nilfs_btree *btree,
160 struct nilfs_btree_node *node,
163 node->bn_level = level;
167 nilfs_btree_node_get_nchildren(const struct nilfs_btree *btree,
168 const struct nilfs_btree_node *node)
170 return le16_to_cpu(node->bn_nchildren);
174 nilfs_btree_node_set_nchildren(struct nilfs_btree *btree,
175 struct nilfs_btree_node *node,
178 node->bn_nchildren = cpu_to_le16(nchildren);
182 nilfs_btree_node_size(const struct nilfs_btree *btree)
184 return 1 << btree->bt_bmap.b_inode->i_blkbits;
188 nilfs_btree_node_nchildren_min(const struct nilfs_btree *btree,
189 const struct nilfs_btree_node *node)
191 return nilfs_btree_node_root(btree, node) ?
192 NILFS_BTREE_ROOT_NCHILDREN_MIN :
193 NILFS_BTREE_NODE_NCHILDREN_MIN(nilfs_btree_node_size(btree));
197 nilfs_btree_node_nchildren_max(const struct nilfs_btree *btree,
198 const struct nilfs_btree_node *node)
200 return nilfs_btree_node_root(btree, node) ?
201 NILFS_BTREE_ROOT_NCHILDREN_MAX :
202 NILFS_BTREE_NODE_NCHILDREN_MAX(nilfs_btree_node_size(btree));
205 static inline __le64 *
206 nilfs_btree_node_dkeys(const struct nilfs_btree *btree,
207 const struct nilfs_btree_node *node)
209 return (__le64 *)((char *)(node + 1) +
210 (nilfs_btree_node_root(btree, node) ?
211 0 : NILFS_BTREE_NODE_EXTRA_PAD_SIZE));
214 static inline __le64 *
215 nilfs_btree_node_dptrs(const struct nilfs_btree *btree,
216 const struct nilfs_btree_node *node)
218 return (__le64 *)(nilfs_btree_node_dkeys(btree, node) +
219 nilfs_btree_node_nchildren_max(btree, node));
223 nilfs_btree_node_get_key(const struct nilfs_btree *btree,
224 const struct nilfs_btree_node *node, int index)
226 return nilfs_bmap_dkey_to_key(*(nilfs_btree_node_dkeys(btree, node) +
231 nilfs_btree_node_set_key(struct nilfs_btree *btree,
232 struct nilfs_btree_node *node, int index, __u64 key)
234 *(nilfs_btree_node_dkeys(btree, node) + index) =
235 nilfs_bmap_key_to_dkey(key);
239 nilfs_btree_node_get_ptr(const struct nilfs_btree *btree,
240 const struct nilfs_btree_node *node,
243 return nilfs_bmap_dptr_to_ptr(*(nilfs_btree_node_dptrs(btree, node) +
248 nilfs_btree_node_set_ptr(struct nilfs_btree *btree,
249 struct nilfs_btree_node *node,
253 *(nilfs_btree_node_dptrs(btree, node) + index) =
254 nilfs_bmap_ptr_to_dptr(ptr);
257 static void nilfs_btree_node_init(struct nilfs_btree *btree,
258 struct nilfs_btree_node *node,
259 int flags, int level, int nchildren,
260 const __u64 *keys, const __u64 *ptrs)
266 nilfs_btree_node_set_flags(btree, node, flags);
267 nilfs_btree_node_set_level(btree, node, level);
268 nilfs_btree_node_set_nchildren(btree, node, nchildren);
270 dkeys = nilfs_btree_node_dkeys(btree, node);
271 dptrs = nilfs_btree_node_dptrs(btree, node);
272 for (i = 0; i < nchildren; i++) {
273 dkeys[i] = nilfs_bmap_key_to_dkey(keys[i]);
274 dptrs[i] = nilfs_bmap_ptr_to_dptr(ptrs[i]);
278 /* Assume the buffer heads corresponding to left and right are locked. */
279 static void nilfs_btree_node_move_left(struct nilfs_btree *btree,
280 struct nilfs_btree_node *left,
281 struct nilfs_btree_node *right,
284 __le64 *ldkeys, *rdkeys;
285 __le64 *ldptrs, *rdptrs;
286 int lnchildren, rnchildren;
288 ldkeys = nilfs_btree_node_dkeys(btree, left);
289 ldptrs = nilfs_btree_node_dptrs(btree, left);
290 lnchildren = nilfs_btree_node_get_nchildren(btree, left);
292 rdkeys = nilfs_btree_node_dkeys(btree, right);
293 rdptrs = nilfs_btree_node_dptrs(btree, right);
294 rnchildren = nilfs_btree_node_get_nchildren(btree, right);
296 memcpy(ldkeys + lnchildren, rdkeys, n * sizeof(*rdkeys));
297 memcpy(ldptrs + lnchildren, rdptrs, n * sizeof(*rdptrs));
298 memmove(rdkeys, rdkeys + n, (rnchildren - n) * sizeof(*rdkeys));
299 memmove(rdptrs, rdptrs + n, (rnchildren - n) * sizeof(*rdptrs));
303 nilfs_btree_node_set_nchildren(btree, left, lnchildren);
304 nilfs_btree_node_set_nchildren(btree, right, rnchildren);
307 /* Assume that the buffer heads corresponding to left and right are locked. */
308 static void nilfs_btree_node_move_right(struct nilfs_btree *btree,
309 struct nilfs_btree_node *left,
310 struct nilfs_btree_node *right,
313 __le64 *ldkeys, *rdkeys;
314 __le64 *ldptrs, *rdptrs;
315 int lnchildren, rnchildren;
317 ldkeys = nilfs_btree_node_dkeys(btree, left);
318 ldptrs = nilfs_btree_node_dptrs(btree, left);
319 lnchildren = nilfs_btree_node_get_nchildren(btree, left);
321 rdkeys = nilfs_btree_node_dkeys(btree, right);
322 rdptrs = nilfs_btree_node_dptrs(btree, right);
323 rnchildren = nilfs_btree_node_get_nchildren(btree, right);
325 memmove(rdkeys + n, rdkeys, rnchildren * sizeof(*rdkeys));
326 memmove(rdptrs + n, rdptrs, rnchildren * sizeof(*rdptrs));
327 memcpy(rdkeys, ldkeys + lnchildren - n, n * sizeof(*rdkeys));
328 memcpy(rdptrs, ldptrs + lnchildren - n, n * sizeof(*rdptrs));
332 nilfs_btree_node_set_nchildren(btree, left, lnchildren);
333 nilfs_btree_node_set_nchildren(btree, right, rnchildren);
336 /* Assume that the buffer head corresponding to node is locked. */
337 static void nilfs_btree_node_insert(struct nilfs_btree *btree,
338 struct nilfs_btree_node *node,
339 __u64 key, __u64 ptr, int index)
345 dkeys = nilfs_btree_node_dkeys(btree, node);
346 dptrs = nilfs_btree_node_dptrs(btree, node);
347 nchildren = nilfs_btree_node_get_nchildren(btree, node);
348 if (index < nchildren) {
349 memmove(dkeys + index + 1, dkeys + index,
350 (nchildren - index) * sizeof(*dkeys));
351 memmove(dptrs + index + 1, dptrs + index,
352 (nchildren - index) * sizeof(*dptrs));
354 dkeys[index] = nilfs_bmap_key_to_dkey(key);
355 dptrs[index] = nilfs_bmap_ptr_to_dptr(ptr);
357 nilfs_btree_node_set_nchildren(btree, node, nchildren);
360 /* Assume that the buffer head corresponding to node is locked. */
361 static void nilfs_btree_node_delete(struct nilfs_btree *btree,
362 struct nilfs_btree_node *node,
363 __u64 *keyp, __u64 *ptrp, int index)
371 dkeys = nilfs_btree_node_dkeys(btree, node);
372 dptrs = nilfs_btree_node_dptrs(btree, node);
373 key = nilfs_bmap_dkey_to_key(dkeys[index]);
374 ptr = nilfs_bmap_dptr_to_ptr(dptrs[index]);
375 nchildren = nilfs_btree_node_get_nchildren(btree, node);
381 if (index < nchildren - 1) {
382 memmove(dkeys + index, dkeys + index + 1,
383 (nchildren - index - 1) * sizeof(*dkeys));
384 memmove(dptrs + index, dptrs + index + 1,
385 (nchildren - index - 1) * sizeof(*dptrs));
388 nilfs_btree_node_set_nchildren(btree, node, nchildren);
391 static int nilfs_btree_node_lookup(const struct nilfs_btree *btree,
392 const struct nilfs_btree_node *node,
393 __u64 key, int *indexp)
396 int index, low, high, s;
400 high = nilfs_btree_node_get_nchildren(btree, node) - 1;
403 while (low <= high) {
404 index = (low + high) / 2;
405 nkey = nilfs_btree_node_get_key(btree, node, index);
409 } else if (nkey < key) {
419 if (nilfs_btree_node_get_level(btree, node) >
420 NILFS_BTREE_LEVEL_NODE_MIN) {
421 if ((s > 0) && (index > 0))
432 static inline struct nilfs_btree_node *
433 nilfs_btree_get_root(const struct nilfs_btree *btree)
435 return (struct nilfs_btree_node *)btree->bt_bmap.b_u.u_data;
438 static inline struct nilfs_btree_node *
439 nilfs_btree_get_nonroot_node(const struct nilfs_btree *btree,
440 const struct nilfs_btree_path *path,
443 return (struct nilfs_btree_node *)path[level].bp_bh->b_data;
446 static inline struct nilfs_btree_node *
447 nilfs_btree_get_sib_node(const struct nilfs_btree *btree,
448 const struct nilfs_btree_path *path,
451 return (struct nilfs_btree_node *)path[level].bp_sib_bh->b_data;
454 static inline int nilfs_btree_height(const struct nilfs_btree *btree)
456 return nilfs_btree_node_get_level(btree, nilfs_btree_get_root(btree))
460 static inline struct nilfs_btree_node *
461 nilfs_btree_get_node(const struct nilfs_btree *btree,
462 const struct nilfs_btree_path *path,
465 return (level == nilfs_btree_height(btree) - 1) ?
466 nilfs_btree_get_root(btree) :
467 nilfs_btree_get_nonroot_node(btree, path, level);
470 static int nilfs_btree_do_lookup(const struct nilfs_btree *btree,
471 struct nilfs_btree_path *path,
472 __u64 key, __u64 *ptrp, int minlevel)
474 struct nilfs_btree_node *node;
476 int level, index, found, ret;
478 node = nilfs_btree_get_root(btree);
479 level = nilfs_btree_node_get_level(btree, node);
480 if ((level < minlevel) ||
481 (nilfs_btree_node_get_nchildren(btree, node) <= 0))
484 found = nilfs_btree_node_lookup(btree, node, key, &index);
485 ptr = nilfs_btree_node_get_ptr(btree, node, index);
486 path[level].bp_bh = NULL;
487 path[level].bp_index = index;
489 for (level--; level >= minlevel; level--) {
490 ret = nilfs_bmap_get_block(&btree->bt_bmap, ptr,
494 node = nilfs_btree_get_nonroot_node(btree, path, level);
495 BUG_ON(level != nilfs_btree_node_get_level(btree, node));
497 found = nilfs_btree_node_lookup(btree, node, key,
501 if (index < nilfs_btree_node_nchildren_max(btree, node))
502 ptr = nilfs_btree_node_get_ptr(btree, node, index);
504 WARN_ON(found || level != NILFS_BTREE_LEVEL_NODE_MIN);
506 ptr = NILFS_BMAP_INVALID_PTR;
508 path[level].bp_index = index;
519 static int nilfs_btree_do_lookup_last(const struct nilfs_btree *btree,
520 struct nilfs_btree_path *path,
521 __u64 *keyp, __u64 *ptrp)
523 struct nilfs_btree_node *node;
525 int index, level, ret;
527 node = nilfs_btree_get_root(btree);
528 index = nilfs_btree_node_get_nchildren(btree, node) - 1;
531 level = nilfs_btree_node_get_level(btree, node);
532 ptr = nilfs_btree_node_get_ptr(btree, node, index);
533 path[level].bp_bh = NULL;
534 path[level].bp_index = index;
536 for (level--; level > 0; level--) {
537 ret = nilfs_bmap_get_block(&btree->bt_bmap, ptr,
541 node = nilfs_btree_get_nonroot_node(btree, path, level);
542 BUG_ON(level != nilfs_btree_node_get_level(btree, node));
543 index = nilfs_btree_node_get_nchildren(btree, node) - 1;
544 ptr = nilfs_btree_node_get_ptr(btree, node, index);
545 path[level].bp_index = index;
549 *keyp = nilfs_btree_node_get_key(btree, node, index);
556 static int nilfs_btree_lookup(const struct nilfs_bmap *bmap,
557 __u64 key, int level, __u64 *ptrp)
559 struct nilfs_btree *btree;
560 struct nilfs_btree_path *path;
564 btree = (struct nilfs_btree *)bmap;
565 path = nilfs_btree_alloc_path(btree);
568 nilfs_btree_init_path(btree, path);
570 ret = nilfs_btree_do_lookup(btree, path, key, &ptr, level);
575 nilfs_btree_clear_path(btree, path);
576 nilfs_btree_free_path(btree, path);
581 static void nilfs_btree_promote_key(struct nilfs_btree *btree,
582 struct nilfs_btree_path *path,
583 int level, __u64 key)
585 if (level < nilfs_btree_height(btree) - 1) {
587 lock_buffer(path[level].bp_bh);
588 nilfs_btree_node_set_key(
590 nilfs_btree_get_nonroot_node(
592 path[level].bp_index, key);
593 if (!buffer_dirty(path[level].bp_bh))
594 nilfs_btnode_mark_dirty(path[level].bp_bh);
595 unlock_buffer(path[level].bp_bh);
596 } while ((path[level].bp_index == 0) &&
597 (++level < nilfs_btree_height(btree) - 1));
601 if (level == nilfs_btree_height(btree) - 1) {
602 nilfs_btree_node_set_key(btree,
603 nilfs_btree_get_root(btree),
604 path[level].bp_index, key);
608 static void nilfs_btree_do_insert(struct nilfs_btree *btree,
609 struct nilfs_btree_path *path,
610 int level, __u64 *keyp, __u64 *ptrp)
612 struct nilfs_btree_node *node;
614 if (level < nilfs_btree_height(btree) - 1) {
615 lock_buffer(path[level].bp_bh);
616 node = nilfs_btree_get_nonroot_node(btree, path, level);
617 nilfs_btree_node_insert(btree, node, *keyp, *ptrp,
618 path[level].bp_index);
619 if (!buffer_dirty(path[level].bp_bh))
620 nilfs_btnode_mark_dirty(path[level].bp_bh);
621 unlock_buffer(path[level].bp_bh);
623 if (path[level].bp_index == 0)
624 nilfs_btree_promote_key(btree, path, level + 1,
625 nilfs_btree_node_get_key(
628 node = nilfs_btree_get_root(btree);
629 nilfs_btree_node_insert(btree, node, *keyp, *ptrp,
630 path[level].bp_index);
634 static void nilfs_btree_carry_left(struct nilfs_btree *btree,
635 struct nilfs_btree_path *path,
636 int level, __u64 *keyp, __u64 *ptrp)
638 struct nilfs_btree_node *node, *left;
639 int nchildren, lnchildren, n, move;
641 lock_buffer(path[level].bp_bh);
642 lock_buffer(path[level].bp_sib_bh);
644 node = nilfs_btree_get_nonroot_node(btree, path, level);
645 left = nilfs_btree_get_sib_node(btree, path, level);
646 nchildren = nilfs_btree_node_get_nchildren(btree, node);
647 lnchildren = nilfs_btree_node_get_nchildren(btree, left);
650 n = (nchildren + lnchildren + 1) / 2 - lnchildren;
651 if (n > path[level].bp_index) {
652 /* move insert point */
657 nilfs_btree_node_move_left(btree, left, node, n);
659 if (!buffer_dirty(path[level].bp_bh))
660 nilfs_btnode_mark_dirty(path[level].bp_bh);
661 if (!buffer_dirty(path[level].bp_sib_bh))
662 nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
664 unlock_buffer(path[level].bp_bh);
665 unlock_buffer(path[level].bp_sib_bh);
667 nilfs_btree_promote_key(btree, path, level + 1,
668 nilfs_btree_node_get_key(btree, node, 0));
671 brelse(path[level].bp_bh);
672 path[level].bp_bh = path[level].bp_sib_bh;
673 path[level].bp_sib_bh = NULL;
674 path[level].bp_index += lnchildren;
675 path[level + 1].bp_index--;
677 brelse(path[level].bp_sib_bh);
678 path[level].bp_sib_bh = NULL;
679 path[level].bp_index -= n;
682 nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
685 static void nilfs_btree_carry_right(struct nilfs_btree *btree,
686 struct nilfs_btree_path *path,
687 int level, __u64 *keyp, __u64 *ptrp)
689 struct nilfs_btree_node *node, *right;
690 int nchildren, rnchildren, n, move;
692 lock_buffer(path[level].bp_bh);
693 lock_buffer(path[level].bp_sib_bh);
695 node = nilfs_btree_get_nonroot_node(btree, path, level);
696 right = nilfs_btree_get_sib_node(btree, path, level);
697 nchildren = nilfs_btree_node_get_nchildren(btree, node);
698 rnchildren = nilfs_btree_node_get_nchildren(btree, right);
701 n = (nchildren + rnchildren + 1) / 2 - rnchildren;
702 if (n > nchildren - path[level].bp_index) {
703 /* move insert point */
708 nilfs_btree_node_move_right(btree, node, right, n);
710 if (!buffer_dirty(path[level].bp_bh))
711 nilfs_btnode_mark_dirty(path[level].bp_bh);
712 if (!buffer_dirty(path[level].bp_sib_bh))
713 nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
715 unlock_buffer(path[level].bp_bh);
716 unlock_buffer(path[level].bp_sib_bh);
718 path[level + 1].bp_index++;
719 nilfs_btree_promote_key(btree, path, level + 1,
720 nilfs_btree_node_get_key(btree, right, 0));
721 path[level + 1].bp_index--;
724 brelse(path[level].bp_bh);
725 path[level].bp_bh = path[level].bp_sib_bh;
726 path[level].bp_sib_bh = NULL;
727 path[level].bp_index -=
728 nilfs_btree_node_get_nchildren(btree, node);
729 path[level + 1].bp_index++;
731 brelse(path[level].bp_sib_bh);
732 path[level].bp_sib_bh = NULL;
735 nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
738 static void nilfs_btree_split(struct nilfs_btree *btree,
739 struct nilfs_btree_path *path,
740 int level, __u64 *keyp, __u64 *ptrp)
742 struct nilfs_btree_node *node, *right;
745 int nchildren, n, move;
747 lock_buffer(path[level].bp_bh);
748 lock_buffer(path[level].bp_sib_bh);
750 node = nilfs_btree_get_nonroot_node(btree, path, level);
751 right = nilfs_btree_get_sib_node(btree, path, level);
752 nchildren = nilfs_btree_node_get_nchildren(btree, node);
755 n = (nchildren + 1) / 2;
756 if (n > nchildren - path[level].bp_index) {
761 nilfs_btree_node_move_right(btree, node, right, n);
763 if (!buffer_dirty(path[level].bp_bh))
764 nilfs_btnode_mark_dirty(path[level].bp_bh);
765 if (!buffer_dirty(path[level].bp_sib_bh))
766 nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
768 unlock_buffer(path[level].bp_bh);
769 unlock_buffer(path[level].bp_sib_bh);
771 newkey = nilfs_btree_node_get_key(btree, right, 0);
772 newptr = path[level].bp_newreq.bpr_ptr;
775 path[level].bp_index -=
776 nilfs_btree_node_get_nchildren(btree, node);
777 nilfs_btree_node_insert(btree, right, *keyp, *ptrp,
778 path[level].bp_index);
780 *keyp = nilfs_btree_node_get_key(btree, right, 0);
781 *ptrp = path[level].bp_newreq.bpr_ptr;
783 brelse(path[level].bp_bh);
784 path[level].bp_bh = path[level].bp_sib_bh;
785 path[level].bp_sib_bh = NULL;
787 nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
789 *keyp = nilfs_btree_node_get_key(btree, right, 0);
790 *ptrp = path[level].bp_newreq.bpr_ptr;
792 brelse(path[level].bp_sib_bh);
793 path[level].bp_sib_bh = NULL;
796 path[level + 1].bp_index++;
799 static void nilfs_btree_grow(struct nilfs_btree *btree,
800 struct nilfs_btree_path *path,
801 int level, __u64 *keyp, __u64 *ptrp)
803 struct nilfs_btree_node *root, *child;
806 lock_buffer(path[level].bp_sib_bh);
808 root = nilfs_btree_get_root(btree);
809 child = nilfs_btree_get_sib_node(btree, path, level);
811 n = nilfs_btree_node_get_nchildren(btree, root);
813 nilfs_btree_node_move_right(btree, root, child, n);
814 nilfs_btree_node_set_level(btree, root, level + 1);
816 if (!buffer_dirty(path[level].bp_sib_bh))
817 nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
819 unlock_buffer(path[level].bp_sib_bh);
821 path[level].bp_bh = path[level].bp_sib_bh;
822 path[level].bp_sib_bh = NULL;
824 nilfs_btree_do_insert(btree, path, level, keyp, ptrp);
826 *keyp = nilfs_btree_node_get_key(btree, child, 0);
827 *ptrp = path[level].bp_newreq.bpr_ptr;
830 static __u64 nilfs_btree_find_near(const struct nilfs_btree *btree,
831 const struct nilfs_btree_path *path)
833 struct nilfs_btree_node *node;
837 return NILFS_BMAP_INVALID_PTR;
840 level = NILFS_BTREE_LEVEL_NODE_MIN;
841 if (path[level].bp_index > 0) {
842 node = nilfs_btree_get_node(btree, path, level);
843 return nilfs_btree_node_get_ptr(btree, node,
844 path[level].bp_index - 1);
848 level = NILFS_BTREE_LEVEL_NODE_MIN + 1;
849 if (level <= nilfs_btree_height(btree) - 1) {
850 node = nilfs_btree_get_node(btree, path, level);
851 return nilfs_btree_node_get_ptr(btree, node,
852 path[level].bp_index);
855 return NILFS_BMAP_INVALID_PTR;
858 static __u64 nilfs_btree_find_target_v(const struct nilfs_btree *btree,
859 const struct nilfs_btree_path *path,
864 ptr = nilfs_bmap_find_target_seq(&btree->bt_bmap, key);
865 if (ptr != NILFS_BMAP_INVALID_PTR)
866 /* sequential access */
869 ptr = nilfs_btree_find_near(btree, path);
870 if (ptr != NILFS_BMAP_INVALID_PTR)
875 return nilfs_bmap_find_target_in_group(&btree->bt_bmap);
878 static void nilfs_btree_set_target_v(struct nilfs_btree *btree, __u64 key,
881 btree->bt_bmap.b_last_allocated_key = key;
882 btree->bt_bmap.b_last_allocated_ptr = ptr;
885 static int nilfs_btree_prepare_insert(struct nilfs_btree *btree,
886 struct nilfs_btree_path *path,
887 int *levelp, __u64 key, __u64 ptr,
888 struct nilfs_bmap_stats *stats)
890 struct buffer_head *bh;
891 struct nilfs_btree_node *node, *parent, *sib;
893 int pindex, level, ret;
895 stats->bs_nblocks = 0;
896 level = NILFS_BTREE_LEVEL_DATA;
898 /* allocate a new ptr for data block */
899 if (btree->bt_ops->btop_find_target != NULL)
900 path[level].bp_newreq.bpr_ptr =
901 btree->bt_ops->btop_find_target(btree, path, key);
903 ret = btree->bt_bmap.b_pops->bpop_prepare_alloc_ptr(
904 &btree->bt_bmap, &path[level].bp_newreq);
908 for (level = NILFS_BTREE_LEVEL_NODE_MIN;
909 level < nilfs_btree_height(btree) - 1;
911 node = nilfs_btree_get_nonroot_node(btree, path, level);
912 if (nilfs_btree_node_get_nchildren(btree, node) <
913 nilfs_btree_node_nchildren_max(btree, node)) {
914 path[level].bp_op = nilfs_btree_do_insert;
919 parent = nilfs_btree_get_node(btree, path, level + 1);
920 pindex = path[level + 1].bp_index;
924 sibptr = nilfs_btree_node_get_ptr(btree, parent,
926 ret = nilfs_bmap_get_block(&btree->bt_bmap, sibptr,
929 goto err_out_child_node;
930 sib = (struct nilfs_btree_node *)bh->b_data;
931 if (nilfs_btree_node_get_nchildren(btree, sib) <
932 nilfs_btree_node_nchildren_max(btree, sib)) {
933 path[level].bp_sib_bh = bh;
934 path[level].bp_op = nilfs_btree_carry_left;
943 nilfs_btree_node_get_nchildren(btree, parent) - 1) {
944 sibptr = nilfs_btree_node_get_ptr(btree, parent,
946 ret = nilfs_bmap_get_block(&btree->bt_bmap, sibptr,
949 goto err_out_child_node;
950 sib = (struct nilfs_btree_node *)bh->b_data;
951 if (nilfs_btree_node_get_nchildren(btree, sib) <
952 nilfs_btree_node_nchildren_max(btree, sib)) {
953 path[level].bp_sib_bh = bh;
954 path[level].bp_op = nilfs_btree_carry_right;
962 path[level].bp_newreq.bpr_ptr =
963 path[level - 1].bp_newreq.bpr_ptr + 1;
964 ret = btree->bt_bmap.b_pops->bpop_prepare_alloc_ptr(
965 &btree->bt_bmap, &path[level].bp_newreq);
967 goto err_out_child_node;
968 ret = nilfs_bmap_get_new_block(&btree->bt_bmap,
969 path[level].bp_newreq.bpr_ptr,
972 goto err_out_curr_node;
977 nilfs_btree_node_init(btree,
978 (struct nilfs_btree_node *)bh->b_data,
979 0, level, 0, NULL, NULL);
981 path[level].bp_sib_bh = bh;
982 path[level].bp_op = nilfs_btree_split;
986 node = nilfs_btree_get_root(btree);
987 if (nilfs_btree_node_get_nchildren(btree, node) <
988 nilfs_btree_node_nchildren_max(btree, node)) {
989 path[level].bp_op = nilfs_btree_do_insert;
995 path[level].bp_newreq.bpr_ptr = path[level - 1].bp_newreq.bpr_ptr + 1;
996 ret = btree->bt_bmap.b_pops->bpop_prepare_alloc_ptr(
997 &btree->bt_bmap, &path[level].bp_newreq);
999 goto err_out_child_node;
1000 ret = nilfs_bmap_get_new_block(&btree->bt_bmap,
1001 path[level].bp_newreq.bpr_ptr, &bh);
1003 goto err_out_curr_node;
1006 nilfs_btree_node_init(btree, (struct nilfs_btree_node *)bh->b_data,
1007 0, level, 0, NULL, NULL);
1009 path[level].bp_sib_bh = bh;
1010 path[level].bp_op = nilfs_btree_grow;
1013 path[level].bp_op = nilfs_btree_do_insert;
1015 /* a newly-created node block and a data block are added */
1016 stats->bs_nblocks += 2;
1025 btree->bt_bmap.b_pops->bpop_abort_alloc_ptr(&btree->bt_bmap,
1026 &path[level].bp_newreq);
1028 for (level--; level > NILFS_BTREE_LEVEL_DATA; level--) {
1029 nilfs_bmap_delete_block(&btree->bt_bmap, path[level].bp_sib_bh);
1030 btree->bt_bmap.b_pops->bpop_abort_alloc_ptr(
1031 &btree->bt_bmap, &path[level].bp_newreq);
1035 btree->bt_bmap.b_pops->bpop_abort_alloc_ptr(&btree->bt_bmap,
1036 &path[level].bp_newreq);
1039 stats->bs_nblocks = 0;
1043 static void nilfs_btree_commit_insert(struct nilfs_btree *btree,
1044 struct nilfs_btree_path *path,
1045 int maxlevel, __u64 key, __u64 ptr)
1049 set_buffer_nilfs_volatile((struct buffer_head *)((unsigned long)ptr));
1050 ptr = path[NILFS_BTREE_LEVEL_DATA].bp_newreq.bpr_ptr;
1051 if (btree->bt_ops->btop_set_target != NULL)
1052 btree->bt_ops->btop_set_target(btree, key, ptr);
1054 for (level = NILFS_BTREE_LEVEL_NODE_MIN; level <= maxlevel; level++) {
1055 if (btree->bt_bmap.b_pops->bpop_commit_alloc_ptr != NULL) {
1056 btree->bt_bmap.b_pops->bpop_commit_alloc_ptr(
1057 &btree->bt_bmap, &path[level - 1].bp_newreq);
1059 path[level].bp_op(btree, path, level, &key, &ptr);
1062 if (!nilfs_bmap_dirty(&btree->bt_bmap))
1063 nilfs_bmap_set_dirty(&btree->bt_bmap);
1066 static int nilfs_btree_insert(struct nilfs_bmap *bmap, __u64 key, __u64 ptr)
1068 struct nilfs_btree *btree;
1069 struct nilfs_btree_path *path;
1070 struct nilfs_bmap_stats stats;
1073 btree = (struct nilfs_btree *)bmap;
1074 path = nilfs_btree_alloc_path(btree);
1077 nilfs_btree_init_path(btree, path);
1079 ret = nilfs_btree_do_lookup(btree, path, key, NULL,
1080 NILFS_BTREE_LEVEL_NODE_MIN);
1081 if (ret != -ENOENT) {
1087 ret = nilfs_btree_prepare_insert(btree, path, &level, key, ptr, &stats);
1090 nilfs_btree_commit_insert(btree, path, level, key, ptr);
1091 nilfs_bmap_add_blocks(bmap, stats.bs_nblocks);
1094 nilfs_btree_clear_path(btree, path);
1095 nilfs_btree_free_path(btree, path);
1099 static void nilfs_btree_do_delete(struct nilfs_btree *btree,
1100 struct nilfs_btree_path *path,
1101 int level, __u64 *keyp, __u64 *ptrp)
1103 struct nilfs_btree_node *node;
1105 if (level < nilfs_btree_height(btree) - 1) {
1106 lock_buffer(path[level].bp_bh);
1107 node = nilfs_btree_get_nonroot_node(btree, path, level);
1108 nilfs_btree_node_delete(btree, node, keyp, ptrp,
1109 path[level].bp_index);
1110 if (!buffer_dirty(path[level].bp_bh))
1111 nilfs_btnode_mark_dirty(path[level].bp_bh);
1112 unlock_buffer(path[level].bp_bh);
1113 if (path[level].bp_index == 0)
1114 nilfs_btree_promote_key(btree, path, level + 1,
1115 nilfs_btree_node_get_key(btree, node, 0));
1117 node = nilfs_btree_get_root(btree);
1118 nilfs_btree_node_delete(btree, node, keyp, ptrp,
1119 path[level].bp_index);
1123 static void nilfs_btree_borrow_left(struct nilfs_btree *btree,
1124 struct nilfs_btree_path *path,
1125 int level, __u64 *keyp, __u64 *ptrp)
1127 struct nilfs_btree_node *node, *left;
1128 int nchildren, lnchildren, n;
1130 nilfs_btree_do_delete(btree, path, level, keyp, ptrp);
1132 lock_buffer(path[level].bp_bh);
1133 lock_buffer(path[level].bp_sib_bh);
1135 node = nilfs_btree_get_nonroot_node(btree, path, level);
1136 left = nilfs_btree_get_sib_node(btree, path, level);
1137 nchildren = nilfs_btree_node_get_nchildren(btree, node);
1138 lnchildren = nilfs_btree_node_get_nchildren(btree, left);
1140 n = (nchildren + lnchildren) / 2 - nchildren;
1142 nilfs_btree_node_move_right(btree, left, node, n);
1144 if (!buffer_dirty(path[level].bp_bh))
1145 nilfs_btnode_mark_dirty(path[level].bp_bh);
1146 if (!buffer_dirty(path[level].bp_sib_bh))
1147 nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
1149 unlock_buffer(path[level].bp_bh);
1150 unlock_buffer(path[level].bp_sib_bh);
1152 nilfs_btree_promote_key(btree, path, level + 1,
1153 nilfs_btree_node_get_key(btree, node, 0));
1155 brelse(path[level].bp_sib_bh);
1156 path[level].bp_sib_bh = NULL;
1157 path[level].bp_index += n;
1160 static void nilfs_btree_borrow_right(struct nilfs_btree *btree,
1161 struct nilfs_btree_path *path,
1162 int level, __u64 *keyp, __u64 *ptrp)
1164 struct nilfs_btree_node *node, *right;
1165 int nchildren, rnchildren, n;
1167 nilfs_btree_do_delete(btree, path, level, keyp, ptrp);
1169 lock_buffer(path[level].bp_bh);
1170 lock_buffer(path[level].bp_sib_bh);
1172 node = nilfs_btree_get_nonroot_node(btree, path, level);
1173 right = nilfs_btree_get_sib_node(btree, path, level);
1174 nchildren = nilfs_btree_node_get_nchildren(btree, node);
1175 rnchildren = nilfs_btree_node_get_nchildren(btree, right);
1177 n = (nchildren + rnchildren) / 2 - nchildren;
1179 nilfs_btree_node_move_left(btree, node, right, n);
1181 if (!buffer_dirty(path[level].bp_bh))
1182 nilfs_btnode_mark_dirty(path[level].bp_bh);
1183 if (!buffer_dirty(path[level].bp_sib_bh))
1184 nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
1186 unlock_buffer(path[level].bp_bh);
1187 unlock_buffer(path[level].bp_sib_bh);
1189 path[level + 1].bp_index++;
1190 nilfs_btree_promote_key(btree, path, level + 1,
1191 nilfs_btree_node_get_key(btree, right, 0));
1192 path[level + 1].bp_index--;
1194 brelse(path[level].bp_sib_bh);
1195 path[level].bp_sib_bh = NULL;
1198 static void nilfs_btree_concat_left(struct nilfs_btree *btree,
1199 struct nilfs_btree_path *path,
1200 int level, __u64 *keyp, __u64 *ptrp)
1202 struct nilfs_btree_node *node, *left;
1205 nilfs_btree_do_delete(btree, path, level, keyp, ptrp);
1207 lock_buffer(path[level].bp_bh);
1208 lock_buffer(path[level].bp_sib_bh);
1210 node = nilfs_btree_get_nonroot_node(btree, path, level);
1211 left = nilfs_btree_get_sib_node(btree, path, level);
1213 n = nilfs_btree_node_get_nchildren(btree, node);
1215 nilfs_btree_node_move_left(btree, left, node, n);
1217 if (!buffer_dirty(path[level].bp_sib_bh))
1218 nilfs_btnode_mark_dirty(path[level].bp_sib_bh);
1220 unlock_buffer(path[level].bp_bh);
1221 unlock_buffer(path[level].bp_sib_bh);
1223 nilfs_bmap_delete_block(&btree->bt_bmap, path[level].bp_bh);
1224 path[level].bp_bh = path[level].bp_sib_bh;
1225 path[level].bp_sib_bh = NULL;
1226 path[level].bp_index += nilfs_btree_node_get_nchildren(btree, left);
1229 static void nilfs_btree_concat_right(struct nilfs_btree *btree,
1230 struct nilfs_btree_path *path,
1231 int level, __u64 *keyp, __u64 *ptrp)
1233 struct nilfs_btree_node *node, *right;
1236 nilfs_btree_do_delete(btree, path, level, keyp, ptrp);
1238 lock_buffer(path[level].bp_bh);
1239 lock_buffer(path[level].bp_sib_bh);
1241 node = nilfs_btree_get_nonroot_node(btree, path, level);
1242 right = nilfs_btree_get_sib_node(btree, path, level);
1244 n = nilfs_btree_node_get_nchildren(btree, right);
1246 nilfs_btree_node_move_left(btree, node, right, n);
1248 if (!buffer_dirty(path[level].bp_bh))
1249 nilfs_btnode_mark_dirty(path[level].bp_bh);
1251 unlock_buffer(path[level].bp_bh);
1252 unlock_buffer(path[level].bp_sib_bh);
1254 nilfs_bmap_delete_block(&btree->bt_bmap, path[level].bp_sib_bh);
1255 path[level].bp_sib_bh = NULL;
1256 path[level + 1].bp_index++;
1259 static void nilfs_btree_shrink(struct nilfs_btree *btree,
1260 struct nilfs_btree_path *path,
1261 int level, __u64 *keyp, __u64 *ptrp)
1263 struct nilfs_btree_node *root, *child;
1266 nilfs_btree_do_delete(btree, path, level, keyp, ptrp);
1268 lock_buffer(path[level].bp_bh);
1269 root = nilfs_btree_get_root(btree);
1270 child = nilfs_btree_get_nonroot_node(btree, path, level);
1272 nilfs_btree_node_delete(btree, root, NULL, NULL, 0);
1273 nilfs_btree_node_set_level(btree, root, level);
1274 n = nilfs_btree_node_get_nchildren(btree, child);
1275 nilfs_btree_node_move_left(btree, root, child, n);
1276 unlock_buffer(path[level].bp_bh);
1278 nilfs_bmap_delete_block(&btree->bt_bmap, path[level].bp_bh);
1279 path[level].bp_bh = NULL;
1283 static int nilfs_btree_prepare_delete(struct nilfs_btree *btree,
1284 struct nilfs_btree_path *path,
1286 struct nilfs_bmap_stats *stats)
1288 struct buffer_head *bh;
1289 struct nilfs_btree_node *node, *parent, *sib;
1291 int pindex, level, ret;
1294 stats->bs_nblocks = 0;
1295 for (level = NILFS_BTREE_LEVEL_NODE_MIN;
1296 level < nilfs_btree_height(btree) - 1;
1298 node = nilfs_btree_get_nonroot_node(btree, path, level);
1299 path[level].bp_oldreq.bpr_ptr =
1300 nilfs_btree_node_get_ptr(btree, node,
1301 path[level].bp_index);
1302 if (btree->bt_bmap.b_pops->bpop_prepare_end_ptr != NULL) {
1303 ret = btree->bt_bmap.b_pops->bpop_prepare_end_ptr(
1304 &btree->bt_bmap, &path[level].bp_oldreq);
1306 goto err_out_child_node;
1309 if (nilfs_btree_node_get_nchildren(btree, node) >
1310 nilfs_btree_node_nchildren_min(btree, node)) {
1311 path[level].bp_op = nilfs_btree_do_delete;
1312 stats->bs_nblocks++;
1316 parent = nilfs_btree_get_node(btree, path, level + 1);
1317 pindex = path[level + 1].bp_index;
1321 sibptr = nilfs_btree_node_get_ptr(btree, parent,
1323 ret = nilfs_bmap_get_block(&btree->bt_bmap, sibptr,
1326 goto err_out_curr_node;
1327 sib = (struct nilfs_btree_node *)bh->b_data;
1328 if (nilfs_btree_node_get_nchildren(btree, sib) >
1329 nilfs_btree_node_nchildren_min(btree, sib)) {
1330 path[level].bp_sib_bh = bh;
1331 path[level].bp_op = nilfs_btree_borrow_left;
1332 stats->bs_nblocks++;
1335 path[level].bp_sib_bh = bh;
1336 path[level].bp_op = nilfs_btree_concat_left;
1337 stats->bs_nblocks++;
1341 nilfs_btree_node_get_nchildren(btree, parent) - 1) {
1343 sibptr = nilfs_btree_node_get_ptr(btree, parent,
1345 ret = nilfs_bmap_get_block(&btree->bt_bmap, sibptr,
1348 goto err_out_curr_node;
1349 sib = (struct nilfs_btree_node *)bh->b_data;
1350 if (nilfs_btree_node_get_nchildren(btree, sib) >
1351 nilfs_btree_node_nchildren_min(btree, sib)) {
1352 path[level].bp_sib_bh = bh;
1353 path[level].bp_op = nilfs_btree_borrow_right;
1354 stats->bs_nblocks++;
1357 path[level].bp_sib_bh = bh;
1358 path[level].bp_op = nilfs_btree_concat_right;
1359 stats->bs_nblocks++;
1364 /* the only child of the root node */
1365 WARN_ON(level != nilfs_btree_height(btree) - 2);
1366 if (nilfs_btree_node_get_nchildren(btree, node) - 1 <=
1367 NILFS_BTREE_ROOT_NCHILDREN_MAX) {
1368 path[level].bp_op = nilfs_btree_shrink;
1369 stats->bs_nblocks += 2;
1371 path[level].bp_op = nilfs_btree_do_delete;
1372 stats->bs_nblocks++;
1380 node = nilfs_btree_get_root(btree);
1381 path[level].bp_oldreq.bpr_ptr =
1382 nilfs_btree_node_get_ptr(btree, node, path[level].bp_index);
1383 if (btree->bt_bmap.b_pops->bpop_prepare_end_ptr != NULL) {
1384 ret = btree->bt_bmap.b_pops->bpop_prepare_end_ptr(
1385 &btree->bt_bmap, &path[level].bp_oldreq);
1387 goto err_out_child_node;
1389 /* child of the root node is deleted */
1390 path[level].bp_op = nilfs_btree_do_delete;
1391 stats->bs_nblocks++;
1400 if (btree->bt_bmap.b_pops->bpop_abort_end_ptr != NULL)
1401 btree->bt_bmap.b_pops->bpop_abort_end_ptr(
1402 &btree->bt_bmap, &path[level].bp_oldreq);
1404 for (level--; level >= NILFS_BTREE_LEVEL_NODE_MIN; level--) {
1405 brelse(path[level].bp_sib_bh);
1406 if (btree->bt_bmap.b_pops->bpop_abort_end_ptr != NULL)
1407 btree->bt_bmap.b_pops->bpop_abort_end_ptr(
1408 &btree->bt_bmap, &path[level].bp_oldreq);
1411 stats->bs_nblocks = 0;
1415 static void nilfs_btree_commit_delete(struct nilfs_btree *btree,
1416 struct nilfs_btree_path *path,
1421 for (level = NILFS_BTREE_LEVEL_NODE_MIN; level <= maxlevel; level++) {
1422 if (btree->bt_bmap.b_pops->bpop_commit_end_ptr != NULL)
1423 btree->bt_bmap.b_pops->bpop_commit_end_ptr(
1424 &btree->bt_bmap, &path[level].bp_oldreq);
1425 path[level].bp_op(btree, path, level, NULL, NULL);
1428 if (!nilfs_bmap_dirty(&btree->bt_bmap))
1429 nilfs_bmap_set_dirty(&btree->bt_bmap);
1432 static int nilfs_btree_delete(struct nilfs_bmap *bmap, __u64 key)
1435 struct nilfs_btree *btree;
1436 struct nilfs_btree_path *path;
1437 struct nilfs_bmap_stats stats;
1440 btree = (struct nilfs_btree *)bmap;
1441 path = nilfs_btree_alloc_path(btree);
1444 nilfs_btree_init_path(btree, path);
1445 ret = nilfs_btree_do_lookup(btree, path, key, NULL,
1446 NILFS_BTREE_LEVEL_NODE_MIN);
1450 ret = nilfs_btree_prepare_delete(btree, path, &level, &stats);
1453 nilfs_btree_commit_delete(btree, path, level);
1454 nilfs_bmap_sub_blocks(bmap, stats.bs_nblocks);
1457 nilfs_btree_clear_path(btree, path);
1458 nilfs_btree_free_path(btree, path);
1462 static int nilfs_btree_last_key(const struct nilfs_bmap *bmap, __u64 *keyp)
1464 struct nilfs_btree *btree;
1465 struct nilfs_btree_path *path;
1468 btree = (struct nilfs_btree *)bmap;
1469 path = nilfs_btree_alloc_path(btree);
1472 nilfs_btree_init_path(btree, path);
1474 ret = nilfs_btree_do_lookup_last(btree, path, keyp, NULL);
1476 nilfs_btree_clear_path(btree, path);
1477 nilfs_btree_free_path(btree, path);
1482 static int nilfs_btree_check_delete(struct nilfs_bmap *bmap, __u64 key)
1484 struct buffer_head *bh;
1485 struct nilfs_btree *btree;
1486 struct nilfs_btree_node *root, *node;
1487 __u64 maxkey, nextmaxkey;
1491 btree = (struct nilfs_btree *)bmap;
1492 root = nilfs_btree_get_root(btree);
1493 switch (nilfs_btree_height(btree)) {
1499 nchildren = nilfs_btree_node_get_nchildren(btree, root);
1502 ptr = nilfs_btree_node_get_ptr(btree, root, nchildren - 1);
1503 ret = nilfs_bmap_get_block(bmap, ptr, &bh);
1506 node = (struct nilfs_btree_node *)bh->b_data;
1512 nchildren = nilfs_btree_node_get_nchildren(btree, node);
1513 maxkey = nilfs_btree_node_get_key(btree, node, nchildren - 1);
1514 nextmaxkey = (nchildren > 1) ?
1515 nilfs_btree_node_get_key(btree, node, nchildren - 2) : 0;
1519 return (maxkey == key) && (nextmaxkey < bmap->b_low);
1522 static int nilfs_btree_gather_data(struct nilfs_bmap *bmap,
1523 __u64 *keys, __u64 *ptrs, int nitems)
1525 struct buffer_head *bh;
1526 struct nilfs_btree *btree;
1527 struct nilfs_btree_node *node, *root;
1531 int nchildren, i, ret;
1533 btree = (struct nilfs_btree *)bmap;
1534 root = nilfs_btree_get_root(btree);
1535 switch (nilfs_btree_height(btree)) {
1541 nchildren = nilfs_btree_node_get_nchildren(btree, root);
1542 WARN_ON(nchildren > 1);
1543 ptr = nilfs_btree_node_get_ptr(btree, root, nchildren - 1);
1544 ret = nilfs_bmap_get_block(bmap, ptr, &bh);
1547 node = (struct nilfs_btree_node *)bh->b_data;
1554 nchildren = nilfs_btree_node_get_nchildren(btree, node);
1555 if (nchildren < nitems)
1557 dkeys = nilfs_btree_node_dkeys(btree, node);
1558 dptrs = nilfs_btree_node_dptrs(btree, node);
1559 for (i = 0; i < nitems; i++) {
1560 keys[i] = nilfs_bmap_dkey_to_key(dkeys[i]);
1561 ptrs[i] = nilfs_bmap_dptr_to_ptr(dptrs[i]);
1571 nilfs_btree_prepare_convert_and_insert(struct nilfs_bmap *bmap, __u64 key,
1572 union nilfs_bmap_ptr_req *dreq,
1573 union nilfs_bmap_ptr_req *nreq,
1574 struct buffer_head **bhp,
1575 struct nilfs_bmap_stats *stats)
1577 struct buffer_head *bh;
1578 struct nilfs_btree *btree;
1581 btree = (struct nilfs_btree *)bmap;
1582 stats->bs_nblocks = 0;
1585 /* cannot find near ptr */
1586 if (btree->bt_ops->btop_find_target != NULL)
1588 = btree->bt_ops->btop_find_target(btree, NULL, key);
1589 ret = bmap->b_pops->bpop_prepare_alloc_ptr(bmap, dreq);
1594 stats->bs_nblocks++;
1596 nreq->bpr_ptr = dreq->bpr_ptr + 1;
1597 ret = bmap->b_pops->bpop_prepare_alloc_ptr(bmap, nreq);
1601 ret = nilfs_bmap_get_new_block(bmap, nreq->bpr_ptr, &bh);
1606 stats->bs_nblocks++;
1614 bmap->b_pops->bpop_abort_alloc_ptr(bmap, nreq);
1616 bmap->b_pops->bpop_abort_alloc_ptr(bmap, dreq);
1617 stats->bs_nblocks = 0;
1623 nilfs_btree_commit_convert_and_insert(struct nilfs_bmap *bmap,
1624 __u64 key, __u64 ptr,
1625 const __u64 *keys, const __u64 *ptrs,
1626 int n, __u64 low, __u64 high,
1627 union nilfs_bmap_ptr_req *dreq,
1628 union nilfs_bmap_ptr_req *nreq,
1629 struct buffer_head *bh)
1631 struct nilfs_btree *btree;
1632 struct nilfs_btree_node *node;
1635 /* free resources */
1636 if (bmap->b_ops->bop_clear != NULL)
1637 bmap->b_ops->bop_clear(bmap);
1639 /* ptr must be a pointer to a buffer head. */
1640 set_buffer_nilfs_volatile((struct buffer_head *)((unsigned long)ptr));
1642 /* convert and insert */
1643 btree = (struct nilfs_btree *)bmap;
1644 nilfs_btree_init(bmap, low, high);
1646 if (bmap->b_pops->bpop_commit_alloc_ptr != NULL) {
1647 bmap->b_pops->bpop_commit_alloc_ptr(bmap, dreq);
1648 bmap->b_pops->bpop_commit_alloc_ptr(bmap, nreq);
1651 /* create child node at level 1 */
1653 node = (struct nilfs_btree_node *)bh->b_data;
1654 nilfs_btree_node_init(btree, node, 0, 1, n, keys, ptrs);
1655 nilfs_btree_node_insert(btree, node,
1656 key, dreq->bpr_ptr, n);
1657 if (!buffer_dirty(bh))
1658 nilfs_btnode_mark_dirty(bh);
1659 if (!nilfs_bmap_dirty(bmap))
1660 nilfs_bmap_set_dirty(bmap);
1665 /* create root node at level 2 */
1666 node = nilfs_btree_get_root(btree);
1667 tmpptr = nreq->bpr_ptr;
1668 nilfs_btree_node_init(btree, node, NILFS_BTREE_NODE_ROOT,
1669 2, 1, &keys[0], &tmpptr);
1671 if (bmap->b_pops->bpop_commit_alloc_ptr != NULL)
1672 bmap->b_pops->bpop_commit_alloc_ptr(bmap, dreq);
1674 /* create root node at level 1 */
1675 node = nilfs_btree_get_root(btree);
1676 nilfs_btree_node_init(btree, node, NILFS_BTREE_NODE_ROOT,
1678 nilfs_btree_node_insert(btree, node,
1679 key, dreq->bpr_ptr, n);
1680 if (!nilfs_bmap_dirty(bmap))
1681 nilfs_bmap_set_dirty(bmap);
1684 if (btree->bt_ops->btop_set_target != NULL)
1685 btree->bt_ops->btop_set_target(btree, key, dreq->bpr_ptr);
1689 * nilfs_btree_convert_and_insert -
1699 int nilfs_btree_convert_and_insert(struct nilfs_bmap *bmap,
1700 __u64 key, __u64 ptr,
1701 const __u64 *keys, const __u64 *ptrs,
1702 int n, __u64 low, __u64 high)
1704 struct buffer_head *bh;
1705 union nilfs_bmap_ptr_req dreq, nreq, *di, *ni;
1706 struct nilfs_bmap_stats stats;
1709 if (n + 1 <= NILFS_BTREE_ROOT_NCHILDREN_MAX) {
1712 } else if ((n + 1) <= NILFS_BTREE_NODE_NCHILDREN_MAX(
1713 1 << bmap->b_inode->i_blkbits)) {
1722 ret = nilfs_btree_prepare_convert_and_insert(bmap, key, di, ni, &bh,
1726 nilfs_btree_commit_convert_and_insert(bmap, key, ptr, keys, ptrs, n,
1727 low, high, di, ni, bh);
1728 nilfs_bmap_add_blocks(bmap, stats.bs_nblocks);
1732 static int nilfs_btree_propagate_p(struct nilfs_btree *btree,
1733 struct nilfs_btree_path *path,
1735 struct buffer_head *bh)
1737 while ((++level < nilfs_btree_height(btree) - 1) &&
1738 !buffer_dirty(path[level].bp_bh))
1739 nilfs_btnode_mark_dirty(path[level].bp_bh);
1744 static int nilfs_btree_prepare_update_v(struct nilfs_btree *btree,
1745 struct nilfs_btree_path *path,
1748 struct nilfs_btree_node *parent;
1751 parent = nilfs_btree_get_node(btree, path, level + 1);
1752 path[level].bp_oldreq.bpr_ptr =
1753 nilfs_btree_node_get_ptr(btree, parent,
1754 path[level + 1].bp_index);
1755 path[level].bp_newreq.bpr_ptr = path[level].bp_oldreq.bpr_ptr + 1;
1756 ret = nilfs_bmap_prepare_update(&btree->bt_bmap,
1757 &path[level].bp_oldreq,
1758 &path[level].bp_newreq);
1762 if (buffer_nilfs_node(path[level].bp_bh)) {
1763 path[level].bp_ctxt.oldkey = path[level].bp_oldreq.bpr_ptr;
1764 path[level].bp_ctxt.newkey = path[level].bp_newreq.bpr_ptr;
1765 path[level].bp_ctxt.bh = path[level].bp_bh;
1766 ret = nilfs_btnode_prepare_change_key(
1767 &NILFS_BMAP_I(&btree->bt_bmap)->i_btnode_cache,
1768 &path[level].bp_ctxt);
1770 nilfs_bmap_abort_update(&btree->bt_bmap,
1771 &path[level].bp_oldreq,
1772 &path[level].bp_newreq);
1780 static void nilfs_btree_commit_update_v(struct nilfs_btree *btree,
1781 struct nilfs_btree_path *path,
1784 struct nilfs_btree_node *parent;
1786 nilfs_bmap_commit_update(&btree->bt_bmap,
1787 &path[level].bp_oldreq,
1788 &path[level].bp_newreq);
1790 if (buffer_nilfs_node(path[level].bp_bh)) {
1791 nilfs_btnode_commit_change_key(
1792 &NILFS_BMAP_I(&btree->bt_bmap)->i_btnode_cache,
1793 &path[level].bp_ctxt);
1794 path[level].bp_bh = path[level].bp_ctxt.bh;
1796 set_buffer_nilfs_volatile(path[level].bp_bh);
1798 parent = nilfs_btree_get_node(btree, path, level + 1);
1799 nilfs_btree_node_set_ptr(btree, parent, path[level + 1].bp_index,
1800 path[level].bp_newreq.bpr_ptr);
1803 static void nilfs_btree_abort_update_v(struct nilfs_btree *btree,
1804 struct nilfs_btree_path *path,
1807 nilfs_bmap_abort_update(&btree->bt_bmap,
1808 &path[level].bp_oldreq,
1809 &path[level].bp_newreq);
1810 if (buffer_nilfs_node(path[level].bp_bh))
1811 nilfs_btnode_abort_change_key(
1812 &NILFS_BMAP_I(&btree->bt_bmap)->i_btnode_cache,
1813 &path[level].bp_ctxt);
1816 static int nilfs_btree_prepare_propagate_v(struct nilfs_btree *btree,
1817 struct nilfs_btree_path *path,
1824 if (!buffer_nilfs_volatile(path[level].bp_bh)) {
1825 ret = nilfs_btree_prepare_update_v(btree, path, level);
1829 while ((++level < nilfs_btree_height(btree) - 1) &&
1830 !buffer_dirty(path[level].bp_bh)) {
1832 WARN_ON(buffer_nilfs_volatile(path[level].bp_bh));
1833 ret = nilfs_btree_prepare_update_v(btree, path, level);
1839 *maxlevelp = level - 1;
1844 while (--level > minlevel)
1845 nilfs_btree_abort_update_v(btree, path, level);
1846 if (!buffer_nilfs_volatile(path[level].bp_bh))
1847 nilfs_btree_abort_update_v(btree, path, level);
1851 static void nilfs_btree_commit_propagate_v(struct nilfs_btree *btree,
1852 struct nilfs_btree_path *path,
1855 struct buffer_head *bh)
1859 if (!buffer_nilfs_volatile(path[minlevel].bp_bh))
1860 nilfs_btree_commit_update_v(btree, path, minlevel);
1862 for (level = minlevel + 1; level <= maxlevel; level++)
1863 nilfs_btree_commit_update_v(btree, path, level);
1866 static int nilfs_btree_propagate_v(struct nilfs_btree *btree,
1867 struct nilfs_btree_path *path,
1869 struct buffer_head *bh)
1872 struct nilfs_btree_node *parent;
1876 path[level].bp_bh = bh;
1877 ret = nilfs_btree_prepare_propagate_v(btree, path, level, &maxlevel);
1881 if (buffer_nilfs_volatile(path[level].bp_bh)) {
1882 parent = nilfs_btree_get_node(btree, path, level + 1);
1883 ptr = nilfs_btree_node_get_ptr(btree, parent,
1884 path[level + 1].bp_index);
1885 ret = nilfs_bmap_mark_dirty(&btree->bt_bmap, ptr);
1890 nilfs_btree_commit_propagate_v(btree, path, level, maxlevel, bh);
1893 brelse(path[level].bp_bh);
1894 path[level].bp_bh = NULL;
1898 static int nilfs_btree_propagate(const struct nilfs_bmap *bmap,
1899 struct buffer_head *bh)
1901 struct nilfs_btree *btree;
1902 struct nilfs_btree_path *path;
1903 struct nilfs_btree_node *node;
1907 WARN_ON(!buffer_dirty(bh));
1909 btree = (struct nilfs_btree *)bmap;
1910 path = nilfs_btree_alloc_path(btree);
1913 nilfs_btree_init_path(btree, path);
1915 if (buffer_nilfs_node(bh)) {
1916 node = (struct nilfs_btree_node *)bh->b_data;
1917 key = nilfs_btree_node_get_key(btree, node, 0);
1918 level = nilfs_btree_node_get_level(btree, node);
1920 key = nilfs_bmap_data_get_key(bmap, bh);
1921 level = NILFS_BTREE_LEVEL_DATA;
1924 ret = nilfs_btree_do_lookup(btree, path, key, NULL, level + 1);
1926 if (unlikely(ret == -ENOENT))
1927 printk(KERN_CRIT "%s: key = %llu, level == %d\n",
1928 __func__, (unsigned long long)key, level);
1932 ret = btree->bt_ops->btop_propagate(btree, path, level, bh);
1935 nilfs_btree_clear_path(btree, path);
1936 nilfs_btree_free_path(btree, path);
1941 static int nilfs_btree_propagate_gc(const struct nilfs_bmap *bmap,
1942 struct buffer_head *bh)
1944 return nilfs_bmap_mark_dirty(bmap, bh->b_blocknr);
1947 static void nilfs_btree_add_dirty_buffer(struct nilfs_btree *btree,
1948 struct list_head *lists,
1949 struct buffer_head *bh)
1951 struct list_head *head;
1952 struct buffer_head *cbh;
1953 struct nilfs_btree_node *node, *cnode;
1958 node = (struct nilfs_btree_node *)bh->b_data;
1959 key = nilfs_btree_node_get_key(btree, node, 0);
1960 level = nilfs_btree_node_get_level(btree, node);
1961 list_for_each(head, &lists[level]) {
1962 cbh = list_entry(head, struct buffer_head, b_assoc_buffers);
1963 cnode = (struct nilfs_btree_node *)cbh->b_data;
1964 ckey = nilfs_btree_node_get_key(btree, cnode, 0);
1968 list_add_tail(&bh->b_assoc_buffers, head);
1971 static void nilfs_btree_lookup_dirty_buffers(struct nilfs_bmap *bmap,
1972 struct list_head *listp)
1974 struct nilfs_btree *btree = (struct nilfs_btree *)bmap;
1975 struct address_space *btcache = &NILFS_BMAP_I(bmap)->i_btnode_cache;
1976 struct list_head lists[NILFS_BTREE_LEVEL_MAX];
1977 struct pagevec pvec;
1978 struct buffer_head *bh, *head;
1982 for (level = NILFS_BTREE_LEVEL_NODE_MIN;
1983 level < NILFS_BTREE_LEVEL_MAX;
1985 INIT_LIST_HEAD(&lists[level]);
1987 pagevec_init(&pvec, 0);
1989 while (pagevec_lookup_tag(&pvec, btcache, &index, PAGECACHE_TAG_DIRTY,
1991 for (i = 0; i < pagevec_count(&pvec); i++) {
1992 bh = head = page_buffers(pvec.pages[i]);
1994 if (buffer_dirty(bh))
1995 nilfs_btree_add_dirty_buffer(btree,
1997 } while ((bh = bh->b_this_page) != head);
1999 pagevec_release(&pvec);
2003 for (level = NILFS_BTREE_LEVEL_NODE_MIN;
2004 level < NILFS_BTREE_LEVEL_MAX;
2006 list_splice(&lists[level], listp->prev);
2009 static int nilfs_btree_assign_p(struct nilfs_btree *btree,
2010 struct nilfs_btree_path *path,
2012 struct buffer_head **bh,
2014 union nilfs_binfo *binfo)
2016 struct nilfs_btree_node *parent;
2021 parent = nilfs_btree_get_node(btree, path, level + 1);
2022 ptr = nilfs_btree_node_get_ptr(btree, parent,
2023 path[level + 1].bp_index);
2024 if (buffer_nilfs_node(*bh)) {
2025 path[level].bp_ctxt.oldkey = ptr;
2026 path[level].bp_ctxt.newkey = blocknr;
2027 path[level].bp_ctxt.bh = *bh;
2028 ret = nilfs_btnode_prepare_change_key(
2029 &NILFS_BMAP_I(&btree->bt_bmap)->i_btnode_cache,
2030 &path[level].bp_ctxt);
2033 nilfs_btnode_commit_change_key(
2034 &NILFS_BMAP_I(&btree->bt_bmap)->i_btnode_cache,
2035 &path[level].bp_ctxt);
2036 *bh = path[level].bp_ctxt.bh;
2039 nilfs_btree_node_set_ptr(btree, parent,
2040 path[level + 1].bp_index, blocknr);
2042 key = nilfs_btree_node_get_key(btree, parent,
2043 path[level + 1].bp_index);
2044 /* on-disk format */
2045 binfo->bi_dat.bi_blkoff = nilfs_bmap_key_to_dkey(key);
2046 binfo->bi_dat.bi_level = level;
2051 static int nilfs_btree_assign_v(struct nilfs_btree *btree,
2052 struct nilfs_btree_path *path,
2054 struct buffer_head **bh,
2056 union nilfs_binfo *binfo)
2058 struct nilfs_btree_node *parent;
2061 union nilfs_bmap_ptr_req req;
2064 parent = nilfs_btree_get_node(btree, path, level + 1);
2065 ptr = nilfs_btree_node_get_ptr(btree, parent,
2066 path[level + 1].bp_index);
2068 ret = nilfs_bmap_start_v(&btree->bt_bmap, &req, blocknr);
2069 if (unlikely(ret < 0))
2072 key = nilfs_btree_node_get_key(btree, parent,
2073 path[level + 1].bp_index);
2074 /* on-disk format */
2075 binfo->bi_v.bi_vblocknr = nilfs_bmap_ptr_to_dptr(ptr);
2076 binfo->bi_v.bi_blkoff = nilfs_bmap_key_to_dkey(key);
2081 static int nilfs_btree_assign(struct nilfs_bmap *bmap,
2082 struct buffer_head **bh,
2084 union nilfs_binfo *binfo)
2086 struct nilfs_btree *btree;
2087 struct nilfs_btree_path *path;
2088 struct nilfs_btree_node *node;
2092 btree = (struct nilfs_btree *)bmap;
2093 path = nilfs_btree_alloc_path(btree);
2096 nilfs_btree_init_path(btree, path);
2098 if (buffer_nilfs_node(*bh)) {
2099 node = (struct nilfs_btree_node *)(*bh)->b_data;
2100 key = nilfs_btree_node_get_key(btree, node, 0);
2101 level = nilfs_btree_node_get_level(btree, node);
2103 key = nilfs_bmap_data_get_key(bmap, *bh);
2104 level = NILFS_BTREE_LEVEL_DATA;
2107 ret = nilfs_btree_do_lookup(btree, path, key, NULL, level + 1);
2109 WARN_ON(ret == -ENOENT);
2113 ret = btree->bt_ops->btop_assign(btree, path, level, bh,
2117 nilfs_btree_clear_path(btree, path);
2118 nilfs_btree_free_path(btree, path);
2123 static int nilfs_btree_assign_gc(struct nilfs_bmap *bmap,
2124 struct buffer_head **bh,
2126 union nilfs_binfo *binfo)
2128 struct nilfs_btree *btree;
2129 struct nilfs_btree_node *node;
2133 btree = (struct nilfs_btree *)bmap;
2134 ret = nilfs_bmap_move_v(bmap, (*bh)->b_blocknr, blocknr);
2138 if (buffer_nilfs_node(*bh)) {
2139 node = (struct nilfs_btree_node *)(*bh)->b_data;
2140 key = nilfs_btree_node_get_key(btree, node, 0);
2142 key = nilfs_bmap_data_get_key(bmap, *bh);
2144 /* on-disk format */
2145 binfo->bi_v.bi_vblocknr = cpu_to_le64((*bh)->b_blocknr);
2146 binfo->bi_v.bi_blkoff = nilfs_bmap_key_to_dkey(key);
2151 static int nilfs_btree_mark(struct nilfs_bmap *bmap, __u64 key, int level)
2153 struct buffer_head *bh;
2154 struct nilfs_btree *btree;
2155 struct nilfs_btree_path *path;
2159 btree = (struct nilfs_btree *)bmap;
2160 path = nilfs_btree_alloc_path(btree);
2163 nilfs_btree_init_path(btree, path);
2165 ret = nilfs_btree_do_lookup(btree, path, key, &ptr, level + 1);
2167 WARN_ON(ret == -ENOENT);
2170 ret = nilfs_bmap_get_block(&btree->bt_bmap, ptr, &bh);
2172 WARN_ON(ret == -ENOENT);
2176 if (!buffer_dirty(bh))
2177 nilfs_btnode_mark_dirty(bh);
2179 if (!nilfs_bmap_dirty(&btree->bt_bmap))
2180 nilfs_bmap_set_dirty(&btree->bt_bmap);
2183 nilfs_btree_clear_path(btree, path);
2184 nilfs_btree_free_path(btree, path);
2188 static const struct nilfs_bmap_operations nilfs_btree_ops = {
2189 .bop_lookup = nilfs_btree_lookup,
2190 .bop_insert = nilfs_btree_insert,
2191 .bop_delete = nilfs_btree_delete,
2194 .bop_propagate = nilfs_btree_propagate,
2196 .bop_lookup_dirty_buffers = nilfs_btree_lookup_dirty_buffers,
2198 .bop_assign = nilfs_btree_assign,
2199 .bop_mark = nilfs_btree_mark,
2201 .bop_last_key = nilfs_btree_last_key,
2202 .bop_check_insert = NULL,
2203 .bop_check_delete = nilfs_btree_check_delete,
2204 .bop_gather_data = nilfs_btree_gather_data,
2207 static const struct nilfs_bmap_operations nilfs_btree_ops_gc = {
2213 .bop_propagate = nilfs_btree_propagate_gc,
2215 .bop_lookup_dirty_buffers = nilfs_btree_lookup_dirty_buffers,
2217 .bop_assign = nilfs_btree_assign_gc,
2220 .bop_last_key = NULL,
2221 .bop_check_insert = NULL,
2222 .bop_check_delete = NULL,
2223 .bop_gather_data = NULL,
2226 static const struct nilfs_btree_operations nilfs_btree_ops_v = {
2227 .btop_find_target = nilfs_btree_find_target_v,
2228 .btop_set_target = nilfs_btree_set_target_v,
2229 .btop_propagate = nilfs_btree_propagate_v,
2230 .btop_assign = nilfs_btree_assign_v,
2233 static const struct nilfs_btree_operations nilfs_btree_ops_p = {
2234 .btop_find_target = NULL,
2235 .btop_set_target = NULL,
2236 .btop_propagate = nilfs_btree_propagate_p,
2237 .btop_assign = nilfs_btree_assign_p,
2240 int nilfs_btree_init(struct nilfs_bmap *bmap, __u64 low, __u64 high)
2242 struct nilfs_btree *btree;
2244 btree = (struct nilfs_btree *)bmap;
2245 bmap->b_ops = &nilfs_btree_ops;
2247 bmap->b_high = high;
2248 switch (bmap->b_inode->i_ino) {
2250 btree->bt_ops = &nilfs_btree_ops_p;
2253 btree->bt_ops = &nilfs_btree_ops_v;
2260 void nilfs_btree_init_gc(struct nilfs_bmap *bmap)
2262 bmap->b_low = NILFS_BMAP_LARGE_LOW;
2263 bmap->b_high = NILFS_BMAP_LARGE_HIGH;
2264 bmap->b_ops = &nilfs_btree_ops_gc;