diff options
author | Mike Marshall <hubcap@omnibond.com> | 2017-02-25 11:12:48 -0500 |
---|---|---|
committer | Mike Marshall <hubcap@omnibond.com> | 2017-02-25 11:12:48 -0500 |
commit | e98bdb3059cbf2b1cd4261e126b08429f64466c3 (patch) | |
tree | e378fc95b495cc6e0e558f247e99bcaa21a6d567 /fs/btrfs/relocation.c | |
parent | eb68d0324dc4d88ab0d6159bdcd98c247a3a8954 (diff) | |
parent | c470abd4fde40ea6a0846a2beab642a578c0b8cd (diff) | |
download | linux-e98bdb3059cbf2b1cd4261e126b08429f64466c3.tar.bz2 |
Merge tag 'v4.10' of git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux into for-next
Linux 4.10
Diffstat (limited to 'fs/btrfs/relocation.c')
-rw-r--r-- | fs/btrfs/relocation.c | 453 |
1 files changed, 213 insertions, 240 deletions
diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index c4af0cdb783d..379711048fb0 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -1288,9 +1288,10 @@ fail: */ static int __must_check __add_reloc_root(struct btrfs_root *root) { + struct btrfs_fs_info *fs_info = root->fs_info; struct rb_node *rb_node; struct mapping_node *node; - struct reloc_control *rc = root->fs_info->reloc_ctl; + struct reloc_control *rc = fs_info->reloc_ctl; node = kmalloc(sizeof(*node), GFP_NOFS); if (!node) @@ -1304,7 +1305,7 @@ static int __must_check __add_reloc_root(struct btrfs_root *root) node->bytenr, &node->rb_node); spin_unlock(&rc->reloc_root_tree.lock); if (rb_node) { - btrfs_panic(root->fs_info, -EEXIST, + btrfs_panic(fs_info, -EEXIST, "Duplicate root found for start=%llu while inserting into relocation tree", node->bytenr); kfree(node); @@ -1321,9 +1322,10 @@ static int __must_check __add_reloc_root(struct btrfs_root *root) */ static void __del_reloc_root(struct btrfs_root *root) { + struct btrfs_fs_info *fs_info = root->fs_info; struct rb_node *rb_node; struct mapping_node *node = NULL; - struct reloc_control *rc = root->fs_info->reloc_ctl; + struct reloc_control *rc = fs_info->reloc_ctl; spin_lock(&rc->reloc_root_tree.lock); rb_node = tree_search(&rc->reloc_root_tree.rb_root, @@ -1338,9 +1340,9 @@ static void __del_reloc_root(struct btrfs_root *root) return; BUG_ON((struct btrfs_root *)node->data != root); - spin_lock(&root->fs_info->trans_lock); + spin_lock(&fs_info->trans_lock); list_del_init(&root->root_list); - spin_unlock(&root->fs_info->trans_lock); + spin_unlock(&fs_info->trans_lock); kfree(node); } @@ -1350,9 +1352,10 @@ static void __del_reloc_root(struct btrfs_root *root) */ static int __update_reloc_root(struct btrfs_root *root, u64 new_bytenr) { + struct btrfs_fs_info *fs_info = root->fs_info; struct rb_node *rb_node; struct mapping_node *node = NULL; - struct reloc_control *rc = root->fs_info->reloc_ctl; + struct reloc_control *rc = fs_info->reloc_ctl; spin_lock(&rc->reloc_root_tree.lock); rb_node = tree_search(&rc->reloc_root_tree.rb_root, @@ -1380,11 +1383,11 @@ static int __update_reloc_root(struct btrfs_root *root, u64 new_bytenr) static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, struct btrfs_root *root, u64 objectid) { + struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_root *reloc_root; struct extent_buffer *eb; struct btrfs_root_item *root_item; struct btrfs_key root_key; - u64 last_snap = 0; int ret; root_item = kmalloc(sizeof(*root_item), GFP_NOFS); @@ -1395,14 +1398,22 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, root_key.offset = objectid; if (root->root_key.objectid == objectid) { + u64 commit_root_gen; + /* called by btrfs_init_reloc_root */ ret = btrfs_copy_root(trans, root, root->commit_root, &eb, BTRFS_TREE_RELOC_OBJECTID); BUG_ON(ret); - - last_snap = btrfs_root_last_snapshot(&root->root_item); - btrfs_set_root_last_snapshot(&root->root_item, - trans->transid - 1); + /* + * Set the last_snapshot field to the generation of the commit + * root - like this ctree.c:btrfs_block_can_be_shared() behaves + * correctly (returns true) when the relocation root is created + * either inside the critical section of a transaction commit + * (through transaction.c:qgroup_account_snapshot()) and when + * it's created before the transaction commit is started. + */ + commit_root_gen = btrfs_header_generation(root->commit_root); + btrfs_set_root_last_snapshot(&root->root_item, commit_root_gen); } else { /* * called by btrfs_reloc_post_snapshot_hook. @@ -1426,23 +1437,17 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, memset(&root_item->drop_progress, 0, sizeof(struct btrfs_disk_key)); root_item->drop_level = 0; - /* - * abuse rtransid, it is safe because it is impossible to - * receive data into a relocation tree. - */ - btrfs_set_root_rtransid(root_item, last_snap); - btrfs_set_root_otransid(root_item, trans->transid); } btrfs_tree_unlock(eb); free_extent_buffer(eb); - ret = btrfs_insert_root(trans, root->fs_info->tree_root, + ret = btrfs_insert_root(trans, fs_info->tree_root, &root_key, root_item); BUG_ON(ret); kfree(root_item); - reloc_root = btrfs_read_fs_root(root->fs_info->tree_root, &root_key); + reloc_root = btrfs_read_fs_root(fs_info->tree_root, &root_key); BUG_ON(IS_ERR(reloc_root)); reloc_root->last_trans = trans->transid; return reloc_root; @@ -1455,8 +1460,9 @@ static struct btrfs_root *create_reloc_root(struct btrfs_trans_handle *trans, int btrfs_init_reloc_root(struct btrfs_trans_handle *trans, struct btrfs_root *root) { + struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_root *reloc_root; - struct reloc_control *rc = root->fs_info->reloc_ctl; + struct reloc_control *rc = fs_info->reloc_ctl; struct btrfs_block_rsv *rsv; int clear_rsv = 0; int ret; @@ -1492,6 +1498,7 @@ int btrfs_init_reloc_root(struct btrfs_trans_handle *trans, int btrfs_update_reloc_root(struct btrfs_trans_handle *trans, struct btrfs_root *root) { + struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_root *reloc_root; struct btrfs_root_item *root_item; int ret; @@ -1502,7 +1509,7 @@ int btrfs_update_reloc_root(struct btrfs_trans_handle *trans, reloc_root = root->reloc_root; root_item = &reloc_root->root_item; - if (root->fs_info->reloc_ctl->merge_reloc_tree && + if (fs_info->reloc_ctl->merge_reloc_tree && btrfs_root_refs(root_item) == 0) { root->reloc_root = NULL; __del_reloc_root(reloc_root); @@ -1514,7 +1521,7 @@ int btrfs_update_reloc_root(struct btrfs_trans_handle *trans, reloc_root->commit_root = btrfs_root_node(reloc_root); } - ret = btrfs_update_root(trans, root->fs_info->tree_root, + ret = btrfs_update_root(trans, fs_info->tree_root, &reloc_root->root_key, root_item); BUG_ON(ret); @@ -1642,6 +1649,7 @@ int replace_file_extents(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct extent_buffer *leaf) { + struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_key key; struct btrfs_file_extent_item *fi; struct inode *inode = NULL; @@ -1698,8 +1706,8 @@ int replace_file_extents(struct btrfs_trans_handle *trans, end = key.offset + btrfs_file_extent_num_bytes(leaf, fi); WARN_ON(!IS_ALIGNED(key.offset, - root->sectorsize)); - WARN_ON(!IS_ALIGNED(end, root->sectorsize)); + fs_info->sectorsize)); + WARN_ON(!IS_ALIGNED(end, fs_info->sectorsize)); end--; ret = try_lock_extent(&BTRFS_I(inode)->io_tree, key.offset, end); @@ -1727,7 +1735,7 @@ int replace_file_extents(struct btrfs_trans_handle *trans, dirty = 1; key.offset -= btrfs_file_extent_offset(leaf, fi); - ret = btrfs_inc_extent_ref(trans, root, new_bytenr, + ret = btrfs_inc_extent_ref(trans, fs_info, new_bytenr, num_bytes, parent, btrfs_header_owner(leaf), key.objectid, key.offset); @@ -1736,7 +1744,7 @@ int replace_file_extents(struct btrfs_trans_handle *trans, break; } - ret = btrfs_free_extent(trans, root, bytenr, num_bytes, + ret = btrfs_free_extent(trans, fs_info, bytenr, num_bytes, parent, btrfs_header_owner(leaf), key.objectid, key.offset); if (ret) { @@ -1777,6 +1785,7 @@ int replace_path(struct btrfs_trans_handle *trans, struct btrfs_path *path, struct btrfs_key *next_key, int lowest_level, int max_level) { + struct btrfs_fs_info *fs_info = dest->fs_info; struct extent_buffer *eb; struct extent_buffer *parent; struct btrfs_key key; @@ -1834,7 +1843,7 @@ again: btrfs_node_key_to_cpu(parent, next_key, slot + 1); old_bytenr = btrfs_node_blockptr(parent, slot); - blocksize = dest->nodesize; + blocksize = fs_info->nodesize; old_ptr_gen = btrfs_node_ptr_generation(parent, slot); if (level <= max_level) { @@ -1860,7 +1869,7 @@ again: break; } - eb = read_tree_block(dest, old_bytenr, old_ptr_gen); + eb = read_tree_block(fs_info, old_bytenr, old_ptr_gen); if (IS_ERR(eb)) { ret = PTR_ERR(eb); break; @@ -1901,6 +1910,29 @@ again: BUG_ON(ret); /* + * Info qgroup to trace both subtrees. + * + * We must trace both trees. + * 1) Tree reloc subtree + * If not traced, we will leak data numbers + * 2) Fs subtree + * If not traced, we will double count old data + * and tree block numbers, if current trans doesn't free + * data reloc tree inode. + */ + ret = btrfs_qgroup_trace_subtree(trans, src, parent, + btrfs_header_generation(parent), + btrfs_header_level(parent)); + if (ret < 0) + break; + ret = btrfs_qgroup_trace_subtree(trans, dest, + path->nodes[level], + btrfs_header_generation(path->nodes[level]), + btrfs_header_level(path->nodes[level])); + if (ret < 0) + break; + + /* * swap blocks in fs tree and reloc tree. */ btrfs_set_node_blockptr(parent, slot, new_bytenr); @@ -1913,21 +1945,21 @@ again: path->slots[level], old_ptr_gen); btrfs_mark_buffer_dirty(path->nodes[level]); - ret = btrfs_inc_extent_ref(trans, src, old_bytenr, blocksize, - path->nodes[level]->start, + ret = btrfs_inc_extent_ref(trans, fs_info, old_bytenr, + blocksize, path->nodes[level]->start, src->root_key.objectid, level - 1, 0); BUG_ON(ret); - ret = btrfs_inc_extent_ref(trans, dest, new_bytenr, blocksize, - 0, dest->root_key.objectid, level - 1, - 0); + ret = btrfs_inc_extent_ref(trans, fs_info, new_bytenr, + blocksize, 0, dest->root_key.objectid, + level - 1, 0); BUG_ON(ret); - ret = btrfs_free_extent(trans, src, new_bytenr, blocksize, + ret = btrfs_free_extent(trans, fs_info, new_bytenr, blocksize, path->nodes[level]->start, src->root_key.objectid, level - 1, 0); BUG_ON(ret); - ret = btrfs_free_extent(trans, dest, old_bytenr, blocksize, + ret = btrfs_free_extent(trans, fs_info, old_bytenr, blocksize, 0, dest->root_key.objectid, level - 1, 0); BUG_ON(ret); @@ -1986,6 +2018,7 @@ static noinline_for_stack int walk_down_reloc_tree(struct btrfs_root *root, struct btrfs_path *path, int *level) { + struct btrfs_fs_info *fs_info = root->fs_info; struct extent_buffer *eb = NULL; int i; u64 bytenr; @@ -2016,7 +2049,7 @@ int walk_down_reloc_tree(struct btrfs_root *root, struct btrfs_path *path, } bytenr = btrfs_node_blockptr(eb, path->slots[i]); - eb = read_tree_block(root, bytenr, ptr_gen); + eb = read_tree_block(fs_info, bytenr, ptr_gen); if (IS_ERR(eb)) { return PTR_ERR(eb); } else if (!extent_buffer_uptodate(eb)) { @@ -2038,6 +2071,7 @@ static int invalidate_extent_cache(struct btrfs_root *root, struct btrfs_key *min_key, struct btrfs_key *max_key) { + struct btrfs_fs_info *fs_info = root->fs_info; struct inode *inode = NULL; u64 objectid; u64 start, end; @@ -2072,7 +2106,7 @@ static int invalidate_extent_cache(struct btrfs_root *root, start = 0; else { start = min_key->offset; - WARN_ON(!IS_ALIGNED(start, root->sectorsize)); + WARN_ON(!IS_ALIGNED(start, fs_info->sectorsize)); } } else { start = 0; @@ -2087,7 +2121,7 @@ static int invalidate_extent_cache(struct btrfs_root *root, if (max_key->offset == 0) continue; end = max_key->offset; - WARN_ON(!IS_ALIGNED(end, root->sectorsize)); + WARN_ON(!IS_ALIGNED(end, fs_info->sectorsize)); end--; } } else { @@ -2127,6 +2161,7 @@ static int find_next_key(struct btrfs_path *path, int level, static noinline_for_stack int merge_reloc_root(struct reloc_control *rc, struct btrfs_root *root) { + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; LIST_HEAD(inode_list); struct btrfs_key key; struct btrfs_key next_key; @@ -2175,7 +2210,7 @@ static noinline_for_stack int merge_reloc_root(struct reloc_control *rc, btrfs_unlock_up_safe(path, 0); } - min_reserved = root->nodesize * (BTRFS_MAX_LEVEL - 1) * 2; + min_reserved = fs_info->nodesize * (BTRFS_MAX_LEVEL - 1) * 2; memset(&next_key, 0, sizeof(next_key)); while (1) { @@ -2236,10 +2271,10 @@ static noinline_for_stack int merge_reloc_root(struct reloc_control *rc, path->slots[level]); root_item->drop_level = level; - btrfs_end_transaction_throttle(trans, root); + btrfs_end_transaction_throttle(trans); trans = NULL; - btrfs_btree_balance_dirty(root); + btrfs_btree_balance_dirty(fs_info); if (replaced && rc->stage == UPDATE_DATA_PTRS) invalidate_extent_cache(root, &key, &next_key); @@ -2267,9 +2302,9 @@ out: } if (trans) - btrfs_end_transaction_throttle(trans, root); + btrfs_end_transaction_throttle(trans); - btrfs_btree_balance_dirty(root); + btrfs_btree_balance_dirty(fs_info); if (replaced && rc->stage == UPDATE_DATA_PTRS) invalidate_extent_cache(root, &key, &next_key); @@ -2281,16 +2316,17 @@ static noinline_for_stack int prepare_to_merge(struct reloc_control *rc, int err) { struct btrfs_root *root = rc->extent_root; + struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_root *reloc_root; struct btrfs_trans_handle *trans; LIST_HEAD(reloc_roots); u64 num_bytes = 0; int ret; - mutex_lock(&root->fs_info->reloc_mutex); - rc->merging_rsv_size += root->nodesize * (BTRFS_MAX_LEVEL - 1) * 2; + mutex_lock(&fs_info->reloc_mutex); + rc->merging_rsv_size += fs_info->nodesize * (BTRFS_MAX_LEVEL - 1) * 2; rc->merging_rsv_size += rc->nodes_relocated * 2; - mutex_unlock(&root->fs_info->reloc_mutex); + mutex_unlock(&fs_info->reloc_mutex); again: if (!err) { @@ -2304,16 +2340,16 @@ again: trans = btrfs_join_transaction(rc->extent_root); if (IS_ERR(trans)) { if (!err) - btrfs_block_rsv_release(rc->extent_root, - rc->block_rsv, num_bytes); + btrfs_block_rsv_release(fs_info, rc->block_rsv, + num_bytes); return PTR_ERR(trans); } if (!err) { if (num_bytes != rc->merging_rsv_size) { - btrfs_end_transaction(trans, rc->extent_root); - btrfs_block_rsv_release(rc->extent_root, - rc->block_rsv, num_bytes); + btrfs_end_transaction(trans); + btrfs_block_rsv_release(fs_info, rc->block_rsv, + num_bytes); goto again; } } @@ -2325,8 +2361,7 @@ again: struct btrfs_root, root_list); list_del_init(&reloc_root->root_list); - root = read_fs_root(reloc_root->fs_info, - reloc_root->root_key.offset); + root = read_fs_root(fs_info, reloc_root->root_key.offset); BUG_ON(IS_ERR(root)); BUG_ON(root->reloc_root != reloc_root); @@ -2344,9 +2379,9 @@ again: list_splice(&reloc_roots, &rc->reloc_roots); if (!err) - btrfs_commit_transaction(trans, rc->extent_root); + btrfs_commit_transaction(trans); else - btrfs_end_transaction(trans, rc->extent_root); + btrfs_end_transaction(trans); return err; } @@ -2369,11 +2404,9 @@ void free_reloc_roots(struct list_head *list) static noinline_for_stack void merge_reloc_roots(struct reloc_control *rc) { + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; struct btrfs_root *root; struct btrfs_root *reloc_root; - u64 last_snap; - u64 otransid; - u64 objectid; LIST_HEAD(reloc_roots); int found = 0; int ret = 0; @@ -2386,9 +2419,9 @@ again: * adding their roots to the list while we are * doing this splice */ - mutex_lock(&root->fs_info->reloc_mutex); + mutex_lock(&fs_info->reloc_mutex); list_splice_init(&rc->reloc_roots, &reloc_roots); - mutex_unlock(&root->fs_info->reloc_mutex); + mutex_unlock(&fs_info->reloc_mutex); while (!list_empty(&reloc_roots)) { found = 1; @@ -2396,7 +2429,7 @@ again: struct btrfs_root, root_list); if (btrfs_root_refs(&reloc_root->root_item) > 0) { - root = read_fs_root(reloc_root->fs_info, + root = read_fs_root(fs_info, reloc_root->root_key.offset); BUG_ON(IS_ERR(root)); BUG_ON(root->reloc_root != reloc_root); @@ -2412,14 +2445,6 @@ again: list_del_init(&reloc_root->root_list); } - /* - * we keep the old last snapshot transid in rtranid when we - * created the relocation tree. - */ - last_snap = btrfs_root_rtransid(&reloc_root->root_item); - otransid = btrfs_root_otransid(&reloc_root->root_item); - objectid = reloc_root->root_key.offset; - ret = btrfs_drop_snapshot(reloc_root, rc->block_rsv, 0, 1); if (ret < 0) { if (list_empty(&reloc_root->root_list)) @@ -2435,14 +2460,14 @@ again: } out: if (ret) { - btrfs_handle_fs_error(root->fs_info, ret, NULL); + btrfs_handle_fs_error(fs_info, ret, NULL); if (!list_empty(&reloc_roots)) free_reloc_roots(&reloc_roots); /* new reloc root may be added */ - mutex_lock(&root->fs_info->reloc_mutex); + mutex_lock(&fs_info->reloc_mutex); list_splice_init(&rc->reloc_roots, &reloc_roots); - mutex_unlock(&root->fs_info->reloc_mutex); + mutex_unlock(&fs_info->reloc_mutex); if (!list_empty(&reloc_roots)) free_reloc_roots(&reloc_roots); } @@ -2464,12 +2489,13 @@ static void free_block_list(struct rb_root *blocks) static int record_reloc_root_in_trans(struct btrfs_trans_handle *trans, struct btrfs_root *reloc_root) { + struct btrfs_fs_info *fs_info = reloc_root->fs_info; struct btrfs_root *root; if (reloc_root->last_trans == trans->transid) return 0; - root = read_fs_root(reloc_root->fs_info, reloc_root->root_key.offset); + root = read_fs_root(fs_info, reloc_root->root_key.offset); BUG_ON(IS_ERR(root)); BUG_ON(root->reloc_root != reloc_root); @@ -2579,6 +2605,7 @@ static noinline_for_stack u64 calcu_metadata_size(struct reloc_control *rc, struct backref_node *node, int reserve) { + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; struct backref_node *next = node; struct backref_edge *edge; struct backref_edge *edges[BTRFS_MAX_LEVEL - 1]; @@ -2593,7 +2620,7 @@ u64 calcu_metadata_size(struct reloc_control *rc, if (next->processed && (reserve || next != node)) break; - num_bytes += rc->extent_root->nodesize; + num_bytes += fs_info->nodesize; if (list_empty(&next->upper)) break; @@ -2613,6 +2640,7 @@ static int reserve_metadata_space(struct btrfs_trans_handle *trans, struct backref_node *node) { struct btrfs_root *root = rc->extent_root; + struct btrfs_fs_info *fs_info = root->fs_info; u64 num_bytes; int ret; u64 tmp; @@ -2630,7 +2658,7 @@ static int reserve_metadata_space(struct btrfs_trans_handle *trans, ret = btrfs_block_rsv_refill(root, rc->block_rsv, num_bytes, BTRFS_RESERVE_FLUSH_LIMIT); if (ret) { - tmp = rc->extent_root->nodesize * RELOCATION_RESERVED_NODES; + tmp = fs_info->nodesize * RELOCATION_RESERVED_NODES; while (tmp <= rc->reserved_bytes) tmp <<= 1; /* @@ -2640,8 +2668,8 @@ static int reserve_metadata_space(struct btrfs_trans_handle *trans, * space for relocation and we will return eailer in * enospc case. */ - rc->block_rsv->size = tmp + rc->extent_root->nodesize * - RELOCATION_RESERVED_NODES; + rc->block_rsv->size = tmp + fs_info->nodesize * + RELOCATION_RESERVED_NODES; return -EAGAIN; } @@ -2661,6 +2689,7 @@ static int do_relocation(struct btrfs_trans_handle *trans, struct btrfs_key *key, struct btrfs_path *path, int lowest) { + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; struct backref_node *upper; struct backref_edge *edge; struct backref_edge *edges[BTRFS_MAX_LEVEL - 1]; @@ -2741,9 +2770,9 @@ static int do_relocation(struct btrfs_trans_handle *trans, goto next; } - blocksize = root->nodesize; + blocksize = root->fs_info->nodesize; generation = btrfs_node_ptr_generation(upper->eb, slot); - eb = read_tree_block(root, bytenr, generation); + eb = read_tree_block(fs_info, bytenr, generation); if (IS_ERR(eb)) { err = PTR_ERR(eb); goto next; @@ -2772,7 +2801,7 @@ static int do_relocation(struct btrfs_trans_handle *trans, trans->transid); btrfs_mark_buffer_dirty(upper->eb); - ret = btrfs_inc_extent_ref(trans, root, + ret = btrfs_inc_extent_ref(trans, root->fs_info, node->eb->start, blocksize, upper->eb->start, btrfs_header_owner(upper->eb), @@ -2854,7 +2883,7 @@ static void __mark_block_processed(struct reloc_control *rc, u32 blocksize; if (node->level == 0 || in_block_group(node->bytenr, rc->block_group)) { - blocksize = rc->extent_root->nodesize; + blocksize = rc->extent_root->fs_info->nodesize; mark_block_processed(rc, node->bytenr, blocksize); } node->processed = 1; @@ -2894,7 +2923,7 @@ static void update_processed_blocks(struct reloc_control *rc, static int tree_block_processed(u64 bytenr, struct reloc_control *rc) { - u32 blocksize = rc->extent_root->nodesize; + u32 blocksize = rc->extent_root->fs_info->nodesize; if (test_range_bit(&rc->processed_blocks, bytenr, bytenr + blocksize - 1, EXTENT_DIRTY, 1, NULL)) @@ -2902,14 +2931,13 @@ static int tree_block_processed(u64 bytenr, struct reloc_control *rc) return 0; } -static int get_tree_block_key(struct reloc_control *rc, +static int get_tree_block_key(struct btrfs_fs_info *fs_info, struct tree_block *block) { struct extent_buffer *eb; BUG_ON(block->key_ready); - eb = read_tree_block(rc->extent_root, block->bytenr, - block->key.offset); + eb = read_tree_block(fs_info, block->bytenr, block->key.offset); if (IS_ERR(eb)) { return PTR_ERR(eb); } else if (!extent_buffer_uptodate(eb)) { @@ -2988,6 +3016,7 @@ static noinline_for_stack int relocate_tree_blocks(struct btrfs_trans_handle *trans, struct reloc_control *rc, struct rb_root *blocks) { + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; struct backref_node *node; struct btrfs_path *path; struct tree_block *block; @@ -3005,7 +3034,7 @@ int relocate_tree_blocks(struct btrfs_trans_handle *trans, while (rb_node) { block = rb_entry(rb_node, struct tree_block, rb_node); if (!block->key_ready) - readahead_tree_block(rc->extent_root, block->bytenr); + readahead_tree_block(fs_info, block->bytenr); rb_node = rb_next(rb_node); } @@ -3013,7 +3042,7 @@ int relocate_tree_blocks(struct btrfs_trans_handle *trans, while (rb_node) { block = rb_entry(rb_node, struct tree_block, rb_node); if (!block->key_ready) { - err = get_tree_block_key(rc, block); + err = get_tree_block_key(fs_info, block); if (err) goto out_free_path; } @@ -3107,7 +3136,7 @@ static noinline_for_stack int setup_extent_mapping(struct inode *inode, u64 start, u64 end, u64 block_start) { - struct btrfs_root *root = BTRFS_I(inode)->root; + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; struct extent_map *em; int ret = 0; @@ -3120,7 +3149,7 @@ int setup_extent_mapping(struct inode *inode, u64 start, u64 end, em->len = end + 1 - start; em->block_len = em->len; em->block_start = block_start; - em->bdev = root->fs_info->fs_devices->latest_bdev; + em->bdev = fs_info->fs_devices->latest_bdev; set_bit(EXTENT_FLAG_PINNED, &em->flags); lock_extent(&BTRFS_I(inode)->io_tree, start, end); @@ -3141,6 +3170,7 @@ int setup_extent_mapping(struct inode *inode, u64 start, u64 end, static int relocate_file_extent_cluster(struct inode *inode, struct file_extent_cluster *cluster) { + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); u64 page_start; u64 page_end; u64 offset = BTRFS_I(inode)->index_cnt; @@ -3236,7 +3266,7 @@ static int relocate_file_extent_cluster(struct inode *inode, index++; balance_dirty_pages_ratelimited(inode->i_mapping); - btrfs_throttle(BTRFS_I(inode)->root); + btrfs_throttle(fs_info); } WARN_ON(nr != cluster->nr); out: @@ -3376,7 +3406,7 @@ static int add_tree_block(struct reloc_control *rc, return -ENOMEM; block->bytenr = extent_key->objectid; - block->key.objectid = rc->extent_root->nodesize; + block->key.objectid = rc->extent_root->fs_info->nodesize; block->key.offset = generation; block->level = level; block->key_ready = 0; @@ -3395,11 +3425,11 @@ static int __add_tree_block(struct reloc_control *rc, u64 bytenr, u32 blocksize, struct rb_root *blocks) { + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; struct btrfs_path *path; struct btrfs_key key; int ret; - bool skinny = btrfs_fs_incompat(rc->extent_root->fs_info, - SKINNY_METADATA); + bool skinny = btrfs_fs_incompat(fs_info, SKINNY_METADATA); if (tree_block_processed(bytenr, rc)) return 0; @@ -3465,7 +3495,7 @@ static int block_use_full_backref(struct reloc_control *rc, btrfs_header_backref_rev(eb) < BTRFS_MIXED_BACKREF_REV) return 1; - ret = btrfs_lookup_extent_info(NULL, rc->extent_root, + ret = btrfs_lookup_extent_info(NULL, rc->extent_root->fs_info, eb->start, btrfs_header_level(eb), 1, NULL, &flags); BUG_ON(ret); @@ -3502,7 +3532,7 @@ static int delete_block_group_cache(struct btrfs_fs_info *fs_info, } truncate: - ret = btrfs_check_trunc_cache_free_space(root, + ret = btrfs_check_trunc_cache_free_space(fs_info, &fs_info->global_block_rsv); if (ret) goto out; @@ -3515,8 +3545,8 @@ truncate: ret = btrfs_truncate_free_space_cache(root, trans, block_group, inode); - btrfs_end_transaction(trans, root); - btrfs_btree_balance_dirty(root); + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty(fs_info); out: iput(inode); return ret; @@ -3532,6 +3562,7 @@ static int find_data_references(struct reloc_control *rc, struct btrfs_extent_data_ref *ref, struct rb_root *blocks) { + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; struct btrfs_path *path; struct tree_block *block; struct btrfs_root *root; @@ -3558,8 +3589,7 @@ static int find_data_references(struct reloc_control *rc, * it and redo the search. */ if (ref_root == BTRFS_ROOT_TREE_OBJECTID) { - ret = delete_block_group_cache(rc->extent_root->fs_info, - rc->block_group, + ret = delete_block_group_cache(fs_info, rc->block_group, NULL, ref_objectid); if (ret != -ENOENT) return ret; @@ -3571,7 +3601,7 @@ static int find_data_references(struct reloc_control *rc, return -ENOMEM; path->reada = READA_FORWARD; - root = read_fs_root(rc->extent_root->fs_info, ref_root); + root = read_fs_root(fs_info, ref_root); if (IS_ERR(root)) { err = PTR_ERR(root); goto out; @@ -3706,7 +3736,7 @@ int add_data_references(struct reloc_control *rc, struct btrfs_extent_inline_ref *iref; unsigned long ptr; unsigned long end; - u32 blocksize = rc->extent_root->nodesize; + u32 blocksize = rc->extent_root->fs_info->nodesize; int ret = 0; int err = 0; @@ -3797,6 +3827,7 @@ static noinline_for_stack int find_next_extent(struct reloc_control *rc, struct btrfs_path *path, struct btrfs_key *extent_key) { + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; struct btrfs_key key; struct extent_buffer *leaf; u64 start, end, last; @@ -3848,7 +3879,7 @@ next: } if (key.type == BTRFS_METADATA_ITEM_KEY && - key.objectid + rc->extent_root->nodesize <= + key.objectid + fs_info->nodesize <= rc->search_start) { path->slots[0]++; goto next; @@ -3866,7 +3897,7 @@ next: rc->search_start = key.objectid + key.offset; else rc->search_start = key.objectid + - rc->extent_root->nodesize; + fs_info->nodesize; memcpy(extent_key, &key, sizeof(key)); return 0; } @@ -3913,7 +3944,7 @@ int prepare_to_relocate(struct reloc_control *rc) struct btrfs_trans_handle *trans; int ret; - rc->block_rsv = btrfs_alloc_block_rsv(rc->extent_root, + rc->block_rsv = btrfs_alloc_block_rsv(rc->extent_root->fs_info, BTRFS_BLOCK_RSV_TEMP); if (!rc->block_rsv) return -ENOMEM; @@ -3924,7 +3955,7 @@ int prepare_to_relocate(struct reloc_control *rc) rc->nodes_relocated = 0; rc->merging_rsv_size = 0; rc->reserved_bytes = 0; - rc->block_rsv->size = rc->extent_root->nodesize * + rc->block_rsv->size = rc->extent_root->fs_info->nodesize * RELOCATION_RESERVED_NODES; ret = btrfs_block_rsv_refill(rc->extent_root, rc->block_rsv, rc->block_rsv->size, @@ -3945,96 +3976,13 @@ int prepare_to_relocate(struct reloc_control *rc) */ return PTR_ERR(trans); } - btrfs_commit_transaction(trans, rc->extent_root); + btrfs_commit_transaction(trans); return 0; } -/* - * Qgroup fixer for data chunk relocation. - * The data relocation is done in the following steps - * 1) Copy data extents into data reloc tree - * 2) Create tree reloc tree(special snapshot) for related subvolumes - * 3) Modify file extents in tree reloc tree - * 4) Merge tree reloc tree with original fs tree, by swapping tree blocks - * - * The problem is, data and tree reloc tree are not accounted to qgroup, - * and 4) will only info qgroup to track tree blocks change, not file extents - * in the tree blocks. - * - * The good news is, related data extents are all in data reloc tree, so we - * only need to info qgroup to track all file extents in data reloc tree - * before commit trans. - */ -static int qgroup_fix_relocated_data_extents(struct btrfs_trans_handle *trans, - struct reloc_control *rc) -{ - struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; - struct inode *inode = rc->data_inode; - struct btrfs_root *data_reloc_root = BTRFS_I(inode)->root; - struct btrfs_path *path; - struct btrfs_key key; - int ret = 0; - - if (!test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) - return 0; - - /* - * Only for stage where we update data pointers the qgroup fix is - * valid. - * For MOVING_DATA stage, we will miss the timing of swapping tree - * blocks, and won't fix it. - */ - if (!(rc->stage == UPDATE_DATA_PTRS && rc->extents_found)) - return 0; - - path = btrfs_alloc_path(); - if (!path) - return -ENOMEM; - key.objectid = btrfs_ino(inode); - key.type = BTRFS_EXTENT_DATA_KEY; - key.offset = 0; - - ret = btrfs_search_slot(NULL, data_reloc_root, &key, path, 0, 0); - if (ret < 0) - goto out; - - lock_extent(&BTRFS_I(inode)->io_tree, 0, (u64)-1); - while (1) { - struct btrfs_file_extent_item *fi; - - btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); - if (key.objectid > btrfs_ino(inode)) - break; - if (key.type != BTRFS_EXTENT_DATA_KEY) - goto next; - fi = btrfs_item_ptr(path->nodes[0], path->slots[0], - struct btrfs_file_extent_item); - if (btrfs_file_extent_type(path->nodes[0], fi) != - BTRFS_FILE_EXTENT_REG) - goto next; - ret = btrfs_qgroup_insert_dirty_extent(trans, fs_info, - btrfs_file_extent_disk_bytenr(path->nodes[0], fi), - btrfs_file_extent_disk_num_bytes(path->nodes[0], fi), - GFP_NOFS); - if (ret < 0) - break; -next: - ret = btrfs_next_item(data_reloc_root, path); - if (ret < 0) - break; - if (ret > 0) { - ret = 0; - break; - } - } - unlock_extent(&BTRFS_I(inode)->io_tree, 0 , (u64)-1); -out: - btrfs_free_path(path); - return ret; -} - static noinline_for_stack int relocate_block_group(struct reloc_control *rc) { + struct btrfs_fs_info *fs_info = rc->extent_root->fs_info; struct rb_root blocks = RB_ROOT; struct btrfs_key key; struct btrfs_trans_handle *trans = NULL; @@ -4075,7 +4023,7 @@ static noinline_for_stack int relocate_block_group(struct reloc_control *rc) } restart: if (update_backref_cache(trans, &rc->backref_cache)) { - btrfs_end_transaction(trans, rc->extent_root); + btrfs_end_transaction(trans); continue; } @@ -4163,8 +4111,8 @@ restart: } } - btrfs_end_transaction_throttle(trans, rc->extent_root); - btrfs_btree_balance_dirty(rc->extent_root); + btrfs_end_transaction_throttle(trans); + btrfs_btree_balance_dirty(fs_info); trans = NULL; if (rc->stage == MOVE_DATA_EXTENTS && @@ -4179,7 +4127,7 @@ restart: } } if (trans && progress && err == -ENOSPC) { - ret = btrfs_force_chunk_alloc(trans, rc->extent_root, + ret = btrfs_force_chunk_alloc(trans, fs_info, rc->block_group->flags); if (ret == 1) { err = 0; @@ -4192,8 +4140,8 @@ restart: clear_extent_bits(&rc->processed_blocks, 0, (u64)-1, EXTENT_DIRTY); if (trans) { - btrfs_end_transaction_throttle(trans, rc->extent_root); - btrfs_btree_balance_dirty(rc->extent_root); + btrfs_end_transaction_throttle(trans); + btrfs_btree_balance_dirty(fs_info); } if (!err) { @@ -4207,7 +4155,7 @@ restart: set_reloc_control(rc); backref_cache_cleanup(&rc->backref_cache); - btrfs_block_rsv_release(rc->extent_root, rc->block_rsv, (u64)-1); + btrfs_block_rsv_release(fs_info, rc->block_rsv, (u64)-1); err = prepare_to_merge(rc, err); @@ -4215,7 +4163,7 @@ restart: rc->merge_reloc_tree = 0; unset_reloc_control(rc); - btrfs_block_rsv_release(rc->extent_root, rc->block_rsv, (u64)-1); + btrfs_block_rsv_release(fs_info, rc->block_rsv, (u64)-1); /* get rid of pinned extents */ trans = btrfs_join_transaction(rc->extent_root); @@ -4223,16 +4171,9 @@ restart: err = PTR_ERR(trans); goto out_free; } - ret = qgroup_fix_relocated_data_extents(trans, rc); - if (ret < 0) { - btrfs_abort_transaction(trans, ret); - if (!err) - err = ret; - goto out_free; - } - btrfs_commit_transaction(trans, rc->extent_root); + btrfs_commit_transaction(trans); out_free: - btrfs_free_block_rsv(rc->extent_root, rc->block_rsv); + btrfs_free_block_rsv(fs_info, rc->block_rsv); btrfs_free_path(path); return err; } @@ -4255,7 +4196,7 @@ static int __insert_orphan_inode(struct btrfs_trans_handle *trans, leaf = path->nodes[0]; item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_inode_item); - memset_extent_buffer(leaf, 0, (unsigned long)item, sizeof(*item)); + memzero_extent_buffer(leaf, (unsigned long)item, sizeof(*item)); btrfs_set_inode_generation(leaf, item, 1); btrfs_set_inode_size(leaf, item, 0); btrfs_set_inode_mode(leaf, item, S_IFREG | 0600); @@ -4300,14 +4241,14 @@ struct inode *create_reloc_inode(struct btrfs_fs_info *fs_info, key.objectid = objectid; key.type = BTRFS_INODE_ITEM_KEY; key.offset = 0; - inode = btrfs_iget(root->fs_info->sb, &key, root, NULL); + inode = btrfs_iget(fs_info->sb, &key, root, NULL); BUG_ON(IS_ERR(inode) || is_bad_inode(inode)); BTRFS_I(inode)->index_cnt = group->key.objectid; err = btrfs_orphan_add(trans, inode); out: - btrfs_end_transaction(trans, root); - btrfs_btree_balance_dirty(root); + btrfs_end_transaction(trans); + btrfs_btree_balance_dirty(fs_info); if (err) { if (inode) iput(inode); @@ -4333,11 +4274,50 @@ static struct reloc_control *alloc_reloc_control(struct btrfs_fs_info *fs_info) } /* + * Print the block group being relocated + */ +static void describe_relocation(struct btrfs_fs_info *fs_info, + struct btrfs_block_group_cache *block_group) +{ + char buf[128]; /* prefixed by a '|' that'll be dropped */ + u64 flags = block_group->flags; + + /* Shouldn't happen */ + if (!flags) { + strcpy(buf, "|NONE"); + } else { + char *bp = buf; + +#define DESCRIBE_FLAG(f, d) \ + if (flags & BTRFS_BLOCK_GROUP_##f) { \ + bp += snprintf(bp, buf - bp + sizeof(buf), "|%s", d); \ + flags &= ~BTRFS_BLOCK_GROUP_##f; \ + } + DESCRIBE_FLAG(DATA, "data"); + DESCRIBE_FLAG(SYSTEM, "system"); + DESCRIBE_FLAG(METADATA, "metadata"); + DESCRIBE_FLAG(RAID0, "raid0"); + DESCRIBE_FLAG(RAID1, "raid1"); + DESCRIBE_FLAG(DUP, "dup"); + DESCRIBE_FLAG(RAID10, "raid10"); + DESCRIBE_FLAG(RAID5, "raid5"); + DESCRIBE_FLAG(RAID6, "raid6"); + if (flags) + snprintf(buf, buf - bp + sizeof(buf), "|0x%llx", flags); +#undef DESCRIBE_FLAG + } + + btrfs_info(fs_info, + "relocating block group %llu flags %s", + block_group->key.objectid, buf + 1); +} + +/* * function to relocate all extents in a block group. */ -int btrfs_relocate_block_group(struct btrfs_root *extent_root, u64 group_start) +int btrfs_relocate_block_group(struct btrfs_fs_info *fs_info, u64 group_start) { - struct btrfs_fs_info *fs_info = extent_root->fs_info; + struct btrfs_root *extent_root = fs_info->extent_root; struct reloc_control *rc; struct inode *inode; struct btrfs_path *path; @@ -4388,9 +4368,7 @@ int btrfs_relocate_block_group(struct btrfs_root *extent_root, u64 group_start) goto out; } - btrfs_info(extent_root->fs_info, - "relocating block group %llu flags %llu", - rc->block_group->key.objectid, rc->block_group->flags); + describe_relocation(fs_info, rc->block_group); btrfs_wait_block_group_reservations(rc->block_group); btrfs_wait_nocow_writers(rc->block_group); @@ -4410,8 +4388,7 @@ int btrfs_relocate_block_group(struct btrfs_root *extent_root, u64 group_start) if (rc->extents_found == 0) break; - btrfs_info(extent_root->fs_info, "found %llu extents", - rc->extents_found); + btrfs_info(fs_info, "found %llu extents", rc->extents_found); if (rc->stage == MOVE_DATA_EXTENTS && rc->found_file_extent) { ret = btrfs_wait_ordered_range(rc->data_inode, 0, @@ -4431,7 +4408,7 @@ int btrfs_relocate_block_group(struct btrfs_root *extent_root, u64 group_start) WARN_ON(btrfs_block_group_used(&rc->block_group->item) > 0); out: if (err && rw) - btrfs_dec_block_group_ro(extent_root, rc->block_group); + btrfs_dec_block_group_ro(rc->block_group); iput(rc->data_inode); btrfs_put_block_group(rc->block_group); kfree(rc); @@ -4440,10 +4417,11 @@ out: static noinline_for_stack int mark_garbage_root(struct btrfs_root *root) { + struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_trans_handle *trans; int ret, err; - trans = btrfs_start_transaction(root->fs_info->tree_root, 0); + trans = btrfs_start_transaction(fs_info->tree_root, 0); if (IS_ERR(trans)) return PTR_ERR(trans); @@ -4451,10 +4429,10 @@ static noinline_for_stack int mark_garbage_root(struct btrfs_root *root) sizeof(root->root_item.drop_progress)); root->root_item.drop_level = 0; btrfs_set_root_refs(&root->root_item, 0); - ret = btrfs_update_root(trans, root->fs_info->tree_root, + ret = btrfs_update_root(trans, fs_info->tree_root, &root->root_key, &root->root_item); - err = btrfs_end_transaction(trans, root->fs_info->tree_root); + err = btrfs_end_transaction(trans); if (err) return err; return ret; @@ -4468,6 +4446,7 @@ static noinline_for_stack int mark_garbage_root(struct btrfs_root *root) */ int btrfs_recover_relocation(struct btrfs_root *root) { + struct btrfs_fs_info *fs_info = root->fs_info; LIST_HEAD(reloc_roots); struct btrfs_key key; struct btrfs_root *fs_root; @@ -4489,7 +4468,7 @@ int btrfs_recover_relocation(struct btrfs_root *root) key.offset = (u64)-1; while (1) { - ret = btrfs_search_slot(NULL, root->fs_info->tree_root, &key, + ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); if (ret < 0) { err = ret; @@ -4517,7 +4496,7 @@ int btrfs_recover_relocation(struct btrfs_root *root) list_add(&reloc_root->root_list, &reloc_roots); if (btrfs_root_refs(&reloc_root->root_item) > 0) { - fs_root = read_fs_root(root->fs_info, + fs_root = read_fs_root(fs_info, reloc_root->root_key.offset); if (IS_ERR(fs_root)) { ret = PTR_ERR(fs_root); @@ -4543,13 +4522,13 @@ int btrfs_recover_relocation(struct btrfs_root *root) if (list_empty(&reloc_roots)) goto out; - rc = alloc_reloc_control(root->fs_info); + rc = alloc_reloc_control(fs_info); if (!rc) { err = -ENOMEM; goto out; } - rc->extent_root = root->fs_info->extent_root; + rc->extent_root = fs_info->extent_root; set_reloc_control(rc); @@ -4573,8 +4552,7 @@ int btrfs_recover_relocation(struct btrfs_root *root) continue; } - fs_root = read_fs_root(root->fs_info, - reloc_root->root_key.offset); + fs_root = read_fs_root(fs_info, reloc_root->root_key.offset); if (IS_ERR(fs_root)) { err = PTR_ERR(fs_root); goto out_free; @@ -4585,7 +4563,7 @@ int btrfs_recover_relocation(struct btrfs_root *root) fs_root->reloc_root = reloc_root; } - err = btrfs_commit_transaction(trans, rc->extent_root); + err = btrfs_commit_transaction(trans); if (err) goto out_free; @@ -4598,12 +4576,7 @@ int btrfs_recover_relocation(struct btrfs_root *root) err = PTR_ERR(trans); goto out_free; } - err = qgroup_fix_relocated_data_extents(trans, rc); - if (err < 0) { - btrfs_abort_transaction(trans, err); - goto out_free; - } - err = btrfs_commit_transaction(trans, rc->extent_root); + err = btrfs_commit_transaction(trans); out_free: kfree(rc); out: @@ -4614,8 +4587,7 @@ out: if (err == 0) { /* cleanup orphan inode in data relocation tree */ - fs_root = read_fs_root(root->fs_info, - BTRFS_DATA_RELOC_TREE_OBJECTID); + fs_root = read_fs_root(fs_info, BTRFS_DATA_RELOC_TREE_OBJECTID); if (IS_ERR(fs_root)) err = PTR_ERR(fs_root); else @@ -4632,9 +4604,9 @@ out: */ int btrfs_reloc_clone_csums(struct inode *inode, u64 file_pos, u64 len) { + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct btrfs_ordered_sum *sums; struct btrfs_ordered_extent *ordered; - struct btrfs_root *root = BTRFS_I(inode)->root; int ret; u64 disk_bytenr; u64 new_bytenr; @@ -4644,7 +4616,7 @@ int btrfs_reloc_clone_csums(struct inode *inode, u64 file_pos, u64 len) BUG_ON(ordered->file_offset != file_pos || ordered->len != len); disk_bytenr = file_pos + BTRFS_I(inode)->index_cnt; - ret = btrfs_lookup_csums_range(root->fs_info->csum_root, disk_bytenr, + ret = btrfs_lookup_csums_range(fs_info->csum_root, disk_bytenr, disk_bytenr + len - 1, &list, 0); if (ret) goto out; @@ -4679,13 +4651,14 @@ int btrfs_reloc_cow_block(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct extent_buffer *buf, struct extent_buffer *cow) { + struct btrfs_fs_info *fs_info = root->fs_info; struct reloc_control *rc; struct backref_node *node; int first_cow = 0; int level; int ret = 0; - rc = root->fs_info->reloc_ctl; + rc = fs_info->reloc_ctl; if (!rc) return 0; |