Lines Matching +full:system +full:- +full:ctl
1 // SPDX-License-Identifier: GPL-2.0
16 #include "disk-io.h"
17 #include "extent-tree.h"
21 #include "rcu-string.h"
22 #include "dev-replace.h"
24 #include "tree-checker.h"
25 #include "space-info.h"
26 #include "block-group.h"
31 #include "uuid-tree.h"
36 #include "raid-stripe-tree.h"
228 size_bp -= ret; \ in btrfs_describe_block_groups()
235 DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_SYSTEM, "system"); in btrfs_describe_block_groups()
246 size_bp -= ret; in btrfs_describe_block_groups()
250 buf[size_buf - size_bp - 1] = '\0'; /* remove last | */ in btrfs_describe_block_groups()
267 * There are several mutexes that protect manipulation of devices and low-level
271 * ------------------------
272 * protects the fs_uuids list that tracks all per-fs fs_devices, resulting from
276 * the mutex can be very coarse and can cover long-running operations
281 * global::fs_devs - add, remove, updates to the global list
287 * btrfs_device::name - renames (write side), read is RCU
289 * fs_devices::device_list_mutex (per-fs, with RCU)
290 * ------------------------------------------------
293 * simple list traversal with read-only actions can be done with RCU protection
302 * -------------
307 * -----------
314 * -------------
334 * - Balance (*)
335 * - Device add
336 * - Device remove
337 * - Device replace (*)
338 * - Resize
342 * - Running state
343 * - Paused state
344 * - Completed state
349 * - ioctl (only Balance can be Paused through ioctl)
350 * - filesystem remounted as read-only
351 * - filesystem unmounted and mounted as read-only
352 * - system power-cycle and filesystem mounted as read-only
353 * - filesystem or device errors leading to forced read-only
358 * either by ioctl (Balance only) or when remounted as read-write.
386 return ERR_PTR(-ENOMEM); in alloc_fs_devices()
388 mutex_init(&fs_devs->device_list_mutex); in alloc_fs_devices()
390 INIT_LIST_HEAD(&fs_devs->devices); in alloc_fs_devices()
391 INIT_LIST_HEAD(&fs_devs->alloc_list); in alloc_fs_devices()
392 INIT_LIST_HEAD(&fs_devs->fs_list); in alloc_fs_devices()
393 INIT_LIST_HEAD(&fs_devs->seed_list); in alloc_fs_devices()
396 memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE); in alloc_fs_devices()
397 memcpy(fs_devs->metadata_uuid, fsid, BTRFS_FSID_SIZE); in alloc_fs_devices()
405 WARN_ON(!list_empty(&device->post_commit_list)); in btrfs_free_device()
406 rcu_string_free(device->name); in btrfs_free_device()
407 extent_io_tree_release(&device->alloc_state); in btrfs_free_device()
416 WARN_ON(fs_devices->opened); in free_fs_devices()
417 while (!list_empty(&fs_devices->devices)) { in free_fs_devices()
418 device = list_entry(fs_devices->devices.next, in free_fs_devices()
420 list_del(&device->dev_list); in free_fs_devices()
433 list_del(&fs_devices->fs_list); in btrfs_cleanup_fs_uuids()
441 if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) != 0) in match_fsid_fs_devices()
447 if (memcmp(metadata_fsid, fs_devices->metadata_uuid, BTRFS_FSID_SIZE) != 0) in match_fsid_fs_devices()
460 /* Handle non-split brain cases */ in find_fsid()
521 * -EBUSY if @devt is a mounted device.
522 * -ENOENT if @devt does not match any device in the list.
537 mutex_lock(&fs_devices->device_list_mutex); in btrfs_free_stale_devices()
539 &fs_devices->devices, dev_list) { in btrfs_free_stale_devices()
542 if (devt && devt != device->devt) in btrfs_free_stale_devices()
544 if (fs_devices->opened) { in btrfs_free_stale_devices()
546 ret = -EBUSY; in btrfs_free_stale_devices()
551 fs_devices->num_devices--; in btrfs_free_stale_devices()
552 list_del(&device->dev_list); in btrfs_free_stale_devices()
557 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_free_stale_devices()
559 if (fs_devices->num_devices == 0) { in btrfs_free_stale_devices()
561 list_del(&fs_devices->fs_list); in btrfs_free_stale_devices()
584 fsid_fs_devices = find_fsid(disk_super->fsid, in find_fsid_by_device()
585 has_metadata_uuid ? disk_super->metadata_uuid : NULL); in find_fsid_by_device()
593 * functions as a multi-device filesystem. So, temp-fsid feature is in find_fsid_by_device()
603 list_for_each_entry(device, &devt_fs_devices->devices, dev_list) { in find_fsid_by_device()
604 if (device->devt == devt) { in find_fsid_by_device()
616 if (devt_fs_devices->opened == 0) { in find_fsid_by_device()
644 * fs_devices->device_list_mutex here.
655 if (device->bdev) in btrfs_open_one_device()
656 return -EINVAL; in btrfs_open_one_device()
657 if (!device->name) in btrfs_open_one_device()
658 return -EINVAL; in btrfs_open_one_device()
660 ret = btrfs_get_bdev_and_sb(device->name->str, flags, holder, 1, in btrfs_open_one_device()
665 devid = btrfs_stack_device_id(&disk_super->dev_item); in btrfs_open_one_device()
666 if (devid != device->devid) in btrfs_open_one_device()
669 if (memcmp(device->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE)) in btrfs_open_one_device()
672 device->generation = btrfs_super_generation(disk_super); in btrfs_open_one_device()
678 "BTRFS: Invalid seeding and uuid-changed device detected\n"); in btrfs_open_one_device()
682 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); in btrfs_open_one_device()
683 fs_devices->seeding = true; in btrfs_open_one_device()
686 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); in btrfs_open_one_device()
688 set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); in btrfs_open_one_device()
692 fs_devices->rotating = true; in btrfs_open_one_device()
695 fs_devices->discardable = true; in btrfs_open_one_device()
697 device->bdev_file = bdev_file; in btrfs_open_one_device()
698 device->bdev = file_bdev(bdev_file); in btrfs_open_one_device()
699 clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); in btrfs_open_one_device()
701 if (device->devt != device->bdev->bd_dev) { in btrfs_open_one_device()
704 device->name->str, MAJOR(device->devt), in btrfs_open_one_device()
705 MINOR(device->devt), MAJOR(device->bdev->bd_dev), in btrfs_open_one_device()
706 MINOR(device->bdev->bd_dev)); in btrfs_open_one_device()
708 device->devt = device->bdev->bd_dev; in btrfs_open_one_device()
711 fs_devices->open_devices++; in btrfs_open_one_device()
712 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) && in btrfs_open_one_device()
713 device->devid != BTRFS_DEV_REPLACE_DEVID) { in btrfs_open_one_device()
714 fs_devices->rw_devices++; in btrfs_open_one_device()
715 list_add_tail(&device->dev_alloc_list, &fs_devices->alloc_list); in btrfs_open_one_device()
725 return -EINVAL; in btrfs_open_one_device()
733 return has_metadata_uuid ? sb->metadata_uuid : sb->fsid; in btrfs_sb_fsid_ptr()
787 ret = -EINVAL; in get_canonical_dev_path()
793 ret = -ENOMEM; in get_canonical_dev_path()
820 if (!device->name) in is_same_device()
828 ret = strscpy(old_path, rcu_str_deref(device->name), PATH_MAX); in is_same_device()
863 u64 devid = btrfs_stack_device_id(&disk_super->dev_item); in device_list_add()
874 return ERR_PTR(-EAGAIN); in device_list_add()
887 fs_devices = alloc_fs_devices(disk_super->fsid); in device_list_add()
892 memcpy(fs_devices->metadata_uuid, in device_list_add()
893 disk_super->metadata_uuid, BTRFS_FSID_SIZE); in device_list_add()
896 generate_random_uuid(fs_devices->fsid); in device_list_add()
897 fs_devices->temp_fsid = true; in device_list_add()
898 pr_info("BTRFS: device %s (%d:%d) using temp-fsid %pU\n", in device_list_add()
900 fs_devices->fsid); in device_list_add()
903 mutex_lock(&fs_devices->device_list_mutex); in device_list_add()
904 list_add(&fs_devices->fs_list, &fs_uuids); in device_list_add()
910 .uuid = disk_super->dev_item.uuid, in device_list_add()
913 mutex_lock(&fs_devices->device_list_mutex); in device_list_add()
916 if (found_transid > fs_devices->latest_generation) { in device_list_add()
917 memcpy(fs_devices->fsid, disk_super->fsid, in device_list_add()
919 memcpy(fs_devices->metadata_uuid, in device_list_add()
927 if (fs_devices->opened) { in device_list_add()
931 fs_devices->fsid, current->comm, in device_list_add()
933 mutex_unlock(&fs_devices->device_list_mutex); in device_list_add()
934 return ERR_PTR(-EBUSY); in device_list_add()
939 disk_super->dev_item.uuid, path); in device_list_add()
942 mutex_unlock(&fs_devices->device_list_mutex); in device_list_add()
947 device->devt = path_devt; in device_list_add()
949 list_add_rcu(&device->dev_list, &fs_devices->devices); in device_list_add()
950 fs_devices->num_devices++; in device_list_add()
952 device->fs_devices = fs_devices; in device_list_add()
955 if (disk_super->label[0]) in device_list_add()
958 disk_super->label, devid, found_transid, path, in device_list_add()
960 current->comm, task_pid_nr(current)); in device_list_add()
964 disk_super->fsid, devid, found_transid, path, in device_list_add()
966 current->comm, task_pid_nr(current)); in device_list_add()
968 } else if (!device->name || !is_same_device(device, path)) { in device_list_add()
971 * 1. If you are here and if the device->name is NULL that in device_list_add()
973 * 2. If you are here and if the device->name is different in device_list_add()
977 * b. The missing-disk-which-was-replaced, has in device_list_add()
995 if (!fs_devices->opened && found_transid < device->generation) { in device_list_add()
1000 * with larger generation number or the last-in if in device_list_add()
1003 mutex_unlock(&fs_devices->device_list_mutex); in device_list_add()
1006 path, found_transid, device->generation); in device_list_add()
1007 return ERR_PTR(-EEXIST); in device_list_add()
1014 * NOTE: the device->fs_info may not be reliable here so pass in device_list_add()
1016 * use-after-free when the fs_info and fs_info->sb are already in device_list_add()
1019 if (device->bdev) { in device_list_add()
1020 if (device->devt != path_devt) { in device_list_add()
1021 mutex_unlock(&fs_devices->device_list_mutex); in device_list_add()
1025 current->comm, in device_list_add()
1027 return ERR_PTR(-EEXIST); in device_list_add()
1032 path, current->comm, in device_list_add()
1038 mutex_unlock(&fs_devices->device_list_mutex); in device_list_add()
1039 return ERR_PTR(-ENOMEM); in device_list_add()
1041 rcu_string_free(device->name); in device_list_add()
1042 rcu_assign_pointer(device->name, name); in device_list_add()
1043 if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) { in device_list_add()
1044 fs_devices->missing_devices--; in device_list_add()
1045 clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state); in device_list_add()
1047 device->devt = path_devt; in device_list_add()
1056 if (!fs_devices->opened) { in device_list_add()
1057 device->generation = found_transid; in device_list_add()
1058 fs_devices->latest_generation = max_t(u64, found_transid, in device_list_add()
1059 fs_devices->latest_generation); in device_list_add()
1062 fs_devices->total_devices = btrfs_super_num_devices(disk_super); in device_list_add()
1064 mutex_unlock(&fs_devices->device_list_mutex); in device_list_add()
1077 fs_devices = alloc_fs_devices(orig->fsid); in clone_fs_devices()
1081 fs_devices->total_devices = orig->total_devices; in clone_fs_devices()
1083 list_for_each_entry(orig_dev, &orig->devices, dev_list) { in clone_fs_devices()
1090 if (orig_dev->name) in clone_fs_devices()
1091 dev_path = orig_dev->name->str; in clone_fs_devices()
1093 device = btrfs_alloc_device(NULL, &orig_dev->devid, in clone_fs_devices()
1094 orig_dev->uuid, dev_path); in clone_fs_devices()
1100 if (orig_dev->zone_info) { in clone_fs_devices()
1106 ret = -ENOMEM; in clone_fs_devices()
1109 device->zone_info = zone_info; in clone_fs_devices()
1112 list_add(&device->dev_list, &fs_devices->devices); in clone_fs_devices()
1113 device->fs_devices = fs_devices; in clone_fs_devices()
1114 fs_devices->num_devices++; in clone_fs_devices()
1128 list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) { in __btrfs_free_extra_devids()
1129 if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state)) { in __btrfs_free_extra_devids()
1131 &device->dev_state) && in __btrfs_free_extra_devids()
1133 &device->dev_state) && in __btrfs_free_extra_devids()
1135 device->generation > (*latest_dev)->generation)) { in __btrfs_free_extra_devids()
1145 if (device->devid == BTRFS_DEV_REPLACE_DEVID) in __btrfs_free_extra_devids()
1148 if (device->bdev_file) { in __btrfs_free_extra_devids()
1149 fput(device->bdev_file); in __btrfs_free_extra_devids()
1150 device->bdev = NULL; in __btrfs_free_extra_devids()
1151 device->bdev_file = NULL; in __btrfs_free_extra_devids()
1152 fs_devices->open_devices--; in __btrfs_free_extra_devids()
1154 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in __btrfs_free_extra_devids()
1155 list_del_init(&device->dev_alloc_list); in __btrfs_free_extra_devids()
1156 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); in __btrfs_free_extra_devids()
1157 fs_devices->rw_devices--; in __btrfs_free_extra_devids()
1159 list_del_init(&device->dev_list); in __btrfs_free_extra_devids()
1160 fs_devices->num_devices--; in __btrfs_free_extra_devids()
1167 * After we have read the system tree and know devids belonging to this
1178 list_for_each_entry(seed_dev, &fs_devices->seed_list, seed_list) in btrfs_free_extra_devids()
1181 fs_devices->latest_dev = latest_dev; in btrfs_free_extra_devids()
1188 if (!device->bdev) in btrfs_close_bdev()
1191 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in btrfs_close_bdev()
1192 sync_blockdev(device->bdev); in btrfs_close_bdev()
1193 invalidate_bdev(device->bdev); in btrfs_close_bdev()
1196 fput(device->bdev_file); in btrfs_close_bdev()
1201 struct btrfs_fs_devices *fs_devices = device->fs_devices; in btrfs_close_one_device()
1203 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) && in btrfs_close_one_device()
1204 device->devid != BTRFS_DEV_REPLACE_DEVID) { in btrfs_close_one_device()
1205 list_del_init(&device->dev_alloc_list); in btrfs_close_one_device()
1206 fs_devices->rw_devices--; in btrfs_close_one_device()
1209 if (device->devid == BTRFS_DEV_REPLACE_DEVID) in btrfs_close_one_device()
1210 clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); in btrfs_close_one_device()
1212 if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) { in btrfs_close_one_device()
1213 clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state); in btrfs_close_one_device()
1214 fs_devices->missing_devices--; in btrfs_close_one_device()
1218 if (device->bdev) { in btrfs_close_one_device()
1219 fs_devices->open_devices--; in btrfs_close_one_device()
1220 device->bdev = NULL; in btrfs_close_one_device()
1221 device->bdev_file = NULL; in btrfs_close_one_device()
1223 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); in btrfs_close_one_device()
1226 device->fs_info = NULL; in btrfs_close_one_device()
1227 atomic_set(&device->dev_stats_ccnt, 0); in btrfs_close_one_device()
1228 extent_io_tree_release(&device->alloc_state); in btrfs_close_one_device()
1236 * (btrfs_check_rw_degradable() should not fail) - if after mounting the in btrfs_close_one_device()
1241 device->last_flush_error = 0; in btrfs_close_one_device()
1244 WARN_ON(test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state)); in btrfs_close_one_device()
1245 WARN_ON(test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)); in btrfs_close_one_device()
1246 WARN_ON(!list_empty(&device->dev_alloc_list)); in btrfs_close_one_device()
1247 WARN_ON(!list_empty(&device->post_commit_list)); in btrfs_close_one_device()
1256 if (--fs_devices->opened > 0) in close_fs_devices()
1259 list_for_each_entry_safe(device, tmp, &fs_devices->devices, dev_list) in close_fs_devices()
1262 WARN_ON(fs_devices->open_devices); in close_fs_devices()
1263 WARN_ON(fs_devices->rw_devices); in close_fs_devices()
1264 fs_devices->opened = 0; in close_fs_devices()
1265 fs_devices->seeding = false; in close_fs_devices()
1266 fs_devices->fs_info = NULL; in close_fs_devices()
1276 if (!fs_devices->opened) { in btrfs_close_devices()
1277 list_splice_init(&fs_devices->seed_list, &list); in btrfs_close_devices()
1281 * other device, it can be re-initialized during the next mount in btrfs_close_devices()
1282 * without the needing device-scan step. Therefore, it can be in btrfs_close_devices()
1285 if (fs_devices->num_devices == 1) { in btrfs_close_devices()
1286 list_del(&fs_devices->fs_list); in btrfs_close_devices()
1294 list_del(&fs_devices->seed_list); in btrfs_close_devices()
1309 list_for_each_entry_safe(device, tmp_device, &fs_devices->devices, in open_fs_devices()
1315 (!latest_dev || device->generation > latest_dev->generation)) { in open_fs_devices()
1317 } else if (ret2 == -ENODATA) { in open_fs_devices()
1318 fs_devices->num_devices--; in open_fs_devices()
1319 list_del(&device->dev_list); in open_fs_devices()
1326 if (fs_devices->open_devices == 0) { in open_fs_devices()
1329 return -EINVAL; in open_fs_devices()
1332 fs_devices->opened = 1; in open_fs_devices()
1333 fs_devices->latest_dev = latest_dev; in open_fs_devices()
1334 fs_devices->total_rw_bytes = 0; in open_fs_devices()
1335 fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_REGULAR; in open_fs_devices()
1337 fs_devices->rr_min_contig_read = BTRFS_DEFAULT_RR_MIN_CONTIG_READ; in open_fs_devices()
1338 fs_devices->read_devid = latest_dev->devid; in open_fs_devices()
1339 fs_devices->read_policy = btrfs_read_policy_to_enum(btrfs_get_mod_read_policy(), in open_fs_devices()
1341 if (fs_devices->read_policy == BTRFS_READ_POLICY_RR) in open_fs_devices()
1342 fs_devices->collect_fs_stats = true; in open_fs_devices()
1345 if (fs_devices->read_policy == BTRFS_READ_POLICY_RR) in open_fs_devices()
1346 fs_devices->rr_min_contig_read = value; in open_fs_devices()
1347 if (fs_devices->read_policy == BTRFS_READ_POLICY_DEVID) in open_fs_devices()
1348 fs_devices->read_devid = value; in open_fs_devices()
1351 fs_devices->read_policy = BTRFS_READ_POLICY_PID; in open_fs_devices()
1365 if (dev1->devid < dev2->devid) in devid_cmp()
1366 return -1; in devid_cmp()
1367 else if (dev1->devid > dev2->devid) in devid_cmp()
1386 if (fs_devices->opened) { in btrfs_open_devices()
1387 fs_devices->opened++; in btrfs_open_devices()
1390 list_sort(NULL, &fs_devices->devices, devid_cmp); in btrfs_open_devices()
1414 return ERR_PTR(-EINVAL); in btrfs_read_disk_super()
1418 return ERR_PTR(-EINVAL); in btrfs_read_disk_super()
1422 if ((bytenr + sizeof(*disk_super) - 1) >> PAGE_SHIFT != index) in btrfs_read_disk_super()
1423 return ERR_PTR(-EINVAL); in btrfs_read_disk_super()
1426 page = read_cache_page_gfp(bdev->bd_mapping, index, GFP_KERNEL); in btrfs_read_disk_super()
1439 return ERR_PTR(-EINVAL); in btrfs_read_disk_super()
1442 if (disk_super->label[0] && disk_super->label[BTRFS_LABEL_SIZE - 1]) in btrfs_read_disk_super()
1443 disk_super->label[BTRFS_LABEL_SIZE - 1] = 0; in btrfs_read_disk_super()
1469 * A successful scan ensures grub2-probe selects the correct device. in btrfs_skip_registration()
1474 mutex_lock(&fs_devices->device_list_mutex); in btrfs_skip_registration()
1476 if (!fs_devices->opened) { in btrfs_skip_registration()
1477 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_skip_registration()
1481 list_for_each_entry(device, &fs_devices->devices, dev_list) { in btrfs_skip_registration()
1482 if (device->bdev && (device->bdev->bd_dev == devt) && in btrfs_skip_registration()
1483 strcmp(device->name->str, path) != 0) { in btrfs_skip_registration()
1484 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_skip_registration()
1490 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_skip_registration()
1506 * the device or return an error. Multi-device and seeding devices are registered
1534 * Avoid an exclusive open here, as the systemd-udev may initiate the in btrfs_scan_one_device()
1551 * using the userspace command 'btrfs check --super'. in btrfs_scan_one_device()
1566 devt = file_bdev(bdev_file)->bd_dev; in btrfs_scan_one_device()
1568 pr_debug("BTRFS: skip registering single non-seed device %s (%d:%d)\n", in btrfs_scan_one_device()
1580 btrfs_free_stale_devices(device->devt, device); in btrfs_scan_one_device()
1601 lockdep_assert_held(&device->fs_info->chunk_mutex); in contains_pending_extent()
1603 if (find_first_extent_bit(&device->alloc_state, *start, in contains_pending_extent()
1609 physical_end + 1 - physical_start)) { in contains_pending_extent()
1619 switch (device->fs_devices->chunk_alloc_policy) { in dev_extent_search_start()
1638 u64 zone_size = device->zone_info->zone_size; in dev_extent_hole_check_zoned()
1650 *hole_size = *hole_start + *hole_size - pos; in dev_extent_hole_check_zoned()
1664 if (ret == -ERANGE) { in dev_extent_hole_check_zoned()
1671 *hole_size -= zone_size; in dev_extent_hole_check_zoned()
1702 *hole_size = hole_end - *hole_start; in dev_extent_hole_check()
1708 switch (device->fs_devices->chunk_alloc_policy) { in dev_extent_hole_check()
1763 struct btrfs_fs_info *fs_info = device->fs_info; in find_free_dev_extent()
1764 struct btrfs_root *root = fs_info->dev_root; in find_free_dev_extent()
1773 u64 search_end = device->total_bytes; in find_free_dev_extent()
1781 WARN_ON(device->zone_info && in find_free_dev_extent()
1782 !IS_ALIGNED(num_bytes, device->zone_info->zone_size)); in find_free_dev_extent()
1786 ret = -ENOMEM; in find_free_dev_extent()
1791 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { in find_free_dev_extent()
1792 ret = -ENOSPC; in find_free_dev_extent()
1796 path->reada = READA_FORWARD; in find_free_dev_extent()
1797 path->search_commit_root = 1; in find_free_dev_extent()
1798 path->skip_locking = 1; in find_free_dev_extent()
1800 key.objectid = device->devid; in find_free_dev_extent()
1809 l = path->nodes[0]; in find_free_dev_extent()
1810 slot = path->slots[0]; in find_free_dev_extent()
1822 if (key.objectid < device->devid) in find_free_dev_extent()
1825 if (key.objectid > device->devid) in find_free_dev_extent()
1835 hole_size = key.offset - search_start; in find_free_dev_extent()
1865 path->slots[0]++; in find_free_dev_extent()
1875 hole_size = search_end - search_start; in find_free_dev_extent()
1890 ret = -ENOSPC; in find_free_dev_extent()
1907 struct btrfs_fs_info *fs_info = device->fs_info; in btrfs_free_dev_extent()
1908 struct btrfs_root *root = fs_info->dev_root; in btrfs_free_dev_extent()
1918 return -ENOMEM; in btrfs_free_dev_extent()
1920 key.objectid = device->devid; in btrfs_free_dev_extent()
1924 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); in btrfs_free_dev_extent()
1930 leaf = path->nodes[0]; in btrfs_free_dev_extent()
1931 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); in btrfs_free_dev_extent()
1932 extent = btrfs_item_ptr(leaf, path->slots[0], in btrfs_free_dev_extent()
1940 leaf = path->nodes[0]; in btrfs_free_dev_extent()
1941 extent = btrfs_item_ptr(leaf, path->slots[0], in btrfs_free_dev_extent()
1951 set_bit(BTRFS_TRANS_HAVE_FREE_BGS, &trans->transaction->flags); in btrfs_free_dev_extent()
1962 read_lock(&fs_info->mapping_tree_lock); in find_next_chunk()
1963 n = rb_last(&fs_info->mapping_tree.rb_root); in find_next_chunk()
1968 ret = map->start + map->chunk_len; in find_next_chunk()
1970 read_unlock(&fs_info->mapping_tree_lock); in find_next_chunk()
1985 return -ENOMEM; in find_next_devid()
1989 key.offset = (u64)-1; in find_next_devid()
1991 ret = btrfs_search_slot(NULL, fs_info->chunk_root, &key, path, 0, 0); in find_next_devid()
1997 btrfs_err(fs_info, "corrupted chunk tree devid -1 matched"); in find_next_devid()
1998 ret = -EUCLEAN; in find_next_devid()
2002 ret = btrfs_previous_item(fs_info->chunk_root, path, in find_next_devid()
2008 btrfs_item_key_to_cpu(path->nodes[0], &found_key, in find_next_devid()
2009 path->slots[0]); in find_next_devid()
2034 return -ENOMEM; in btrfs_add_dev_item()
2038 key.offset = device->devid; in btrfs_add_dev_item()
2041 ret = btrfs_insert_empty_item(trans, trans->fs_info->chunk_root, path, in btrfs_add_dev_item()
2047 leaf = path->nodes[0]; in btrfs_add_dev_item()
2048 dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item); in btrfs_add_dev_item()
2050 btrfs_set_device_id(leaf, dev_item, device->devid); in btrfs_add_dev_item()
2052 btrfs_set_device_type(leaf, dev_item, device->type); in btrfs_add_dev_item()
2053 btrfs_set_device_io_align(leaf, dev_item, device->io_align); in btrfs_add_dev_item()
2054 btrfs_set_device_io_width(leaf, dev_item, device->io_width); in btrfs_add_dev_item()
2055 btrfs_set_device_sector_size(leaf, dev_item, device->sector_size); in btrfs_add_dev_item()
2066 write_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE); in btrfs_add_dev_item()
2068 write_extent_buffer(leaf, trans->fs_info->fs_devices->metadata_uuid, in btrfs_add_dev_item()
2099 struct btrfs_root *root = device->fs_info->chunk_root; in btrfs_rm_dev_item()
2106 return -ENOMEM; in btrfs_rm_dev_item()
2110 key.offset = device->devid; in btrfs_rm_dev_item()
2113 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); in btrfs_rm_dev_item()
2117 ret = -ENOENT; in btrfs_rm_dev_item()
2140 seq = read_seqbegin(&fs_info->profiles_lock); in btrfs_check_raid_min_devices()
2142 all_avail = fs_info->avail_data_alloc_bits | in btrfs_check_raid_min_devices()
2143 fs_info->avail_system_alloc_bits | in btrfs_check_raid_min_devices()
2144 fs_info->avail_metadata_alloc_bits; in btrfs_check_raid_min_devices()
2145 } while (read_seqretry(&fs_info->profiles_lock, seq)); in btrfs_check_raid_min_devices()
2163 list_for_each_entry(next_device, &fs_devs->devices, dev_list) { in btrfs_find_next_active_device()
2165 !test_bit(BTRFS_DEV_STATE_MISSING, &next_device->dev_state) in btrfs_find_next_active_device()
2166 && next_device->bdev) in btrfs_find_next_active_device()
2182 struct btrfs_fs_info *fs_info = device->fs_info; in btrfs_assign_next_active_device()
2185 next_device = btrfs_find_next_active_device(fs_info->fs_devices, in btrfs_assign_next_active_device()
2189 if (fs_info->sb->s_bdev && in btrfs_assign_next_active_device()
2190 (fs_info->sb->s_bdev == device->bdev)) in btrfs_assign_next_active_device()
2191 fs_info->sb->s_bdev = next_device->bdev; in btrfs_assign_next_active_device()
2193 if (fs_info->fs_devices->latest_dev->bdev == device->bdev) in btrfs_assign_next_active_device()
2194 fs_info->fs_devices->latest_dev = next_device; in btrfs_assign_next_active_device()
2203 u64 num_devices = fs_info->fs_devices->num_devices; in btrfs_num_devices()
2205 down_read(&fs_info->dev_replace.rwsem); in btrfs_num_devices()
2206 if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace)) { in btrfs_num_devices()
2208 num_devices--; in btrfs_num_devices()
2210 up_read(&fs_info->dev_replace.rwsem); in btrfs_num_devices()
2219 const size_t len = sizeof(disk_super->magic); in btrfs_scratch_superblock()
2227 memset(&disk_super->magic, 0, len); in btrfs_scratch_superblock()
2231 ret = sync_blockdev_range(bdev, bytenr, bytenr + len - 1); in btrfs_scratch_superblock()
2240 struct block_device *bdev = device->bdev; in btrfs_scratch_superblocks()
2256 update_dev_time(device->name->str); in btrfs_scratch_superblocks()
2266 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in btrfs_rm_device()
2272 return -EINVAL; in btrfs_rm_device()
2282 ret = btrfs_check_raid_min_devices(fs_info, num_devices - 1); in btrfs_rm_device()
2286 device = btrfs_find_device(fs_info->fs_devices, args); in btrfs_rm_device()
2288 if (args->missing) in btrfs_rm_device()
2291 ret = -ENOENT; in btrfs_rm_device()
2298 btrfs_dev_name(device), device->devid); in btrfs_rm_device()
2299 return -ETXTBSY; in btrfs_rm_device()
2302 if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) in btrfs_rm_device()
2305 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) && in btrfs_rm_device()
2306 fs_info->fs_devices->rw_devices == 1) in btrfs_rm_device()
2309 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in btrfs_rm_device()
2310 mutex_lock(&fs_info->chunk_mutex); in btrfs_rm_device()
2311 list_del_init(&device->dev_alloc_list); in btrfs_rm_device()
2312 device->fs_devices->rw_devices--; in btrfs_rm_device()
2313 mutex_unlock(&fs_info->chunk_mutex); in btrfs_rm_device()
2320 trans = btrfs_start_transaction(fs_info->chunk_root, 0); in btrfs_rm_device()
2331 device->devid, ret); in btrfs_rm_device()
2337 clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); in btrfs_rm_device()
2353 * its own fs_devices listed under the fs_devices->seed_list. in btrfs_rm_device()
2355 cur_devices = device->fs_devices; in btrfs_rm_device()
2356 mutex_lock(&fs_devices->device_list_mutex); in btrfs_rm_device()
2357 list_del_rcu(&device->dev_list); in btrfs_rm_device()
2359 cur_devices->num_devices--; in btrfs_rm_device()
2360 cur_devices->total_devices--; in btrfs_rm_device()
2363 fs_devices->total_devices--; in btrfs_rm_device()
2365 if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) in btrfs_rm_device()
2366 cur_devices->missing_devices--; in btrfs_rm_device()
2370 if (device->bdev_file) { in btrfs_rm_device()
2371 cur_devices->open_devices--; in btrfs_rm_device()
2376 num_devices = btrfs_super_num_devices(fs_info->super_copy) - 1; in btrfs_rm_device()
2377 btrfs_set_super_num_devices(fs_info->super_copy, num_devices); in btrfs_rm_device()
2378 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_rm_device()
2387 * ->open_mutex on the block device and it's dependencies. Instead in btrfs_rm_device()
2390 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in btrfs_rm_device()
2392 if (device->bdev) { in btrfs_rm_device()
2393 sync_blockdev(device->bdev); in btrfs_rm_device()
2394 invalidate_bdev(device->bdev); in btrfs_rm_device()
2398 *bdev_file = device->bdev_file; in btrfs_rm_device()
2406 * seed_devices, we can simply decrement cur_devices->opened and then in btrfs_rm_device()
2409 if (cur_devices->num_devices == 0) { in btrfs_rm_device()
2410 list_del_init(&cur_devices->seed_list); in btrfs_rm_device()
2411 ASSERT(cur_devices->opened == 1); in btrfs_rm_device()
2412 cur_devices->opened--; in btrfs_rm_device()
2421 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in btrfs_rm_device()
2422 mutex_lock(&fs_info->chunk_mutex); in btrfs_rm_device()
2423 list_add(&device->dev_alloc_list, in btrfs_rm_device()
2424 &fs_devices->alloc_list); in btrfs_rm_device()
2425 device->fs_devices->rw_devices++; in btrfs_rm_device()
2426 mutex_unlock(&fs_info->chunk_mutex); in btrfs_rm_device()
2435 lockdep_assert_held(&srcdev->fs_info->fs_devices->device_list_mutex); in btrfs_rm_dev_replace_remove_srcdev()
2438 * in case of fs with no seed, srcdev->fs_devices will point in btrfs_rm_dev_replace_remove_srcdev()
2443 fs_devices = srcdev->fs_devices; in btrfs_rm_dev_replace_remove_srcdev()
2445 list_del_rcu(&srcdev->dev_list); in btrfs_rm_dev_replace_remove_srcdev()
2446 list_del(&srcdev->dev_alloc_list); in btrfs_rm_dev_replace_remove_srcdev()
2447 fs_devices->num_devices--; in btrfs_rm_dev_replace_remove_srcdev()
2448 if (test_bit(BTRFS_DEV_STATE_MISSING, &srcdev->dev_state)) in btrfs_rm_dev_replace_remove_srcdev()
2449 fs_devices->missing_devices--; in btrfs_rm_dev_replace_remove_srcdev()
2451 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state)) in btrfs_rm_dev_replace_remove_srcdev()
2452 fs_devices->rw_devices--; in btrfs_rm_dev_replace_remove_srcdev()
2454 if (srcdev->bdev) in btrfs_rm_dev_replace_remove_srcdev()
2455 fs_devices->open_devices--; in btrfs_rm_dev_replace_remove_srcdev()
2460 struct btrfs_fs_devices *fs_devices = srcdev->fs_devices; in btrfs_rm_dev_replace_free_srcdev()
2469 if (!fs_devices->num_devices) { in btrfs_rm_dev_replace_free_srcdev()
2476 ASSERT(fs_devices->seeding); in btrfs_rm_dev_replace_free_srcdev()
2478 list_del_init(&fs_devices->seed_list); in btrfs_rm_dev_replace_free_srcdev()
2487 struct btrfs_fs_devices *fs_devices = tgtdev->fs_info->fs_devices; in btrfs_destroy_dev_replace_tgtdev()
2489 mutex_lock(&fs_devices->device_list_mutex); in btrfs_destroy_dev_replace_tgtdev()
2493 if (tgtdev->bdev) in btrfs_destroy_dev_replace_tgtdev()
2494 fs_devices->open_devices--; in btrfs_destroy_dev_replace_tgtdev()
2496 fs_devices->num_devices--; in btrfs_destroy_dev_replace_tgtdev()
2500 list_del_rcu(&tgtdev->dev_list); in btrfs_destroy_dev_replace_tgtdev()
2502 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_destroy_dev_replace_tgtdev()
2504 btrfs_scratch_superblocks(tgtdev->fs_info, tgtdev); in btrfs_destroy_dev_replace_tgtdev()
2527 * Return: 0 for success, -errno for failure
2538 return -EINVAL; in btrfs_get_dev_args_from_path()
2540 args->missing = true; in btrfs_get_dev_args_from_path()
2544 args->uuid = kzalloc(BTRFS_UUID_SIZE, GFP_KERNEL); in btrfs_get_dev_args_from_path()
2545 args->fsid = kzalloc(BTRFS_FSID_SIZE, GFP_KERNEL); in btrfs_get_dev_args_from_path()
2546 if (!args->uuid || !args->fsid) { in btrfs_get_dev_args_from_path()
2548 return -ENOMEM; in btrfs_get_dev_args_from_path()
2558 args->devid = btrfs_stack_device_id(&disk_super->dev_item); in btrfs_get_dev_args_from_path()
2559 memcpy(args->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE); in btrfs_get_dev_args_from_path()
2561 memcpy(args->fsid, disk_super->metadata_uuid, BTRFS_FSID_SIZE); in btrfs_get_dev_args_from_path()
2563 memcpy(args->fsid, disk_super->fsid, BTRFS_FSID_SIZE); in btrfs_get_dev_args_from_path()
2571 * allocate our ->uuid and ->fsid pointers, everybody else uses local variables
2576 kfree(args->uuid); in btrfs_put_dev_args_from_path()
2577 kfree(args->fsid); in btrfs_put_dev_args_from_path()
2578 args->uuid = NULL; in btrfs_put_dev_args_from_path()
2579 args->fsid = NULL; in btrfs_put_dev_args_from_path()
2592 device = btrfs_find_device(fs_info->fs_devices, &args); in btrfs_find_device_by_devspec()
2594 return ERR_PTR(-ENOENT); in btrfs_find_device_by_devspec()
2601 device = btrfs_find_device(fs_info->fs_devices, &args); in btrfs_find_device_by_devspec()
2604 return ERR_PTR(-ENOENT); in btrfs_find_device_by_devspec()
2610 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in btrfs_init_sprout()
2615 if (!fs_devices->seeding) in btrfs_init_sprout()
2616 return ERR_PTR(-EINVAL); in btrfs_init_sprout()
2620 * fs_info->fs_devices->seed_list in btrfs_init_sprout()
2638 list_add(&old_devices->fs_list, &fs_uuids); in btrfs_init_sprout()
2641 seed_devices->opened = 1; in btrfs_init_sprout()
2642 INIT_LIST_HEAD(&seed_devices->devices); in btrfs_init_sprout()
2643 INIT_LIST_HEAD(&seed_devices->alloc_list); in btrfs_init_sprout()
2644 mutex_init(&seed_devices->device_list_mutex); in btrfs_init_sprout()
2651 * Generate a new fsid for the sprouted read-write filesystem.
2656 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in btrfs_setup_sprout()
2657 struct btrfs_super_block *disk_super = fs_info->super_copy; in btrfs_setup_sprout()
2670 * - All device ops and balance - as we are in btrfs_exclop_start. in btrfs_setup_sprout()
2671 * - Various dev_list readers - are using RCU. in btrfs_setup_sprout()
2672 * - btrfs_ioctl_fitrim() - is using RCU. in btrfs_setup_sprout()
2674 * For-read threads as below are using device_list_mutex: in btrfs_setup_sprout()
2675 * - Readonly scrub btrfs_scrub_dev() in btrfs_setup_sprout()
2676 * - Readonly scrub btrfs_scrub_progress() in btrfs_setup_sprout()
2677 * - btrfs_get_dev_stats() in btrfs_setup_sprout()
2679 lockdep_assert_held(&fs_devices->device_list_mutex); in btrfs_setup_sprout()
2681 list_splice_init_rcu(&fs_devices->devices, &seed_devices->devices, in btrfs_setup_sprout()
2683 list_for_each_entry(device, &seed_devices->devices, dev_list) in btrfs_setup_sprout()
2684 device->fs_devices = seed_devices; in btrfs_setup_sprout()
2686 fs_devices->seeding = false; in btrfs_setup_sprout()
2687 fs_devices->num_devices = 0; in btrfs_setup_sprout()
2688 fs_devices->open_devices = 0; in btrfs_setup_sprout()
2689 fs_devices->missing_devices = 0; in btrfs_setup_sprout()
2690 fs_devices->rotating = false; in btrfs_setup_sprout()
2691 list_add(&seed_devices->seed_list, &fs_devices->seed_list); in btrfs_setup_sprout()
2693 generate_random_uuid(fs_devices->fsid); in btrfs_setup_sprout()
2694 memcpy(fs_devices->metadata_uuid, fs_devices->fsid, BTRFS_FSID_SIZE); in btrfs_setup_sprout()
2695 memcpy(disk_super->fsid, fs_devices->fsid, BTRFS_FSID_SIZE); in btrfs_setup_sprout()
2708 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_finish_sprout()
2709 struct btrfs_root *root = fs_info->chunk_root; in btrfs_finish_sprout()
2721 return -ENOMEM; in btrfs_finish_sprout()
2734 leaf = path->nodes[0]; in btrfs_finish_sprout()
2736 if (path->slots[0] >= btrfs_header_nritems(leaf)) { in btrfs_finish_sprout()
2742 leaf = path->nodes[0]; in btrfs_finish_sprout()
2743 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); in btrfs_finish_sprout()
2748 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); in btrfs_finish_sprout()
2753 dev_item = btrfs_item_ptr(leaf, path->slots[0], in btrfs_finish_sprout()
2762 device = btrfs_find_device(fs_info->fs_devices, &args); in btrfs_finish_sprout()
2765 if (device->fs_devices->seeding) in btrfs_finish_sprout()
2767 device->generation); in btrfs_finish_sprout()
2769 path->slots[0]++; in btrfs_finish_sprout()
2780 struct btrfs_root *root = fs_info->dev_root; in btrfs_init_new_device()
2784 struct super_block *sb = fs_info->sb; in btrfs_init_new_device()
2785 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in btrfs_init_new_device()
2793 if (sb_rdonly(sb) && !fs_devices->seeding) in btrfs_init_new_device()
2794 return -EROFS; in btrfs_init_new_device()
2797 fs_info->bdev_holder, NULL); in btrfs_init_new_device()
2802 ret = -EINVAL; in btrfs_init_new_device()
2806 if (fs_devices->seeding) { in btrfs_init_new_device()
2808 down_write(&sb->s_umount); in btrfs_init_new_device()
2816 list_for_each_entry_rcu(device, &fs_devices->devices, dev_list) { in btrfs_init_new_device()
2817 if (device->bdev == file_bdev(bdev_file)) { in btrfs_init_new_device()
2818 ret = -EEXIST; in btrfs_init_new_device()
2832 device->fs_info = fs_info; in btrfs_init_new_device()
2833 device->bdev_file = bdev_file; in btrfs_init_new_device()
2834 device->bdev = file_bdev(bdev_file); in btrfs_init_new_device()
2835 ret = lookup_bdev(device_path, &device->devt); in btrfs_init_new_device()
2849 set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); in btrfs_init_new_device()
2850 device->generation = trans->transid; in btrfs_init_new_device()
2851 device->io_width = fs_info->sectorsize; in btrfs_init_new_device()
2852 device->io_align = fs_info->sectorsize; in btrfs_init_new_device()
2853 device->sector_size = fs_info->sectorsize; in btrfs_init_new_device()
2854 device->total_bytes = in btrfs_init_new_device()
2855 round_down(bdev_nr_bytes(device->bdev), fs_info->sectorsize); in btrfs_init_new_device()
2856 device->disk_total_bytes = device->total_bytes; in btrfs_init_new_device()
2857 device->commit_total_bytes = device->total_bytes; in btrfs_init_new_device()
2858 set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); in btrfs_init_new_device()
2859 clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); in btrfs_init_new_device()
2860 device->dev_stats_valid = 1; in btrfs_init_new_device()
2861 set_blocksize(device->bdev_file, BTRFS_BDEV_BLOCKSIZE); in btrfs_init_new_device()
2873 mutex_lock(&fs_devices->device_list_mutex); in btrfs_init_new_device()
2876 btrfs_assign_next_active_device(fs_info->fs_devices->latest_dev, in btrfs_init_new_device()
2880 device->fs_devices = fs_devices; in btrfs_init_new_device()
2882 mutex_lock(&fs_info->chunk_mutex); in btrfs_init_new_device()
2883 list_add_rcu(&device->dev_list, &fs_devices->devices); in btrfs_init_new_device()
2884 list_add(&device->dev_alloc_list, &fs_devices->alloc_list); in btrfs_init_new_device()
2885 fs_devices->num_devices++; in btrfs_init_new_device()
2886 fs_devices->open_devices++; in btrfs_init_new_device()
2887 fs_devices->rw_devices++; in btrfs_init_new_device()
2888 fs_devices->total_devices++; in btrfs_init_new_device()
2889 fs_devices->total_rw_bytes += device->total_bytes; in btrfs_init_new_device()
2891 atomic64_add(device->total_bytes, &fs_info->free_chunk_space); in btrfs_init_new_device()
2893 if (!bdev_nonrot(device->bdev)) in btrfs_init_new_device()
2894 fs_devices->rotating = true; in btrfs_init_new_device()
2896 orig_super_total_bytes = btrfs_super_total_bytes(fs_info->super_copy); in btrfs_init_new_device()
2897 btrfs_set_super_total_bytes(fs_info->super_copy, in btrfs_init_new_device()
2898 round_down(orig_super_total_bytes + device->total_bytes, in btrfs_init_new_device()
2899 fs_info->sectorsize)); in btrfs_init_new_device()
2901 orig_super_num_devices = btrfs_super_num_devices(fs_info->super_copy); in btrfs_init_new_device()
2902 btrfs_set_super_num_devices(fs_info->super_copy, in btrfs_init_new_device()
2911 mutex_unlock(&fs_info->chunk_mutex); in btrfs_init_new_device()
2916 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_init_new_device()
2919 mutex_lock(&fs_info->chunk_mutex); in btrfs_init_new_device()
2921 mutex_unlock(&fs_info->chunk_mutex); in btrfs_init_new_device()
2952 up_write(&sb->s_umount); in btrfs_init_new_device()
2964 if (PTR_ERR(trans) == -ENOENT) in btrfs_init_new_device()
2977 * We can ignore the return value as it typically returns -EINVAL and in btrfs_init_new_device()
2980 btrfs_forget_devices(device->devt); in btrfs_init_new_device()
2989 mutex_lock(&fs_info->fs_devices->device_list_mutex); in btrfs_init_new_device()
2990 mutex_lock(&fs_info->chunk_mutex); in btrfs_init_new_device()
2991 list_del_rcu(&device->dev_list); in btrfs_init_new_device()
2992 list_del(&device->dev_alloc_list); in btrfs_init_new_device()
2993 fs_info->fs_devices->num_devices--; in btrfs_init_new_device()
2994 fs_info->fs_devices->open_devices--; in btrfs_init_new_device()
2995 fs_info->fs_devices->rw_devices--; in btrfs_init_new_device()
2996 fs_info->fs_devices->total_devices--; in btrfs_init_new_device()
2997 fs_info->fs_devices->total_rw_bytes -= device->total_bytes; in btrfs_init_new_device()
2998 atomic64_sub(device->total_bytes, &fs_info->free_chunk_space); in btrfs_init_new_device()
2999 btrfs_set_super_total_bytes(fs_info->super_copy, in btrfs_init_new_device()
3001 btrfs_set_super_num_devices(fs_info->super_copy, in btrfs_init_new_device()
3003 mutex_unlock(&fs_info->chunk_mutex); in btrfs_init_new_device()
3004 mutex_unlock(&fs_info->fs_devices->device_list_mutex); in btrfs_init_new_device()
3016 up_write(&sb->s_umount); in btrfs_init_new_device()
3026 struct btrfs_root *root = device->fs_info->chunk_root; in btrfs_update_device()
3033 return -ENOMEM; in btrfs_update_device()
3037 key.offset = device->devid; in btrfs_update_device()
3044 ret = -ENOENT; in btrfs_update_device()
3048 leaf = path->nodes[0]; in btrfs_update_device()
3049 dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item); in btrfs_update_device()
3051 btrfs_set_device_id(leaf, dev_item, device->devid); in btrfs_update_device()
3052 btrfs_set_device_type(leaf, dev_item, device->type); in btrfs_update_device()
3053 btrfs_set_device_io_align(leaf, dev_item, device->io_align); in btrfs_update_device()
3054 btrfs_set_device_io_width(leaf, dev_item, device->io_width); in btrfs_update_device()
3055 btrfs_set_device_sector_size(leaf, dev_item, device->sector_size); in btrfs_update_device()
3068 struct btrfs_fs_info *fs_info = device->fs_info; in btrfs_grow_device()
3069 struct btrfs_super_block *super_copy = fs_info->super_copy; in btrfs_grow_device()
3074 if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) in btrfs_grow_device()
3075 return -EACCES; in btrfs_grow_device()
3077 new_size = round_down(new_size, fs_info->sectorsize); in btrfs_grow_device()
3079 mutex_lock(&fs_info->chunk_mutex); in btrfs_grow_device()
3081 diff = round_down(new_size - device->total_bytes, fs_info->sectorsize); in btrfs_grow_device()
3083 if (new_size <= device->total_bytes || in btrfs_grow_device()
3084 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { in btrfs_grow_device()
3085 mutex_unlock(&fs_info->chunk_mutex); in btrfs_grow_device()
3086 return -EINVAL; in btrfs_grow_device()
3090 round_down(old_total + diff, fs_info->sectorsize)); in btrfs_grow_device()
3091 device->fs_devices->total_rw_bytes += diff; in btrfs_grow_device()
3092 atomic64_add(diff, &fs_info->free_chunk_space); in btrfs_grow_device()
3096 btrfs_clear_space_info_full(device->fs_info); in btrfs_grow_device()
3097 if (list_empty(&device->post_commit_list)) in btrfs_grow_device()
3098 list_add_tail(&device->post_commit_list, in btrfs_grow_device()
3099 &trans->transaction->dev_update_list); in btrfs_grow_device()
3100 mutex_unlock(&fs_info->chunk_mutex); in btrfs_grow_device()
3111 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_free_chunk()
3112 struct btrfs_root *root = fs_info->chunk_root; in btrfs_free_chunk()
3119 return -ENOMEM; in btrfs_free_chunk()
3125 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); in btrfs_free_chunk()
3131 btrfs_abort_transaction(trans, -ENOENT); in btrfs_free_chunk()
3132 ret = -EUCLEAN; in btrfs_free_chunk()
3149 struct btrfs_super_block *super_copy = fs_info->super_copy; in btrfs_del_sys_chunk()
3160 lockdep_assert_held(&fs_info->chunk_mutex); in btrfs_del_sys_chunk()
3163 ptr = super_copy->sys_chunk_array; in btrfs_del_sys_chunk()
3177 ret = -EIO; in btrfs_del_sys_chunk()
3182 memmove(ptr, ptr + len, array_size - (cur + len)); in btrfs_del_sys_chunk()
3183 array_size -= len; in btrfs_del_sys_chunk()
3196 struct rb_node *node = fs_info->mapping_tree.rb_root.rb_node; in btrfs_find_chunk_map_nolock()
3207 if (logical < map->start) { in btrfs_find_chunk_map_nolock()
3208 node = node->rb_left; in btrfs_find_chunk_map_nolock()
3209 } else if (logical >= map->start + map->chunk_len) { in btrfs_find_chunk_map_nolock()
3210 node = node->rb_right; in btrfs_find_chunk_map_nolock()
3212 refcount_inc(&map->refs); in btrfs_find_chunk_map_nolock()
3221 while (prev && logical >= prev_map->start + prev_map->chunk_len) { in btrfs_find_chunk_map_nolock()
3229 while (prev && logical < prev_map->start) { in btrfs_find_chunk_map_nolock()
3246 if (end > prev_map->start && in btrfs_find_chunk_map_nolock()
3247 logical < prev_map->start + prev_map->chunk_len) { in btrfs_find_chunk_map_nolock()
3248 refcount_inc(&prev_map->refs); in btrfs_find_chunk_map_nolock()
3261 read_lock(&fs_info->mapping_tree_lock); in btrfs_find_chunk_map()
3263 read_unlock(&fs_info->mapping_tree_lock); in btrfs_find_chunk_map()
3287 return ERR_PTR(-EINVAL); in btrfs_get_chunk_map()
3290 if (unlikely(map->start > logical || map->start + map->chunk_len <= logical)) { in btrfs_get_chunk_map()
3292 "found a bad chunk map, wanted %llu-%llu, found %llu-%llu", in btrfs_get_chunk_map()
3293 logical, logical + length, map->start, in btrfs_get_chunk_map()
3294 map->start + map->chunk_len); in btrfs_get_chunk_map()
3296 return ERR_PTR(-EINVAL); in btrfs_get_chunk_map()
3313 lockdep_assert_held(&trans->fs_info->chunk_mutex); in remove_chunk_item()
3315 for (i = 0; i < map->num_stripes; i++) { in remove_chunk_item()
3318 ret = btrfs_update_device(trans, map->stripes[i].dev); in remove_chunk_item()
3328 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_remove_chunk()
3332 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in btrfs_remove_chunk()
3349 * fs_info->chunk_mutex. Note that here we do not acquire the chunk_mutex in btrfs_remove_chunk()
3351 * items from the devices btree - COWing an extent buffer from the btree in btrfs_remove_chunk()
3353 * lock again fs_info->chunk_mutex. in btrfs_remove_chunk()
3355 mutex_lock(&fs_devices->device_list_mutex); in btrfs_remove_chunk()
3356 for (i = 0; i < map->num_stripes; i++) { in btrfs_remove_chunk()
3357 struct btrfs_device *device = map->stripes[i].dev; in btrfs_remove_chunk()
3359 map->stripes[i].physical, in btrfs_remove_chunk()
3362 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_remove_chunk()
3367 if (device->bytes_used > 0) { in btrfs_remove_chunk()
3368 mutex_lock(&fs_info->chunk_mutex); in btrfs_remove_chunk()
3370 device->bytes_used - dev_extent_len); in btrfs_remove_chunk()
3371 atomic64_add(dev_extent_len, &fs_info->free_chunk_space); in btrfs_remove_chunk()
3373 mutex_unlock(&fs_info->chunk_mutex); in btrfs_remove_chunk()
3376 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_remove_chunk()
3379 * We acquire fs_info->chunk_mutex for 2 reasons: in btrfs_remove_chunk()
3382 * reserve system space, do all chunk btree updates and deletions, and in btrfs_remove_chunk()
3383 * update the system chunk array in the superblock while holding this in btrfs_remove_chunk()
3391 * (dev-replace.c:btrfs_dev_replace_finishing()), so we could grab the in btrfs_remove_chunk()
3399 trans->removing_chunk = true; in btrfs_remove_chunk()
3400 mutex_lock(&fs_info->chunk_mutex); in btrfs_remove_chunk()
3402 check_system_chunk(trans, map->type); in btrfs_remove_chunk()
3406 * Normally we should not get -ENOSPC since we reserved space before in btrfs_remove_chunk()
3409 * Despite our system space_info having enough free space, we may not in btrfs_remove_chunk()
3411 * an incompatible profile, which will force us to allocate a new system in btrfs_remove_chunk()
3413 * check_system_space() above, a scrub turned the only system block group in btrfs_remove_chunk()
3417 * So if we get -ENOSPC, allocate a new system chunk and retry once. in btrfs_remove_chunk()
3419 if (ret == -ENOSPC) { in btrfs_remove_chunk()
3446 trace_btrfs_chunk_free(fs_info, map, chunk_offset, map->chunk_len); in btrfs_remove_chunk()
3448 if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) { in btrfs_remove_chunk()
3456 mutex_unlock(&fs_info->chunk_mutex); in btrfs_remove_chunk()
3457 trans->removing_chunk = false; in btrfs_remove_chunk()
3461 * system space we previously reserved (with check_system_chunk()). in btrfs_remove_chunk()
3472 if (trans->removing_chunk) { in btrfs_remove_chunk()
3473 mutex_unlock(&fs_info->chunk_mutex); in btrfs_remove_chunk()
3474 trans->removing_chunk = false; in btrfs_remove_chunk()
3483 struct btrfs_root *root = fs_info->chunk_root; in btrfs_relocate_chunk()
3492 return -EINVAL; in btrfs_relocate_chunk()
3507 lockdep_assert_held(&fs_info->reclaim_bgs_lock); in btrfs_relocate_chunk()
3525 return -ENOENT; in btrfs_relocate_chunk()
3526 btrfs_discard_cancel_work(&fs_info->discard_ctl, block_group); in btrfs_relocate_chunk()
3527 length = block_group->length; in btrfs_relocate_chunk()
3531 * On a zoned file system, discard the whole block group, this will in btrfs_relocate_chunk()
3544 trans = btrfs_start_trans_remove_block_group(root->fs_info, in btrfs_relocate_chunk()
3548 btrfs_handle_fs_error(root->fs_info, ret, NULL); in btrfs_relocate_chunk()
3563 struct btrfs_root *chunk_root = fs_info->chunk_root; in btrfs_relocate_sys_chunks()
3576 return -ENOMEM; in btrfs_relocate_sys_chunks()
3580 key.offset = (u64)-1; in btrfs_relocate_sys_chunks()
3584 mutex_lock(&fs_info->reclaim_bgs_lock); in btrfs_relocate_sys_chunks()
3587 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_relocate_sys_chunks()
3593 * offset -1, which is not possible. On subsequent in btrfs_relocate_sys_chunks()
3598 ret = -EUCLEAN; in btrfs_relocate_sys_chunks()
3599 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_relocate_sys_chunks()
3606 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_relocate_sys_chunks()
3612 leaf = path->nodes[0]; in btrfs_relocate_sys_chunks()
3613 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); in btrfs_relocate_sys_chunks()
3615 chunk = btrfs_item_ptr(leaf, path->slots[0], in btrfs_relocate_sys_chunks()
3622 if (ret == -ENOSPC) in btrfs_relocate_sys_chunks()
3627 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_relocate_sys_chunks()
3631 key.offset = found_key.offset - 1; in btrfs_relocate_sys_chunks()
3639 ret = -ENOSPC; in btrfs_relocate_sys_chunks()
3660 chunk_type = cache->flags; in btrfs_may_alloc_data_chunk()
3666 spin_lock(&fs_info->data_sinfo->lock); in btrfs_may_alloc_data_chunk()
3667 bytes_used = fs_info->data_sinfo->bytes_used; in btrfs_may_alloc_data_chunk()
3668 spin_unlock(&fs_info->data_sinfo->lock); in btrfs_may_alloc_data_chunk()
3674 trans = btrfs_join_transaction(fs_info->tree_root); in btrfs_may_alloc_data_chunk()
3693 cpu->profiles = le64_to_cpu(disk->profiles); in btrfs_disk_balance_args_to_cpu()
3694 cpu->usage = le64_to_cpu(disk->usage); in btrfs_disk_balance_args_to_cpu()
3695 cpu->devid = le64_to_cpu(disk->devid); in btrfs_disk_balance_args_to_cpu()
3696 cpu->pstart = le64_to_cpu(disk->pstart); in btrfs_disk_balance_args_to_cpu()
3697 cpu->pend = le64_to_cpu(disk->pend); in btrfs_disk_balance_args_to_cpu()
3698 cpu->vstart = le64_to_cpu(disk->vstart); in btrfs_disk_balance_args_to_cpu()
3699 cpu->vend = le64_to_cpu(disk->vend); in btrfs_disk_balance_args_to_cpu()
3700 cpu->target = le64_to_cpu(disk->target); in btrfs_disk_balance_args_to_cpu()
3701 cpu->flags = le64_to_cpu(disk->flags); in btrfs_disk_balance_args_to_cpu()
3702 cpu->limit = le64_to_cpu(disk->limit); in btrfs_disk_balance_args_to_cpu()
3703 cpu->stripes_min = le32_to_cpu(disk->stripes_min); in btrfs_disk_balance_args_to_cpu()
3704 cpu->stripes_max = le32_to_cpu(disk->stripes_max); in btrfs_disk_balance_args_to_cpu()
3712 disk->profiles = cpu_to_le64(cpu->profiles); in btrfs_cpu_balance_args_to_disk()
3713 disk->usage = cpu_to_le64(cpu->usage); in btrfs_cpu_balance_args_to_disk()
3714 disk->devid = cpu_to_le64(cpu->devid); in btrfs_cpu_balance_args_to_disk()
3715 disk->pstart = cpu_to_le64(cpu->pstart); in btrfs_cpu_balance_args_to_disk()
3716 disk->pend = cpu_to_le64(cpu->pend); in btrfs_cpu_balance_args_to_disk()
3717 disk->vstart = cpu_to_le64(cpu->vstart); in btrfs_cpu_balance_args_to_disk()
3718 disk->vend = cpu_to_le64(cpu->vend); in btrfs_cpu_balance_args_to_disk()
3719 disk->target = cpu_to_le64(cpu->target); in btrfs_cpu_balance_args_to_disk()
3720 disk->flags = cpu_to_le64(cpu->flags); in btrfs_cpu_balance_args_to_disk()
3721 disk->limit = cpu_to_le64(cpu->limit); in btrfs_cpu_balance_args_to_disk()
3722 disk->stripes_min = cpu_to_le32(cpu->stripes_min); in btrfs_cpu_balance_args_to_disk()
3723 disk->stripes_max = cpu_to_le32(cpu->stripes_max); in btrfs_cpu_balance_args_to_disk()
3729 struct btrfs_root *root = fs_info->tree_root; in insert_balance_item()
3740 return -ENOMEM; in insert_balance_item()
3757 leaf = path->nodes[0]; in insert_balance_item()
3758 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item); in insert_balance_item()
3762 btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->data); in insert_balance_item()
3764 btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->meta); in insert_balance_item()
3766 btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->sys); in insert_balance_item()
3768 btrfs_set_balance_flags(leaf, item, bctl->flags); in insert_balance_item()
3779 struct btrfs_root *root = fs_info->tree_root; in del_balance_item()
3787 return -ENOMEM; in del_balance_item()
3799 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); in del_balance_item()
3803 ret = -ENOENT; in del_balance_item()
3825 if (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) in update_balance_args()
3826 bctl->data.flags |= BTRFS_BALANCE_ARGS_SOFT; in update_balance_args()
3827 if (bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) in update_balance_args()
3828 bctl->sys.flags |= BTRFS_BALANCE_ARGS_SOFT; in update_balance_args()
3829 if (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) in update_balance_args()
3830 bctl->meta.flags |= BTRFS_BALANCE_ARGS_SOFT; in update_balance_args()
3836 * converted - that will keep us from relocating unconverted in update_balance_args()
3839 if (!(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE) && in update_balance_args()
3840 !(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) && in update_balance_args()
3841 !(bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)) { in update_balance_args()
3842 bctl->data.flags |= BTRFS_BALANCE_ARGS_USAGE; in update_balance_args()
3843 bctl->data.usage = 90; in update_balance_args()
3845 if (!(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE) && in update_balance_args()
3846 !(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) && in update_balance_args()
3847 !(bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)) { in update_balance_args()
3848 bctl->sys.flags |= BTRFS_BALANCE_ARGS_USAGE; in update_balance_args()
3849 bctl->sys.usage = 90; in update_balance_args()
3851 if (!(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE) && in update_balance_args()
3852 !(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) && in update_balance_args()
3853 !(bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)) { in update_balance_args()
3854 bctl->meta.flags |= BTRFS_BALANCE_ARGS_USAGE; in update_balance_args()
3855 bctl->meta.usage = 90; in update_balance_args()
3864 struct btrfs_balance_control *bctl = fs_info->balance_ctl; in reset_balance_state()
3867 ASSERT(fs_info->balance_ctl); in reset_balance_state()
3869 spin_lock(&fs_info->balance_lock); in reset_balance_state()
3870 fs_info->balance_ctl = NULL; in reset_balance_state()
3871 spin_unlock(&fs_info->balance_lock); in reset_balance_state()
3889 if (bargs->profiles & chunk_type) in chunk_profiles_filter()
3905 chunk_used = cache->used; in chunk_usage_range_filter()
3907 if (bargs->usage_min == 0) in chunk_usage_range_filter()
3910 user_thresh_min = mult_perc(cache->length, bargs->usage_min); in chunk_usage_range_filter()
3912 if (bargs->usage_max == 0) in chunk_usage_range_filter()
3914 else if (bargs->usage_max > 100) in chunk_usage_range_filter()
3915 user_thresh_max = cache->length; in chunk_usage_range_filter()
3917 user_thresh_max = mult_perc(cache->length, bargs->usage_max); in chunk_usage_range_filter()
3934 chunk_used = cache->used; in chunk_usage_filter()
3936 if (bargs->usage_min == 0) in chunk_usage_filter()
3938 else if (bargs->usage > 100) in chunk_usage_filter()
3939 user_thresh = cache->length; in chunk_usage_filter()
3941 user_thresh = mult_perc(cache->length, bargs->usage); in chunk_usage_filter()
3960 if (btrfs_stripe_devid(leaf, stripe) == bargs->devid) in chunk_devid_filter()
3973 return (num_stripes - nparity) / ncopies; in calc_data_stripes()
3989 if (!(bargs->flags & BTRFS_BALANCE_ARGS_DEVID)) in chunk_drange_filter()
3997 if (btrfs_stripe_devid(leaf, stripe) != bargs->devid) in chunk_drange_filter()
4004 if (stripe_offset < bargs->pend && in chunk_drange_filter()
4005 stripe_offset + stripe_length > bargs->pstart) in chunk_drange_filter()
4018 if (chunk_offset < bargs->vend && in chunk_vrange_filter()
4019 chunk_offset + btrfs_chunk_length(leaf, chunk) > bargs->vstart) in chunk_vrange_filter()
4032 if (bargs->stripes_min <= num_stripes in chunk_stripes_range_filter()
4033 && num_stripes <= bargs->stripes_max) in chunk_stripes_range_filter()
4042 if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT)) in chunk_soft_convert_filter()
4048 if (bargs->target == chunk_type) in chunk_soft_convert_filter()
4057 struct btrfs_fs_info *fs_info = leaf->fs_info; in should_balance_chunk()
4058 struct btrfs_balance_control *bctl = fs_info->balance_ctl; in should_balance_chunk()
4064 (bctl->flags & BTRFS_BALANCE_TYPE_MASK))) { in should_balance_chunk()
4069 bargs = &bctl->data; in should_balance_chunk()
4071 bargs = &bctl->sys; in should_balance_chunk()
4073 bargs = &bctl->meta; in should_balance_chunk()
4076 if ((bargs->flags & BTRFS_BALANCE_ARGS_PROFILES) && in should_balance_chunk()
4082 if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE) && in should_balance_chunk()
4085 } else if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) && in should_balance_chunk()
4091 if ((bargs->flags & BTRFS_BALANCE_ARGS_DEVID) && in should_balance_chunk()
4097 if ((bargs->flags & BTRFS_BALANCE_ARGS_DRANGE) && in should_balance_chunk()
4103 if ((bargs->flags & BTRFS_BALANCE_ARGS_VRANGE) && in should_balance_chunk()
4109 if ((bargs->flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE) && in should_balance_chunk()
4115 if ((bargs->flags & BTRFS_BALANCE_ARGS_SOFT) && in should_balance_chunk()
4123 if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT)) { in should_balance_chunk()
4124 if (bargs->limit == 0) in should_balance_chunk()
4127 bargs->limit--; in should_balance_chunk()
4128 } else if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)) { in should_balance_chunk()
4134 if (bargs->limit_max == 0) in should_balance_chunk()
4137 bargs->limit_max--; in should_balance_chunk()
4145 struct btrfs_balance_control *bctl = fs_info->balance_ctl; in __btrfs_balance()
4146 struct btrfs_root *chunk_root = fs_info->chunk_root; in __btrfs_balance()
4158 u64 limit_data = bctl->data.limit; in __btrfs_balance()
4159 u64 limit_meta = bctl->meta.limit; in __btrfs_balance()
4160 u64 limit_sys = bctl->sys.limit; in __btrfs_balance()
4168 ret = -ENOMEM; in __btrfs_balance()
4173 spin_lock(&fs_info->balance_lock); in __btrfs_balance()
4174 memset(&bctl->stat, 0, sizeof(bctl->stat)); in __btrfs_balance()
4175 spin_unlock(&fs_info->balance_lock); in __btrfs_balance()
4182 bctl->data.limit = limit_data; in __btrfs_balance()
4183 bctl->meta.limit = limit_meta; in __btrfs_balance()
4184 bctl->sys.limit = limit_sys; in __btrfs_balance()
4187 key.offset = (u64)-1; in __btrfs_balance()
4191 if ((!counting && atomic_read(&fs_info->balance_pause_req)) || in __btrfs_balance()
4192 atomic_read(&fs_info->balance_cancel_req)) { in __btrfs_balance()
4193 ret = -ECANCELED; in __btrfs_balance()
4197 mutex_lock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4200 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4214 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4219 leaf = path->nodes[0]; in __btrfs_balance()
4220 slot = path->slots[0]; in __btrfs_balance()
4224 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4232 spin_lock(&fs_info->balance_lock); in __btrfs_balance()
4233 bctl->stat.considered++; in __btrfs_balance()
4234 spin_unlock(&fs_info->balance_lock); in __btrfs_balance()
4241 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4246 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4247 spin_lock(&fs_info->balance_lock); in __btrfs_balance()
4248 bctl->stat.expected++; in __btrfs_balance()
4249 spin_unlock(&fs_info->balance_lock); in __btrfs_balance()
4266 count_data < bctl->data.limit_min) in __btrfs_balance()
4268 count_meta < bctl->meta.limit_min) in __btrfs_balance()
4270 count_sys < bctl->sys.limit_min)) { in __btrfs_balance()
4271 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4285 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4293 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4294 if (ret == -ENOSPC) { in __btrfs_balance()
4296 } else if (ret == -ETXTBSY) { in __btrfs_balance()
4304 spin_lock(&fs_info->balance_lock); in __btrfs_balance()
4305 bctl->stat.completed++; in __btrfs_balance()
4306 spin_unlock(&fs_info->balance_lock); in __btrfs_balance()
4311 key.offset = found_key.offset - 1; in __btrfs_balance()
4325 ret = -ENOSPC; in __btrfs_balance()
4363 if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT)) in validate_convert_profile()
4367 if (alloc_profile_is_valid(bargs->target, 1) && in validate_convert_profile()
4368 (bargs->target & ~allowed) == 0) in validate_convert_profile()
4372 type, btrfs_bg_type_to_raid_name(bargs->target)); in validate_convert_profile()
4387 u64 flags = bargs->flags; in describe_balance_args()
4398 size_bp -= ret; \ in describe_balance_args()
4407 size_bp -= ret; \ in describe_balance_args()
4416 size_bp -= ret; \ in describe_balance_args()
4422 btrfs_bg_type_to_raid_name(bargs->target)); in describe_balance_args()
4428 btrfs_describe_block_groups(bargs->profiles, tmp_buf, in describe_balance_args()
4434 CHECK_APPEND_1ARG("usage=%llu,", bargs->usage); in describe_balance_args()
4438 bargs->usage_min, bargs->usage_max); in describe_balance_args()
4441 CHECK_APPEND_1ARG("devid=%llu,", bargs->devid); in describe_balance_args()
4445 bargs->pstart, bargs->pend); in describe_balance_args()
4449 bargs->vstart, bargs->vend); in describe_balance_args()
4452 CHECK_APPEND_1ARG("limit=%llu,", bargs->limit); in describe_balance_args()
4456 bargs->limit_min, bargs->limit_max); in describe_balance_args()
4460 bargs->stripes_min, bargs->stripes_max); in describe_balance_args()
4469 buf[size_buf - size_bp - 1] = '\0'; /* remove last , */ in describe_balance_args()
4482 struct btrfs_balance_control *bctl = fs_info->balance_ctl; in describe_balance_start_or_resume()
4495 size_bp -= ret; \ in describe_balance_start_or_resume()
4499 if (bctl->flags & BTRFS_BALANCE_FORCE) in describe_balance_start_or_resume()
4500 CHECK_APPEND_1ARG("%s", "-f "); in describe_balance_start_or_resume()
4502 if (bctl->flags & BTRFS_BALANCE_DATA) { in describe_balance_start_or_resume()
4503 describe_balance_args(&bctl->data, tmp_buf, sizeof(tmp_buf)); in describe_balance_start_or_resume()
4504 CHECK_APPEND_1ARG("-d%s ", tmp_buf); in describe_balance_start_or_resume()
4507 if (bctl->flags & BTRFS_BALANCE_METADATA) { in describe_balance_start_or_resume()
4508 describe_balance_args(&bctl->meta, tmp_buf, sizeof(tmp_buf)); in describe_balance_start_or_resume()
4509 CHECK_APPEND_1ARG("-m%s ", tmp_buf); in describe_balance_start_or_resume()
4512 if (bctl->flags & BTRFS_BALANCE_SYSTEM) { in describe_balance_start_or_resume()
4513 describe_balance_args(&bctl->sys, tmp_buf, sizeof(tmp_buf)); in describe_balance_start_or_resume()
4514 CHECK_APPEND_1ARG("-s%s ", tmp_buf); in describe_balance_start_or_resume()
4522 buf[size_buf - size_bp - 1] = '\0'; /* remove last " " */ in describe_balance_start_or_resume()
4524 (bctl->flags & BTRFS_BALANCE_RESUME) ? in describe_balance_start_or_resume()
4548 atomic_read(&fs_info->balance_pause_req) || in btrfs_balance()
4550 ret = -EINVAL; in btrfs_balance()
4554 allowed = btrfs_super_incompat_flags(fs_info->super_copy); in btrfs_balance()
4563 if (mixed && (bctl->flags & allowed)) { in btrfs_balance()
4564 if (!(bctl->flags & BTRFS_BALANCE_DATA) || in btrfs_balance()
4565 !(bctl->flags & BTRFS_BALANCE_METADATA) || in btrfs_balance()
4566 memcmp(&bctl->data, &bctl->meta, sizeof(bctl->data))) { in btrfs_balance()
4569 ret = -EINVAL; in btrfs_balance()
4578 num_devices = fs_info->fs_devices->rw_devices; in btrfs_balance()
4581 * SINGLE profile on-disk has no profile bit, but in-memory we have a in btrfs_balance()
4590 if (!validate_convert_profile(fs_info, &bctl->data, allowed, "data") || in btrfs_balance()
4591 !validate_convert_profile(fs_info, &bctl->meta, allowed, "metadata") || in btrfs_balance()
4592 !validate_convert_profile(fs_info, &bctl->sys, allowed, "system")) { in btrfs_balance()
4593 ret = -EINVAL; in btrfs_balance()
4598 * Allow to reduce metadata or system integrity only if force set for in btrfs_balance()
4608 seq = read_seqbegin(&fs_info->profiles_lock); in btrfs_balance()
4610 if (((bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) && in btrfs_balance()
4611 (fs_info->avail_system_alloc_bits & allowed) && in btrfs_balance()
4612 !(bctl->sys.target & allowed)) || in btrfs_balance()
4613 ((bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) && in btrfs_balance()
4614 (fs_info->avail_metadata_alloc_bits & allowed) && in btrfs_balance()
4615 !(bctl->meta.target & allowed))) in btrfs_balance()
4621 meta_target = (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) ? in btrfs_balance()
4622 bctl->meta.target : fs_info->avail_metadata_alloc_bits; in btrfs_balance()
4623 data_target = (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) ? in btrfs_balance()
4624 bctl->data.target : fs_info->avail_data_alloc_bits; in btrfs_balance()
4625 } while (read_seqretry(&fs_info->profiles_lock, seq)); in btrfs_balance()
4628 if (bctl->flags & BTRFS_BALANCE_FORCE) { in btrfs_balance()
4633 "balance: reduces metadata redundancy, use --force if you want this"); in btrfs_balance()
4634 ret = -EINVAL; in btrfs_balance()
4648 if (ret && ret != -EEXIST) in btrfs_balance()
4651 if (!(bctl->flags & BTRFS_BALANCE_RESUME)) { in btrfs_balance()
4652 BUG_ON(ret == -EEXIST); in btrfs_balance()
4653 BUG_ON(fs_info->balance_ctl); in btrfs_balance()
4654 spin_lock(&fs_info->balance_lock); in btrfs_balance()
4655 fs_info->balance_ctl = bctl; in btrfs_balance()
4656 spin_unlock(&fs_info->balance_lock); in btrfs_balance()
4658 BUG_ON(ret != -EEXIST); in btrfs_balance()
4659 spin_lock(&fs_info->balance_lock); in btrfs_balance()
4661 spin_unlock(&fs_info->balance_lock); in btrfs_balance()
4664 ASSERT(!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)); in btrfs_balance()
4665 set_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags); in btrfs_balance()
4667 mutex_unlock(&fs_info->balance_mutex); in btrfs_balance()
4671 mutex_lock(&fs_info->balance_mutex); in btrfs_balance()
4672 if (ret == -ECANCELED && atomic_read(&fs_info->balance_pause_req)) { in btrfs_balance()
4680 * - Regular cancel request in btrfs_balance()
4681 * Then ret == -ECANCELED and balance_cancel_req > 0 in btrfs_balance()
4683 * - Fatal signal to "btrfs" process in btrfs_balance()
4685 * got -EINTR, or caught by btrfs_should_cancel_balance() and in btrfs_balance()
4686 * got -ECANCELED. in btrfs_balance()
4688 * ret == -EINTR or ret == -ECANCELED. in btrfs_balance()
4692 else if (ret == -ECANCELED || ret == -EINTR) in btrfs_balance()
4697 clear_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags); in btrfs_balance()
4710 wake_up(&fs_info->balance_wait_q); in btrfs_balance()
4714 if (bctl->flags & BTRFS_BALANCE_RESUME) in btrfs_balance()
4728 sb_start_write(fs_info->sb); in balance_kthread()
4729 mutex_lock(&fs_info->balance_mutex); in balance_kthread()
4730 if (fs_info->balance_ctl) in balance_kthread()
4731 ret = btrfs_balance(fs_info, fs_info->balance_ctl, NULL); in balance_kthread()
4732 mutex_unlock(&fs_info->balance_mutex); in balance_kthread()
4733 sb_end_write(fs_info->sb); in balance_kthread()
4742 mutex_lock(&fs_info->balance_mutex); in btrfs_resume_balance_async()
4743 if (!fs_info->balance_ctl) { in btrfs_resume_balance_async()
4744 mutex_unlock(&fs_info->balance_mutex); in btrfs_resume_balance_async()
4747 mutex_unlock(&fs_info->balance_mutex); in btrfs_resume_balance_async()
4754 spin_lock(&fs_info->super_lock); in btrfs_resume_balance_async()
4755 ASSERT(fs_info->exclusive_operation == BTRFS_EXCLOP_BALANCE_PAUSED); in btrfs_resume_balance_async()
4756 fs_info->exclusive_operation = BTRFS_EXCLOP_BALANCE; in btrfs_resume_balance_async()
4757 spin_unlock(&fs_info->super_lock); in btrfs_resume_balance_async()
4759 * A ro->rw remount sequence should continue with the paused balance in btrfs_resume_balance_async()
4760 * regardless of who pauses it, system or the user as of now, so set in btrfs_resume_balance_async()
4763 spin_lock(&fs_info->balance_lock); in btrfs_resume_balance_async()
4764 fs_info->balance_ctl->flags |= BTRFS_BALANCE_RESUME; in btrfs_resume_balance_async()
4765 spin_unlock(&fs_info->balance_lock); in btrfs_resume_balance_async()
4767 tsk = kthread_run(balance_kthread, fs_info, "btrfs-balance"); in btrfs_resume_balance_async()
4783 return -ENOMEM; in btrfs_recover_balance()
4789 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); in btrfs_recover_balance()
4792 if (ret > 0) { /* ret = -ENOENT; */ in btrfs_recover_balance()
4799 ret = -ENOMEM; in btrfs_recover_balance()
4803 leaf = path->nodes[0]; in btrfs_recover_balance()
4804 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item); in btrfs_recover_balance()
4806 bctl->flags = btrfs_balance_flags(leaf, item); in btrfs_recover_balance()
4807 bctl->flags |= BTRFS_BALANCE_RESUME; in btrfs_recover_balance()
4810 btrfs_disk_balance_args_to_cpu(&bctl->data, &disk_bargs); in btrfs_recover_balance()
4812 btrfs_disk_balance_args_to_cpu(&bctl->meta, &disk_bargs); in btrfs_recover_balance()
4814 btrfs_disk_balance_args_to_cpu(&bctl->sys, &disk_bargs); in btrfs_recover_balance()
4832 mutex_lock(&fs_info->balance_mutex); in btrfs_recover_balance()
4833 BUG_ON(fs_info->balance_ctl); in btrfs_recover_balance()
4834 spin_lock(&fs_info->balance_lock); in btrfs_recover_balance()
4835 fs_info->balance_ctl = bctl; in btrfs_recover_balance()
4836 spin_unlock(&fs_info->balance_lock); in btrfs_recover_balance()
4837 mutex_unlock(&fs_info->balance_mutex); in btrfs_recover_balance()
4847 mutex_lock(&fs_info->balance_mutex); in btrfs_pause_balance()
4848 if (!fs_info->balance_ctl) { in btrfs_pause_balance()
4849 mutex_unlock(&fs_info->balance_mutex); in btrfs_pause_balance()
4850 return -ENOTCONN; in btrfs_pause_balance()
4853 if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) { in btrfs_pause_balance()
4854 atomic_inc(&fs_info->balance_pause_req); in btrfs_pause_balance()
4855 mutex_unlock(&fs_info->balance_mutex); in btrfs_pause_balance()
4857 wait_event(fs_info->balance_wait_q, in btrfs_pause_balance()
4858 !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)); in btrfs_pause_balance()
4860 mutex_lock(&fs_info->balance_mutex); in btrfs_pause_balance()
4862 BUG_ON(test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)); in btrfs_pause_balance()
4863 atomic_dec(&fs_info->balance_pause_req); in btrfs_pause_balance()
4865 ret = -ENOTCONN; in btrfs_pause_balance()
4868 mutex_unlock(&fs_info->balance_mutex); in btrfs_pause_balance()
4874 mutex_lock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4875 if (!fs_info->balance_ctl) { in btrfs_cancel_balance()
4876 mutex_unlock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4877 return -ENOTCONN; in btrfs_cancel_balance()
4882 * mount time if the mount is read-write. Otherwise it's still paused in btrfs_cancel_balance()
4885 if (sb_rdonly(fs_info->sb)) { in btrfs_cancel_balance()
4886 mutex_unlock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4887 return -EROFS; in btrfs_cancel_balance()
4890 atomic_inc(&fs_info->balance_cancel_req); in btrfs_cancel_balance()
4895 if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) { in btrfs_cancel_balance()
4896 mutex_unlock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4897 wait_event(fs_info->balance_wait_q, in btrfs_cancel_balance()
4898 !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)); in btrfs_cancel_balance()
4899 mutex_lock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4901 mutex_unlock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4906 mutex_lock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4908 if (fs_info->balance_ctl) { in btrfs_cancel_balance()
4915 ASSERT(!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)); in btrfs_cancel_balance()
4916 atomic_dec(&fs_info->balance_cancel_req); in btrfs_cancel_balance()
4917 mutex_unlock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4928 struct btrfs_fs_info *fs_info = device->fs_info; in btrfs_shrink_device()
4929 struct btrfs_root *root = fs_info->dev_root; in btrfs_shrink_device()
4941 struct btrfs_super_block *super_copy = fs_info->super_copy; in btrfs_shrink_device()
4948 new_size = round_down(new_size, fs_info->sectorsize); in btrfs_shrink_device()
4950 diff = round_down(old_size - new_size, fs_info->sectorsize); in btrfs_shrink_device()
4952 if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) in btrfs_shrink_device()
4953 return -EINVAL; in btrfs_shrink_device()
4957 return -ENOMEM; in btrfs_shrink_device()
4959 path->reada = READA_BACK; in btrfs_shrink_device()
4967 mutex_lock(&fs_info->chunk_mutex); in btrfs_shrink_device()
4970 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in btrfs_shrink_device()
4971 device->fs_devices->total_rw_bytes -= diff; in btrfs_shrink_device()
4974 * The new free_chunk_space is new_size - used, so we have to in btrfs_shrink_device()
4976 * old_size - used. If used > new_size then just subtract this in btrfs_shrink_device()
4979 if (device->bytes_used < new_size) in btrfs_shrink_device()
4980 free_diff = (old_size - device->bytes_used) - in btrfs_shrink_device()
4981 (new_size - device->bytes_used); in btrfs_shrink_device()
4983 free_diff = old_size - device->bytes_used; in btrfs_shrink_device()
4984 atomic64_sub(free_diff, &fs_info->free_chunk_space); in btrfs_shrink_device()
4989 * in-memory chunks are synced to disk so that the loop below sees them in btrfs_shrink_device()
4993 mutex_unlock(&fs_info->chunk_mutex); in btrfs_shrink_device()
4998 mutex_unlock(&fs_info->chunk_mutex); in btrfs_shrink_device()
5003 key.objectid = device->devid; in btrfs_shrink_device()
5004 key.offset = (u64)-1; in btrfs_shrink_device()
5008 mutex_lock(&fs_info->reclaim_bgs_lock); in btrfs_shrink_device()
5011 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_shrink_device()
5017 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_shrink_device()
5025 l = path->nodes[0]; in btrfs_shrink_device()
5026 slot = path->slots[0]; in btrfs_shrink_device()
5027 btrfs_item_key_to_cpu(l, &key, path->slots[0]); in btrfs_shrink_device()
5029 if (key.objectid != device->devid) { in btrfs_shrink_device()
5030 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_shrink_device()
5039 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_shrink_device()
5055 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_shrink_device()
5060 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_shrink_device()
5061 if (ret == -ENOSPC) { in btrfs_shrink_device()
5064 if (ret == -ETXTBSY) { in btrfs_shrink_device()
5071 } while (key.offset-- > 0); in btrfs_shrink_device()
5078 ret = -ENOSPC; in btrfs_shrink_device()
5089 mutex_lock(&fs_info->chunk_mutex); in btrfs_shrink_device()
5091 clear_extent_bits(&device->alloc_state, new_size, (u64)-1, in btrfs_shrink_device()
5095 if (list_empty(&device->post_commit_list)) in btrfs_shrink_device()
5096 list_add_tail(&device->post_commit_list, in btrfs_shrink_device()
5097 &trans->transaction->dev_update_list); in btrfs_shrink_device()
5101 round_down(old_total - diff, fs_info->sectorsize)); in btrfs_shrink_device()
5102 mutex_unlock(&fs_info->chunk_mutex); in btrfs_shrink_device()
5105 /* Now btrfs_update_device() will change the on-disk size. */ in btrfs_shrink_device()
5117 mutex_lock(&fs_info->chunk_mutex); in btrfs_shrink_device()
5119 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in btrfs_shrink_device()
5120 device->fs_devices->total_rw_bytes += diff; in btrfs_shrink_device()
5121 atomic64_add(free_diff, &fs_info->free_chunk_space); in btrfs_shrink_device()
5123 mutex_unlock(&fs_info->chunk_mutex); in btrfs_shrink_device()
5132 struct btrfs_super_block *super_copy = fs_info->super_copy; in btrfs_add_system_chunk()
5137 lockdep_assert_held(&fs_info->chunk_mutex); in btrfs_add_system_chunk()
5142 return -EFBIG; in btrfs_add_system_chunk()
5144 ptr = super_copy->sys_chunk_array + array_size; in btrfs_add_system_chunk()
5163 if (di_a->max_avail > di_b->max_avail) in btrfs_cmp_device_info()
5164 return -1; in btrfs_cmp_device_info()
5165 if (di_a->max_avail < di_b->max_avail) in btrfs_cmp_device_info()
5167 if (di_a->total_avail > di_b->total_avail) in btrfs_cmp_device_info()
5168 return -1; in btrfs_cmp_device_info()
5169 if (di_a->total_avail < di_b->total_avail) in btrfs_cmp_device_info()
5223 struct alloc_chunk_ctl *ctl) in init_alloc_chunk_ctl_policy_regular() argument
5227 space_info = btrfs_find_space_info(fs_devices->fs_info, ctl->type); in init_alloc_chunk_ctl_policy_regular()
5230 ctl->max_chunk_size = READ_ONCE(space_info->chunk_size); in init_alloc_chunk_ctl_policy_regular()
5231 ctl->max_stripe_size = min_t(u64, ctl->max_chunk_size, SZ_1G); in init_alloc_chunk_ctl_policy_regular()
5233 if (ctl->type & BTRFS_BLOCK_GROUP_SYSTEM) in init_alloc_chunk_ctl_policy_regular()
5234 ctl->devs_max = min_t(int, ctl->devs_max, BTRFS_MAX_DEVS_SYS_CHUNK); in init_alloc_chunk_ctl_policy_regular()
5237 ctl->max_chunk_size = min(mult_perc(fs_devices->total_rw_bytes, 10), in init_alloc_chunk_ctl_policy_regular()
5238 ctl->max_chunk_size); in init_alloc_chunk_ctl_policy_regular()
5239 ctl->dev_extent_min = btrfs_stripe_nr_to_offset(ctl->dev_stripes); in init_alloc_chunk_ctl_policy_regular()
5244 struct alloc_chunk_ctl *ctl) in init_alloc_chunk_ctl_policy_zoned() argument
5246 u64 zone_size = fs_devices->fs_info->zone_size; in init_alloc_chunk_ctl_policy_zoned()
5248 int min_num_stripes = ctl->devs_min * ctl->dev_stripes; in init_alloc_chunk_ctl_policy_zoned()
5249 int min_data_stripes = (min_num_stripes - ctl->nparity) / ctl->ncopies; in init_alloc_chunk_ctl_policy_zoned()
5251 u64 type = ctl->type; in init_alloc_chunk_ctl_policy_zoned()
5253 ctl->max_stripe_size = zone_size; in init_alloc_chunk_ctl_policy_zoned()
5255 ctl->max_chunk_size = round_down(BTRFS_MAX_DATA_CHUNK_SIZE, in init_alloc_chunk_ctl_policy_zoned()
5258 ctl->max_chunk_size = ctl->max_stripe_size; in init_alloc_chunk_ctl_policy_zoned()
5260 ctl->max_chunk_size = 2 * ctl->max_stripe_size; in init_alloc_chunk_ctl_policy_zoned()
5261 ctl->devs_max = min_t(int, ctl->devs_max, in init_alloc_chunk_ctl_policy_zoned()
5268 limit = max(round_down(mult_perc(fs_devices->total_rw_bytes, 10), in init_alloc_chunk_ctl_policy_zoned()
5271 ctl->max_chunk_size = min(limit, ctl->max_chunk_size); in init_alloc_chunk_ctl_policy_zoned()
5272 ctl->dev_extent_min = zone_size * ctl->dev_stripes; in init_alloc_chunk_ctl_policy_zoned()
5276 struct alloc_chunk_ctl *ctl) in init_alloc_chunk_ctl() argument
5278 int index = btrfs_bg_flags_to_raid_index(ctl->type); in init_alloc_chunk_ctl()
5280 ctl->sub_stripes = btrfs_raid_array[index].sub_stripes; in init_alloc_chunk_ctl()
5281 ctl->dev_stripes = btrfs_raid_array[index].dev_stripes; in init_alloc_chunk_ctl()
5282 ctl->devs_max = btrfs_raid_array[index].devs_max; in init_alloc_chunk_ctl()
5283 if (!ctl->devs_max) in init_alloc_chunk_ctl()
5284 ctl->devs_max = BTRFS_MAX_DEVS(fs_devices->fs_info); in init_alloc_chunk_ctl()
5285 ctl->devs_min = btrfs_raid_array[index].devs_min; in init_alloc_chunk_ctl()
5286 ctl->devs_increment = btrfs_raid_array[index].devs_increment; in init_alloc_chunk_ctl()
5287 ctl->ncopies = btrfs_raid_array[index].ncopies; in init_alloc_chunk_ctl()
5288 ctl->nparity = btrfs_raid_array[index].nparity; in init_alloc_chunk_ctl()
5289 ctl->ndevs = 0; in init_alloc_chunk_ctl()
5291 switch (fs_devices->chunk_alloc_policy) { in init_alloc_chunk_ctl()
5293 init_alloc_chunk_ctl_policy_regular(fs_devices, ctl); in init_alloc_chunk_ctl()
5296 init_alloc_chunk_ctl_policy_zoned(fs_devices, ctl); in init_alloc_chunk_ctl()
5304 struct alloc_chunk_ctl *ctl, in gather_device_info() argument
5307 struct btrfs_fs_info *info = fs_devices->fs_info; in gather_device_info()
5310 u64 dev_extent_want = ctl->max_stripe_size * ctl->dev_stripes; in gather_device_info()
5320 list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) { in gather_device_info()
5321 if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in gather_device_info()
5323 "BTRFS: read-only device in alloc_list\n"); in gather_device_info()
5328 &device->dev_state) || in gather_device_info()
5329 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) in gather_device_info()
5332 if (device->total_bytes > device->bytes_used) in gather_device_info()
5333 total_avail = device->total_bytes - device->bytes_used; in gather_device_info()
5338 if (total_avail < ctl->dev_extent_min) in gather_device_info()
5343 if (ret && ret != -ENOSPC) in gather_device_info()
5349 if (max_avail < ctl->dev_extent_min) { in gather_device_info()
5353 __func__, device->devid, max_avail, in gather_device_info()
5354 ctl->dev_extent_min); in gather_device_info()
5358 if (ndevs == fs_devices->rw_devices) { in gather_device_info()
5360 __func__, fs_devices->rw_devices); in gather_device_info()
5369 ctl->ndevs = ndevs; in gather_device_info()
5380 static int decide_stripe_size_regular(struct alloc_chunk_ctl *ctl, in decide_stripe_size_regular() argument
5393 ctl->stripe_size = div_u64(devices_info[ctl->ndevs - 1].max_avail, in decide_stripe_size_regular()
5394 ctl->dev_stripes); in decide_stripe_size_regular()
5395 ctl->num_stripes = ctl->ndevs * ctl->dev_stripes; in decide_stripe_size_regular()
5398 data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies; in decide_stripe_size_regular()
5406 if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) { in decide_stripe_size_regular()
5412 ctl->stripe_size = min(round_up(div_u64(ctl->max_chunk_size, in decide_stripe_size_regular()
5414 ctl->stripe_size); in decide_stripe_size_regular()
5418 ctl->stripe_size = min_t(u64, ctl->stripe_size, SZ_1G); in decide_stripe_size_regular()
5421 ctl->stripe_size = round_down(ctl->stripe_size, BTRFS_STRIPE_LEN); in decide_stripe_size_regular()
5422 ctl->chunk_size = ctl->stripe_size * data_stripes; in decide_stripe_size_regular()
5427 static int decide_stripe_size_zoned(struct alloc_chunk_ctl *ctl, in decide_stripe_size_zoned() argument
5430 u64 zone_size = devices_info[0].dev->zone_info->zone_size; in decide_stripe_size_zoned()
5438 ASSERT(devices_info[ctl->ndevs - 1].max_avail == ctl->dev_extent_min); in decide_stripe_size_zoned()
5440 ctl->stripe_size = zone_size; in decide_stripe_size_zoned()
5441 ctl->num_stripes = ctl->ndevs * ctl->dev_stripes; in decide_stripe_size_zoned()
5442 data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies; in decide_stripe_size_zoned()
5445 if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) { in decide_stripe_size_zoned()
5446 ctl->ndevs = div_u64(div_u64(ctl->max_chunk_size * ctl->ncopies, in decide_stripe_size_zoned()
5447 ctl->stripe_size) + ctl->nparity, in decide_stripe_size_zoned()
5448 ctl->dev_stripes); in decide_stripe_size_zoned()
5449 ctl->num_stripes = ctl->ndevs * ctl->dev_stripes; in decide_stripe_size_zoned()
5450 data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies; in decide_stripe_size_zoned()
5451 ASSERT(ctl->stripe_size * data_stripes <= ctl->max_chunk_size); in decide_stripe_size_zoned()
5454 ctl->chunk_size = ctl->stripe_size * data_stripes; in decide_stripe_size_zoned()
5460 struct alloc_chunk_ctl *ctl, in decide_stripe_size() argument
5463 struct btrfs_fs_info *info = fs_devices->fs_info; in decide_stripe_size()
5470 ctl->ndevs = rounddown(ctl->ndevs, ctl->devs_increment); in decide_stripe_size()
5472 if (ctl->ndevs < ctl->devs_min) { in decide_stripe_size()
5476 __func__, ctl->ndevs, ctl->devs_min); in decide_stripe_size()
5478 return -ENOSPC; in decide_stripe_size()
5481 ctl->ndevs = min(ctl->ndevs, ctl->devs_max); in decide_stripe_size()
5483 switch (fs_devices->chunk_alloc_policy) { in decide_stripe_size()
5485 return decide_stripe_size_regular(ctl, devices_info); in decide_stripe_size()
5487 return decide_stripe_size_zoned(ctl, devices_info); in decide_stripe_size()
5495 for (int i = 0; i < map->num_stripes; i++) { in chunk_map_device_set_bits()
5496 struct btrfs_io_stripe *stripe = &map->stripes[i]; in chunk_map_device_set_bits()
5497 struct btrfs_device *device = stripe->dev; in chunk_map_device_set_bits()
5499 set_extent_bit(&device->alloc_state, stripe->physical, in chunk_map_device_set_bits()
5500 stripe->physical + map->stripe_size - 1, in chunk_map_device_set_bits()
5507 for (int i = 0; i < map->num_stripes; i++) { in chunk_map_device_clear_bits()
5508 struct btrfs_io_stripe *stripe = &map->stripes[i]; in chunk_map_device_clear_bits()
5509 struct btrfs_device *device = stripe->dev; in chunk_map_device_clear_bits()
5511 __clear_extent_bit(&device->alloc_state, stripe->physical, in chunk_map_device_clear_bits()
5512 stripe->physical + map->stripe_size - 1, in chunk_map_device_clear_bits()
5520 write_lock(&fs_info->mapping_tree_lock); in btrfs_remove_chunk_map()
5521 rb_erase_cached(&map->rb_node, &fs_info->mapping_tree); in btrfs_remove_chunk_map()
5522 RB_CLEAR_NODE(&map->rb_node); in btrfs_remove_chunk_map()
5524 write_unlock(&fs_info->mapping_tree_lock); in btrfs_remove_chunk_map()
5538 if (new_map->start == exist_map->start) in btrfs_chunk_map_cmp()
5540 if (new_map->start < exist_map->start) in btrfs_chunk_map_cmp()
5541 return -1; in btrfs_chunk_map_cmp()
5550 write_lock(&fs_info->mapping_tree_lock); in btrfs_add_chunk_map()
5551 exist = rb_find_add_cached(&map->rb_node, &fs_info->mapping_tree, in btrfs_add_chunk_map()
5555 write_unlock(&fs_info->mapping_tree_lock); in btrfs_add_chunk_map()
5556 return -EEXIST; in btrfs_add_chunk_map()
5560 write_unlock(&fs_info->mapping_tree_lock); in btrfs_add_chunk_map()
5574 refcount_set(&map->refs, 1); in btrfs_alloc_chunk_map()
5575 RB_CLEAR_NODE(&map->rb_node); in btrfs_alloc_chunk_map()
5581 struct alloc_chunk_ctl *ctl, in create_chunk() argument
5584 struct btrfs_fs_info *info = trans->fs_info; in create_chunk()
5587 u64 start = ctl->start; in create_chunk()
5588 u64 type = ctl->type; in create_chunk()
5591 map = btrfs_alloc_chunk_map(ctl->num_stripes, GFP_NOFS); in create_chunk()
5593 return ERR_PTR(-ENOMEM); in create_chunk()
5595 map->start = start; in create_chunk()
5596 map->chunk_len = ctl->chunk_size; in create_chunk()
5597 map->stripe_size = ctl->stripe_size; in create_chunk()
5598 map->type = type; in create_chunk()
5599 map->io_align = BTRFS_STRIPE_LEN; in create_chunk()
5600 map->io_width = BTRFS_STRIPE_LEN; in create_chunk()
5601 map->sub_stripes = ctl->sub_stripes; in create_chunk()
5602 map->num_stripes = ctl->num_stripes; in create_chunk()
5604 for (int i = 0; i < ctl->ndevs; i++) { in create_chunk()
5605 for (int j = 0; j < ctl->dev_stripes; j++) { in create_chunk()
5606 int s = i * ctl->dev_stripes + j; in create_chunk()
5607 map->stripes[s].dev = devices_info[i].dev; in create_chunk()
5608 map->stripes[s].physical = devices_info[i].dev_offset + in create_chunk()
5609 j * ctl->stripe_size; in create_chunk()
5613 trace_btrfs_chunk_alloc(info, map, start, ctl->chunk_size); in create_chunk()
5621 block_group = btrfs_make_block_group(trans, type, start, ctl->chunk_size); in create_chunk()
5627 for (int i = 0; i < map->num_stripes; i++) { in create_chunk()
5628 struct btrfs_device *dev = map->stripes[i].dev; in create_chunk()
5631 dev->bytes_used + ctl->stripe_size); in create_chunk()
5632 if (list_empty(&dev->post_commit_list)) in create_chunk()
5633 list_add_tail(&dev->post_commit_list, in create_chunk()
5634 &trans->transaction->dev_update_list); in create_chunk()
5637 atomic64_sub(ctl->stripe_size * map->num_stripes, in create_chunk()
5638 &info->free_chunk_space); in create_chunk()
5649 struct btrfs_fs_info *info = trans->fs_info; in btrfs_create_chunk()
5650 struct btrfs_fs_devices *fs_devices = info->fs_devices; in btrfs_create_chunk()
5652 struct alloc_chunk_ctl ctl; in btrfs_create_chunk() local
5656 lockdep_assert_held(&info->chunk_mutex); in btrfs_create_chunk()
5660 return ERR_PTR(-EINVAL); in btrfs_create_chunk()
5663 if (list_empty(&fs_devices->alloc_list)) { in btrfs_create_chunk()
5666 return ERR_PTR(-ENOSPC); in btrfs_create_chunk()
5672 return ERR_PTR(-EINVAL); in btrfs_create_chunk()
5675 ctl.start = find_next_chunk(info); in btrfs_create_chunk()
5676 ctl.type = type; in btrfs_create_chunk()
5677 init_alloc_chunk_ctl(fs_devices, &ctl); in btrfs_create_chunk()
5679 devices_info = kcalloc(fs_devices->rw_devices, sizeof(*devices_info), in btrfs_create_chunk()
5682 return ERR_PTR(-ENOMEM); in btrfs_create_chunk()
5684 ret = gather_device_info(fs_devices, &ctl, devices_info); in btrfs_create_chunk()
5690 ret = decide_stripe_size(fs_devices, &ctl, devices_info); in btrfs_create_chunk()
5696 block_group = create_chunk(trans, &ctl, devices_info); in btrfs_create_chunk()
5705 * phase 1 of chunk allocation. It belongs to phase 2 only when allocating system
5714 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_chunk_alloc_add_chunk_item()
5715 struct btrfs_root *chunk_root = fs_info->chunk_root; in btrfs_chunk_alloc_add_chunk_item()
5728 * the chunk_mutex, as well as updating the system chunk array in the in btrfs_chunk_alloc_add_chunk_item()
5736 * (dev-replace.c:btrfs_dev_replace_finishing()), so we could grab the in btrfs_chunk_alloc_add_chunk_item()
5742 * acquires both mutexes - first the device_list_mutex and then the in btrfs_chunk_alloc_add_chunk_item()
5746 lockdep_assert_held(&fs_info->chunk_mutex); in btrfs_chunk_alloc_add_chunk_item()
5748 map = btrfs_get_chunk_map(fs_info, bg->start, bg->length); in btrfs_chunk_alloc_add_chunk_item()
5755 item_size = btrfs_chunk_item_size(map->num_stripes); in btrfs_chunk_alloc_add_chunk_item()
5759 ret = -ENOMEM; in btrfs_chunk_alloc_add_chunk_item()
5764 for (i = 0; i < map->num_stripes; i++) { in btrfs_chunk_alloc_add_chunk_item()
5765 struct btrfs_device *device = map->stripes[i].dev; in btrfs_chunk_alloc_add_chunk_item()
5772 stripe = &chunk->stripe; in btrfs_chunk_alloc_add_chunk_item()
5773 for (i = 0; i < map->num_stripes; i++) { in btrfs_chunk_alloc_add_chunk_item()
5774 struct btrfs_device *device = map->stripes[i].dev; in btrfs_chunk_alloc_add_chunk_item()
5775 const u64 dev_offset = map->stripes[i].physical; in btrfs_chunk_alloc_add_chunk_item()
5777 btrfs_set_stack_stripe_devid(stripe, device->devid); in btrfs_chunk_alloc_add_chunk_item()
5779 memcpy(stripe->dev_uuid, device->uuid, BTRFS_UUID_SIZE); in btrfs_chunk_alloc_add_chunk_item()
5783 btrfs_set_stack_chunk_length(chunk, bg->length); in btrfs_chunk_alloc_add_chunk_item()
5786 btrfs_set_stack_chunk_type(chunk, map->type); in btrfs_chunk_alloc_add_chunk_item()
5787 btrfs_set_stack_chunk_num_stripes(chunk, map->num_stripes); in btrfs_chunk_alloc_add_chunk_item()
5790 btrfs_set_stack_chunk_sector_size(chunk, fs_info->sectorsize); in btrfs_chunk_alloc_add_chunk_item()
5791 btrfs_set_stack_chunk_sub_stripes(chunk, map->sub_stripes); in btrfs_chunk_alloc_add_chunk_item()
5795 key.offset = bg->start; in btrfs_chunk_alloc_add_chunk_item()
5801 set_bit(BLOCK_GROUP_FLAG_CHUNK_ITEM_INSERTED, &bg->runtime_flags); in btrfs_chunk_alloc_add_chunk_item()
5803 if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) { in btrfs_chunk_alloc_add_chunk_item()
5817 struct btrfs_fs_info *fs_info = trans->fs_info; in init_first_rw_device()
5823 * When adding a new device for sprouting, the seed device is read-only in init_first_rw_device()
5824 * so we must first allocate a metadata and a system chunk. But before in init_first_rw_device()
5829 * otherwise we would get -ENOSPC since the block groups from the in init_first_rw_device()
5830 * seed device are read-only; in init_first_rw_device()
5832 * 2) Add the device item for the new sprout device - finishing the setup in init_first_rw_device()
5835 * ensures this does not fail with -ENOSPC. in init_first_rw_device()
5858 const int index = btrfs_bg_flags_to_raid_index(map->type); in btrfs_chunk_max_errors()
5874 for (i = 0; i < map->num_stripes; i++) { in btrfs_chunk_writeable()
5876 &map->stripes[i].dev->dev_state)) { in btrfs_chunk_writeable()
5881 &map->stripes[i].dev->dev_state)) { in btrfs_chunk_writeable()
5900 write_lock(&fs_info->mapping_tree_lock); in btrfs_mapping_tree_free()
5901 while (!RB_EMPTY_ROOT(&fs_info->mapping_tree.rb_root)) { in btrfs_mapping_tree_free()
5905 node = rb_first_cached(&fs_info->mapping_tree); in btrfs_mapping_tree_free()
5907 rb_erase_cached(&map->rb_node, &fs_info->mapping_tree); in btrfs_mapping_tree_free()
5908 RB_CLEAR_NODE(&map->rb_node); in btrfs_mapping_tree_free()
5912 cond_resched_rwlock_write(&fs_info->mapping_tree_lock); in btrfs_mapping_tree_free()
5914 write_unlock(&fs_info->mapping_tree_lock); in btrfs_mapping_tree_free()
5919 enum btrfs_raid_types index = btrfs_bg_flags_to_raid_index(map->type); in btrfs_chunk_map_num_copies()
5921 if (map->type & BTRFS_BLOCK_GROUP_RAID5) in btrfs_chunk_map_num_copies()
5931 if (map->type & BTRFS_BLOCK_GROUP_RAID6) in btrfs_chunk_map_num_copies()
5932 return map->num_stripes; in btrfs_chunk_map_num_copies()
5934 /* Non-RAID56, use their ncopies from btrfs_raid_array. */ in btrfs_chunk_map_num_copies()
5962 unsigned long len = fs_info->sectorsize; in btrfs_full_stripe_len()
5970 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) in btrfs_full_stripe_len()
5981 const struct btrfs_device *device = map->stripes[index].dev; in btrfs_read_preferred()
5983 if (device->devid == READ_ONCE(device->fs_devices->read_devid)) in btrfs_read_preferred()
5987 /* If no read-preferred device is set use the first stripe. */ in btrfs_read_preferred()
6001 if (s1->devid < s2->devid) in btrfs_cmp_devid()
6002 return -1; in btrfs_cmp_devid()
6003 if (s1->devid > s2->devid) in btrfs_cmp_devid()
6009 * Select a stripe for reading using the round-robin algorithm.
6024 struct btrfs_device *device = map->stripes[first].dev; in btrfs_read_rr()
6025 struct btrfs_fs_info *fs_info = device->fs_devices->fs_info; in btrfs_read_rr()
6030 total_reads = percpu_counter_sum(&fs_info->stats_read_blocks); in btrfs_read_rr()
6031 min_reads_per_dev = READ_ONCE(fs_info->fs_devices->rr_min_contig_read) >> in btrfs_read_rr()
6032 fs_info->sectorsize_bits; in btrfs_read_rr()
6035 stripes[index].devid = map->stripes[i].dev->devid; in btrfs_read_rr()
6051 const enum btrfs_read_policy policy = READ_ONCE(fs_info->fs_devices->read_policy); in find_live_mirror()
6058 ASSERT((map->type & in find_live_mirror()
6061 if (map->type & BTRFS_BLOCK_GROUP_RAID10) in find_live_mirror()
6062 num_stripes = map->sub_stripes; in find_live_mirror()
6064 num_stripes = map->num_stripes; in find_live_mirror()
6071 WRITE_ONCE(fs_info->fs_devices->read_policy, BTRFS_READ_POLICY_PID); in find_live_mirror()
6074 preferred_mirror = first + (current->pid % num_stripes); in find_live_mirror()
6087 fs_info->dev_replace.cont_reading_from_srcdev_mode == in find_live_mirror()
6089 srcdev = fs_info->dev_replace.srcdev; in find_live_mirror()
6095 * dev-replace procedure, only choose it if no other non-missing in find_live_mirror()
6099 if (map->stripes[preferred_mirror].dev->bdev && in find_live_mirror()
6100 (tolerance || map->stripes[preferred_mirror].dev != srcdev)) in find_live_mirror()
6103 if (map->stripes[i].dev->bdev && in find_live_mirror()
6104 (tolerance || map->stripes[i].dev != srcdev)) in find_live_mirror()
6131 refcount_set(&bioc->refs, 1); in alloc_btrfs_io_context()
6133 bioc->fs_info = fs_info; in alloc_btrfs_io_context()
6134 bioc->replace_stripe_src = -1; in alloc_btrfs_io_context()
6135 bioc->full_stripe_logical = (u64)-1; in alloc_btrfs_io_context()
6136 bioc->logical = logical; in alloc_btrfs_io_context()
6143 WARN_ON(!refcount_read(&bioc->refs)); in btrfs_get_bioc()
6144 refcount_inc(&bioc->refs); in btrfs_get_bioc()
6151 if (refcount_dec_and_test(&bioc->refs)) in btrfs_put_bioc()
6186 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { in btrfs_map_discard()
6187 ret = -EOPNOTSUPP; in btrfs_map_discard()
6191 offset = logical - map->start; in btrfs_map_discard()
6192 length = min_t(u64, map->start + map->chunk_len - logical, length); in btrfs_map_discard()
6202 stripe_offset = offset - btrfs_stripe_nr_to_offset(stripe_nr); in btrfs_map_discard()
6206 stripe_cnt = stripe_nr_end - stripe_nr; in btrfs_map_discard()
6207 stripe_end_offset = btrfs_stripe_nr_to_offset(stripe_nr_end) - in btrfs_map_discard()
6216 if (map->type & (BTRFS_BLOCK_GROUP_RAID0 | in btrfs_map_discard()
6218 if (map->type & BTRFS_BLOCK_GROUP_RAID0) in btrfs_map_discard()
6221 sub_stripes = map->sub_stripes; in btrfs_map_discard()
6223 factor = map->num_stripes / sub_stripes; in btrfs_map_discard()
6224 *num_stripes = min_t(u64, map->num_stripes, in btrfs_map_discard()
6232 last_stripe = ((stripe_nr_end - 1) % factor) * sub_stripes; in btrfs_map_discard()
6233 } else if (map->type & (BTRFS_BLOCK_GROUP_RAID1_MASK | in btrfs_map_discard()
6235 *num_stripes = map->num_stripes; in btrfs_map_discard()
6237 stripe_index = stripe_nr % map->num_stripes; in btrfs_map_discard()
6238 stripe_nr /= map->num_stripes; in btrfs_map_discard()
6243 ret = -ENOMEM; in btrfs_map_discard()
6249 map->stripes[stripe_index].physical + in btrfs_map_discard()
6251 stripes[i].dev = map->stripes[stripe_index].dev; in btrfs_map_discard()
6253 if (map->type & (BTRFS_BLOCK_GROUP_RAID0 | in btrfs_map_discard()
6264 * |-------|...|-------| in btrfs_map_discard()
6265 * |----------| in btrfs_map_discard()
6269 stripes[i].length -= stripe_offset; in btrfs_map_discard()
6273 sub_stripes - 1)) in btrfs_map_discard()
6274 stripes[i].length -= stripe_end_offset; in btrfs_map_discard()
6276 if (i == sub_stripes - 1) in btrfs_map_discard()
6283 if (stripe_index == map->num_stripes) { in btrfs_map_discard()
6307 ret = test_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags); in is_block_group_to_copy()
6318 u64 srcdev_devid = dev_replace->srcdev->devid; in handle_ops_on_dev_replace()
6323 int num_stripes = io_geom->num_stripes; in handle_ops_on_dev_replace()
6324 int max_errors = io_geom->max_errors; in handle_ops_on_dev_replace()
6330 * the dev-replace process. We can avoid cloning IO here. in handle_ops_on_dev_replace()
6332 if (is_block_group_to_copy(dev_replace->srcdev->fs_info, logical)) in handle_ops_on_dev_replace()
6336 * Duplicate the write operations while the dev-replace procedure is in handle_ops_on_dev_replace()
6342 * Note that device->missing is handled by the caller, and that the in handle_ops_on_dev_replace()
6346 struct btrfs_io_stripe *old = &bioc->stripes[i]; in handle_ops_on_dev_replace()
6347 struct btrfs_io_stripe *new = &bioc->stripes[num_stripes + nr_extra_stripes]; in handle_ops_on_dev_replace()
6349 if (old->dev->devid != srcdev_devid) in handle_ops_on_dev_replace()
6352 new->physical = old->physical; in handle_ops_on_dev_replace()
6353 new->dev = dev_replace->tgtdev; in handle_ops_on_dev_replace()
6354 if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) in handle_ops_on_dev_replace()
6355 bioc->replace_stripe_src = i; in handle_ops_on_dev_replace()
6366 if (io_geom->op == BTRFS_MAP_GET_READ_MIRRORS && nr_extra_stripes == 2) { in handle_ops_on_dev_replace()
6367 struct btrfs_io_stripe *first = &bioc->stripes[num_stripes]; in handle_ops_on_dev_replace()
6368 struct btrfs_io_stripe *second = &bioc->stripes[num_stripes + 1]; in handle_ops_on_dev_replace()
6371 ASSERT(bioc->map_type & BTRFS_BLOCK_GROUP_DUP); in handle_ops_on_dev_replace()
6377 if (first->physical > second->physical) { in handle_ops_on_dev_replace()
6378 swap(second->physical, first->physical); in handle_ops_on_dev_replace()
6379 swap(second->dev, first->dev); in handle_ops_on_dev_replace()
6380 nr_extra_stripes--; in handle_ops_on_dev_replace()
6384 io_geom->num_stripes = num_stripes + nr_extra_stripes; in handle_ops_on_dev_replace()
6385 io_geom->max_errors = max_errors + nr_extra_stripes; in handle_ops_on_dev_replace()
6386 bioc->replace_nr_stripes = nr_extra_stripes; in handle_ops_on_dev_replace()
6396 io_geom->stripe_offset = offset & BTRFS_STRIPE_LEN_MASK; in btrfs_max_io_len()
6397 io_geom->stripe_nr = offset >> BTRFS_STRIPE_LEN_SHIFT; in btrfs_max_io_len()
6398 ASSERT(io_geom->stripe_offset < U32_MAX); in btrfs_max_io_len()
6400 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { in btrfs_max_io_len()
6413 io_geom->raid56_full_stripe_start = btrfs_stripe_nr_to_offset( in btrfs_max_io_len()
6414 rounddown(io_geom->stripe_nr, nr_data_stripes(map))); in btrfs_max_io_len()
6416 ASSERT(io_geom->raid56_full_stripe_start + full_stripe_len > offset); in btrfs_max_io_len()
6417 ASSERT(io_geom->raid56_full_stripe_start <= offset); in btrfs_max_io_len()
6422 if (io_geom->op == BTRFS_MAP_WRITE) in btrfs_max_io_len()
6423 return full_stripe_len - (offset - io_geom->raid56_full_stripe_start); in btrfs_max_io_len()
6430 if (map->type & BTRFS_BLOCK_GROUP_STRIPE_MASK) in btrfs_max_io_len()
6431 return BTRFS_STRIPE_LEN - io_geom->stripe_offset; in btrfs_max_io_len()
6440 dst->dev = map->stripes[io_geom->stripe_index].dev; in set_io_stripe()
6442 if (io_geom->op == BTRFS_MAP_READ && io_geom->use_rst) in set_io_stripe()
6444 map->type, in set_io_stripe()
6445 io_geom->stripe_index, dst); in set_io_stripe()
6447 dst->physical = map->stripes[io_geom->stripe_index].physical + in set_io_stripe()
6448 io_geom->stripe_offset + in set_io_stripe()
6449 btrfs_stripe_nr_to_offset(io_geom->stripe_nr); in set_io_stripe()
6465 if (io_geom->use_rst && io_geom->op != BTRFS_MAP_READ) in is_single_device_io()
6468 if ((map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) && io_geom->mirror_num > 1) in is_single_device_io()
6477 io_geom->stripe_index = io_geom->stripe_nr % map->num_stripes; in map_blocks_raid0()
6478 io_geom->stripe_nr /= map->num_stripes; in map_blocks_raid0()
6479 if (io_geom->op == BTRFS_MAP_READ) in map_blocks_raid0()
6480 io_geom->mirror_num = 1; in map_blocks_raid0()
6488 if (io_geom->op != BTRFS_MAP_READ) { in map_blocks_raid1()
6489 io_geom->num_stripes = map->num_stripes; in map_blocks_raid1()
6493 if (io_geom->mirror_num) { in map_blocks_raid1()
6494 io_geom->stripe_index = io_geom->mirror_num - 1; in map_blocks_raid1()
6498 io_geom->stripe_index = find_live_mirror(fs_info, map, 0, in map_blocks_raid1()
6500 io_geom->mirror_num = io_geom->stripe_index + 1; in map_blocks_raid1()
6506 if (io_geom->op != BTRFS_MAP_READ) { in map_blocks_dup()
6507 io_geom->num_stripes = map->num_stripes; in map_blocks_dup()
6511 if (io_geom->mirror_num) { in map_blocks_dup()
6512 io_geom->stripe_index = io_geom->mirror_num - 1; in map_blocks_dup()
6516 io_geom->mirror_num = 1; in map_blocks_dup()
6524 u32 factor = map->num_stripes / map->sub_stripes; in map_blocks_raid10()
6527 io_geom->stripe_index = (io_geom->stripe_nr % factor) * map->sub_stripes; in map_blocks_raid10()
6528 io_geom->stripe_nr /= factor; in map_blocks_raid10()
6530 if (io_geom->op != BTRFS_MAP_READ) { in map_blocks_raid10()
6531 io_geom->num_stripes = map->sub_stripes; in map_blocks_raid10()
6535 if (io_geom->mirror_num) { in map_blocks_raid10()
6536 io_geom->stripe_index += io_geom->mirror_num - 1; in map_blocks_raid10()
6540 old_stripe_index = io_geom->stripe_index; in map_blocks_raid10()
6541 io_geom->stripe_index = find_live_mirror(fs_info, map, in map_blocks_raid10()
6542 io_geom->stripe_index, in map_blocks_raid10()
6544 io_geom->mirror_num = io_geom->stripe_index - old_stripe_index + 1; in map_blocks_raid10()
6563 io_geom->stripe_nr /= data_stripes; in map_blocks_raid56_write()
6566 io_geom->num_stripes = map->num_stripes; in map_blocks_raid56_write()
6567 io_geom->max_errors = btrfs_chunk_max_errors(map); in map_blocks_raid56_write()
6571 io_geom->raid56_full_stripe_start + map->start + in map_blocks_raid56_write()
6572 btrfs_stripe_nr_to_offset(data_stripes)) - in map_blocks_raid56_write()
6574 io_geom->stripe_index = 0; in map_blocks_raid56_write()
6575 io_geom->stripe_offset = 0; in map_blocks_raid56_write()
6583 ASSERT(io_geom->mirror_num <= 1); in map_blocks_raid56_read()
6585 io_geom->stripe_index = io_geom->stripe_nr % data_stripes; in map_blocks_raid56_read()
6586 io_geom->stripe_nr /= data_stripes; in map_blocks_raid56_read()
6589 io_geom->stripe_index = in map_blocks_raid56_read()
6590 (io_geom->stripe_nr + io_geom->stripe_index) % map->num_stripes; in map_blocks_raid56_read()
6592 if (io_geom->op == BTRFS_MAP_READ && io_geom->mirror_num < 1) in map_blocks_raid56_read()
6593 io_geom->mirror_num = 1; in map_blocks_raid56_read()
6599 io_geom->stripe_index = io_geom->stripe_nr % map->num_stripes; in map_blocks_single()
6600 io_geom->stripe_nr /= map->num_stripes; in map_blocks_single()
6601 io_geom->mirror_num = io_geom->stripe_index + 1; in map_blocks_single()
6628 * For non-RAID56 profiles, non-zero mirror_num means
6650 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; in btrfs_map_block()
6668 return -EINVAL; in btrfs_map_block()
6670 map_offset = logical - map->start; in btrfs_map_block()
6671 io_geom.raid56_full_stripe_start = (u64)-1; in btrfs_map_block()
6673 *length = min_t(u64, map->chunk_len - map_offset, max_len); in btrfs_map_block()
6674 io_geom.use_rst = btrfs_need_stripe_tree_update(fs_info, map->type); in btrfs_map_block()
6676 if (dev_replace->replace_task != current) in btrfs_map_block()
6677 down_read(&dev_replace->rwsem); in btrfs_map_block()
6684 if (!dev_replace_is_ongoing && dev_replace->replace_task != current) in btrfs_map_block()
6685 up_read(&dev_replace->rwsem); in btrfs_map_block()
6687 switch (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) { in btrfs_map_block()
6718 if (io_geom.stripe_index >= map->num_stripes) { in btrfs_map_block()
6721 io_geom.stripe_index, map->num_stripes); in btrfs_map_block()
6722 ret = -EINVAL; in btrfs_map_block()
6727 if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL && in btrfs_map_block()
6753 ret = -ENOMEM; in btrfs_map_block()
6756 bioc->map_type = map->type; in btrfs_map_block()
6757 bioc->use_rst = io_geom.use_rst; in btrfs_map_block()
6766 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK && in btrfs_map_block()
6776 bioc->full_stripe_logical = map->start + in btrfs_map_block()
6780 struct btrfs_io_stripe *dst = &bioc->stripes[i]; in btrfs_map_block()
6784 dst->dev = map->stripes[stripe_index].dev; in btrfs_map_block()
6785 dst->physical = in btrfs_map_block()
6786 map->stripes[stripe_index].physical + in btrfs_map_block()
6792 * For all other non-RAID56 profiles, just copy the target in btrfs_map_block()
6797 &bioc->stripes[i], map, &io_geom); in btrfs_map_block()
6813 if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL && in btrfs_map_block()
6819 bioc->num_stripes = io_geom.num_stripes; in btrfs_map_block()
6820 bioc->max_errors = io_geom.max_errors; in btrfs_map_block()
6821 bioc->mirror_num = io_geom.mirror_num; in btrfs_map_block()
6824 if (dev_replace_is_ongoing && dev_replace->replace_task != current) { in btrfs_map_block()
6825 lockdep_assert_held(&dev_replace->rwsem); in btrfs_map_block()
6827 up_read(&dev_replace->rwsem); in btrfs_map_block()
6836 if (args->fsid == NULL) in dev_args_match_fs_devices()
6838 if (memcmp(fs_devices->metadata_uuid, args->fsid, BTRFS_FSID_SIZE) == 0) in dev_args_match_fs_devices()
6846 if (args->missing) { in dev_args_match_device()
6847 if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state) && in dev_args_match_device()
6848 !device->bdev) in dev_args_match_device()
6853 if (device->devid != args->devid) in dev_args_match_device()
6855 if (args->uuid && memcmp(device->uuid, args->uuid, BTRFS_UUID_SIZE) != 0) in dev_args_match_device()
6874 list_for_each_entry(device, &fs_devices->devices, dev_list) { in btrfs_find_device()
6880 list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) { in btrfs_find_device()
6883 list_for_each_entry(device, &seed_devs->devices, dev_list) { in btrfs_find_device()
6911 list_add(&device->dev_list, &fs_devices->devices); in add_missing_dev()
6912 device->fs_devices = fs_devices; in add_missing_dev()
6913 fs_devices->num_devices++; in add_missing_dev()
6915 set_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state); in add_missing_dev()
6916 fs_devices->missing_devices++; in add_missing_dev()
6944 return ERR_PTR(-EINVAL); in btrfs_alloc_device()
6948 return ERR_PTR(-ENOMEM); in btrfs_alloc_device()
6950 INIT_LIST_HEAD(&dev->dev_list); in btrfs_alloc_device()
6951 INIT_LIST_HEAD(&dev->dev_alloc_list); in btrfs_alloc_device()
6952 INIT_LIST_HEAD(&dev->post_commit_list); in btrfs_alloc_device()
6954 atomic_set(&dev->dev_stats_ccnt, 0); in btrfs_alloc_device()
6956 extent_io_tree_init(fs_info, &dev->alloc_state, IO_TREE_DEVICE_ALLOC_STATE); in btrfs_alloc_device()
6969 dev->devid = tmp; in btrfs_alloc_device()
6972 memcpy(dev->uuid, uuid, BTRFS_UUID_SIZE); in btrfs_alloc_device()
6974 generate_random_uuid(dev->uuid); in btrfs_alloc_device()
6982 return ERR_PTR(-ENOMEM); in btrfs_alloc_device()
6984 rcu_assign_pointer(dev->name, name); in btrfs_alloc_device()
7003 const int data_stripes = calc_data_stripes(map->type, map->num_stripes); in btrfs_calc_stripe_length()
7005 return div_u64(map->chunk_len, data_stripes); in btrfs_calc_stripe_length()
7026 return -EOVERFLOW; in check_32bit_meta_chunk()
7055 return ERR_PTR(-ENOENT); in handle_missing_device()
7058 dev = add_missing_dev(fs_info->fs_devices, devid, uuid); in handle_missing_device()
7073 struct btrfs_fs_info *fs_info = leaf->fs_info; in read_one_chunk()
7085 logical = key->offset; in read_one_chunk()
7101 if (map && map->start <= logical && map->start + map->chunk_len > logical) { in read_one_chunk()
7110 return -ENOMEM; in read_one_chunk()
7112 map->start = logical; in read_one_chunk()
7113 map->chunk_len = length; in read_one_chunk()
7114 map->num_stripes = num_stripes; in read_one_chunk()
7115 map->io_width = btrfs_chunk_io_width(leaf, chunk); in read_one_chunk()
7116 map->io_align = btrfs_chunk_io_align(leaf, chunk); in read_one_chunk()
7117 map->type = type; in read_one_chunk()
7122 * In that case, it can cause divide-by-zero errors later. in read_one_chunk()
7126 map->sub_stripes = btrfs_raid_array[index].sub_stripes; in read_one_chunk()
7127 map->verified_stripes = 0; in read_one_chunk()
7128 map->stripe_size = btrfs_calc_stripe_length(map); in read_one_chunk()
7130 map->stripes[i].physical = in read_one_chunk()
7138 map->stripes[i].dev = btrfs_find_device(fs_info->fs_devices, &args); in read_one_chunk()
7139 if (!map->stripes[i].dev) { in read_one_chunk()
7140 map->stripes[i].dev = handle_missing_device(fs_info, in read_one_chunk()
7142 if (IS_ERR(map->stripes[i].dev)) { in read_one_chunk()
7143 ret = PTR_ERR(map->stripes[i].dev); in read_one_chunk()
7150 &(map->stripes[i].dev->dev_state)); in read_one_chunk()
7157 map->start, map->chunk_len, ret); in read_one_chunk()
7170 device->devid = btrfs_device_id(leaf, dev_item); in fill_device_from_item()
7171 device->disk_total_bytes = btrfs_device_total_bytes(leaf, dev_item); in fill_device_from_item()
7172 device->total_bytes = device->disk_total_bytes; in fill_device_from_item()
7173 device->commit_total_bytes = device->disk_total_bytes; in fill_device_from_item()
7174 device->bytes_used = btrfs_device_bytes_used(leaf, dev_item); in fill_device_from_item()
7175 device->commit_bytes_used = device->bytes_used; in fill_device_from_item()
7176 device->type = btrfs_device_type(leaf, dev_item); in fill_device_from_item()
7177 device->io_align = btrfs_device_io_align(leaf, dev_item); in fill_device_from_item()
7178 device->io_width = btrfs_device_io_width(leaf, dev_item); in fill_device_from_item()
7179 device->sector_size = btrfs_device_sector_size(leaf, dev_item); in fill_device_from_item()
7180 WARN_ON(device->devid == BTRFS_DEV_REPLACE_DEVID); in fill_device_from_item()
7181 clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); in fill_device_from_item()
7184 read_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE); in fill_device_from_item()
7196 /* This will match only for multi-device seed fs */ in open_seed_devices()
7197 list_for_each_entry(fs_devices, &fs_info->fs_devices->seed_list, seed_list) in open_seed_devices()
7198 if (!memcmp(fs_devices->fsid, fsid, BTRFS_FSID_SIZE)) in open_seed_devices()
7208 return ERR_PTR(-ENOENT); in open_seed_devices()
7215 fs_devices->seeding = true; in open_seed_devices()
7216 fs_devices->opened = 1; in open_seed_devices()
7222 * respective fs_devices and anchor it at fs_info->fs_devices->seed_list in open_seed_devices()
7228 ret = open_fs_devices(fs_devices, BLK_OPEN_READ, fs_info->bdev_holder); in open_seed_devices()
7234 if (!fs_devices->seeding) { in open_seed_devices()
7237 return ERR_PTR(-EINVAL); in open_seed_devices()
7240 list_add(&fs_devices->seed_list, &fs_info->fs_devices->seed_list); in open_seed_devices()
7249 struct btrfs_fs_info *fs_info = leaf->fs_info; in read_one_dev()
7250 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in read_one_dev()
7266 if (memcmp(fs_uuid, fs_devices->metadata_uuid, BTRFS_FSID_SIZE)) { in read_one_dev()
7272 device = btrfs_find_device(fs_info->fs_devices, &args); in read_one_dev()
7277 return -ENOENT; in read_one_dev()
7289 if (!device->bdev) { in read_one_dev()
7293 return -ENOENT; in read_one_dev()
7299 if (!device->bdev && in read_one_dev()
7300 !test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) { in read_one_dev()
7304 * device->bdev is NULL, and so we have to set in read_one_dev()
7305 * device->missing to one here in read_one_dev()
7307 device->fs_devices->missing_devices++; in read_one_dev()
7308 set_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state); in read_one_dev()
7312 if (device->fs_devices != fs_devices) { in read_one_dev()
7314 &device->dev_state)); in read_one_dev()
7316 list_move(&device->dev_list, &fs_devices->devices); in read_one_dev()
7317 device->fs_devices->num_devices--; in read_one_dev()
7318 fs_devices->num_devices++; in read_one_dev()
7320 device->fs_devices->missing_devices--; in read_one_dev()
7321 fs_devices->missing_devices++; in read_one_dev()
7323 device->fs_devices = fs_devices; in read_one_dev()
7327 if (device->fs_devices != fs_info->fs_devices) { in read_one_dev()
7328 BUG_ON(test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)); in read_one_dev()
7329 if (device->generation != in read_one_dev()
7331 return -EINVAL; in read_one_dev()
7335 if (device->bdev) { in read_one_dev()
7336 u64 max_total_bytes = bdev_nr_bytes(device->bdev); in read_one_dev()
7338 if (device->total_bytes > max_total_bytes) { in read_one_dev()
7341 max_total_bytes, device->total_bytes); in read_one_dev()
7342 return -EINVAL; in read_one_dev()
7345 set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); in read_one_dev()
7346 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) && in read_one_dev()
7347 !test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { in read_one_dev()
7348 device->fs_devices->total_rw_bytes += device->total_bytes; in read_one_dev()
7349 atomic64_add(device->total_bytes - device->bytes_used, in read_one_dev()
7350 &fs_info->free_chunk_space); in read_one_dev()
7358 struct btrfs_super_block *super_copy = fs_info->super_copy; in btrfs_read_sys_array()
7367 ASSERT(BTRFS_SUPER_INFO_SIZE <= fs_info->nodesize); in btrfs_read_sys_array()
7372 * that's fine, we will not go beyond system chunk array anyway. in btrfs_read_sys_array()
7376 return -ENOMEM; in btrfs_read_sys_array()
7382 array_ptr = super_copy->sys_chunk_array; in btrfs_read_sys_array()
7426 * Check if all chunks in the fs are OK for read-write degraded mount
7453 map->type); in btrfs_check_rw_degradable()
7454 for (i = 0; i < map->num_stripes; i++) { in btrfs_check_rw_degradable()
7455 struct btrfs_device *dev = map->stripes[i].dev; in btrfs_check_rw_degradable()
7457 if (!dev || !dev->bdev || in btrfs_check_rw_degradable()
7458 test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) || in btrfs_check_rw_degradable()
7459 dev->last_flush_error) in btrfs_check_rw_degradable()
7468 map->start, missing, max_tolerated); in btrfs_check_rw_degradable()
7473 next_start = map->start + map->chunk_len; in btrfs_check_rw_degradable()
7476 map = btrfs_find_chunk_map(fs_info, next_start, U64_MAX - next_start); in btrfs_check_rw_degradable()
7493 struct btrfs_root *root = fs_info->chunk_root; in btrfs_read_chunk_tree()
7506 return -ENOMEM; in btrfs_read_chunk_tree()
7520 fs_info->fs_devices->total_rw_bytes = 0; in btrfs_read_chunk_tree()
7532 ASSERT(!test_bit(BTRFS_FS_OPEN, &fs_info->flags)); in btrfs_read_chunk_tree()
7533 path->skip_locking = 1; in btrfs_read_chunk_tree()
7539 * item - BTRFS_FIRST_CHUNK_TREE_OBJECTID). in btrfs_read_chunk_tree()
7545 struct extent_buffer *node = path->nodes[1]; in btrfs_read_chunk_tree()
7547 leaf = path->nodes[0]; in btrfs_read_chunk_tree()
7548 slot = path->slots[0]; in btrfs_read_chunk_tree()
7551 if (last_ra_node != node->start) { in btrfs_read_chunk_tree()
7553 last_ra_node = node->start; in btrfs_read_chunk_tree()
7569 * fs_info->chunk_mutex. Plus, to avoid lockdep warnings, in btrfs_read_chunk_tree()
7570 * we always lock first fs_info->chunk_mutex before in btrfs_read_chunk_tree()
7591 if (total_dev != fs_info->fs_devices->total_devices) { in btrfs_read_chunk_tree()
7594 btrfs_super_num_devices(fs_info->super_copy), in btrfs_read_chunk_tree()
7596 fs_info->fs_devices->total_devices = total_dev; in btrfs_read_chunk_tree()
7597 btrfs_set_super_num_devices(fs_info->super_copy, total_dev); in btrfs_read_chunk_tree()
7599 if (btrfs_super_total_bytes(fs_info->super_copy) < in btrfs_read_chunk_tree()
7600 fs_info->fs_devices->total_rw_bytes) { in btrfs_read_chunk_tree()
7603 btrfs_super_total_bytes(fs_info->super_copy), in btrfs_read_chunk_tree()
7604 fs_info->fs_devices->total_rw_bytes); in btrfs_read_chunk_tree()
7605 ret = -EINVAL; in btrfs_read_chunk_tree()
7618 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs; in btrfs_init_devices_late()
7622 mutex_lock(&fs_devices->device_list_mutex); in btrfs_init_devices_late()
7623 list_for_each_entry(device, &fs_devices->devices, dev_list) in btrfs_init_devices_late()
7624 device->fs_info = fs_info; in btrfs_init_devices_late()
7626 list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) { in btrfs_init_devices_late()
7627 list_for_each_entry(device, &seed_devs->devices, dev_list) { in btrfs_init_devices_late()
7628 device->fs_info = fs_info; in btrfs_init_devices_late()
7634 seed_devs->fs_info = fs_info; in btrfs_init_devices_late()
7636 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_init_devices_late()
7673 if (!device->fs_info->dev_root) in btrfs_device_init_dev_stats()
7678 key.offset = device->devid; in btrfs_device_init_dev_stats()
7679 ret = btrfs_search_slot(NULL, device->fs_info->dev_root, &key, path, 0, 0); in btrfs_device_init_dev_stats()
7683 device->dev_stats_valid = 1; in btrfs_device_init_dev_stats()
7687 slot = path->slots[0]; in btrfs_device_init_dev_stats()
7688 eb = path->nodes[0]; in btrfs_device_init_dev_stats()
7701 device->dev_stats_valid = 1; in btrfs_device_init_dev_stats()
7710 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs; in btrfs_init_dev_stats()
7717 return -ENOMEM; in btrfs_init_dev_stats()
7719 mutex_lock(&fs_devices->device_list_mutex); in btrfs_init_dev_stats()
7720 list_for_each_entry(device, &fs_devices->devices, dev_list) { in btrfs_init_dev_stats()
7725 list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) { in btrfs_init_dev_stats()
7726 list_for_each_entry(device, &seed_devs->devices, dev_list) { in btrfs_init_dev_stats()
7733 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_init_dev_stats()
7742 struct btrfs_fs_info *fs_info = trans->fs_info; in update_dev_stat_item()
7743 struct btrfs_root *dev_root = fs_info->dev_root; in update_dev_stat_item()
7753 key.offset = device->devid; in update_dev_stat_item()
7757 return -ENOMEM; in update_dev_stat_item()
7758 ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1); in update_dev_stat_item()
7767 btrfs_item_size(path->nodes[0], path->slots[0]) < sizeof(*ptr)) { in update_dev_stat_item()
7792 eb = path->nodes[0]; in update_dev_stat_item()
7793 ptr = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dev_stats_item); in update_dev_stat_item()
7807 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_run_dev_stats()
7808 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in btrfs_run_dev_stats()
7813 mutex_lock(&fs_devices->device_list_mutex); in btrfs_run_dev_stats()
7814 list_for_each_entry(device, &fs_devices->devices, dev_list) { in btrfs_run_dev_stats()
7815 stats_cnt = atomic_read(&device->dev_stats_ccnt); in btrfs_run_dev_stats()
7816 if (!device->dev_stats_valid || stats_cnt == 0) in btrfs_run_dev_stats()
7821 * There is a LOAD-LOAD control dependency between the value of in btrfs_run_dev_stats()
7822 * dev_stats_ccnt and updating the on-disk values which requires in btrfs_run_dev_stats()
7823 * reading the in-memory counters. Such control dependencies in btrfs_run_dev_stats()
7835 atomic_sub(stats_cnt, &device->dev_stats_ccnt); in btrfs_run_dev_stats()
7837 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_run_dev_stats()
7846 if (!dev->dev_stats_valid) in btrfs_dev_stat_inc_and_print()
7848 btrfs_err_rl_in_rcu(dev->fs_info, in btrfs_dev_stat_inc_and_print()
7868 btrfs_info_in_rcu(dev->fs_info, in btrfs_dev_stat_print_on_load()
7883 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in btrfs_get_dev_stats()
7886 mutex_lock(&fs_devices->device_list_mutex); in btrfs_get_dev_stats()
7887 args.devid = stats->devid; in btrfs_get_dev_stats()
7888 dev = btrfs_find_device(fs_info->fs_devices, &args); in btrfs_get_dev_stats()
7889 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_get_dev_stats()
7893 return -ENODEV; in btrfs_get_dev_stats()
7894 } else if (!dev->dev_stats_valid) { in btrfs_get_dev_stats()
7896 return -ENODEV; in btrfs_get_dev_stats()
7897 } else if (stats->flags & BTRFS_DEV_STATS_RESET) { in btrfs_get_dev_stats()
7899 if (stats->nr_items > i) in btrfs_get_dev_stats()
7900 stats->values[i] = in btrfs_get_dev_stats()
7906 current->comm, task_pid_nr(current)); in btrfs_get_dev_stats()
7909 if (stats->nr_items > i) in btrfs_get_dev_stats()
7910 stats->values[i] = btrfs_dev_stat_read(dev, i); in btrfs_get_dev_stats()
7912 if (stats->nr_items > BTRFS_DEV_STAT_VALUES_MAX) in btrfs_get_dev_stats()
7913 stats->nr_items = BTRFS_DEV_STAT_VALUES_MAX; in btrfs_get_dev_stats()
7928 ASSERT(trans->state == TRANS_STATE_COMMIT_DOING); in btrfs_commit_device_sizes()
7930 if (list_empty(&trans->dev_update_list)) in btrfs_commit_device_sizes()
7938 mutex_lock(&trans->fs_info->chunk_mutex); in btrfs_commit_device_sizes()
7939 list_for_each_entry_safe(curr, next, &trans->dev_update_list, in btrfs_commit_device_sizes()
7941 list_del_init(&curr->post_commit_list); in btrfs_commit_device_sizes()
7942 curr->commit_total_bytes = curr->disk_total_bytes; in btrfs_commit_device_sizes()
7943 curr->commit_bytes_used = curr->bytes_used; in btrfs_commit_device_sizes()
7945 mutex_unlock(&trans->fs_info->chunk_mutex); in btrfs_commit_device_sizes()
7949 * Multiplicity factor for simple profiles: DUP, RAID1-like and RAID10.
7977 ret = -EUCLEAN; in verify_one_dev_extent()
7985 physical_offset, devid, map->start, physical_len, in verify_one_dev_extent()
7987 ret = -EUCLEAN; in verify_one_dev_extent()
8001 for (i = 0; i < map->num_stripes; i++) { in verify_one_dev_extent()
8002 if (map->stripes[i].dev->devid == devid && in verify_one_dev_extent()
8003 map->stripes[i].physical == physical_offset) { in verify_one_dev_extent()
8005 if (map->verified_stripes >= map->num_stripes) { in verify_one_dev_extent()
8008 map->start); in verify_one_dev_extent()
8009 ret = -EUCLEAN; in verify_one_dev_extent()
8012 map->verified_stripes++; in verify_one_dev_extent()
8020 ret = -EUCLEAN; in verify_one_dev_extent()
8024 dev = btrfs_find_device(fs_info->fs_devices, &args); in verify_one_dev_extent()
8027 ret = -EUCLEAN; in verify_one_dev_extent()
8031 if (physical_offset + physical_len > dev->disk_total_bytes) { in verify_one_dev_extent()
8035 dev->disk_total_bytes); in verify_one_dev_extent()
8036 ret = -EUCLEAN; in verify_one_dev_extent()
8040 if (dev->zone_info) { in verify_one_dev_extent()
8041 u64 zone_size = dev->zone_info->zone_size; in verify_one_dev_extent()
8048 ret = -EUCLEAN; in verify_one_dev_extent()
8063 read_lock(&fs_info->mapping_tree_lock); in verify_chunk_dev_extent_mapping()
8064 for (node = rb_first_cached(&fs_info->mapping_tree); node; node = rb_next(node)) { in verify_chunk_dev_extent_mapping()
8068 if (map->num_stripes != map->verified_stripes) { in verify_chunk_dev_extent_mapping()
8071 map->start, map->verified_stripes, map->num_stripes); in verify_chunk_dev_extent_mapping()
8072 ret = -EUCLEAN; in verify_chunk_dev_extent_mapping()
8077 read_unlock(&fs_info->mapping_tree_lock); in verify_chunk_dev_extent_mapping()
8091 struct btrfs_root *root = fs_info->dev_root; in btrfs_verify_dev_extents()
8116 return -ENOMEM; in btrfs_verify_dev_extents()
8118 path->reada = READA_FORWARD; in btrfs_verify_dev_extents()
8123 if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { in btrfs_verify_dev_extents()
8129 ret = -EUCLEAN; in btrfs_verify_dev_extents()
8134 struct extent_buffer *leaf = path->nodes[0]; in btrfs_verify_dev_extents()
8136 int slot = path->slots[0]; in btrfs_verify_dev_extents()
8157 ret = -EUCLEAN; in btrfs_verify_dev_extents()
8193 spin_lock(&fs_info->swapfile_pins_lock); in btrfs_pinned_by_swapfile()
8194 node = fs_info->swapfile_pins.rb_node; in btrfs_pinned_by_swapfile()
8197 if (ptr < sp->ptr) in btrfs_pinned_by_swapfile()
8198 node = node->rb_left; in btrfs_pinned_by_swapfile()
8199 else if (ptr > sp->ptr) in btrfs_pinned_by_swapfile()
8200 node = node->rb_right; in btrfs_pinned_by_swapfile()
8204 spin_unlock(&fs_info->swapfile_pins_lock); in btrfs_pinned_by_swapfile()
8211 struct btrfs_fs_info *fs_info = cache->fs_info; in relocating_repair_kthread()
8215 target = cache->start; in relocating_repair_kthread()
8218 sb_start_write(fs_info->sb); in relocating_repair_kthread()
8223 sb_end_write(fs_info->sb); in relocating_repair_kthread()
8224 return -EBUSY; in relocating_repair_kthread()
8227 mutex_lock(&fs_info->reclaim_bgs_lock); in relocating_repair_kthread()
8234 if (!test_bit(BLOCK_GROUP_FLAG_RELOCATING_REPAIR, &cache->runtime_flags)) in relocating_repair_kthread()
8249 mutex_unlock(&fs_info->reclaim_bgs_lock); in relocating_repair_kthread()
8251 sb_end_write(fs_info->sb); in relocating_repair_kthread()
8271 if (test_and_set_bit(BLOCK_GROUP_FLAG_RELOCATING_REPAIR, &cache->runtime_flags)) { in btrfs_repair_one_zone()
8277 "btrfs-relocating-repair"); in btrfs_repair_one_zone()
8290 u64 stripe_start = bioc->full_stripe_logical + in map_raid56_repair_block()
8298 smap->dev = bioc->stripes[i].dev; in map_raid56_repair_block()
8299 smap->physical = bioc->stripes[i].physical + in map_raid56_repair_block()
8300 ((logical - bioc->full_stripe_logical) & in map_raid56_repair_block()
8313 * - Call btrfs_bio_counter_inc_blocked() first
8314 * - The range does not cross stripe boundary
8315 * - Has a valid @mirror_num passed in.
8340 /* Map the RAID56 multi-stripe writes to a single one. */ in btrfs_map_repair_block()
8341 if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) { in btrfs_map_repair_block()
8346 ASSERT(mirror_num <= bioc->num_stripes); in btrfs_map_repair_block()
8347 smap->dev = bioc->stripes[mirror_num - 1].dev; in btrfs_map_repair_block()
8348 smap->physical = bioc->stripes[mirror_num - 1].physical; in btrfs_map_repair_block()
8351 ASSERT(smap->dev); in btrfs_map_repair_block()