1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (C) STRATO AG 2012. All rights reserved. 4 */ 5 6#include <linux/sched.h> 7#include <linux/bio.h> 8#include <linux/slab.h> 9#include <linux/blkdev.h> 10#include <linux/kthread.h> 11#include <linux/math64.h> 12#include "misc.h" 13#include "ctree.h" 14#include "extent_map.h" 15#include "disk-io.h" 16#include "transaction.h" 17#include "print-tree.h" 18#include "volumes.h" 19#include "async-thread.h" 20#include "check-integrity.h" 21#include "rcu-string.h" 22#include "dev-replace.h" 23#include "sysfs.h" 24 25/* 26 * Device replace overview 27 * 28 * [Objective] 29 * To copy all extents (both new and on-disk) from source device to target 30 * device, while still keeping the filesystem read-write. 31 * 32 * [Method] 33 * There are two main methods involved: 34 * 35 * - Write duplication 36 * 37 * All new writes will be written to both target and source devices, so even 38 * if replace gets canceled, sources device still contans up-to-date data. 39 * 40 * Location: handle_ops_on_dev_replace() from __btrfs_map_block() 41 * Start: btrfs_dev_replace_start() 42 * End: btrfs_dev_replace_finishing() 43 * Content: Latest data/metadata 44 * 45 * - Copy existing extents 46 * 47 * This happens by re-using scrub facility, as scrub also iterates through 48 * existing extents from commit root. 49 * 50 * Location: scrub_write_block_to_dev_replace() from 51 * scrub_block_complete() 52 * Content: Data/meta from commit root. 53 * 54 * Due to the content difference, we need to avoid nocow write when dev-replace 55 * is happening. This is done by marking the block group read-only and waiting 56 * for NOCOW writes. 57 * 58 * After replace is done, the finishing part is done by swapping the target and 59 * source devices. 60 * 61 * Location: btrfs_dev_replace_update_device_in_mapping_tree() from 62 * btrfs_dev_replace_finishing() 63 */ 64 65static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info, 66 int scrub_ret); 67static int btrfs_dev_replace_kthread(void *data); 68 69int btrfs_init_dev_replace(struct btrfs_fs_info *fs_info) 70{ 71 struct btrfs_key key; 72 struct btrfs_root *dev_root = fs_info->dev_root; 73 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; 74 struct extent_buffer *eb; 75 int slot; 76 int ret = 0; 77 struct btrfs_path *path = NULL; 78 int item_size; 79 struct btrfs_dev_replace_item *ptr; 80 u64 src_devid; 81 82 path = btrfs_alloc_path(); 83 if (!path) { 84 ret = -ENOMEM; 85 goto out; 86 } 87 88 key.objectid = 0; 89 key.type = BTRFS_DEV_REPLACE_KEY; 90 key.offset = 0; 91 ret = btrfs_search_slot(NULL, dev_root, &key, path, 0, 0); 92 if (ret) { 93no_valid_dev_replace_entry_found: 94 /* 95 * We don't have a replace item or it's corrupted. If there is 96 * a replace target, fail the mount. 97 */ 98 if (btrfs_find_device(fs_info->fs_devices, 99 BTRFS_DEV_REPLACE_DEVID, NULL, NULL, false)) { 100 btrfs_err(fs_info, 101 "found replace target device without a valid replace item"); 102 ret = -EUCLEAN; 103 goto out; 104 } 105 ret = 0; 106 dev_replace->replace_state = 107 BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED; 108 dev_replace->cont_reading_from_srcdev_mode = 109 BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_ALWAYS; 110 dev_replace->time_started = 0; 111 dev_replace->time_stopped = 0; 112 atomic64_set(&dev_replace->num_write_errors, 0); 113 atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0); 114 dev_replace->cursor_left = 0; 115 dev_replace->committed_cursor_left = 0; 116 dev_replace->cursor_left_last_write_of_item = 0; 117 dev_replace->cursor_right = 0; 118 dev_replace->srcdev = NULL; 119 dev_replace->tgtdev = NULL; 120 dev_replace->is_valid = 0; 121 dev_replace->item_needs_writeback = 0; 122 goto out; 123 } 124 slot = path->slots[0]; 125 eb = path->nodes[0]; 126 item_size = btrfs_item_size_nr(eb, slot); 127 ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_replace_item); 128 129 if (item_size != sizeof(struct btrfs_dev_replace_item)) { 130 btrfs_warn(fs_info, 131 "dev_replace entry found has unexpected size, ignore entry"); 132 goto no_valid_dev_replace_entry_found; 133 } 134 135 src_devid = btrfs_dev_replace_src_devid(eb, ptr); 136 dev_replace->cont_reading_from_srcdev_mode = 137 btrfs_dev_replace_cont_reading_from_srcdev_mode(eb, ptr); 138 dev_replace->replace_state = btrfs_dev_replace_replace_state(eb, ptr); 139 dev_replace->time_started = btrfs_dev_replace_time_started(eb, ptr); 140 dev_replace->time_stopped = 141 btrfs_dev_replace_time_stopped(eb, ptr); 142 atomic64_set(&dev_replace->num_write_errors, 143 btrfs_dev_replace_num_write_errors(eb, ptr)); 144 atomic64_set(&dev_replace->num_uncorrectable_read_errors, 145 btrfs_dev_replace_num_uncorrectable_read_errors(eb, ptr)); 146 dev_replace->cursor_left = btrfs_dev_replace_cursor_left(eb, ptr); 147 dev_replace->committed_cursor_left = dev_replace->cursor_left; 148 dev_replace->cursor_left_last_write_of_item = dev_replace->cursor_left; 149 dev_replace->cursor_right = btrfs_dev_replace_cursor_right(eb, ptr); 150 dev_replace->is_valid = 1; 151 152 dev_replace->item_needs_writeback = 0; 153 switch (dev_replace->replace_state) { 154 case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: 155 case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED: 156 case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED: 157 /* 158 * We don't have an active replace item but if there is a 159 * replace target, fail the mount. 160 */ 161 if (btrfs_find_device(fs_info->fs_devices, 162 BTRFS_DEV_REPLACE_DEVID, NULL, NULL, false)) { 163 btrfs_err(fs_info, 164"replace without active item, run 'device scan --forget' on the target device"); 165 ret = -EUCLEAN; 166 } else { 167 dev_replace->srcdev = NULL; 168 dev_replace->tgtdev = NULL; 169 } 170 break; 171 case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED: 172 case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED: 173 dev_replace->srcdev = btrfs_find_device(fs_info->fs_devices, 174 src_devid, NULL, NULL, true); 175 dev_replace->tgtdev = btrfs_find_device(fs_info->fs_devices, 176 BTRFS_DEV_REPLACE_DEVID, 177 NULL, NULL, true); 178 /* 179 * allow 'btrfs dev replace_cancel' if src/tgt device is 180 * missing 181 */ 182 if (!dev_replace->srcdev && 183 !btrfs_test_opt(fs_info, DEGRADED)) { 184 ret = -EIO; 185 btrfs_warn(fs_info, 186 "cannot mount because device replace operation is ongoing and"); 187 btrfs_warn(fs_info, 188 "srcdev (devid %llu) is missing, need to run 'btrfs dev scan'?", 189 src_devid); 190 } 191 if (!dev_replace->tgtdev && 192 !btrfs_test_opt(fs_info, DEGRADED)) { 193 ret = -EIO; 194 btrfs_warn(fs_info, 195 "cannot mount because device replace operation is ongoing and"); 196 btrfs_warn(fs_info, 197 "tgtdev (devid %llu) is missing, need to run 'btrfs dev scan'?", 198 BTRFS_DEV_REPLACE_DEVID); 199 } 200 if (dev_replace->tgtdev) { 201 if (dev_replace->srcdev) { 202 dev_replace->tgtdev->total_bytes = 203 dev_replace->srcdev->total_bytes; 204 dev_replace->tgtdev->disk_total_bytes = 205 dev_replace->srcdev->disk_total_bytes; 206 dev_replace->tgtdev->commit_total_bytes = 207 dev_replace->srcdev->commit_total_bytes; 208 dev_replace->tgtdev->bytes_used = 209 dev_replace->srcdev->bytes_used; 210 dev_replace->tgtdev->commit_bytes_used = 211 dev_replace->srcdev->commit_bytes_used; 212 } 213 set_bit(BTRFS_DEV_STATE_REPLACE_TGT, 214 &dev_replace->tgtdev->dev_state); 215 216 WARN_ON(fs_info->fs_devices->rw_devices == 0); 217 dev_replace->tgtdev->io_width = fs_info->sectorsize; 218 dev_replace->tgtdev->io_align = fs_info->sectorsize; 219 dev_replace->tgtdev->sector_size = fs_info->sectorsize; 220 dev_replace->tgtdev->fs_info = fs_info; 221 set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, 222 &dev_replace->tgtdev->dev_state); 223 } 224 break; 225 } 226 227out: 228 btrfs_free_path(path); 229 return ret; 230} 231 232/* 233 * Initialize a new device for device replace target from a given source dev 234 * and path. 235 * 236 * Return 0 and new device in @device_out, otherwise return < 0 237 */ 238static int btrfs_init_dev_replace_tgtdev(struct btrfs_fs_info *fs_info, 239 const char *device_path, 240 struct btrfs_device *srcdev, 241 struct btrfs_device **device_out) 242{ 243 struct btrfs_device *device; 244 struct block_device *bdev; 245 struct rcu_string *name; 246 u64 devid = BTRFS_DEV_REPLACE_DEVID; 247 int ret = 0; 248 249 *device_out = NULL; 250 if (srcdev->fs_devices->seeding) { 251 btrfs_err(fs_info, "the filesystem is a seed filesystem!"); 252 return -EINVAL; 253 } 254 255 bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL, 256 fs_info->bdev_holder); 257 if (IS_ERR(bdev)) { 258 btrfs_err(fs_info, "target device %s is invalid!", device_path); 259 return PTR_ERR(bdev); 260 } 261 262 sync_blockdev(bdev); 263 264 list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) { 265 if (device->bdev == bdev) { 266 btrfs_err(fs_info, 267 "target device is in the filesystem!"); 268 ret = -EEXIST; 269 goto error; 270 } 271 } 272 273 274 if (i_size_read(bdev->bd_inode) < 275 btrfs_device_get_total_bytes(srcdev)) { 276 btrfs_err(fs_info, 277 "target device is smaller than source device!"); 278 ret = -EINVAL; 279 goto error; 280 } 281 282 283 device = btrfs_alloc_device(NULL, &devid, NULL); 284 if (IS_ERR(device)) { 285 ret = PTR_ERR(device); 286 goto error; 287 } 288 289 name = rcu_string_strdup(device_path, GFP_KERNEL); 290 if (!name) { 291 btrfs_free_device(device); 292 ret = -ENOMEM; 293 goto error; 294 } 295 rcu_assign_pointer(device->name, name); 296 297 set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); 298 device->generation = 0; 299 device->io_width = fs_info->sectorsize; 300 device->io_align = fs_info->sectorsize; 301 device->sector_size = fs_info->sectorsize; 302 device->total_bytes = btrfs_device_get_total_bytes(srcdev); 303 device->disk_total_bytes = btrfs_device_get_disk_total_bytes(srcdev); 304 device->bytes_used = btrfs_device_get_bytes_used(srcdev); 305 device->commit_total_bytes = srcdev->commit_total_bytes; 306 device->commit_bytes_used = device->bytes_used; 307 device->fs_info = fs_info; 308 device->bdev = bdev; 309 set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); 310 set_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); 311 device->mode = FMODE_EXCL; 312 device->dev_stats_valid = 1; 313 set_blocksize(device->bdev, BTRFS_BDEV_BLOCKSIZE); 314 device->fs_devices = fs_info->fs_devices; 315 316 mutex_lock(&fs_info->fs_devices->device_list_mutex); 317 list_add(&device->dev_list, &fs_info->fs_devices->devices); 318 fs_info->fs_devices->num_devices++; 319 fs_info->fs_devices->open_devices++; 320 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 321 322 *device_out = device; 323 return 0; 324 325error: 326 blkdev_put(bdev, FMODE_EXCL); 327 return ret; 328} 329 330/* 331 * called from commit_transaction. Writes changed device replace state to 332 * disk. 333 */ 334int btrfs_run_dev_replace(struct btrfs_trans_handle *trans) 335{ 336 struct btrfs_fs_info *fs_info = trans->fs_info; 337 int ret; 338 struct btrfs_root *dev_root = fs_info->dev_root; 339 struct btrfs_path *path; 340 struct btrfs_key key; 341 struct extent_buffer *eb; 342 struct btrfs_dev_replace_item *ptr; 343 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; 344 345 down_read(&dev_replace->rwsem); 346 if (!dev_replace->is_valid || 347 !dev_replace->item_needs_writeback) { 348 up_read(&dev_replace->rwsem); 349 return 0; 350 } 351 up_read(&dev_replace->rwsem); 352 353 key.objectid = 0; 354 key.type = BTRFS_DEV_REPLACE_KEY; 355 key.offset = 0; 356 357 path = btrfs_alloc_path(); 358 if (!path) { 359 ret = -ENOMEM; 360 goto out; 361 } 362 ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1); 363 if (ret < 0) { 364 btrfs_warn(fs_info, 365 "error %d while searching for dev_replace item!", 366 ret); 367 goto out; 368 } 369 370 if (ret == 0 && 371 btrfs_item_size_nr(path->nodes[0], path->slots[0]) < sizeof(*ptr)) { 372 /* 373 * need to delete old one and insert a new one. 374 * Since no attempt is made to recover any old state, if the 375 * dev_replace state is 'running', the data on the target 376 * drive is lost. 377 * It would be possible to recover the state: just make sure 378 * that the beginning of the item is never changed and always 379 * contains all the essential information. Then read this 380 * minimal set of information and use it as a base for the 381 * new state. 382 */ 383 ret = btrfs_del_item(trans, dev_root, path); 384 if (ret != 0) { 385 btrfs_warn(fs_info, 386 "delete too small dev_replace item failed %d!", 387 ret); 388 goto out; 389 } 390 ret = 1; 391 } 392 393 if (ret == 1) { 394 /* need to insert a new item */ 395 btrfs_release_path(path); 396 ret = btrfs_insert_empty_item(trans, dev_root, path, 397 &key, sizeof(*ptr)); 398 if (ret < 0) { 399 btrfs_warn(fs_info, 400 "insert dev_replace item failed %d!", ret); 401 goto out; 402 } 403 } 404 405 eb = path->nodes[0]; 406 ptr = btrfs_item_ptr(eb, path->slots[0], 407 struct btrfs_dev_replace_item); 408 409 down_write(&dev_replace->rwsem); 410 if (dev_replace->srcdev) 411 btrfs_set_dev_replace_src_devid(eb, ptr, 412 dev_replace->srcdev->devid); 413 else 414 btrfs_set_dev_replace_src_devid(eb, ptr, (u64)-1); 415 btrfs_set_dev_replace_cont_reading_from_srcdev_mode(eb, ptr, 416 dev_replace->cont_reading_from_srcdev_mode); 417 btrfs_set_dev_replace_replace_state(eb, ptr, 418 dev_replace->replace_state); 419 btrfs_set_dev_replace_time_started(eb, ptr, dev_replace->time_started); 420 btrfs_set_dev_replace_time_stopped(eb, ptr, dev_replace->time_stopped); 421 btrfs_set_dev_replace_num_write_errors(eb, ptr, 422 atomic64_read(&dev_replace->num_write_errors)); 423 btrfs_set_dev_replace_num_uncorrectable_read_errors(eb, ptr, 424 atomic64_read(&dev_replace->num_uncorrectable_read_errors)); 425 dev_replace->cursor_left_last_write_of_item = 426 dev_replace->cursor_left; 427 btrfs_set_dev_replace_cursor_left(eb, ptr, 428 dev_replace->cursor_left_last_write_of_item); 429 btrfs_set_dev_replace_cursor_right(eb, ptr, 430 dev_replace->cursor_right); 431 dev_replace->item_needs_writeback = 0; 432 up_write(&dev_replace->rwsem); 433 434 btrfs_mark_buffer_dirty(eb); 435 436out: 437 btrfs_free_path(path); 438 439 return ret; 440} 441 442static char* btrfs_dev_name(struct btrfs_device *device) 443{ 444 if (!device || test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) 445 return "<missing disk>"; 446 else 447 return rcu_str_deref(device->name); 448} 449 450static int btrfs_dev_replace_start(struct btrfs_fs_info *fs_info, 451 const char *tgtdev_name, u64 srcdevid, const char *srcdev_name, 452 int read_src) 453{ 454 struct btrfs_root *root = fs_info->dev_root; 455 struct btrfs_trans_handle *trans; 456 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; 457 int ret; 458 struct btrfs_device *tgt_device = NULL; 459 struct btrfs_device *src_device = NULL; 460 461 src_device = btrfs_find_device_by_devspec(fs_info, srcdevid, 462 srcdev_name); 463 if (IS_ERR(src_device)) 464 return PTR_ERR(src_device); 465 466 if (btrfs_pinned_by_swapfile(fs_info, src_device)) { 467 btrfs_warn_in_rcu(fs_info, 468 "cannot replace device %s (devid %llu) due to active swapfile", 469 btrfs_dev_name(src_device), src_device->devid); 470 return -ETXTBSY; 471 } 472 473 /* 474 * Here we commit the transaction to make sure commit_total_bytes 475 * of all the devices are updated. 476 */ 477 trans = btrfs_attach_transaction(root); 478 if (!IS_ERR(trans)) { 479 ret = btrfs_commit_transaction(trans); 480 if (ret) 481 return ret; 482 } else if (PTR_ERR(trans) != -ENOENT) { 483 return PTR_ERR(trans); 484 } 485 486 ret = btrfs_init_dev_replace_tgtdev(fs_info, tgtdev_name, 487 src_device, &tgt_device); 488 if (ret) 489 return ret; 490 491 down_write(&dev_replace->rwsem); 492 switch (dev_replace->replace_state) { 493 case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: 494 case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED: 495 case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED: 496 break; 497 case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED: 498 case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED: 499 ASSERT(0); 500 ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_ALREADY_STARTED; 501 up_write(&dev_replace->rwsem); 502 goto leave; 503 } 504 505 dev_replace->cont_reading_from_srcdev_mode = read_src; 506 dev_replace->srcdev = src_device; 507 dev_replace->tgtdev = tgt_device; 508 509 btrfs_info_in_rcu(fs_info, 510 "dev_replace from %s (devid %llu) to %s started", 511 btrfs_dev_name(src_device), 512 src_device->devid, 513 rcu_str_deref(tgt_device->name)); 514 515 /* 516 * from now on, the writes to the srcdev are all duplicated to 517 * go to the tgtdev as well (refer to btrfs_map_block()). 518 */ 519 dev_replace->replace_state = BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED; 520 dev_replace->time_started = ktime_get_real_seconds(); 521 dev_replace->cursor_left = 0; 522 dev_replace->committed_cursor_left = 0; 523 dev_replace->cursor_left_last_write_of_item = 0; 524 dev_replace->cursor_right = 0; 525 dev_replace->is_valid = 1; 526 dev_replace->item_needs_writeback = 1; 527 atomic64_set(&dev_replace->num_write_errors, 0); 528 atomic64_set(&dev_replace->num_uncorrectable_read_errors, 0); 529 up_write(&dev_replace->rwsem); 530 531 ret = btrfs_sysfs_add_device(tgt_device); 532 if (ret) 533 btrfs_err(fs_info, "kobj add dev failed %d", ret); 534 535 btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1); 536 537 /* Commit dev_replace state and reserve 1 item for it. */ 538 trans = btrfs_start_transaction(root, 1); 539 if (IS_ERR(trans)) { 540 ret = PTR_ERR(trans); 541 down_write(&dev_replace->rwsem); 542 dev_replace->replace_state = 543 BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED; 544 dev_replace->srcdev = NULL; 545 dev_replace->tgtdev = NULL; 546 up_write(&dev_replace->rwsem); 547 goto leave; 548 } 549 550 ret = btrfs_commit_transaction(trans); 551 WARN_ON(ret); 552 553 /* the disk copy procedure reuses the scrub code */ 554 ret = btrfs_scrub_dev(fs_info, src_device->devid, 0, 555 btrfs_device_get_total_bytes(src_device), 556 &dev_replace->scrub_progress, 0, 1); 557 558 ret = btrfs_dev_replace_finishing(fs_info, ret); 559 if (ret == -EINPROGRESS) 560 ret = BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS; 561 562 return ret; 563 564leave: 565 btrfs_destroy_dev_replace_tgtdev(tgt_device); 566 return ret; 567} 568 569int btrfs_dev_replace_by_ioctl(struct btrfs_fs_info *fs_info, 570 struct btrfs_ioctl_dev_replace_args *args) 571{ 572 int ret; 573 574 switch (args->start.cont_reading_from_srcdev_mode) { 575 case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_ALWAYS: 576 case BTRFS_IOCTL_DEV_REPLACE_CONT_READING_FROM_SRCDEV_MODE_AVOID: 577 break; 578 default: 579 return -EINVAL; 580 } 581 582 if ((args->start.srcdevid == 0 && args->start.srcdev_name[0] == '\0') || 583 args->start.tgtdev_name[0] == '\0') 584 return -EINVAL; 585 586 ret = btrfs_dev_replace_start(fs_info, args->start.tgtdev_name, 587 args->start.srcdevid, 588 args->start.srcdev_name, 589 args->start.cont_reading_from_srcdev_mode); 590 args->result = ret; 591 /* don't warn if EINPROGRESS, someone else might be running scrub */ 592 if (ret == BTRFS_IOCTL_DEV_REPLACE_RESULT_SCRUB_INPROGRESS || 593 ret == BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR) 594 return 0; 595 596 return ret; 597} 598 599/* 600 * blocked until all in-flight bios operations are finished. 601 */ 602static void btrfs_rm_dev_replace_blocked(struct btrfs_fs_info *fs_info) 603{ 604 set_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state); 605 wait_event(fs_info->dev_replace.replace_wait, !percpu_counter_sum( 606 &fs_info->dev_replace.bio_counter)); 607} 608 609/* 610 * we have removed target device, it is safe to allow new bios request. 611 */ 612static void btrfs_rm_dev_replace_unblocked(struct btrfs_fs_info *fs_info) 613{ 614 clear_bit(BTRFS_FS_STATE_DEV_REPLACING, &fs_info->fs_state); 615 wake_up(&fs_info->dev_replace.replace_wait); 616} 617 618/* 619 * When finishing the device replace, before swapping the source device with the 620 * target device we must update the chunk allocation state in the target device, 621 * as it is empty because replace works by directly copying the chunks and not 622 * through the normal chunk allocation path. 623 */ 624static int btrfs_set_target_alloc_state(struct btrfs_device *srcdev, 625 struct btrfs_device *tgtdev) 626{ 627 struct extent_state *cached_state = NULL; 628 u64 start = 0; 629 u64 found_start; 630 u64 found_end; 631 int ret = 0; 632 633 lockdep_assert_held(&srcdev->fs_info->chunk_mutex); 634 635 while (!find_first_extent_bit(&srcdev->alloc_state, start, 636 &found_start, &found_end, 637 CHUNK_ALLOCATED, &cached_state)) { 638 ret = set_extent_bits(&tgtdev->alloc_state, found_start, 639 found_end, CHUNK_ALLOCATED); 640 if (ret) 641 break; 642 start = found_end + 1; 643 } 644 645 free_extent_state(cached_state); 646 return ret; 647} 648 649static void btrfs_dev_replace_update_device_in_mapping_tree( 650 struct btrfs_fs_info *fs_info, 651 struct btrfs_device *srcdev, 652 struct btrfs_device *tgtdev) 653{ 654 struct extent_map_tree *em_tree = &fs_info->mapping_tree; 655 struct extent_map *em; 656 struct map_lookup *map; 657 u64 start = 0; 658 int i; 659 660 write_lock(&em_tree->lock); 661 do { 662 em = lookup_extent_mapping(em_tree, start, (u64)-1); 663 if (!em) 664 break; 665 map = em->map_lookup; 666 for (i = 0; i < map->num_stripes; i++) 667 if (srcdev == map->stripes[i].dev) 668 map->stripes[i].dev = tgtdev; 669 start = em->start + em->len; 670 free_extent_map(em); 671 } while (start); 672 write_unlock(&em_tree->lock); 673} 674 675static int btrfs_dev_replace_finishing(struct btrfs_fs_info *fs_info, 676 int scrub_ret) 677{ 678 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; 679 struct btrfs_device *tgt_device; 680 struct btrfs_device *src_device; 681 struct btrfs_root *root = fs_info->tree_root; 682 u8 uuid_tmp[BTRFS_UUID_SIZE]; 683 struct btrfs_trans_handle *trans; 684 int ret = 0; 685 686 /* don't allow cancel or unmount to disturb the finishing procedure */ 687 mutex_lock(&dev_replace->lock_finishing_cancel_unmount); 688 689 down_read(&dev_replace->rwsem); 690 /* was the operation canceled, or is it finished? */ 691 if (dev_replace->replace_state != 692 BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED) { 693 up_read(&dev_replace->rwsem); 694 mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); 695 return 0; 696 } 697 698 tgt_device = dev_replace->tgtdev; 699 src_device = dev_replace->srcdev; 700 up_read(&dev_replace->rwsem); 701 702 /* 703 * flush all outstanding I/O and inode extent mappings before the 704 * copy operation is declared as being finished 705 */ 706 ret = btrfs_start_delalloc_roots(fs_info, U64_MAX, false); 707 if (ret) { 708 mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); 709 return ret; 710 } 711 btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1); 712 713 if (!scrub_ret) 714 btrfs_reada_remove_dev(src_device); 715 716 /* 717 * We have to use this loop approach because at this point src_device 718 * has to be available for transaction commit to complete, yet new 719 * chunks shouldn't be allocated on the device. 720 */ 721 while (1) { 722 trans = btrfs_start_transaction(root, 0); 723 if (IS_ERR(trans)) { 724 btrfs_reada_undo_remove_dev(src_device); 725 mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); 726 return PTR_ERR(trans); 727 } 728 ret = btrfs_commit_transaction(trans); 729 WARN_ON(ret); 730 731 /* Prevent write_all_supers() during the finishing procedure */ 732 mutex_lock(&fs_info->fs_devices->device_list_mutex); 733 /* Prevent new chunks being allocated on the source device */ 734 mutex_lock(&fs_info->chunk_mutex); 735 736 if (!list_empty(&src_device->post_commit_list)) { 737 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 738 mutex_unlock(&fs_info->chunk_mutex); 739 } else { 740 break; 741 } 742 } 743 744 down_write(&dev_replace->rwsem); 745 dev_replace->replace_state = 746 scrub_ret ? BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED 747 : BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED; 748 dev_replace->tgtdev = NULL; 749 dev_replace->srcdev = NULL; 750 dev_replace->time_stopped = ktime_get_real_seconds(); 751 dev_replace->item_needs_writeback = 1; 752 753 /* 754 * Update allocation state in the new device and replace the old device 755 * with the new one in the mapping tree. 756 */ 757 if (!scrub_ret) { 758 scrub_ret = btrfs_set_target_alloc_state(src_device, tgt_device); 759 if (scrub_ret) 760 goto error; 761 btrfs_dev_replace_update_device_in_mapping_tree(fs_info, 762 src_device, 763 tgt_device); 764 } else { 765 if (scrub_ret != -ECANCELED) 766 btrfs_err_in_rcu(fs_info, 767 "btrfs_scrub_dev(%s, %llu, %s) failed %d", 768 btrfs_dev_name(src_device), 769 src_device->devid, 770 rcu_str_deref(tgt_device->name), scrub_ret); 771error: 772 up_write(&dev_replace->rwsem); 773 mutex_unlock(&fs_info->chunk_mutex); 774 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 775 btrfs_reada_undo_remove_dev(src_device); 776 btrfs_rm_dev_replace_blocked(fs_info); 777 if (tgt_device) 778 btrfs_destroy_dev_replace_tgtdev(tgt_device); 779 btrfs_rm_dev_replace_unblocked(fs_info); 780 mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); 781 782 return scrub_ret; 783 } 784 785 btrfs_info_in_rcu(fs_info, 786 "dev_replace from %s (devid %llu) to %s finished", 787 btrfs_dev_name(src_device), 788 src_device->devid, 789 rcu_str_deref(tgt_device->name)); 790 clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &tgt_device->dev_state); 791 tgt_device->devid = src_device->devid; 792 src_device->devid = BTRFS_DEV_REPLACE_DEVID; 793 memcpy(uuid_tmp, tgt_device->uuid, sizeof(uuid_tmp)); 794 memcpy(tgt_device->uuid, src_device->uuid, sizeof(tgt_device->uuid)); 795 memcpy(src_device->uuid, uuid_tmp, sizeof(src_device->uuid)); 796 btrfs_device_set_total_bytes(tgt_device, src_device->total_bytes); 797 btrfs_device_set_disk_total_bytes(tgt_device, 798 src_device->disk_total_bytes); 799 btrfs_device_set_bytes_used(tgt_device, src_device->bytes_used); 800 tgt_device->commit_bytes_used = src_device->bytes_used; 801 802 btrfs_assign_next_active_device(src_device, tgt_device); 803 804 list_add(&tgt_device->dev_alloc_list, &fs_info->fs_devices->alloc_list); 805 fs_info->fs_devices->rw_devices++; 806 807 up_write(&dev_replace->rwsem); 808 btrfs_rm_dev_replace_blocked(fs_info); 809 810 btrfs_rm_dev_replace_remove_srcdev(src_device); 811 812 btrfs_rm_dev_replace_unblocked(fs_info); 813 814 /* 815 * Increment dev_stats_ccnt so that btrfs_run_dev_stats() will 816 * update on-disk dev stats value during commit transaction 817 */ 818 atomic_inc(&tgt_device->dev_stats_ccnt); 819 820 /* 821 * this is again a consistent state where no dev_replace procedure 822 * is running, the target device is part of the filesystem, the 823 * source device is not part of the filesystem anymore and its 1st 824 * superblock is scratched out so that it is no longer marked to 825 * belong to this filesystem. 826 */ 827 mutex_unlock(&fs_info->chunk_mutex); 828 mutex_unlock(&fs_info->fs_devices->device_list_mutex); 829 830 /* replace the sysfs entry */ 831 btrfs_sysfs_remove_device(src_device); 832 btrfs_sysfs_update_devid(tgt_device); 833 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &src_device->dev_state)) 834 btrfs_scratch_superblocks(fs_info, src_device->bdev, 835 src_device->name->str); 836 837 /* write back the superblocks */ 838 trans = btrfs_start_transaction(root, 0); 839 if (!IS_ERR(trans)) 840 btrfs_commit_transaction(trans); 841 842 mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); 843 844 btrfs_rm_dev_replace_free_srcdev(src_device); 845 846 return 0; 847} 848 849/* 850 * Read progress of device replace status according to the state and last 851 * stored position. The value format is the same as for 852 * btrfs_dev_replace::progress_1000 853 */ 854static u64 btrfs_dev_replace_progress(struct btrfs_fs_info *fs_info) 855{ 856 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; 857 u64 ret = 0; 858 859 switch (dev_replace->replace_state) { 860 case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: 861 case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED: 862 ret = 0; 863 break; 864 case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED: 865 ret = 1000; 866 break; 867 case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED: 868 case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED: 869 ret = div64_u64(dev_replace->cursor_left, 870 div_u64(btrfs_device_get_total_bytes( 871 dev_replace->srcdev), 1000)); 872 break; 873 } 874 875 return ret; 876} 877 878void btrfs_dev_replace_status(struct btrfs_fs_info *fs_info, 879 struct btrfs_ioctl_dev_replace_args *args) 880{ 881 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; 882 883 down_read(&dev_replace->rwsem); 884 /* even if !dev_replace_is_valid, the values are good enough for 885 * the replace_status ioctl */ 886 args->result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR; 887 args->status.replace_state = dev_replace->replace_state; 888 args->status.time_started = dev_replace->time_started; 889 args->status.time_stopped = dev_replace->time_stopped; 890 args->status.num_write_errors = 891 atomic64_read(&dev_replace->num_write_errors); 892 args->status.num_uncorrectable_read_errors = 893 atomic64_read(&dev_replace->num_uncorrectable_read_errors); 894 args->status.progress_1000 = btrfs_dev_replace_progress(fs_info); 895 up_read(&dev_replace->rwsem); 896} 897 898int btrfs_dev_replace_cancel(struct btrfs_fs_info *fs_info) 899{ 900 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; 901 struct btrfs_device *tgt_device = NULL; 902 struct btrfs_device *src_device = NULL; 903 struct btrfs_trans_handle *trans; 904 struct btrfs_root *root = fs_info->tree_root; 905 int result; 906 int ret; 907 908 if (sb_rdonly(fs_info->sb)) 909 return -EROFS; 910 911 mutex_lock(&dev_replace->lock_finishing_cancel_unmount); 912 down_write(&dev_replace->rwsem); 913 switch (dev_replace->replace_state) { 914 case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: 915 case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED: 916 case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED: 917 result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NOT_STARTED; 918 up_write(&dev_replace->rwsem); 919 break; 920 case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED: 921 tgt_device = dev_replace->tgtdev; 922 src_device = dev_replace->srcdev; 923 up_write(&dev_replace->rwsem); 924 ret = btrfs_scrub_cancel(fs_info); 925 if (ret < 0) { 926 result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NOT_STARTED; 927 } else { 928 result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR; 929 /* 930 * btrfs_dev_replace_finishing() will handle the 931 * cleanup part 932 */ 933 btrfs_info_in_rcu(fs_info, 934 "dev_replace from %s (devid %llu) to %s canceled", 935 btrfs_dev_name(src_device), src_device->devid, 936 btrfs_dev_name(tgt_device)); 937 } 938 break; 939 case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED: 940 /* 941 * Scrub doing the replace isn't running so we need to do the 942 * cleanup step of btrfs_dev_replace_finishing() here 943 */ 944 result = BTRFS_IOCTL_DEV_REPLACE_RESULT_NO_ERROR; 945 tgt_device = dev_replace->tgtdev; 946 src_device = dev_replace->srcdev; 947 dev_replace->tgtdev = NULL; 948 dev_replace->srcdev = NULL; 949 dev_replace->replace_state = 950 BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED; 951 dev_replace->time_stopped = ktime_get_real_seconds(); 952 dev_replace->item_needs_writeback = 1; 953 954 up_write(&dev_replace->rwsem); 955 956 /* Scrub for replace must not be running in suspended state */ 957 btrfs_scrub_cancel(fs_info); 958 959 trans = btrfs_start_transaction(root, 0); 960 if (IS_ERR(trans)) { 961 mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); 962 return PTR_ERR(trans); 963 } 964 ret = btrfs_commit_transaction(trans); 965 WARN_ON(ret); 966 967 btrfs_info_in_rcu(fs_info, 968 "suspended dev_replace from %s (devid %llu) to %s canceled", 969 btrfs_dev_name(src_device), src_device->devid, 970 btrfs_dev_name(tgt_device)); 971 972 if (tgt_device) 973 btrfs_destroy_dev_replace_tgtdev(tgt_device); 974 break; 975 default: 976 up_write(&dev_replace->rwsem); 977 result = -EINVAL; 978 } 979 980 mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); 981 return result; 982} 983 984void btrfs_dev_replace_suspend_for_unmount(struct btrfs_fs_info *fs_info) 985{ 986 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; 987 988 mutex_lock(&dev_replace->lock_finishing_cancel_unmount); 989 down_write(&dev_replace->rwsem); 990 991 switch (dev_replace->replace_state) { 992 case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: 993 case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED: 994 case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED: 995 case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED: 996 break; 997 case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED: 998 dev_replace->replace_state = 999 BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED; 1000 dev_replace->time_stopped = ktime_get_real_seconds(); 1001 dev_replace->item_needs_writeback = 1; 1002 btrfs_info(fs_info, "suspending dev_replace for unmount"); 1003 break; 1004 } 1005 1006 up_write(&dev_replace->rwsem); 1007 mutex_unlock(&dev_replace->lock_finishing_cancel_unmount); 1008} 1009 1010/* resume dev_replace procedure that was interrupted by unmount */ 1011int btrfs_resume_dev_replace_async(struct btrfs_fs_info *fs_info) 1012{ 1013 struct task_struct *task; 1014 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; 1015 1016 down_write(&dev_replace->rwsem); 1017 1018 switch (dev_replace->replace_state) { 1019 case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: 1020 case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED: 1021 case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED: 1022 up_write(&dev_replace->rwsem); 1023 return 0; 1024 case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED: 1025 break; 1026 case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED: 1027 dev_replace->replace_state = 1028 BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED; 1029 break; 1030 } 1031 if (!dev_replace->tgtdev || !dev_replace->tgtdev->bdev) { 1032 btrfs_info(fs_info, 1033 "cannot continue dev_replace, tgtdev is missing"); 1034 btrfs_info(fs_info, 1035 "you may cancel the operation after 'mount -o degraded'"); 1036 dev_replace->replace_state = 1037 BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED; 1038 up_write(&dev_replace->rwsem); 1039 return 0; 1040 } 1041 up_write(&dev_replace->rwsem); 1042 1043 /* 1044 * This could collide with a paused balance, but the exclusive op logic 1045 * should never allow both to start and pause. We don't want to allow 1046 * dev-replace to start anyway. 1047 */ 1048 if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REPLACE)) { 1049 down_write(&dev_replace->rwsem); 1050 dev_replace->replace_state = 1051 BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED; 1052 up_write(&dev_replace->rwsem); 1053 btrfs_info(fs_info, 1054 "cannot resume dev-replace, other exclusive operation running"); 1055 return 0; 1056 } 1057 1058 task = kthread_run(btrfs_dev_replace_kthread, fs_info, "btrfs-devrepl"); 1059 return PTR_ERR_OR_ZERO(task); 1060} 1061 1062static int btrfs_dev_replace_kthread(void *data) 1063{ 1064 struct btrfs_fs_info *fs_info = data; 1065 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; 1066 u64 progress; 1067 int ret; 1068 1069 progress = btrfs_dev_replace_progress(fs_info); 1070 progress = div_u64(progress, 10); 1071 btrfs_info_in_rcu(fs_info, 1072 "continuing dev_replace from %s (devid %llu) to target %s @%u%%", 1073 btrfs_dev_name(dev_replace->srcdev), 1074 dev_replace->srcdev->devid, 1075 btrfs_dev_name(dev_replace->tgtdev), 1076 (unsigned int)progress); 1077 1078 ret = btrfs_scrub_dev(fs_info, dev_replace->srcdev->devid, 1079 dev_replace->committed_cursor_left, 1080 btrfs_device_get_total_bytes(dev_replace->srcdev), 1081 &dev_replace->scrub_progress, 0, 1); 1082 ret = btrfs_dev_replace_finishing(fs_info, ret); 1083 WARN_ON(ret && ret != -ECANCELED); 1084 1085 btrfs_exclop_finish(fs_info); 1086 return 0; 1087} 1088 1089int __pure btrfs_dev_replace_is_ongoing(struct btrfs_dev_replace *dev_replace) 1090{ 1091 if (!dev_replace->is_valid) 1092 return 0; 1093 1094 switch (dev_replace->replace_state) { 1095 case BTRFS_IOCTL_DEV_REPLACE_STATE_NEVER_STARTED: 1096 case BTRFS_IOCTL_DEV_REPLACE_STATE_FINISHED: 1097 case BTRFS_IOCTL_DEV_REPLACE_STATE_CANCELED: 1098 return 0; 1099 case BTRFS_IOCTL_DEV_REPLACE_STATE_STARTED: 1100 case BTRFS_IOCTL_DEV_REPLACE_STATE_SUSPENDED: 1101 /* 1102 * return true even if tgtdev is missing (this is 1103 * something that can happen if the dev_replace 1104 * procedure is suspended by an umount and then 1105 * the tgtdev is missing (or "btrfs dev scan") was 1106 * not called and the filesystem is remounted 1107 * in degraded state. This does not stop the 1108 * dev_replace procedure. It needs to be canceled 1109 * manually if the cancellation is wanted. 1110 */ 1111 break; 1112 } 1113 return 1; 1114} 1115 1116void btrfs_bio_counter_inc_noblocked(struct btrfs_fs_info *fs_info) 1117{ 1118 percpu_counter_inc(&fs_info->dev_replace.bio_counter); 1119} 1120 1121void btrfs_bio_counter_sub(struct btrfs_fs_info *fs_info, s64 amount) 1122{ 1123 percpu_counter_sub(&fs_info->dev_replace.bio_counter, amount); 1124 cond_wake_up_nomb(&fs_info->dev_replace.replace_wait); 1125} 1126 1127void btrfs_bio_counter_inc_blocked(struct btrfs_fs_info *fs_info) 1128{ 1129 while (1) { 1130 percpu_counter_inc(&fs_info->dev_replace.bio_counter); 1131 if (likely(!test_bit(BTRFS_FS_STATE_DEV_REPLACING, 1132 &fs_info->fs_state))) 1133 break; 1134 1135 btrfs_bio_counter_dec(fs_info); 1136 wait_event(fs_info->dev_replace.replace_wait, 1137 !test_bit(BTRFS_FS_STATE_DEV_REPLACING, 1138 &fs_info->fs_state)); 1139 } 1140} 1141