1// SPDX-License-Identifier: GPL-2.0 2 3#include <linux/bitops.h> 4#include <linux/slab.h> 5#include <linux/blkdev.h> 6#include <linux/sched/mm.h> 7#include <linux/atomic.h> 8#include <linux/vmalloc.h> 9#include "ctree.h" 10#include "volumes.h" 11#include "zoned.h" 12#include "rcu-string.h" 13#include "disk-io.h" 14#include "block-group.h" 15#include "transaction.h" 16#include "dev-replace.h" 17#include "space-info.h" 18#include "super.h" 19#include "fs.h" 20#include "accessors.h" 21#include "bio.h" 22 23/* Maximum number of zones to report per blkdev_report_zones() call */ 24#define BTRFS_REPORT_NR_ZONES 4096 25/* Invalid allocation pointer value for missing devices */ 26#define WP_MISSING_DEV ((u64)-1) 27/* Pseudo write pointer value for conventional zone */ 28#define WP_CONVENTIONAL ((u64)-2) 29 30/* 31 * Location of the first zone of superblock logging zone pairs. 32 * 33 * - primary superblock: 0B (zone 0) 34 * - first copy: 512G (zone starting at that offset) 35 * - second copy: 4T (zone starting at that offset) 36 */ 37#define BTRFS_SB_LOG_PRIMARY_OFFSET (0ULL) 38#define BTRFS_SB_LOG_FIRST_OFFSET (512ULL * SZ_1G) 39#define BTRFS_SB_LOG_SECOND_OFFSET (4096ULL * SZ_1G) 40 41#define BTRFS_SB_LOG_FIRST_SHIFT const_ilog2(BTRFS_SB_LOG_FIRST_OFFSET) 42#define BTRFS_SB_LOG_SECOND_SHIFT const_ilog2(BTRFS_SB_LOG_SECOND_OFFSET) 43 44/* Number of superblock log zones */ 45#define BTRFS_NR_SB_LOG_ZONES 2 46 47/* 48 * Minimum of active zones we need: 49 * 50 * - BTRFS_SUPER_MIRROR_MAX zones for superblock mirrors 51 * - 3 zones to ensure at least one zone per SYSTEM, META and DATA block group 52 * - 1 zone for tree-log dedicated block group 53 * - 1 zone for relocation 54 */ 55#define BTRFS_MIN_ACTIVE_ZONES (BTRFS_SUPER_MIRROR_MAX + 5) 56 57/* 58 * Minimum / maximum supported zone size. Currently, SMR disks have a zone 59 * size of 256MiB, and we are expecting ZNS drives to be in the 1-4GiB range. 60 * We do not expect the zone size to become larger than 8GiB or smaller than 61 * 4MiB in the near future. 62 */ 63#define BTRFS_MAX_ZONE_SIZE SZ_8G 64#define BTRFS_MIN_ZONE_SIZE SZ_4M 65 66#define SUPER_INFO_SECTORS ((u64)BTRFS_SUPER_INFO_SIZE >> SECTOR_SHIFT) 67 68static void wait_eb_writebacks(struct btrfs_block_group *block_group); 69static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_written); 70 71static inline bool sb_zone_is_full(const struct blk_zone *zone) 72{ 73 return (zone->cond == BLK_ZONE_COND_FULL) || 74 (zone->wp + SUPER_INFO_SECTORS > zone->start + zone->capacity); 75} 76 77static int copy_zone_info_cb(struct blk_zone *zone, unsigned int idx, void *data) 78{ 79 struct blk_zone *zones = data; 80 81 memcpy(&zones[idx], zone, sizeof(*zone)); 82 83 return 0; 84} 85 86static int sb_write_pointer(struct block_device *bdev, struct blk_zone *zones, 87 u64 *wp_ret) 88{ 89 bool empty[BTRFS_NR_SB_LOG_ZONES]; 90 bool full[BTRFS_NR_SB_LOG_ZONES]; 91 sector_t sector; 92 int i; 93 94 for (i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) { 95 ASSERT(zones[i].type != BLK_ZONE_TYPE_CONVENTIONAL); 96 empty[i] = (zones[i].cond == BLK_ZONE_COND_EMPTY); 97 full[i] = sb_zone_is_full(&zones[i]); 98 } 99 100 /* 101 * Possible states of log buffer zones 102 * 103 * Empty[0] In use[0] Full[0] 104 * Empty[1] * 0 1 105 * In use[1] x x 1 106 * Full[1] 0 0 C 107 * 108 * Log position: 109 * *: Special case, no superblock is written 110 * 0: Use write pointer of zones[0] 111 * 1: Use write pointer of zones[1] 112 * C: Compare super blocks from zones[0] and zones[1], use the latest 113 * one determined by generation 114 * x: Invalid state 115 */ 116 117 if (empty[0] && empty[1]) { 118 /* Special case to distinguish no superblock to read */ 119 *wp_ret = zones[0].start << SECTOR_SHIFT; 120 return -ENOENT; 121 } else if (full[0] && full[1]) { 122 /* Compare two super blocks */ 123 struct address_space *mapping = bdev->bd_inode->i_mapping; 124 struct page *page[BTRFS_NR_SB_LOG_ZONES]; 125 struct btrfs_super_block *super[BTRFS_NR_SB_LOG_ZONES]; 126 int i; 127 128 for (i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) { 129 u64 zone_end = (zones[i].start + zones[i].capacity) << SECTOR_SHIFT; 130 u64 bytenr = ALIGN_DOWN(zone_end, BTRFS_SUPER_INFO_SIZE) - 131 BTRFS_SUPER_INFO_SIZE; 132 133 page[i] = read_cache_page_gfp(mapping, 134 bytenr >> PAGE_SHIFT, GFP_NOFS); 135 if (IS_ERR(page[i])) { 136 if (i == 1) 137 btrfs_release_disk_super(super[0]); 138 return PTR_ERR(page[i]); 139 } 140 super[i] = page_address(page[i]); 141 } 142 143 if (btrfs_super_generation(super[0]) > 144 btrfs_super_generation(super[1])) 145 sector = zones[1].start; 146 else 147 sector = zones[0].start; 148 149 for (i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) 150 btrfs_release_disk_super(super[i]); 151 } else if (!full[0] && (empty[1] || full[1])) { 152 sector = zones[0].wp; 153 } else if (full[0]) { 154 sector = zones[1].wp; 155 } else { 156 return -EUCLEAN; 157 } 158 *wp_ret = sector << SECTOR_SHIFT; 159 return 0; 160} 161 162/* 163 * Get the first zone number of the superblock mirror 164 */ 165static inline u32 sb_zone_number(int shift, int mirror) 166{ 167 u64 zone = U64_MAX; 168 169 ASSERT(mirror < BTRFS_SUPER_MIRROR_MAX); 170 switch (mirror) { 171 case 0: zone = 0; break; 172 case 1: zone = 1ULL << (BTRFS_SB_LOG_FIRST_SHIFT - shift); break; 173 case 2: zone = 1ULL << (BTRFS_SB_LOG_SECOND_SHIFT - shift); break; 174 } 175 176 ASSERT(zone <= U32_MAX); 177 178 return (u32)zone; 179} 180 181static inline sector_t zone_start_sector(u32 zone_number, 182 struct block_device *bdev) 183{ 184 return (sector_t)zone_number << ilog2(bdev_zone_sectors(bdev)); 185} 186 187static inline u64 zone_start_physical(u32 zone_number, 188 struct btrfs_zoned_device_info *zone_info) 189{ 190 return (u64)zone_number << zone_info->zone_size_shift; 191} 192 193/* 194 * Emulate blkdev_report_zones() for a non-zoned device. It slices up the block 195 * device into static sized chunks and fake a conventional zone on each of 196 * them. 197 */ 198static int emulate_report_zones(struct btrfs_device *device, u64 pos, 199 struct blk_zone *zones, unsigned int nr_zones) 200{ 201 const sector_t zone_sectors = device->fs_info->zone_size >> SECTOR_SHIFT; 202 sector_t bdev_size = bdev_nr_sectors(device->bdev); 203 unsigned int i; 204 205 pos >>= SECTOR_SHIFT; 206 for (i = 0; i < nr_zones; i++) { 207 zones[i].start = i * zone_sectors + pos; 208 zones[i].len = zone_sectors; 209 zones[i].capacity = zone_sectors; 210 zones[i].wp = zones[i].start + zone_sectors; 211 zones[i].type = BLK_ZONE_TYPE_CONVENTIONAL; 212 zones[i].cond = BLK_ZONE_COND_NOT_WP; 213 214 if (zones[i].wp >= bdev_size) { 215 i++; 216 break; 217 } 218 } 219 220 return i; 221} 222 223static int btrfs_get_dev_zones(struct btrfs_device *device, u64 pos, 224 struct blk_zone *zones, unsigned int *nr_zones) 225{ 226 struct btrfs_zoned_device_info *zinfo = device->zone_info; 227 int ret; 228 229 if (!*nr_zones) 230 return 0; 231 232 if (!bdev_is_zoned(device->bdev)) { 233 ret = emulate_report_zones(device, pos, zones, *nr_zones); 234 *nr_zones = ret; 235 return 0; 236 } 237 238 /* Check cache */ 239 if (zinfo->zone_cache) { 240 unsigned int i; 241 u32 zno; 242 243 ASSERT(IS_ALIGNED(pos, zinfo->zone_size)); 244 zno = pos >> zinfo->zone_size_shift; 245 /* 246 * We cannot report zones beyond the zone end. So, it is OK to 247 * cap *nr_zones to at the end. 248 */ 249 *nr_zones = min_t(u32, *nr_zones, zinfo->nr_zones - zno); 250 251 for (i = 0; i < *nr_zones; i++) { 252 struct blk_zone *zone_info; 253 254 zone_info = &zinfo->zone_cache[zno + i]; 255 if (!zone_info->len) 256 break; 257 } 258 259 if (i == *nr_zones) { 260 /* Cache hit on all the zones */ 261 memcpy(zones, zinfo->zone_cache + zno, 262 sizeof(*zinfo->zone_cache) * *nr_zones); 263 return 0; 264 } 265 } 266 267 ret = blkdev_report_zones(device->bdev, pos >> SECTOR_SHIFT, *nr_zones, 268 copy_zone_info_cb, zones); 269 if (ret < 0) { 270 btrfs_err_in_rcu(device->fs_info, 271 "zoned: failed to read zone %llu on %s (devid %llu)", 272 pos, rcu_str_deref(device->name), 273 device->devid); 274 return ret; 275 } 276 *nr_zones = ret; 277 if (!ret) 278 return -EIO; 279 280 /* Populate cache */ 281 if (zinfo->zone_cache) { 282 u32 zno = pos >> zinfo->zone_size_shift; 283 284 memcpy(zinfo->zone_cache + zno, zones, 285 sizeof(*zinfo->zone_cache) * *nr_zones); 286 } 287 288 return 0; 289} 290 291/* The emulated zone size is determined from the size of device extent */ 292static int calculate_emulated_zone_size(struct btrfs_fs_info *fs_info) 293{ 294 struct btrfs_path *path; 295 struct btrfs_root *root = fs_info->dev_root; 296 struct btrfs_key key; 297 struct extent_buffer *leaf; 298 struct btrfs_dev_extent *dext; 299 int ret = 0; 300 301 key.objectid = 1; 302 key.type = BTRFS_DEV_EXTENT_KEY; 303 key.offset = 0; 304 305 path = btrfs_alloc_path(); 306 if (!path) 307 return -ENOMEM; 308 309 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 310 if (ret < 0) 311 goto out; 312 313 if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { 314 ret = btrfs_next_leaf(root, path); 315 if (ret < 0) 316 goto out; 317 /* No dev extents at all? Not good */ 318 if (ret > 0) { 319 ret = -EUCLEAN; 320 goto out; 321 } 322 } 323 324 leaf = path->nodes[0]; 325 dext = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_extent); 326 fs_info->zone_size = btrfs_dev_extent_length(leaf, dext); 327 ret = 0; 328 329out: 330 btrfs_free_path(path); 331 332 return ret; 333} 334 335int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info) 336{ 337 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; 338 struct btrfs_device *device; 339 int ret = 0; 340 341 /* fs_info->zone_size might not set yet. Use the incomapt flag here. */ 342 if (!btrfs_fs_incompat(fs_info, ZONED)) 343 return 0; 344 345 mutex_lock(&fs_devices->device_list_mutex); 346 list_for_each_entry(device, &fs_devices->devices, dev_list) { 347 /* We can skip reading of zone info for missing devices */ 348 if (!device->bdev) 349 continue; 350 351 ret = btrfs_get_dev_zone_info(device, true); 352 if (ret) 353 break; 354 } 355 mutex_unlock(&fs_devices->device_list_mutex); 356 357 return ret; 358} 359 360int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache) 361{ 362 struct btrfs_fs_info *fs_info = device->fs_info; 363 struct btrfs_zoned_device_info *zone_info = NULL; 364 struct block_device *bdev = device->bdev; 365 unsigned int max_active_zones; 366 unsigned int nactive; 367 sector_t nr_sectors; 368 sector_t sector = 0; 369 struct blk_zone *zones = NULL; 370 unsigned int i, nreported = 0, nr_zones; 371 sector_t zone_sectors; 372 char *model, *emulated; 373 int ret; 374 375 /* 376 * Cannot use btrfs_is_zoned here, since fs_info::zone_size might not 377 * yet be set. 378 */ 379 if (!btrfs_fs_incompat(fs_info, ZONED)) 380 return 0; 381 382 if (device->zone_info) 383 return 0; 384 385 zone_info = kzalloc(sizeof(*zone_info), GFP_KERNEL); 386 if (!zone_info) 387 return -ENOMEM; 388 389 device->zone_info = zone_info; 390 391 if (!bdev_is_zoned(bdev)) { 392 if (!fs_info->zone_size) { 393 ret = calculate_emulated_zone_size(fs_info); 394 if (ret) 395 goto out; 396 } 397 398 ASSERT(fs_info->zone_size); 399 zone_sectors = fs_info->zone_size >> SECTOR_SHIFT; 400 } else { 401 zone_sectors = bdev_zone_sectors(bdev); 402 } 403 404 ASSERT(is_power_of_two_u64(zone_sectors)); 405 zone_info->zone_size = zone_sectors << SECTOR_SHIFT; 406 407 /* We reject devices with a zone size larger than 8GB */ 408 if (zone_info->zone_size > BTRFS_MAX_ZONE_SIZE) { 409 btrfs_err_in_rcu(fs_info, 410 "zoned: %s: zone size %llu larger than supported maximum %llu", 411 rcu_str_deref(device->name), 412 zone_info->zone_size, BTRFS_MAX_ZONE_SIZE); 413 ret = -EINVAL; 414 goto out; 415 } else if (zone_info->zone_size < BTRFS_MIN_ZONE_SIZE) { 416 btrfs_err_in_rcu(fs_info, 417 "zoned: %s: zone size %llu smaller than supported minimum %u", 418 rcu_str_deref(device->name), 419 zone_info->zone_size, BTRFS_MIN_ZONE_SIZE); 420 ret = -EINVAL; 421 goto out; 422 } 423 424 nr_sectors = bdev_nr_sectors(bdev); 425 zone_info->zone_size_shift = ilog2(zone_info->zone_size); 426 zone_info->nr_zones = nr_sectors >> ilog2(zone_sectors); 427 if (!IS_ALIGNED(nr_sectors, zone_sectors)) 428 zone_info->nr_zones++; 429 430 max_active_zones = bdev_max_active_zones(bdev); 431 if (max_active_zones && max_active_zones < BTRFS_MIN_ACTIVE_ZONES) { 432 btrfs_err_in_rcu(fs_info, 433"zoned: %s: max active zones %u is too small, need at least %u active zones", 434 rcu_str_deref(device->name), max_active_zones, 435 BTRFS_MIN_ACTIVE_ZONES); 436 ret = -EINVAL; 437 goto out; 438 } 439 zone_info->max_active_zones = max_active_zones; 440 441 zone_info->seq_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); 442 if (!zone_info->seq_zones) { 443 ret = -ENOMEM; 444 goto out; 445 } 446 447 zone_info->empty_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); 448 if (!zone_info->empty_zones) { 449 ret = -ENOMEM; 450 goto out; 451 } 452 453 zone_info->active_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); 454 if (!zone_info->active_zones) { 455 ret = -ENOMEM; 456 goto out; 457 } 458 459 zones = kvcalloc(BTRFS_REPORT_NR_ZONES, sizeof(struct blk_zone), GFP_KERNEL); 460 if (!zones) { 461 ret = -ENOMEM; 462 goto out; 463 } 464 465 /* 466 * Enable zone cache only for a zoned device. On a non-zoned device, we 467 * fill the zone info with emulated CONVENTIONAL zones, so no need to 468 * use the cache. 469 */ 470 if (populate_cache && bdev_is_zoned(device->bdev)) { 471 zone_info->zone_cache = vcalloc(zone_info->nr_zones, 472 sizeof(struct blk_zone)); 473 if (!zone_info->zone_cache) { 474 btrfs_err_in_rcu(device->fs_info, 475 "zoned: failed to allocate zone cache for %s", 476 rcu_str_deref(device->name)); 477 ret = -ENOMEM; 478 goto out; 479 } 480 } 481 482 /* Get zones type */ 483 nactive = 0; 484 while (sector < nr_sectors) { 485 nr_zones = BTRFS_REPORT_NR_ZONES; 486 ret = btrfs_get_dev_zones(device, sector << SECTOR_SHIFT, zones, 487 &nr_zones); 488 if (ret) 489 goto out; 490 491 for (i = 0; i < nr_zones; i++) { 492 if (zones[i].type == BLK_ZONE_TYPE_SEQWRITE_REQ) 493 __set_bit(nreported, zone_info->seq_zones); 494 switch (zones[i].cond) { 495 case BLK_ZONE_COND_EMPTY: 496 __set_bit(nreported, zone_info->empty_zones); 497 break; 498 case BLK_ZONE_COND_IMP_OPEN: 499 case BLK_ZONE_COND_EXP_OPEN: 500 case BLK_ZONE_COND_CLOSED: 501 __set_bit(nreported, zone_info->active_zones); 502 nactive++; 503 break; 504 } 505 nreported++; 506 } 507 sector = zones[nr_zones - 1].start + zones[nr_zones - 1].len; 508 } 509 510 if (nreported != zone_info->nr_zones) { 511 btrfs_err_in_rcu(device->fs_info, 512 "inconsistent number of zones on %s (%u/%u)", 513 rcu_str_deref(device->name), nreported, 514 zone_info->nr_zones); 515 ret = -EIO; 516 goto out; 517 } 518 519 if (max_active_zones) { 520 if (nactive > max_active_zones) { 521 btrfs_err_in_rcu(device->fs_info, 522 "zoned: %u active zones on %s exceeds max_active_zones %u", 523 nactive, rcu_str_deref(device->name), 524 max_active_zones); 525 ret = -EIO; 526 goto out; 527 } 528 atomic_set(&zone_info->active_zones_left, 529 max_active_zones - nactive); 530 set_bit(BTRFS_FS_ACTIVE_ZONE_TRACKING, &fs_info->flags); 531 } 532 533 /* Validate superblock log */ 534 nr_zones = BTRFS_NR_SB_LOG_ZONES; 535 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { 536 u32 sb_zone; 537 u64 sb_wp; 538 int sb_pos = BTRFS_NR_SB_LOG_ZONES * i; 539 540 sb_zone = sb_zone_number(zone_info->zone_size_shift, i); 541 if (sb_zone + 1 >= zone_info->nr_zones) 542 continue; 543 544 ret = btrfs_get_dev_zones(device, 545 zone_start_physical(sb_zone, zone_info), 546 &zone_info->sb_zones[sb_pos], 547 &nr_zones); 548 if (ret) 549 goto out; 550 551 if (nr_zones != BTRFS_NR_SB_LOG_ZONES) { 552 btrfs_err_in_rcu(device->fs_info, 553 "zoned: failed to read super block log zone info at devid %llu zone %u", 554 device->devid, sb_zone); 555 ret = -EUCLEAN; 556 goto out; 557 } 558 559 /* 560 * If zones[0] is conventional, always use the beginning of the 561 * zone to record superblock. No need to validate in that case. 562 */ 563 if (zone_info->sb_zones[BTRFS_NR_SB_LOG_ZONES * i].type == 564 BLK_ZONE_TYPE_CONVENTIONAL) 565 continue; 566 567 ret = sb_write_pointer(device->bdev, 568 &zone_info->sb_zones[sb_pos], &sb_wp); 569 if (ret != -ENOENT && ret) { 570 btrfs_err_in_rcu(device->fs_info, 571 "zoned: super block log zone corrupted devid %llu zone %u", 572 device->devid, sb_zone); 573 ret = -EUCLEAN; 574 goto out; 575 } 576 } 577 578 579 kvfree(zones); 580 581 switch (bdev_zoned_model(bdev)) { 582 case BLK_ZONED_HM: 583 model = "host-managed zoned"; 584 emulated = ""; 585 break; 586 case BLK_ZONED_HA: 587 model = "host-aware zoned"; 588 emulated = ""; 589 break; 590 case BLK_ZONED_NONE: 591 model = "regular"; 592 emulated = "emulated "; 593 break; 594 default: 595 /* Just in case */ 596 btrfs_err_in_rcu(fs_info, "zoned: unsupported model %d on %s", 597 bdev_zoned_model(bdev), 598 rcu_str_deref(device->name)); 599 ret = -EOPNOTSUPP; 600 goto out_free_zone_info; 601 } 602 603 btrfs_info_in_rcu(fs_info, 604 "%s block device %s, %u %szones of %llu bytes", 605 model, rcu_str_deref(device->name), zone_info->nr_zones, 606 emulated, zone_info->zone_size); 607 608 return 0; 609 610out: 611 kvfree(zones); 612out_free_zone_info: 613 btrfs_destroy_dev_zone_info(device); 614 615 return ret; 616} 617 618void btrfs_destroy_dev_zone_info(struct btrfs_device *device) 619{ 620 struct btrfs_zoned_device_info *zone_info = device->zone_info; 621 622 if (!zone_info) 623 return; 624 625 bitmap_free(zone_info->active_zones); 626 bitmap_free(zone_info->seq_zones); 627 bitmap_free(zone_info->empty_zones); 628 vfree(zone_info->zone_cache); 629 kfree(zone_info); 630 device->zone_info = NULL; 631} 632 633struct btrfs_zoned_device_info *btrfs_clone_dev_zone_info(struct btrfs_device *orig_dev) 634{ 635 struct btrfs_zoned_device_info *zone_info; 636 637 zone_info = kmemdup(orig_dev->zone_info, sizeof(*zone_info), GFP_KERNEL); 638 if (!zone_info) 639 return NULL; 640 641 zone_info->seq_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); 642 if (!zone_info->seq_zones) 643 goto out; 644 645 bitmap_copy(zone_info->seq_zones, orig_dev->zone_info->seq_zones, 646 zone_info->nr_zones); 647 648 zone_info->empty_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); 649 if (!zone_info->empty_zones) 650 goto out; 651 652 bitmap_copy(zone_info->empty_zones, orig_dev->zone_info->empty_zones, 653 zone_info->nr_zones); 654 655 zone_info->active_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); 656 if (!zone_info->active_zones) 657 goto out; 658 659 bitmap_copy(zone_info->active_zones, orig_dev->zone_info->active_zones, 660 zone_info->nr_zones); 661 zone_info->zone_cache = NULL; 662 663 return zone_info; 664 665out: 666 bitmap_free(zone_info->seq_zones); 667 bitmap_free(zone_info->empty_zones); 668 bitmap_free(zone_info->active_zones); 669 kfree(zone_info); 670 return NULL; 671} 672 673int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, 674 struct blk_zone *zone) 675{ 676 unsigned int nr_zones = 1; 677 int ret; 678 679 ret = btrfs_get_dev_zones(device, pos, zone, &nr_zones); 680 if (ret != 0 || !nr_zones) 681 return ret ? ret : -EIO; 682 683 return 0; 684} 685 686static int btrfs_check_for_zoned_device(struct btrfs_fs_info *fs_info) 687{ 688 struct btrfs_device *device; 689 690 list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) { 691 if (device->bdev && 692 bdev_zoned_model(device->bdev) == BLK_ZONED_HM) { 693 btrfs_err(fs_info, 694 "zoned: mode not enabled but zoned device found: %pg", 695 device->bdev); 696 return -EINVAL; 697 } 698 } 699 700 return 0; 701} 702 703int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) 704{ 705 struct queue_limits *lim = &fs_info->limits; 706 struct btrfs_device *device; 707 u64 zone_size = 0; 708 int ret; 709 710 /* 711 * Host-Managed devices can't be used without the ZONED flag. With the 712 * ZONED all devices can be used, using zone emulation if required. 713 */ 714 if (!btrfs_fs_incompat(fs_info, ZONED)) 715 return btrfs_check_for_zoned_device(fs_info); 716 717 blk_set_stacking_limits(lim); 718 719 list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) { 720 struct btrfs_zoned_device_info *zone_info = device->zone_info; 721 722 if (!device->bdev) 723 continue; 724 725 if (!zone_size) { 726 zone_size = zone_info->zone_size; 727 } else if (zone_info->zone_size != zone_size) { 728 btrfs_err(fs_info, 729 "zoned: unequal block device zone sizes: have %llu found %llu", 730 zone_info->zone_size, zone_size); 731 return -EINVAL; 732 } 733 734 /* 735 * With the zoned emulation, we can have non-zoned device on the 736 * zoned mode. In this case, we don't have a valid max zone 737 * append size. 738 */ 739 if (bdev_is_zoned(device->bdev)) { 740 blk_stack_limits(lim, 741 &bdev_get_queue(device->bdev)->limits, 742 0); 743 } 744 } 745 746 /* 747 * stripe_size is always aligned to BTRFS_STRIPE_LEN in 748 * btrfs_create_chunk(). Since we want stripe_len == zone_size, 749 * check the alignment here. 750 */ 751 if (!IS_ALIGNED(zone_size, BTRFS_STRIPE_LEN)) { 752 btrfs_err(fs_info, 753 "zoned: zone size %llu not aligned to stripe %u", 754 zone_size, BTRFS_STRIPE_LEN); 755 return -EINVAL; 756 } 757 758 if (btrfs_fs_incompat(fs_info, MIXED_GROUPS)) { 759 btrfs_err(fs_info, "zoned: mixed block groups not supported"); 760 return -EINVAL; 761 } 762 763 fs_info->zone_size = zone_size; 764 /* 765 * Also limit max_zone_append_size by max_segments * PAGE_SIZE. 766 * Technically, we can have multiple pages per segment. But, since 767 * we add the pages one by one to a bio, and cannot increase the 768 * metadata reservation even if it increases the number of extents, it 769 * is safe to stick with the limit. 770 */ 771 fs_info->max_zone_append_size = ALIGN_DOWN( 772 min3((u64)lim->max_zone_append_sectors << SECTOR_SHIFT, 773 (u64)lim->max_sectors << SECTOR_SHIFT, 774 (u64)lim->max_segments << PAGE_SHIFT), 775 fs_info->sectorsize); 776 fs_info->fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_ZONED; 777 if (fs_info->max_zone_append_size < fs_info->max_extent_size) 778 fs_info->max_extent_size = fs_info->max_zone_append_size; 779 780 /* 781 * Check mount options here, because we might change fs_info->zoned 782 * from fs_info->zone_size. 783 */ 784 ret = btrfs_check_mountopts_zoned(fs_info); 785 if (ret) 786 return ret; 787 788 btrfs_info(fs_info, "zoned mode enabled with zone size %llu", zone_size); 789 return 0; 790} 791 792int btrfs_check_mountopts_zoned(struct btrfs_fs_info *info) 793{ 794 if (!btrfs_is_zoned(info)) 795 return 0; 796 797 /* 798 * Space cache writing is not COWed. Disable that to avoid write errors 799 * in sequential zones. 800 */ 801 if (btrfs_test_opt(info, SPACE_CACHE)) { 802 btrfs_err(info, "zoned: space cache v1 is not supported"); 803 return -EINVAL; 804 } 805 806 if (btrfs_test_opt(info, NODATACOW)) { 807 btrfs_err(info, "zoned: NODATACOW not supported"); 808 return -EINVAL; 809 } 810 811 btrfs_clear_and_info(info, DISCARD_ASYNC, 812 "zoned: async discard ignored and disabled for zoned mode"); 813 814 return 0; 815} 816 817static int sb_log_location(struct block_device *bdev, struct blk_zone *zones, 818 int rw, u64 *bytenr_ret) 819{ 820 u64 wp; 821 int ret; 822 823 if (zones[0].type == BLK_ZONE_TYPE_CONVENTIONAL) { 824 *bytenr_ret = zones[0].start << SECTOR_SHIFT; 825 return 0; 826 } 827 828 ret = sb_write_pointer(bdev, zones, &wp); 829 if (ret != -ENOENT && ret < 0) 830 return ret; 831 832 if (rw == WRITE) { 833 struct blk_zone *reset = NULL; 834 835 if (wp == zones[0].start << SECTOR_SHIFT) 836 reset = &zones[0]; 837 else if (wp == zones[1].start << SECTOR_SHIFT) 838 reset = &zones[1]; 839 840 if (reset && reset->cond != BLK_ZONE_COND_EMPTY) { 841 ASSERT(sb_zone_is_full(reset)); 842 843 ret = blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET, 844 reset->start, reset->len, 845 GFP_NOFS); 846 if (ret) 847 return ret; 848 849 reset->cond = BLK_ZONE_COND_EMPTY; 850 reset->wp = reset->start; 851 } 852 } else if (ret != -ENOENT) { 853 /* 854 * For READ, we want the previous one. Move write pointer to 855 * the end of a zone, if it is at the head of a zone. 856 */ 857 u64 zone_end = 0; 858 859 if (wp == zones[0].start << SECTOR_SHIFT) 860 zone_end = zones[1].start + zones[1].capacity; 861 else if (wp == zones[1].start << SECTOR_SHIFT) 862 zone_end = zones[0].start + zones[0].capacity; 863 if (zone_end) 864 wp = ALIGN_DOWN(zone_end << SECTOR_SHIFT, 865 BTRFS_SUPER_INFO_SIZE); 866 867 wp -= BTRFS_SUPER_INFO_SIZE; 868 } 869 870 *bytenr_ret = wp; 871 return 0; 872 873} 874 875int btrfs_sb_log_location_bdev(struct block_device *bdev, int mirror, int rw, 876 u64 *bytenr_ret) 877{ 878 struct blk_zone zones[BTRFS_NR_SB_LOG_ZONES]; 879 sector_t zone_sectors; 880 u32 sb_zone; 881 int ret; 882 u8 zone_sectors_shift; 883 sector_t nr_sectors; 884 u32 nr_zones; 885 886 if (!bdev_is_zoned(bdev)) { 887 *bytenr_ret = btrfs_sb_offset(mirror); 888 return 0; 889 } 890 891 ASSERT(rw == READ || rw == WRITE); 892 893 zone_sectors = bdev_zone_sectors(bdev); 894 if (!is_power_of_2(zone_sectors)) 895 return -EINVAL; 896 zone_sectors_shift = ilog2(zone_sectors); 897 nr_sectors = bdev_nr_sectors(bdev); 898 nr_zones = nr_sectors >> zone_sectors_shift; 899 900 sb_zone = sb_zone_number(zone_sectors_shift + SECTOR_SHIFT, mirror); 901 if (sb_zone + 1 >= nr_zones) 902 return -ENOENT; 903 904 ret = blkdev_report_zones(bdev, zone_start_sector(sb_zone, bdev), 905 BTRFS_NR_SB_LOG_ZONES, copy_zone_info_cb, 906 zones); 907 if (ret < 0) 908 return ret; 909 if (ret != BTRFS_NR_SB_LOG_ZONES) 910 return -EIO; 911 912 return sb_log_location(bdev, zones, rw, bytenr_ret); 913} 914 915int btrfs_sb_log_location(struct btrfs_device *device, int mirror, int rw, 916 u64 *bytenr_ret) 917{ 918 struct btrfs_zoned_device_info *zinfo = device->zone_info; 919 u32 zone_num; 920 921 /* 922 * For a zoned filesystem on a non-zoned block device, use the same 923 * super block locations as regular filesystem. Doing so, the super 924 * block can always be retrieved and the zoned flag of the volume 925 * detected from the super block information. 926 */ 927 if (!bdev_is_zoned(device->bdev)) { 928 *bytenr_ret = btrfs_sb_offset(mirror); 929 return 0; 930 } 931 932 zone_num = sb_zone_number(zinfo->zone_size_shift, mirror); 933 if (zone_num + 1 >= zinfo->nr_zones) 934 return -ENOENT; 935 936 return sb_log_location(device->bdev, 937 &zinfo->sb_zones[BTRFS_NR_SB_LOG_ZONES * mirror], 938 rw, bytenr_ret); 939} 940 941static inline bool is_sb_log_zone(struct btrfs_zoned_device_info *zinfo, 942 int mirror) 943{ 944 u32 zone_num; 945 946 if (!zinfo) 947 return false; 948 949 zone_num = sb_zone_number(zinfo->zone_size_shift, mirror); 950 if (zone_num + 1 >= zinfo->nr_zones) 951 return false; 952 953 if (!test_bit(zone_num, zinfo->seq_zones)) 954 return false; 955 956 return true; 957} 958 959int btrfs_advance_sb_log(struct btrfs_device *device, int mirror) 960{ 961 struct btrfs_zoned_device_info *zinfo = device->zone_info; 962 struct blk_zone *zone; 963 int i; 964 965 if (!is_sb_log_zone(zinfo, mirror)) 966 return 0; 967 968 zone = &zinfo->sb_zones[BTRFS_NR_SB_LOG_ZONES * mirror]; 969 for (i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) { 970 /* Advance the next zone */ 971 if (zone->cond == BLK_ZONE_COND_FULL) { 972 zone++; 973 continue; 974 } 975 976 if (zone->cond == BLK_ZONE_COND_EMPTY) 977 zone->cond = BLK_ZONE_COND_IMP_OPEN; 978 979 zone->wp += SUPER_INFO_SECTORS; 980 981 if (sb_zone_is_full(zone)) { 982 /* 983 * No room left to write new superblock. Since 984 * superblock is written with REQ_SYNC, it is safe to 985 * finish the zone now. 986 * 987 * If the write pointer is exactly at the capacity, 988 * explicit ZONE_FINISH is not necessary. 989 */ 990 if (zone->wp != zone->start + zone->capacity) { 991 int ret; 992 993 ret = blkdev_zone_mgmt(device->bdev, 994 REQ_OP_ZONE_FINISH, zone->start, 995 zone->len, GFP_NOFS); 996 if (ret) 997 return ret; 998 } 999 1000 zone->wp = zone->start + zone->len; 1001 zone->cond = BLK_ZONE_COND_FULL; 1002 } 1003 return 0; 1004 } 1005 1006 /* All the zones are FULL. Should not reach here. */ 1007 ASSERT(0); 1008 return -EIO; 1009} 1010 1011int btrfs_reset_sb_log_zones(struct block_device *bdev, int mirror) 1012{ 1013 sector_t zone_sectors; 1014 sector_t nr_sectors; 1015 u8 zone_sectors_shift; 1016 u32 sb_zone; 1017 u32 nr_zones; 1018 1019 zone_sectors = bdev_zone_sectors(bdev); 1020 zone_sectors_shift = ilog2(zone_sectors); 1021 nr_sectors = bdev_nr_sectors(bdev); 1022 nr_zones = nr_sectors >> zone_sectors_shift; 1023 1024 sb_zone = sb_zone_number(zone_sectors_shift + SECTOR_SHIFT, mirror); 1025 if (sb_zone + 1 >= nr_zones) 1026 return -ENOENT; 1027 1028 return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET, 1029 zone_start_sector(sb_zone, bdev), 1030 zone_sectors * BTRFS_NR_SB_LOG_ZONES, GFP_NOFS); 1031} 1032 1033/* 1034 * Find allocatable zones within a given region. 1035 * 1036 * @device: the device to allocate a region on 1037 * @hole_start: the position of the hole to allocate the region 1038 * @num_bytes: size of wanted region 1039 * @hole_end: the end of the hole 1040 * @return: position of allocatable zones 1041 * 1042 * Allocatable region should not contain any superblock locations. 1043 */ 1044u64 btrfs_find_allocatable_zones(struct btrfs_device *device, u64 hole_start, 1045 u64 hole_end, u64 num_bytes) 1046{ 1047 struct btrfs_zoned_device_info *zinfo = device->zone_info; 1048 const u8 shift = zinfo->zone_size_shift; 1049 u64 nzones = num_bytes >> shift; 1050 u64 pos = hole_start; 1051 u64 begin, end; 1052 bool have_sb; 1053 int i; 1054 1055 ASSERT(IS_ALIGNED(hole_start, zinfo->zone_size)); 1056 ASSERT(IS_ALIGNED(num_bytes, zinfo->zone_size)); 1057 1058 while (pos < hole_end) { 1059 begin = pos >> shift; 1060 end = begin + nzones; 1061 1062 if (end > zinfo->nr_zones) 1063 return hole_end; 1064 1065 /* Check if zones in the region are all empty */ 1066 if (btrfs_dev_is_sequential(device, pos) && 1067 !bitmap_test_range_all_set(zinfo->empty_zones, begin, nzones)) { 1068 pos += zinfo->zone_size; 1069 continue; 1070 } 1071 1072 have_sb = false; 1073 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { 1074 u32 sb_zone; 1075 u64 sb_pos; 1076 1077 sb_zone = sb_zone_number(shift, i); 1078 if (!(end <= sb_zone || 1079 sb_zone + BTRFS_NR_SB_LOG_ZONES <= begin)) { 1080 have_sb = true; 1081 pos = zone_start_physical( 1082 sb_zone + BTRFS_NR_SB_LOG_ZONES, zinfo); 1083 break; 1084 } 1085 1086 /* We also need to exclude regular superblock positions */ 1087 sb_pos = btrfs_sb_offset(i); 1088 if (!(pos + num_bytes <= sb_pos || 1089 sb_pos + BTRFS_SUPER_INFO_SIZE <= pos)) { 1090 have_sb = true; 1091 pos = ALIGN(sb_pos + BTRFS_SUPER_INFO_SIZE, 1092 zinfo->zone_size); 1093 break; 1094 } 1095 } 1096 if (!have_sb) 1097 break; 1098 } 1099 1100 return pos; 1101} 1102 1103static bool btrfs_dev_set_active_zone(struct btrfs_device *device, u64 pos) 1104{ 1105 struct btrfs_zoned_device_info *zone_info = device->zone_info; 1106 unsigned int zno = (pos >> zone_info->zone_size_shift); 1107 1108 /* We can use any number of zones */ 1109 if (zone_info->max_active_zones == 0) 1110 return true; 1111 1112 if (!test_bit(zno, zone_info->active_zones)) { 1113 /* Active zone left? */ 1114 if (atomic_dec_if_positive(&zone_info->active_zones_left) < 0) 1115 return false; 1116 if (test_and_set_bit(zno, zone_info->active_zones)) { 1117 /* Someone already set the bit */ 1118 atomic_inc(&zone_info->active_zones_left); 1119 } 1120 } 1121 1122 return true; 1123} 1124 1125static void btrfs_dev_clear_active_zone(struct btrfs_device *device, u64 pos) 1126{ 1127 struct btrfs_zoned_device_info *zone_info = device->zone_info; 1128 unsigned int zno = (pos >> zone_info->zone_size_shift); 1129 1130 /* We can use any number of zones */ 1131 if (zone_info->max_active_zones == 0) 1132 return; 1133 1134 if (test_and_clear_bit(zno, zone_info->active_zones)) 1135 atomic_inc(&zone_info->active_zones_left); 1136} 1137 1138int btrfs_reset_device_zone(struct btrfs_device *device, u64 physical, 1139 u64 length, u64 *bytes) 1140{ 1141 int ret; 1142 1143 *bytes = 0; 1144 ret = blkdev_zone_mgmt(device->bdev, REQ_OP_ZONE_RESET, 1145 physical >> SECTOR_SHIFT, length >> SECTOR_SHIFT, 1146 GFP_NOFS); 1147 if (ret) 1148 return ret; 1149 1150 *bytes = length; 1151 while (length) { 1152 btrfs_dev_set_zone_empty(device, physical); 1153 btrfs_dev_clear_active_zone(device, physical); 1154 physical += device->zone_info->zone_size; 1155 length -= device->zone_info->zone_size; 1156 } 1157 1158 return 0; 1159} 1160 1161int btrfs_ensure_empty_zones(struct btrfs_device *device, u64 start, u64 size) 1162{ 1163 struct btrfs_zoned_device_info *zinfo = device->zone_info; 1164 const u8 shift = zinfo->zone_size_shift; 1165 unsigned long begin = start >> shift; 1166 unsigned long nbits = size >> shift; 1167 u64 pos; 1168 int ret; 1169 1170 ASSERT(IS_ALIGNED(start, zinfo->zone_size)); 1171 ASSERT(IS_ALIGNED(size, zinfo->zone_size)); 1172 1173 if (begin + nbits > zinfo->nr_zones) 1174 return -ERANGE; 1175 1176 /* All the zones are conventional */ 1177 if (bitmap_test_range_all_zero(zinfo->seq_zones, begin, nbits)) 1178 return 0; 1179 1180 /* All the zones are sequential and empty */ 1181 if (bitmap_test_range_all_set(zinfo->seq_zones, begin, nbits) && 1182 bitmap_test_range_all_set(zinfo->empty_zones, begin, nbits)) 1183 return 0; 1184 1185 for (pos = start; pos < start + size; pos += zinfo->zone_size) { 1186 u64 reset_bytes; 1187 1188 if (!btrfs_dev_is_sequential(device, pos) || 1189 btrfs_dev_is_empty_zone(device, pos)) 1190 continue; 1191 1192 /* Free regions should be empty */ 1193 btrfs_warn_in_rcu( 1194 device->fs_info, 1195 "zoned: resetting device %s (devid %llu) zone %llu for allocation", 1196 rcu_str_deref(device->name), device->devid, pos >> shift); 1197 WARN_ON_ONCE(1); 1198 1199 ret = btrfs_reset_device_zone(device, pos, zinfo->zone_size, 1200 &reset_bytes); 1201 if (ret) 1202 return ret; 1203 } 1204 1205 return 0; 1206} 1207 1208/* 1209 * Calculate an allocation pointer from the extent allocation information 1210 * for a block group consist of conventional zones. It is pointed to the 1211 * end of the highest addressed extent in the block group as an allocation 1212 * offset. 1213 */ 1214static int calculate_alloc_pointer(struct btrfs_block_group *cache, 1215 u64 *offset_ret, bool new) 1216{ 1217 struct btrfs_fs_info *fs_info = cache->fs_info; 1218 struct btrfs_root *root; 1219 struct btrfs_path *path; 1220 struct btrfs_key key; 1221 struct btrfs_key found_key; 1222 int ret; 1223 u64 length; 1224 1225 /* 1226 * Avoid tree lookups for a new block group, there's no use for it. 1227 * It must always be 0. 1228 * 1229 * Also, we have a lock chain of extent buffer lock -> chunk mutex. 1230 * For new a block group, this function is called from 1231 * btrfs_make_block_group() which is already taking the chunk mutex. 1232 * Thus, we cannot call calculate_alloc_pointer() which takes extent 1233 * buffer locks to avoid deadlock. 1234 */ 1235 if (new) { 1236 *offset_ret = 0; 1237 return 0; 1238 } 1239 1240 path = btrfs_alloc_path(); 1241 if (!path) 1242 return -ENOMEM; 1243 1244 key.objectid = cache->start + cache->length; 1245 key.type = 0; 1246 key.offset = 0; 1247 1248 root = btrfs_extent_root(fs_info, key.objectid); 1249 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 1250 /* We should not find the exact match */ 1251 if (!ret) 1252 ret = -EUCLEAN; 1253 if (ret < 0) 1254 goto out; 1255 1256 ret = btrfs_previous_extent_item(root, path, cache->start); 1257 if (ret) { 1258 if (ret == 1) { 1259 ret = 0; 1260 *offset_ret = 0; 1261 } 1262 goto out; 1263 } 1264 1265 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]); 1266 1267 if (found_key.type == BTRFS_EXTENT_ITEM_KEY) 1268 length = found_key.offset; 1269 else 1270 length = fs_info->nodesize; 1271 1272 if (!(found_key.objectid >= cache->start && 1273 found_key.objectid + length <= cache->start + cache->length)) { 1274 ret = -EUCLEAN; 1275 goto out; 1276 } 1277 *offset_ret = found_key.objectid + length - cache->start; 1278 ret = 0; 1279 1280out: 1281 btrfs_free_path(path); 1282 return ret; 1283} 1284 1285int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) 1286{ 1287 struct btrfs_fs_info *fs_info = cache->fs_info; 1288 struct extent_map_tree *em_tree = &fs_info->mapping_tree; 1289 struct extent_map *em; 1290 struct map_lookup *map; 1291 struct btrfs_device *device; 1292 u64 logical = cache->start; 1293 u64 length = cache->length; 1294 int ret; 1295 int i; 1296 unsigned int nofs_flag; 1297 u64 *alloc_offsets = NULL; 1298 u64 *caps = NULL; 1299 u64 *physical = NULL; 1300 unsigned long *active = NULL; 1301 u64 last_alloc = 0; 1302 u32 num_sequential = 0, num_conventional = 0; 1303 1304 if (!btrfs_is_zoned(fs_info)) 1305 return 0; 1306 1307 /* Sanity check */ 1308 if (!IS_ALIGNED(length, fs_info->zone_size)) { 1309 btrfs_err(fs_info, 1310 "zoned: block group %llu len %llu unaligned to zone size %llu", 1311 logical, length, fs_info->zone_size); 1312 return -EIO; 1313 } 1314 1315 /* Get the chunk mapping */ 1316 read_lock(&em_tree->lock); 1317 em = lookup_extent_mapping(em_tree, logical, length); 1318 read_unlock(&em_tree->lock); 1319 1320 if (!em) 1321 return -EINVAL; 1322 1323 map = em->map_lookup; 1324 1325 cache->physical_map = kmemdup(map, map_lookup_size(map->num_stripes), GFP_NOFS); 1326 if (!cache->physical_map) { 1327 ret = -ENOMEM; 1328 goto out; 1329 } 1330 1331 alloc_offsets = kcalloc(map->num_stripes, sizeof(*alloc_offsets), GFP_NOFS); 1332 if (!alloc_offsets) { 1333 ret = -ENOMEM; 1334 goto out; 1335 } 1336 1337 caps = kcalloc(map->num_stripes, sizeof(*caps), GFP_NOFS); 1338 if (!caps) { 1339 ret = -ENOMEM; 1340 goto out; 1341 } 1342 1343 physical = kcalloc(map->num_stripes, sizeof(*physical), GFP_NOFS); 1344 if (!physical) { 1345 ret = -ENOMEM; 1346 goto out; 1347 } 1348 1349 active = bitmap_zalloc(map->num_stripes, GFP_NOFS); 1350 if (!active) { 1351 ret = -ENOMEM; 1352 goto out; 1353 } 1354 1355 for (i = 0; i < map->num_stripes; i++) { 1356 bool is_sequential; 1357 struct blk_zone zone; 1358 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; 1359 int dev_replace_is_ongoing = 0; 1360 1361 device = map->stripes[i].dev; 1362 physical[i] = map->stripes[i].physical; 1363 1364 if (device->bdev == NULL) { 1365 alloc_offsets[i] = WP_MISSING_DEV; 1366 continue; 1367 } 1368 1369 is_sequential = btrfs_dev_is_sequential(device, physical[i]); 1370 if (is_sequential) 1371 num_sequential++; 1372 else 1373 num_conventional++; 1374 1375 /* 1376 * Consider a zone as active if we can allow any number of 1377 * active zones. 1378 */ 1379 if (!device->zone_info->max_active_zones) 1380 __set_bit(i, active); 1381 1382 if (!is_sequential) { 1383 alloc_offsets[i] = WP_CONVENTIONAL; 1384 continue; 1385 } 1386 1387 /* 1388 * This zone will be used for allocation, so mark this zone 1389 * non-empty. 1390 */ 1391 btrfs_dev_clear_zone_empty(device, physical[i]); 1392 1393 down_read(&dev_replace->rwsem); 1394 dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace); 1395 if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL) 1396 btrfs_dev_clear_zone_empty(dev_replace->tgtdev, physical[i]); 1397 up_read(&dev_replace->rwsem); 1398 1399 /* 1400 * The group is mapped to a sequential zone. Get the zone write 1401 * pointer to determine the allocation offset within the zone. 1402 */ 1403 WARN_ON(!IS_ALIGNED(physical[i], fs_info->zone_size)); 1404 nofs_flag = memalloc_nofs_save(); 1405 ret = btrfs_get_dev_zone(device, physical[i], &zone); 1406 memalloc_nofs_restore(nofs_flag); 1407 if (ret == -EIO || ret == -EOPNOTSUPP) { 1408 ret = 0; 1409 alloc_offsets[i] = WP_MISSING_DEV; 1410 continue; 1411 } else if (ret) { 1412 goto out; 1413 } 1414 1415 if (zone.type == BLK_ZONE_TYPE_CONVENTIONAL) { 1416 btrfs_err_in_rcu(fs_info, 1417 "zoned: unexpected conventional zone %llu on device %s (devid %llu)", 1418 zone.start << SECTOR_SHIFT, 1419 rcu_str_deref(device->name), device->devid); 1420 ret = -EIO; 1421 goto out; 1422 } 1423 1424 caps[i] = (zone.capacity << SECTOR_SHIFT); 1425 1426 switch (zone.cond) { 1427 case BLK_ZONE_COND_OFFLINE: 1428 case BLK_ZONE_COND_READONLY: 1429 btrfs_err(fs_info, 1430 "zoned: offline/readonly zone %llu on device %s (devid %llu)", 1431 physical[i] >> device->zone_info->zone_size_shift, 1432 rcu_str_deref(device->name), device->devid); 1433 alloc_offsets[i] = WP_MISSING_DEV; 1434 break; 1435 case BLK_ZONE_COND_EMPTY: 1436 alloc_offsets[i] = 0; 1437 break; 1438 case BLK_ZONE_COND_FULL: 1439 alloc_offsets[i] = caps[i]; 1440 break; 1441 default: 1442 /* Partially used zone */ 1443 alloc_offsets[i] = 1444 ((zone.wp - zone.start) << SECTOR_SHIFT); 1445 __set_bit(i, active); 1446 break; 1447 } 1448 } 1449 1450 if (num_sequential > 0) 1451 set_bit(BLOCK_GROUP_FLAG_SEQUENTIAL_ZONE, &cache->runtime_flags); 1452 1453 if (num_conventional > 0) { 1454 /* Zone capacity is always zone size in emulation */ 1455 cache->zone_capacity = cache->length; 1456 ret = calculate_alloc_pointer(cache, &last_alloc, new); 1457 if (ret) { 1458 btrfs_err(fs_info, 1459 "zoned: failed to determine allocation offset of bg %llu", 1460 cache->start); 1461 goto out; 1462 } else if (map->num_stripes == num_conventional) { 1463 cache->alloc_offset = last_alloc; 1464 set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &cache->runtime_flags); 1465 goto out; 1466 } 1467 } 1468 1469 switch (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) { 1470 case 0: /* single */ 1471 if (alloc_offsets[0] == WP_MISSING_DEV) { 1472 btrfs_err(fs_info, 1473 "zoned: cannot recover write pointer for zone %llu", 1474 physical[0]); 1475 ret = -EIO; 1476 goto out; 1477 } 1478 cache->alloc_offset = alloc_offsets[0]; 1479 cache->zone_capacity = caps[0]; 1480 if (test_bit(0, active)) 1481 set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &cache->runtime_flags); 1482 break; 1483 case BTRFS_BLOCK_GROUP_DUP: 1484 if (map->type & BTRFS_BLOCK_GROUP_DATA) { 1485 btrfs_err(fs_info, "zoned: profile DUP not yet supported on data bg"); 1486 ret = -EINVAL; 1487 goto out; 1488 } 1489 if (alloc_offsets[0] == WP_MISSING_DEV) { 1490 btrfs_err(fs_info, 1491 "zoned: cannot recover write pointer for zone %llu", 1492 physical[0]); 1493 ret = -EIO; 1494 goto out; 1495 } 1496 if (alloc_offsets[1] == WP_MISSING_DEV) { 1497 btrfs_err(fs_info, 1498 "zoned: cannot recover write pointer for zone %llu", 1499 physical[1]); 1500 ret = -EIO; 1501 goto out; 1502 } 1503 if (alloc_offsets[0] != alloc_offsets[1]) { 1504 btrfs_err(fs_info, 1505 "zoned: write pointer offset mismatch of zones in DUP profile"); 1506 ret = -EIO; 1507 goto out; 1508 } 1509 if (test_bit(0, active) != test_bit(1, active)) { 1510 if (!btrfs_zone_activate(cache)) { 1511 ret = -EIO; 1512 goto out; 1513 } 1514 } else { 1515 if (test_bit(0, active)) 1516 set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, 1517 &cache->runtime_flags); 1518 } 1519 cache->alloc_offset = alloc_offsets[0]; 1520 cache->zone_capacity = min(caps[0], caps[1]); 1521 break; 1522 case BTRFS_BLOCK_GROUP_RAID1: 1523 case BTRFS_BLOCK_GROUP_RAID0: 1524 case BTRFS_BLOCK_GROUP_RAID10: 1525 case BTRFS_BLOCK_GROUP_RAID5: 1526 case BTRFS_BLOCK_GROUP_RAID6: 1527 /* non-single profiles are not supported yet */ 1528 default: 1529 btrfs_err(fs_info, "zoned: profile %s not yet supported", 1530 btrfs_bg_type_to_raid_name(map->type)); 1531 ret = -EINVAL; 1532 goto out; 1533 } 1534 1535out: 1536 if (cache->alloc_offset > fs_info->zone_size) { 1537 btrfs_err(fs_info, 1538 "zoned: invalid write pointer %llu in block group %llu", 1539 cache->alloc_offset, cache->start); 1540 ret = -EIO; 1541 } 1542 1543 if (cache->alloc_offset > cache->zone_capacity) { 1544 btrfs_err(fs_info, 1545"zoned: invalid write pointer %llu (larger than zone capacity %llu) in block group %llu", 1546 cache->alloc_offset, cache->zone_capacity, 1547 cache->start); 1548 ret = -EIO; 1549 } 1550 1551 /* An extent is allocated after the write pointer */ 1552 if (!ret && num_conventional && last_alloc > cache->alloc_offset) { 1553 btrfs_err(fs_info, 1554 "zoned: got wrong write pointer in BG %llu: %llu > %llu", 1555 logical, last_alloc, cache->alloc_offset); 1556 ret = -EIO; 1557 } 1558 1559 if (!ret) { 1560 cache->meta_write_pointer = cache->alloc_offset + cache->start; 1561 if (test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &cache->runtime_flags)) { 1562 btrfs_get_block_group(cache); 1563 spin_lock(&fs_info->zone_active_bgs_lock); 1564 list_add_tail(&cache->active_bg_list, 1565 &fs_info->zone_active_bgs); 1566 spin_unlock(&fs_info->zone_active_bgs_lock); 1567 } 1568 } else { 1569 kfree(cache->physical_map); 1570 cache->physical_map = NULL; 1571 } 1572 bitmap_free(active); 1573 kfree(physical); 1574 kfree(caps); 1575 kfree(alloc_offsets); 1576 free_extent_map(em); 1577 1578 return ret; 1579} 1580 1581void btrfs_calc_zone_unusable(struct btrfs_block_group *cache) 1582{ 1583 u64 unusable, free; 1584 1585 if (!btrfs_is_zoned(cache->fs_info)) 1586 return; 1587 1588 WARN_ON(cache->bytes_super != 0); 1589 unusable = (cache->alloc_offset - cache->used) + 1590 (cache->length - cache->zone_capacity); 1591 free = cache->zone_capacity - cache->alloc_offset; 1592 1593 /* We only need ->free_space in ALLOC_SEQ block groups */ 1594 cache->cached = BTRFS_CACHE_FINISHED; 1595 cache->free_space_ctl->free_space = free; 1596 cache->zone_unusable = unusable; 1597} 1598 1599void btrfs_redirty_list_add(struct btrfs_transaction *trans, 1600 struct extent_buffer *eb) 1601{ 1602 if (!btrfs_is_zoned(eb->fs_info) || 1603 btrfs_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN)) 1604 return; 1605 1606 ASSERT(!test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)); 1607 1608 memzero_extent_buffer(eb, 0, eb->len); 1609 set_bit(EXTENT_BUFFER_NO_CHECK, &eb->bflags); 1610 set_extent_buffer_dirty(eb); 1611 set_extent_bit(&trans->dirty_pages, eb->start, eb->start + eb->len - 1, 1612 EXTENT_DIRTY | EXTENT_NOWAIT, NULL); 1613} 1614 1615bool btrfs_use_zone_append(struct btrfs_bio *bbio) 1616{ 1617 u64 start = (bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT); 1618 struct btrfs_inode *inode = bbio->inode; 1619 struct btrfs_fs_info *fs_info = bbio->fs_info; 1620 struct btrfs_block_group *cache; 1621 bool ret = false; 1622 1623 if (!btrfs_is_zoned(fs_info)) 1624 return false; 1625 1626 if (!inode || !is_data_inode(&inode->vfs_inode)) 1627 return false; 1628 1629 if (btrfs_op(&bbio->bio) != BTRFS_MAP_WRITE) 1630 return false; 1631 1632 /* 1633 * Using REQ_OP_ZONE_APPNED for relocation can break assumptions on the 1634 * extent layout the relocation code has. 1635 * Furthermore we have set aside own block-group from which only the 1636 * relocation "process" can allocate and make sure only one process at a 1637 * time can add pages to an extent that gets relocated, so it's safe to 1638 * use regular REQ_OP_WRITE for this special case. 1639 */ 1640 if (btrfs_is_data_reloc_root(inode->root)) 1641 return false; 1642 1643 cache = btrfs_lookup_block_group(fs_info, start); 1644 ASSERT(cache); 1645 if (!cache) 1646 return false; 1647 1648 ret = !!test_bit(BLOCK_GROUP_FLAG_SEQUENTIAL_ZONE, &cache->runtime_flags); 1649 btrfs_put_block_group(cache); 1650 1651 return ret; 1652} 1653 1654void btrfs_record_physical_zoned(struct btrfs_bio *bbio) 1655{ 1656 const u64 physical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT; 1657 struct btrfs_ordered_sum *sum = bbio->sums; 1658 1659 if (physical < bbio->orig_physical) 1660 sum->logical -= bbio->orig_physical - physical; 1661 else 1662 sum->logical += physical - bbio->orig_physical; 1663} 1664 1665static void btrfs_rewrite_logical_zoned(struct btrfs_ordered_extent *ordered, 1666 u64 logical) 1667{ 1668 struct extent_map_tree *em_tree = &BTRFS_I(ordered->inode)->extent_tree; 1669 struct extent_map *em; 1670 1671 ordered->disk_bytenr = logical; 1672 1673 write_lock(&em_tree->lock); 1674 em = search_extent_mapping(em_tree, ordered->file_offset, 1675 ordered->num_bytes); 1676 em->block_start = logical; 1677 free_extent_map(em); 1678 write_unlock(&em_tree->lock); 1679} 1680 1681static bool btrfs_zoned_split_ordered(struct btrfs_ordered_extent *ordered, 1682 u64 logical, u64 len) 1683{ 1684 struct btrfs_ordered_extent *new; 1685 1686 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags) && 1687 split_extent_map(BTRFS_I(ordered->inode), ordered->file_offset, 1688 ordered->num_bytes, len, logical)) 1689 return false; 1690 1691 new = btrfs_split_ordered_extent(ordered, len); 1692 if (IS_ERR(new)) 1693 return false; 1694 new->disk_bytenr = logical; 1695 btrfs_finish_one_ordered(new); 1696 return true; 1697} 1698 1699void btrfs_finish_ordered_zoned(struct btrfs_ordered_extent *ordered) 1700{ 1701 struct btrfs_inode *inode = BTRFS_I(ordered->inode); 1702 struct btrfs_fs_info *fs_info = inode->root->fs_info; 1703 struct btrfs_ordered_sum *sum; 1704 u64 logical, len; 1705 1706 /* 1707 * Write to pre-allocated region is for the data relocation, and so 1708 * it should use WRITE operation. No split/rewrite are necessary. 1709 */ 1710 if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered->flags)) 1711 return; 1712 1713 ASSERT(!list_empty(&ordered->list)); 1714 /* The ordered->list can be empty in the above pre-alloc case. */ 1715 sum = list_first_entry(&ordered->list, struct btrfs_ordered_sum, list); 1716 logical = sum->logical; 1717 len = sum->len; 1718 1719 while (len < ordered->disk_num_bytes) { 1720 sum = list_next_entry(sum, list); 1721 if (sum->logical == logical + len) { 1722 len += sum->len; 1723 continue; 1724 } 1725 if (!btrfs_zoned_split_ordered(ordered, logical, len)) { 1726 set_bit(BTRFS_ORDERED_IOERR, &ordered->flags); 1727 btrfs_err(fs_info, "failed to split ordered extent"); 1728 goto out; 1729 } 1730 logical = sum->logical; 1731 len = sum->len; 1732 } 1733 1734 if (ordered->disk_bytenr != logical) 1735 btrfs_rewrite_logical_zoned(ordered, logical); 1736 1737out: 1738 /* 1739 * If we end up here for nodatasum I/O, the btrfs_ordered_sum structures 1740 * were allocated by btrfs_alloc_dummy_sum only to record the logical 1741 * addresses and don't contain actual checksums. We thus must free them 1742 * here so that we don't attempt to log the csums later. 1743 */ 1744 if ((inode->flags & BTRFS_INODE_NODATASUM) || 1745 test_bit(BTRFS_FS_STATE_NO_CSUMS, &fs_info->fs_state)) { 1746 while ((sum = list_first_entry_or_null(&ordered->list, 1747 typeof(*sum), list))) { 1748 list_del(&sum->list); 1749 kfree(sum); 1750 } 1751 } 1752} 1753 1754static bool check_bg_is_active(struct btrfs_eb_write_context *ctx, 1755 struct btrfs_block_group **active_bg) 1756{ 1757 const struct writeback_control *wbc = ctx->wbc; 1758 struct btrfs_block_group *block_group = ctx->zoned_bg; 1759 struct btrfs_fs_info *fs_info = block_group->fs_info; 1760 1761 if (test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags)) 1762 return true; 1763 1764 if (fs_info->treelog_bg == block_group->start) { 1765 if (!btrfs_zone_activate(block_group)) { 1766 int ret_fin = btrfs_zone_finish_one_bg(fs_info); 1767 1768 if (ret_fin != 1 || !btrfs_zone_activate(block_group)) 1769 return false; 1770 } 1771 } else if (*active_bg != block_group) { 1772 struct btrfs_block_group *tgt = *active_bg; 1773 1774 /* zoned_meta_io_lock protects fs_info->active_{meta,system}_bg. */ 1775 lockdep_assert_held(&fs_info->zoned_meta_io_lock); 1776 1777 if (tgt) { 1778 /* 1779 * If there is an unsent IO left in the allocated area, 1780 * we cannot wait for them as it may cause a deadlock. 1781 */ 1782 if (tgt->meta_write_pointer < tgt->start + tgt->alloc_offset) { 1783 if (wbc->sync_mode == WB_SYNC_NONE || 1784 (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)) 1785 return false; 1786 } 1787 1788 /* Pivot active metadata/system block group. */ 1789 btrfs_zoned_meta_io_unlock(fs_info); 1790 wait_eb_writebacks(tgt); 1791 do_zone_finish(tgt, true); 1792 btrfs_zoned_meta_io_lock(fs_info); 1793 if (*active_bg == tgt) { 1794 btrfs_put_block_group(tgt); 1795 *active_bg = NULL; 1796 } 1797 } 1798 if (!btrfs_zone_activate(block_group)) 1799 return false; 1800 if (*active_bg != block_group) { 1801 ASSERT(*active_bg == NULL); 1802 *active_bg = block_group; 1803 btrfs_get_block_group(block_group); 1804 } 1805 } 1806 1807 return true; 1808} 1809 1810/* 1811 * Check if @ctx->eb is aligned to the write pointer. 1812 * 1813 * Return: 1814 * 0: @ctx->eb is at the write pointer. You can write it. 1815 * -EAGAIN: There is a hole. The caller should handle the case. 1816 * -EBUSY: There is a hole, but the caller can just bail out. 1817 */ 1818int btrfs_check_meta_write_pointer(struct btrfs_fs_info *fs_info, 1819 struct btrfs_eb_write_context *ctx) 1820{ 1821 const struct writeback_control *wbc = ctx->wbc; 1822 const struct extent_buffer *eb = ctx->eb; 1823 struct btrfs_block_group *block_group = ctx->zoned_bg; 1824 1825 if (!btrfs_is_zoned(fs_info)) 1826 return 0; 1827 1828 if (block_group) { 1829 if (block_group->start > eb->start || 1830 block_group->start + block_group->length <= eb->start) { 1831 btrfs_put_block_group(block_group); 1832 block_group = NULL; 1833 ctx->zoned_bg = NULL; 1834 } 1835 } 1836 1837 if (!block_group) { 1838 block_group = btrfs_lookup_block_group(fs_info, eb->start); 1839 if (!block_group) 1840 return 0; 1841 ctx->zoned_bg = block_group; 1842 } 1843 1844 if (block_group->meta_write_pointer == eb->start) { 1845 struct btrfs_block_group **tgt; 1846 1847 if (!test_bit(BTRFS_FS_ACTIVE_ZONE_TRACKING, &fs_info->flags)) 1848 return 0; 1849 1850 if (block_group->flags & BTRFS_BLOCK_GROUP_SYSTEM) 1851 tgt = &fs_info->active_system_bg; 1852 else 1853 tgt = &fs_info->active_meta_bg; 1854 if (check_bg_is_active(ctx, tgt)) 1855 return 0; 1856 } 1857 1858 /* 1859 * Since we may release fs_info->zoned_meta_io_lock, someone can already 1860 * start writing this eb. In that case, we can just bail out. 1861 */ 1862 if (block_group->meta_write_pointer > eb->start) 1863 return -EBUSY; 1864 1865 /* If for_sync, this hole will be filled with trasnsaction commit. */ 1866 if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) 1867 return -EAGAIN; 1868 return -EBUSY; 1869} 1870 1871int btrfs_zoned_issue_zeroout(struct btrfs_device *device, u64 physical, u64 length) 1872{ 1873 if (!btrfs_dev_is_sequential(device, physical)) 1874 return -EOPNOTSUPP; 1875 1876 return blkdev_issue_zeroout(device->bdev, physical >> SECTOR_SHIFT, 1877 length >> SECTOR_SHIFT, GFP_NOFS, 0); 1878} 1879 1880static int read_zone_info(struct btrfs_fs_info *fs_info, u64 logical, 1881 struct blk_zone *zone) 1882{ 1883 struct btrfs_io_context *bioc = NULL; 1884 u64 mapped_length = PAGE_SIZE; 1885 unsigned int nofs_flag; 1886 int nmirrors; 1887 int i, ret; 1888 1889 ret = btrfs_map_block(fs_info, BTRFS_MAP_GET_READ_MIRRORS, logical, 1890 &mapped_length, &bioc, NULL, NULL, 1); 1891 if (ret || !bioc || mapped_length < PAGE_SIZE) { 1892 ret = -EIO; 1893 goto out_put_bioc; 1894 } 1895 1896 if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) { 1897 ret = -EINVAL; 1898 goto out_put_bioc; 1899 } 1900 1901 nofs_flag = memalloc_nofs_save(); 1902 nmirrors = (int)bioc->num_stripes; 1903 for (i = 0; i < nmirrors; i++) { 1904 u64 physical = bioc->stripes[i].physical; 1905 struct btrfs_device *dev = bioc->stripes[i].dev; 1906 1907 /* Missing device */ 1908 if (!dev->bdev) 1909 continue; 1910 1911 ret = btrfs_get_dev_zone(dev, physical, zone); 1912 /* Failing device */ 1913 if (ret == -EIO || ret == -EOPNOTSUPP) 1914 continue; 1915 break; 1916 } 1917 memalloc_nofs_restore(nofs_flag); 1918out_put_bioc: 1919 btrfs_put_bioc(bioc); 1920 return ret; 1921} 1922 1923/* 1924 * Synchronize write pointer in a zone at @physical_start on @tgt_dev, by 1925 * filling zeros between @physical_pos to a write pointer of dev-replace 1926 * source device. 1927 */ 1928int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, u64 logical, 1929 u64 physical_start, u64 physical_pos) 1930{ 1931 struct btrfs_fs_info *fs_info = tgt_dev->fs_info; 1932 struct blk_zone zone; 1933 u64 length; 1934 u64 wp; 1935 int ret; 1936 1937 if (!btrfs_dev_is_sequential(tgt_dev, physical_pos)) 1938 return 0; 1939 1940 ret = read_zone_info(fs_info, logical, &zone); 1941 if (ret) 1942 return ret; 1943 1944 wp = physical_start + ((zone.wp - zone.start) << SECTOR_SHIFT); 1945 1946 if (physical_pos == wp) 1947 return 0; 1948 1949 if (physical_pos > wp) 1950 return -EUCLEAN; 1951 1952 length = wp - physical_pos; 1953 return btrfs_zoned_issue_zeroout(tgt_dev, physical_pos, length); 1954} 1955 1956/* 1957 * Activate block group and underlying device zones 1958 * 1959 * @block_group: the block group to activate 1960 * 1961 * Return: true on success, false otherwise 1962 */ 1963bool btrfs_zone_activate(struct btrfs_block_group *block_group) 1964{ 1965 struct btrfs_fs_info *fs_info = block_group->fs_info; 1966 struct map_lookup *map; 1967 struct btrfs_device *device; 1968 u64 physical; 1969 const bool is_data = (block_group->flags & BTRFS_BLOCK_GROUP_DATA); 1970 bool ret; 1971 int i; 1972 1973 if (!btrfs_is_zoned(block_group->fs_info)) 1974 return true; 1975 1976 map = block_group->physical_map; 1977 1978 spin_lock(&fs_info->zone_active_bgs_lock); 1979 spin_lock(&block_group->lock); 1980 if (test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags)) { 1981 ret = true; 1982 goto out_unlock; 1983 } 1984 1985 /* No space left */ 1986 if (btrfs_zoned_bg_is_full(block_group)) { 1987 ret = false; 1988 goto out_unlock; 1989 } 1990 1991 for (i = 0; i < map->num_stripes; i++) { 1992 struct btrfs_zoned_device_info *zinfo; 1993 int reserved = 0; 1994 1995 device = map->stripes[i].dev; 1996 physical = map->stripes[i].physical; 1997 zinfo = device->zone_info; 1998 1999 if (zinfo->max_active_zones == 0) 2000 continue; 2001 2002 if (is_data) 2003 reserved = zinfo->reserved_active_zones; 2004 /* 2005 * For the data block group, leave active zones for one 2006 * metadata block group and one system block group. 2007 */ 2008 if (atomic_read(&zinfo->active_zones_left) <= reserved) { 2009 ret = false; 2010 goto out_unlock; 2011 } 2012 2013 if (!btrfs_dev_set_active_zone(device, physical)) { 2014 /* Cannot activate the zone */ 2015 ret = false; 2016 goto out_unlock; 2017 } 2018 if (!is_data) 2019 zinfo->reserved_active_zones--; 2020 } 2021 2022 /* Successfully activated all the zones */ 2023 set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags); 2024 spin_unlock(&block_group->lock); 2025 2026 /* For the active block group list */ 2027 btrfs_get_block_group(block_group); 2028 list_add_tail(&block_group->active_bg_list, &fs_info->zone_active_bgs); 2029 spin_unlock(&fs_info->zone_active_bgs_lock); 2030 2031 return true; 2032 2033out_unlock: 2034 spin_unlock(&block_group->lock); 2035 spin_unlock(&fs_info->zone_active_bgs_lock); 2036 return ret; 2037} 2038 2039static void wait_eb_writebacks(struct btrfs_block_group *block_group) 2040{ 2041 struct btrfs_fs_info *fs_info = block_group->fs_info; 2042 const u64 end = block_group->start + block_group->length; 2043 struct radix_tree_iter iter; 2044 struct extent_buffer *eb; 2045 void __rcu **slot; 2046 2047 rcu_read_lock(); 2048 radix_tree_for_each_slot(slot, &fs_info->buffer_radix, &iter, 2049 block_group->start >> fs_info->sectorsize_bits) { 2050 eb = radix_tree_deref_slot(slot); 2051 if (!eb) 2052 continue; 2053 if (radix_tree_deref_retry(eb)) { 2054 slot = radix_tree_iter_retry(&iter); 2055 continue; 2056 } 2057 2058 if (eb->start < block_group->start) 2059 continue; 2060 if (eb->start >= end) 2061 break; 2062 2063 slot = radix_tree_iter_resume(slot, &iter); 2064 rcu_read_unlock(); 2065 wait_on_extent_buffer_writeback(eb); 2066 rcu_read_lock(); 2067 } 2068 rcu_read_unlock(); 2069} 2070 2071static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_written) 2072{ 2073 struct btrfs_fs_info *fs_info = block_group->fs_info; 2074 struct map_lookup *map; 2075 const bool is_metadata = (block_group->flags & 2076 (BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_SYSTEM)); 2077 int ret = 0; 2078 int i; 2079 2080 spin_lock(&block_group->lock); 2081 if (!test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags)) { 2082 spin_unlock(&block_group->lock); 2083 return 0; 2084 } 2085 2086 /* Check if we have unwritten allocated space */ 2087 if (is_metadata && 2088 block_group->start + block_group->alloc_offset > block_group->meta_write_pointer) { 2089 spin_unlock(&block_group->lock); 2090 return -EAGAIN; 2091 } 2092 2093 /* 2094 * If we are sure that the block group is full (= no more room left for 2095 * new allocation) and the IO for the last usable block is completed, we 2096 * don't need to wait for the other IOs. This holds because we ensure 2097 * the sequential IO submissions using the ZONE_APPEND command for data 2098 * and block_group->meta_write_pointer for metadata. 2099 */ 2100 if (!fully_written) { 2101 if (test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags)) { 2102 spin_unlock(&block_group->lock); 2103 return -EAGAIN; 2104 } 2105 spin_unlock(&block_group->lock); 2106 2107 ret = btrfs_inc_block_group_ro(block_group, false); 2108 if (ret) 2109 return ret; 2110 2111 /* Ensure all writes in this block group finish */ 2112 btrfs_wait_block_group_reservations(block_group); 2113 /* No need to wait for NOCOW writers. Zoned mode does not allow that */ 2114 btrfs_wait_ordered_roots(fs_info, U64_MAX, block_group->start, 2115 block_group->length); 2116 /* Wait for extent buffers to be written. */ 2117 if (is_metadata) 2118 wait_eb_writebacks(block_group); 2119 2120 spin_lock(&block_group->lock); 2121 2122 /* 2123 * Bail out if someone already deactivated the block group, or 2124 * allocated space is left in the block group. 2125 */ 2126 if (!test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, 2127 &block_group->runtime_flags)) { 2128 spin_unlock(&block_group->lock); 2129 btrfs_dec_block_group_ro(block_group); 2130 return 0; 2131 } 2132 2133 if (block_group->reserved || 2134 test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, 2135 &block_group->runtime_flags)) { 2136 spin_unlock(&block_group->lock); 2137 btrfs_dec_block_group_ro(block_group); 2138 return -EAGAIN; 2139 } 2140 } 2141 2142 clear_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags); 2143 block_group->alloc_offset = block_group->zone_capacity; 2144 if (block_group->flags & (BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_SYSTEM)) 2145 block_group->meta_write_pointer = block_group->start + 2146 block_group->zone_capacity; 2147 block_group->free_space_ctl->free_space = 0; 2148 btrfs_clear_treelog_bg(block_group); 2149 btrfs_clear_data_reloc_bg(block_group); 2150 spin_unlock(&block_group->lock); 2151 2152 map = block_group->physical_map; 2153 for (i = 0; i < map->num_stripes; i++) { 2154 struct btrfs_device *device = map->stripes[i].dev; 2155 const u64 physical = map->stripes[i].physical; 2156 struct btrfs_zoned_device_info *zinfo = device->zone_info; 2157 2158 if (zinfo->max_active_zones == 0) 2159 continue; 2160 2161 ret = blkdev_zone_mgmt(device->bdev, REQ_OP_ZONE_FINISH, 2162 physical >> SECTOR_SHIFT, 2163 zinfo->zone_size >> SECTOR_SHIFT, 2164 GFP_NOFS); 2165 2166 if (ret) 2167 return ret; 2168 2169 if (!(block_group->flags & BTRFS_BLOCK_GROUP_DATA)) 2170 zinfo->reserved_active_zones++; 2171 btrfs_dev_clear_active_zone(device, physical); 2172 } 2173 2174 if (!fully_written) 2175 btrfs_dec_block_group_ro(block_group); 2176 2177 spin_lock(&fs_info->zone_active_bgs_lock); 2178 ASSERT(!list_empty(&block_group->active_bg_list)); 2179 list_del_init(&block_group->active_bg_list); 2180 spin_unlock(&fs_info->zone_active_bgs_lock); 2181 2182 /* For active_bg_list */ 2183 btrfs_put_block_group(block_group); 2184 2185 clear_and_wake_up_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags); 2186 2187 return 0; 2188} 2189 2190int btrfs_zone_finish(struct btrfs_block_group *block_group) 2191{ 2192 if (!btrfs_is_zoned(block_group->fs_info)) 2193 return 0; 2194 2195 return do_zone_finish(block_group, false); 2196} 2197 2198bool btrfs_can_activate_zone(struct btrfs_fs_devices *fs_devices, u64 flags) 2199{ 2200 struct btrfs_fs_info *fs_info = fs_devices->fs_info; 2201 struct btrfs_device *device; 2202 bool ret = false; 2203 2204 if (!btrfs_is_zoned(fs_info)) 2205 return true; 2206 2207 /* Check if there is a device with active zones left */ 2208 mutex_lock(&fs_info->chunk_mutex); 2209 spin_lock(&fs_info->zone_active_bgs_lock); 2210 list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) { 2211 struct btrfs_zoned_device_info *zinfo = device->zone_info; 2212 int reserved = 0; 2213 2214 if (!device->bdev) 2215 continue; 2216 2217 if (!zinfo->max_active_zones) { 2218 ret = true; 2219 break; 2220 } 2221 2222 if (flags & BTRFS_BLOCK_GROUP_DATA) 2223 reserved = zinfo->reserved_active_zones; 2224 2225 switch (flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) { 2226 case 0: /* single */ 2227 ret = (atomic_read(&zinfo->active_zones_left) >= (1 + reserved)); 2228 break; 2229 case BTRFS_BLOCK_GROUP_DUP: 2230 ret = (atomic_read(&zinfo->active_zones_left) >= (2 + reserved)); 2231 break; 2232 } 2233 if (ret) 2234 break; 2235 } 2236 spin_unlock(&fs_info->zone_active_bgs_lock); 2237 mutex_unlock(&fs_info->chunk_mutex); 2238 2239 if (!ret) 2240 set_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags); 2241 2242 return ret; 2243} 2244 2245void btrfs_zone_finish_endio(struct btrfs_fs_info *fs_info, u64 logical, u64 length) 2246{ 2247 struct btrfs_block_group *block_group; 2248 u64 min_alloc_bytes; 2249 2250 if (!btrfs_is_zoned(fs_info)) 2251 return; 2252 2253 block_group = btrfs_lookup_block_group(fs_info, logical); 2254 ASSERT(block_group); 2255 2256 /* No MIXED_BG on zoned btrfs. */ 2257 if (block_group->flags & BTRFS_BLOCK_GROUP_DATA) 2258 min_alloc_bytes = fs_info->sectorsize; 2259 else 2260 min_alloc_bytes = fs_info->nodesize; 2261 2262 /* Bail out if we can allocate more data from this block group. */ 2263 if (logical + length + min_alloc_bytes <= 2264 block_group->start + block_group->zone_capacity) 2265 goto out; 2266 2267 do_zone_finish(block_group, true); 2268 2269out: 2270 btrfs_put_block_group(block_group); 2271} 2272 2273static void btrfs_zone_finish_endio_workfn(struct work_struct *work) 2274{ 2275 struct btrfs_block_group *bg = 2276 container_of(work, struct btrfs_block_group, zone_finish_work); 2277 2278 wait_on_extent_buffer_writeback(bg->last_eb); 2279 free_extent_buffer(bg->last_eb); 2280 btrfs_zone_finish_endio(bg->fs_info, bg->start, bg->length); 2281 btrfs_put_block_group(bg); 2282} 2283 2284void btrfs_schedule_zone_finish_bg(struct btrfs_block_group *bg, 2285 struct extent_buffer *eb) 2286{ 2287 if (!test_bit(BLOCK_GROUP_FLAG_SEQUENTIAL_ZONE, &bg->runtime_flags) || 2288 eb->start + eb->len * 2 <= bg->start + bg->zone_capacity) 2289 return; 2290 2291 if (WARN_ON(bg->zone_finish_work.func == btrfs_zone_finish_endio_workfn)) { 2292 btrfs_err(bg->fs_info, "double scheduling of bg %llu zone finishing", 2293 bg->start); 2294 return; 2295 } 2296 2297 /* For the work */ 2298 btrfs_get_block_group(bg); 2299 atomic_inc(&eb->refs); 2300 bg->last_eb = eb; 2301 INIT_WORK(&bg->zone_finish_work, btrfs_zone_finish_endio_workfn); 2302 queue_work(system_unbound_wq, &bg->zone_finish_work); 2303} 2304 2305void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg) 2306{ 2307 struct btrfs_fs_info *fs_info = bg->fs_info; 2308 2309 spin_lock(&fs_info->relocation_bg_lock); 2310 if (fs_info->data_reloc_bg == bg->start) 2311 fs_info->data_reloc_bg = 0; 2312 spin_unlock(&fs_info->relocation_bg_lock); 2313} 2314 2315void btrfs_free_zone_cache(struct btrfs_fs_info *fs_info) 2316{ 2317 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; 2318 struct btrfs_device *device; 2319 2320 if (!btrfs_is_zoned(fs_info)) 2321 return; 2322 2323 mutex_lock(&fs_devices->device_list_mutex); 2324 list_for_each_entry(device, &fs_devices->devices, dev_list) { 2325 if (device->zone_info) { 2326 vfree(device->zone_info->zone_cache); 2327 device->zone_info->zone_cache = NULL; 2328 } 2329 } 2330 mutex_unlock(&fs_devices->device_list_mutex); 2331} 2332 2333bool btrfs_zoned_should_reclaim(struct btrfs_fs_info *fs_info) 2334{ 2335 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; 2336 struct btrfs_device *device; 2337 u64 used = 0; 2338 u64 total = 0; 2339 u64 factor; 2340 2341 ASSERT(btrfs_is_zoned(fs_info)); 2342 2343 if (fs_info->bg_reclaim_threshold == 0) 2344 return false; 2345 2346 mutex_lock(&fs_devices->device_list_mutex); 2347 list_for_each_entry(device, &fs_devices->devices, dev_list) { 2348 if (!device->bdev) 2349 continue; 2350 2351 total += device->disk_total_bytes; 2352 used += device->bytes_used; 2353 } 2354 mutex_unlock(&fs_devices->device_list_mutex); 2355 2356 factor = div64_u64(used * 100, total); 2357 return factor >= fs_info->bg_reclaim_threshold; 2358} 2359 2360void btrfs_zoned_release_data_reloc_bg(struct btrfs_fs_info *fs_info, u64 logical, 2361 u64 length) 2362{ 2363 struct btrfs_block_group *block_group; 2364 2365 if (!btrfs_is_zoned(fs_info)) 2366 return; 2367 2368 block_group = btrfs_lookup_block_group(fs_info, logical); 2369 /* It should be called on a previous data relocation block group. */ 2370 ASSERT(block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA)); 2371 2372 spin_lock(&block_group->lock); 2373 if (!test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags)) 2374 goto out; 2375 2376 /* All relocation extents are written. */ 2377 if (block_group->start + block_group->alloc_offset == logical + length) { 2378 /* 2379 * Now, release this block group for further allocations and 2380 * zone finish. 2381 */ 2382 clear_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, 2383 &block_group->runtime_flags); 2384 } 2385 2386out: 2387 spin_unlock(&block_group->lock); 2388 btrfs_put_block_group(block_group); 2389} 2390 2391int btrfs_zone_finish_one_bg(struct btrfs_fs_info *fs_info) 2392{ 2393 struct btrfs_block_group *block_group; 2394 struct btrfs_block_group *min_bg = NULL; 2395 u64 min_avail = U64_MAX; 2396 int ret; 2397 2398 spin_lock(&fs_info->zone_active_bgs_lock); 2399 list_for_each_entry(block_group, &fs_info->zone_active_bgs, 2400 active_bg_list) { 2401 u64 avail; 2402 2403 spin_lock(&block_group->lock); 2404 if (block_group->reserved || block_group->alloc_offset == 0 || 2405 (block_group->flags & BTRFS_BLOCK_GROUP_SYSTEM) || 2406 test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags)) { 2407 spin_unlock(&block_group->lock); 2408 continue; 2409 } 2410 2411 avail = block_group->zone_capacity - block_group->alloc_offset; 2412 if (min_avail > avail) { 2413 if (min_bg) 2414 btrfs_put_block_group(min_bg); 2415 min_bg = block_group; 2416 min_avail = avail; 2417 btrfs_get_block_group(min_bg); 2418 } 2419 spin_unlock(&block_group->lock); 2420 } 2421 spin_unlock(&fs_info->zone_active_bgs_lock); 2422 2423 if (!min_bg) 2424 return 0; 2425 2426 ret = btrfs_zone_finish(min_bg); 2427 btrfs_put_block_group(min_bg); 2428 2429 return ret < 0 ? ret : 1; 2430} 2431 2432int btrfs_zoned_activate_one_bg(struct btrfs_fs_info *fs_info, 2433 struct btrfs_space_info *space_info, 2434 bool do_finish) 2435{ 2436 struct btrfs_block_group *bg; 2437 int index; 2438 2439 if (!btrfs_is_zoned(fs_info) || (space_info->flags & BTRFS_BLOCK_GROUP_DATA)) 2440 return 0; 2441 2442 for (;;) { 2443 int ret; 2444 bool need_finish = false; 2445 2446 down_read(&space_info->groups_sem); 2447 for (index = 0; index < BTRFS_NR_RAID_TYPES; index++) { 2448 list_for_each_entry(bg, &space_info->block_groups[index], 2449 list) { 2450 if (!spin_trylock(&bg->lock)) 2451 continue; 2452 if (btrfs_zoned_bg_is_full(bg) || 2453 test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, 2454 &bg->runtime_flags)) { 2455 spin_unlock(&bg->lock); 2456 continue; 2457 } 2458 spin_unlock(&bg->lock); 2459 2460 if (btrfs_zone_activate(bg)) { 2461 up_read(&space_info->groups_sem); 2462 return 1; 2463 } 2464 2465 need_finish = true; 2466 } 2467 } 2468 up_read(&space_info->groups_sem); 2469 2470 if (!do_finish || !need_finish) 2471 break; 2472 2473 ret = btrfs_zone_finish_one_bg(fs_info); 2474 if (ret == 0) 2475 break; 2476 if (ret < 0) 2477 return ret; 2478 } 2479 2480 return 0; 2481} 2482 2483/* 2484 * Reserve zones for one metadata block group, one tree-log block group, and one 2485 * system block group. 2486 */ 2487void btrfs_check_active_zone_reservation(struct btrfs_fs_info *fs_info) 2488{ 2489 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; 2490 struct btrfs_block_group *block_group; 2491 struct btrfs_device *device; 2492 /* Reserve zones for normal SINGLE metadata and tree-log block group. */ 2493 unsigned int metadata_reserve = 2; 2494 /* Reserve a zone for SINGLE system block group. */ 2495 unsigned int system_reserve = 1; 2496 2497 if (!test_bit(BTRFS_FS_ACTIVE_ZONE_TRACKING, &fs_info->flags)) 2498 return; 2499 2500 /* 2501 * This function is called from the mount context. So, there is no 2502 * parallel process touching the bits. No need for read_seqretry(). 2503 */ 2504 if (fs_info->avail_metadata_alloc_bits & BTRFS_BLOCK_GROUP_DUP) 2505 metadata_reserve = 4; 2506 if (fs_info->avail_system_alloc_bits & BTRFS_BLOCK_GROUP_DUP) 2507 system_reserve = 2; 2508 2509 /* Apply the reservation on all the devices. */ 2510 mutex_lock(&fs_devices->device_list_mutex); 2511 list_for_each_entry(device, &fs_devices->devices, dev_list) { 2512 if (!device->bdev) 2513 continue; 2514 2515 device->zone_info->reserved_active_zones = 2516 metadata_reserve + system_reserve; 2517 } 2518 mutex_unlock(&fs_devices->device_list_mutex); 2519 2520 /* Release reservation for currently active block groups. */ 2521 spin_lock(&fs_info->zone_active_bgs_lock); 2522 list_for_each_entry(block_group, &fs_info->zone_active_bgs, active_bg_list) { 2523 struct map_lookup *map = block_group->physical_map; 2524 2525 if (!(block_group->flags & 2526 (BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_SYSTEM))) 2527 continue; 2528 2529 for (int i = 0; i < map->num_stripes; i++) 2530 map->stripes[i].dev->zone_info->reserved_active_zones--; 2531 } 2532 spin_unlock(&fs_info->zone_active_bgs_lock); 2533} 2534