1/* 2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited. 3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. 4 * 5 * This file is released under the GPL. 6 */ 7 8#include "dm-core.h" 9#include "dm-rq.h" 10#include "dm-uevent.h" 11 12#include <linux/init.h> 13#include <linux/module.h> 14#include <linux/mutex.h> 15#include <linux/sched/mm.h> 16#include <linux/sched/signal.h> 17#include <linux/blkpg.h> 18#include <linux/bio.h> 19#include <linux/mempool.h> 20#include <linux/dax.h> 21#include <linux/slab.h> 22#include <linux/idr.h> 23#include <linux/uio.h> 24#include <linux/hdreg.h> 25#include <linux/delay.h> 26#include <linux/wait.h> 27#include <linux/pr.h> 28#include <linux/refcount.h> 29#include <linux/part_stat.h> 30#include <linux/blk-crypto.h> 31 32#define DM_MSG_PREFIX "core" 33 34/* 35 * Cookies are numeric values sent with CHANGE and REMOVE 36 * uevents while resuming, removing or renaming the device. 37 */ 38#define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE" 39#define DM_COOKIE_LENGTH 24 40 41static const char *_name = DM_NAME; 42 43static unsigned int major = 0; 44static unsigned int _major = 0; 45 46static DEFINE_IDR(_minor_idr); 47 48static DEFINE_SPINLOCK(_minor_lock); 49 50static void do_deferred_remove(struct work_struct *w); 51 52static DECLARE_WORK(deferred_remove_work, do_deferred_remove); 53 54static struct workqueue_struct *deferred_remove_workqueue; 55 56atomic_t dm_global_event_nr = ATOMIC_INIT(0); 57DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq); 58 59void dm_issue_global_event(void) 60{ 61 atomic_inc(&dm_global_event_nr); 62 wake_up(&dm_global_eventq); 63} 64 65/* 66 * One of these is allocated (on-stack) per original bio. 67 */ 68struct clone_info { 69 struct dm_table *map; 70 struct bio *bio; 71 struct dm_io *io; 72 sector_t sector; 73 unsigned sector_count; 74}; 75 76/* 77 * One of these is allocated per clone bio. 78 */ 79#define DM_TIO_MAGIC 7282014 80struct dm_target_io { 81 unsigned magic; 82 struct dm_io *io; 83 struct dm_target *ti; 84 unsigned target_bio_nr; 85 unsigned *len_ptr; 86 bool inside_dm_io; 87 struct bio clone; 88}; 89 90/* 91 * One of these is allocated per original bio. 92 * It contains the first clone used for that original. 93 */ 94#define DM_IO_MAGIC 5191977 95struct dm_io { 96 unsigned magic; 97 struct mapped_device *md; 98 blk_status_t status; 99 atomic_t io_count; 100 struct bio *orig_bio; 101 unsigned long start_time; 102 spinlock_t endio_lock; 103 struct dm_stats_aux stats_aux; 104 /* last member of dm_target_io is 'struct bio' */ 105 struct dm_target_io tio; 106}; 107 108void *dm_per_bio_data(struct bio *bio, size_t data_size) 109{ 110 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 111 if (!tio->inside_dm_io) 112 return (char *)bio - offsetof(struct dm_target_io, clone) - data_size; 113 return (char *)bio - offsetof(struct dm_target_io, clone) - offsetof(struct dm_io, tio) - data_size; 114} 115EXPORT_SYMBOL_GPL(dm_per_bio_data); 116 117struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size) 118{ 119 struct dm_io *io = (struct dm_io *)((char *)data + data_size); 120 if (io->magic == DM_IO_MAGIC) 121 return (struct bio *)((char *)io + offsetof(struct dm_io, tio) + offsetof(struct dm_target_io, clone)); 122 BUG_ON(io->magic != DM_TIO_MAGIC); 123 return (struct bio *)((char *)io + offsetof(struct dm_target_io, clone)); 124} 125EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data); 126 127unsigned dm_bio_get_target_bio_nr(const struct bio *bio) 128{ 129 return container_of(bio, struct dm_target_io, clone)->target_bio_nr; 130} 131EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr); 132 133#define MINOR_ALLOCED ((void *)-1) 134 135/* 136 * Bits for the md->flags field. 137 */ 138#define DMF_BLOCK_IO_FOR_SUSPEND 0 139#define DMF_SUSPENDED 1 140#define DMF_FROZEN 2 141#define DMF_FREEING 3 142#define DMF_DELETING 4 143#define DMF_NOFLUSH_SUSPENDING 5 144#define DMF_DEFERRED_REMOVE 6 145#define DMF_SUSPENDED_INTERNALLY 7 146#define DMF_POST_SUSPENDING 8 147 148#define DM_NUMA_NODE NUMA_NO_NODE 149static int dm_numa_node = DM_NUMA_NODE; 150 151#define DEFAULT_SWAP_BIOS (8 * 1048576 / PAGE_SIZE) 152static int swap_bios = DEFAULT_SWAP_BIOS; 153static int get_swap_bios(void) 154{ 155 int latch = READ_ONCE(swap_bios); 156 if (unlikely(latch <= 0)) 157 latch = DEFAULT_SWAP_BIOS; 158 return latch; 159} 160 161/* 162 * For mempools pre-allocation at the table loading time. 163 */ 164struct dm_md_mempools { 165 struct bio_set bs; 166 struct bio_set io_bs; 167}; 168 169struct table_device { 170 struct list_head list; 171 refcount_t count; 172 struct dm_dev dm_dev; 173}; 174 175/* 176 * Bio-based DM's mempools' reserved IOs set by the user. 177 */ 178#define RESERVED_BIO_BASED_IOS 16 179static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS; 180 181static int __dm_get_module_param_int(int *module_param, int min, int max) 182{ 183 int param = READ_ONCE(*module_param); 184 int modified_param = 0; 185 bool modified = true; 186 187 if (param < min) 188 modified_param = min; 189 else if (param > max) 190 modified_param = max; 191 else 192 modified = false; 193 194 if (modified) { 195 (void)cmpxchg(module_param, param, modified_param); 196 param = modified_param; 197 } 198 199 return param; 200} 201 202unsigned __dm_get_module_param(unsigned *module_param, 203 unsigned def, unsigned max) 204{ 205 unsigned param = READ_ONCE(*module_param); 206 unsigned modified_param = 0; 207 208 if (!param) 209 modified_param = def; 210 else if (param > max) 211 modified_param = max; 212 213 if (modified_param) { 214 (void)cmpxchg(module_param, param, modified_param); 215 param = modified_param; 216 } 217 218 return param; 219} 220 221unsigned dm_get_reserved_bio_based_ios(void) 222{ 223 return __dm_get_module_param(&reserved_bio_based_ios, 224 RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS); 225} 226EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios); 227 228static unsigned dm_get_numa_node(void) 229{ 230 return __dm_get_module_param_int(&dm_numa_node, 231 DM_NUMA_NODE, num_online_nodes() - 1); 232} 233 234static int __init local_init(void) 235{ 236 int r; 237 238 r = dm_uevent_init(); 239 if (r) 240 return r; 241 242 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1); 243 if (!deferred_remove_workqueue) { 244 r = -ENOMEM; 245 goto out_uevent_exit; 246 } 247 248 _major = major; 249 r = register_blkdev(_major, _name); 250 if (r < 0) 251 goto out_free_workqueue; 252 253 if (!_major) 254 _major = r; 255 256 return 0; 257 258out_free_workqueue: 259 destroy_workqueue(deferred_remove_workqueue); 260out_uevent_exit: 261 dm_uevent_exit(); 262 263 return r; 264} 265 266static void local_exit(void) 267{ 268 destroy_workqueue(deferred_remove_workqueue); 269 270 unregister_blkdev(_major, _name); 271 dm_uevent_exit(); 272 273 _major = 0; 274 275 DMINFO("cleaned up"); 276} 277 278static int (*_inits[])(void) __initdata = { 279 local_init, 280 dm_target_init, 281 dm_linear_init, 282 dm_stripe_init, 283 dm_io_init, 284 dm_kcopyd_init, 285 dm_interface_init, 286 dm_statistics_init, 287}; 288 289static void (*_exits[])(void) = { 290 local_exit, 291 dm_target_exit, 292 dm_linear_exit, 293 dm_stripe_exit, 294 dm_io_exit, 295 dm_kcopyd_exit, 296 dm_interface_exit, 297 dm_statistics_exit, 298}; 299 300static int __init dm_init(void) 301{ 302 const int count = ARRAY_SIZE(_inits); 303 304 int r, i; 305 306 for (i = 0; i < count; i++) { 307 r = _inits[i](); 308 if (r) 309 goto bad; 310 } 311 312 return 0; 313 314 bad: 315 while (i--) 316 _exits[i](); 317 318 return r; 319} 320 321static void __exit dm_exit(void) 322{ 323 int i = ARRAY_SIZE(_exits); 324 325 while (i--) 326 _exits[i](); 327 328 /* 329 * Should be empty by this point. 330 */ 331 idr_destroy(&_minor_idr); 332} 333 334/* 335 * Block device functions 336 */ 337int dm_deleting_md(struct mapped_device *md) 338{ 339 return test_bit(DMF_DELETING, &md->flags); 340} 341 342static int dm_blk_open(struct block_device *bdev, fmode_t mode) 343{ 344 struct mapped_device *md; 345 346 spin_lock(&_minor_lock); 347 348 md = bdev->bd_disk->private_data; 349 if (!md) 350 goto out; 351 352 if (test_bit(DMF_FREEING, &md->flags) || 353 dm_deleting_md(md)) { 354 md = NULL; 355 goto out; 356 } 357 358 dm_get(md); 359 atomic_inc(&md->open_count); 360out: 361 spin_unlock(&_minor_lock); 362 363 return md ? 0 : -ENXIO; 364} 365 366static void dm_blk_close(struct gendisk *disk, fmode_t mode) 367{ 368 struct mapped_device *md; 369 370 spin_lock(&_minor_lock); 371 372 md = disk->private_data; 373 if (WARN_ON(!md)) 374 goto out; 375 376 if (atomic_dec_and_test(&md->open_count) && 377 (test_bit(DMF_DEFERRED_REMOVE, &md->flags))) 378 queue_work(deferred_remove_workqueue, &deferred_remove_work); 379 380 dm_put(md); 381out: 382 spin_unlock(&_minor_lock); 383} 384 385int dm_open_count(struct mapped_device *md) 386{ 387 return atomic_read(&md->open_count); 388} 389 390/* 391 * Guarantees nothing is using the device before it's deleted. 392 */ 393int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred) 394{ 395 int r = 0; 396 397 spin_lock(&_minor_lock); 398 399 if (dm_open_count(md)) { 400 r = -EBUSY; 401 if (mark_deferred) 402 set_bit(DMF_DEFERRED_REMOVE, &md->flags); 403 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags)) 404 r = -EEXIST; 405 else 406 set_bit(DMF_DELETING, &md->flags); 407 408 spin_unlock(&_minor_lock); 409 410 return r; 411} 412 413int dm_cancel_deferred_remove(struct mapped_device *md) 414{ 415 int r = 0; 416 417 spin_lock(&_minor_lock); 418 419 if (test_bit(DMF_DELETING, &md->flags)) 420 r = -EBUSY; 421 else 422 clear_bit(DMF_DEFERRED_REMOVE, &md->flags); 423 424 spin_unlock(&_minor_lock); 425 426 return r; 427} 428 429static void do_deferred_remove(struct work_struct *w) 430{ 431 dm_deferred_remove(); 432} 433 434static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo) 435{ 436 struct mapped_device *md = bdev->bd_disk->private_data; 437 438 return dm_get_geometry(md, geo); 439} 440 441#ifdef CONFIG_BLK_DEV_ZONED 442int dm_report_zones_cb(struct blk_zone *zone, unsigned int idx, void *data) 443{ 444 struct dm_report_zones_args *args = data; 445 sector_t sector_diff = args->tgt->begin - args->start; 446 447 /* 448 * Ignore zones beyond the target range. 449 */ 450 if (zone->start >= args->start + args->tgt->len) 451 return 0; 452 453 /* 454 * Remap the start sector and write pointer position of the zone 455 * to match its position in the target range. 456 */ 457 zone->start += sector_diff; 458 if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) { 459 if (zone->cond == BLK_ZONE_COND_FULL) 460 zone->wp = zone->start + zone->len; 461 else if (zone->cond == BLK_ZONE_COND_EMPTY) 462 zone->wp = zone->start; 463 else 464 zone->wp += sector_diff; 465 } 466 467 args->next_sector = zone->start + zone->len; 468 return args->orig_cb(zone, args->zone_idx++, args->orig_data); 469} 470EXPORT_SYMBOL_GPL(dm_report_zones_cb); 471 472static int dm_blk_report_zones(struct gendisk *disk, sector_t sector, 473 unsigned int nr_zones, report_zones_cb cb, void *data) 474{ 475 struct mapped_device *md = disk->private_data; 476 struct dm_table *map; 477 int srcu_idx, ret; 478 struct dm_report_zones_args args = { 479 .next_sector = sector, 480 .orig_data = data, 481 .orig_cb = cb, 482 }; 483 484 if (dm_suspended_md(md)) 485 return -EAGAIN; 486 487 map = dm_get_live_table(md, &srcu_idx); 488 if (!map) { 489 ret = -EIO; 490 goto out; 491 } 492 493 do { 494 struct dm_target *tgt; 495 496 tgt = dm_table_find_target(map, args.next_sector); 497 if (WARN_ON_ONCE(!tgt->type->report_zones)) { 498 ret = -EIO; 499 goto out; 500 } 501 502 args.tgt = tgt; 503 ret = tgt->type->report_zones(tgt, &args, 504 nr_zones - args.zone_idx); 505 if (ret < 0) 506 goto out; 507 } while (args.zone_idx < nr_zones && 508 args.next_sector < get_capacity(disk)); 509 510 ret = args.zone_idx; 511out: 512 dm_put_live_table(md, srcu_idx); 513 return ret; 514} 515#else 516#define dm_blk_report_zones NULL 517#endif /* CONFIG_BLK_DEV_ZONED */ 518 519static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx, 520 struct block_device **bdev) 521{ 522 struct dm_target *tgt; 523 struct dm_table *map; 524 int r; 525 526retry: 527 r = -ENOTTY; 528 map = dm_get_live_table(md, srcu_idx); 529 if (!map || !dm_table_get_size(map)) 530 return r; 531 532 /* We only support devices that have a single target */ 533 if (dm_table_get_num_targets(map) != 1) 534 return r; 535 536 tgt = dm_table_get_target(map, 0); 537 if (!tgt->type->prepare_ioctl) 538 return r; 539 540 if (dm_suspended_md(md)) 541 return -EAGAIN; 542 543 r = tgt->type->prepare_ioctl(tgt, bdev); 544 if (r == -ENOTCONN && !fatal_signal_pending(current)) { 545 dm_put_live_table(md, *srcu_idx); 546 msleep(10); 547 goto retry; 548 } 549 550 return r; 551} 552 553static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx) 554{ 555 dm_put_live_table(md, srcu_idx); 556} 557 558static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode, 559 unsigned int cmd, unsigned long arg) 560{ 561 struct mapped_device *md = bdev->bd_disk->private_data; 562 int r, srcu_idx; 563 564 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 565 if (r < 0) 566 goto out; 567 568 if (r > 0) { 569 /* 570 * Target determined this ioctl is being issued against a 571 * subset of the parent bdev; require extra privileges. 572 */ 573 if (!capable(CAP_SYS_RAWIO)) { 574 DMDEBUG_LIMIT( 575 "%s: sending ioctl %x to DM device without required privilege.", 576 current->comm, cmd); 577 r = -ENOIOCTLCMD; 578 goto out; 579 } 580 } 581 582 r = __blkdev_driver_ioctl(bdev, mode, cmd, arg); 583out: 584 dm_unprepare_ioctl(md, srcu_idx); 585 return r; 586} 587 588u64 dm_start_time_ns_from_clone(struct bio *bio) 589{ 590 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 591 struct dm_io *io = tio->io; 592 593 return jiffies_to_nsecs(io->start_time); 594} 595EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone); 596 597static void start_io_acct(struct dm_io *io) 598{ 599 struct mapped_device *md = io->md; 600 struct bio *bio = io->orig_bio; 601 602 io->start_time = bio_start_io_acct(bio); 603 if (unlikely(dm_stats_used(&md->stats))) 604 dm_stats_account_io(&md->stats, bio_data_dir(bio), 605 bio->bi_iter.bi_sector, bio_sectors(bio), 606 false, 0, &io->stats_aux); 607} 608 609static void end_io_acct(struct mapped_device *md, struct bio *bio, 610 unsigned long start_time, struct dm_stats_aux *stats_aux) 611{ 612 unsigned long duration = jiffies - start_time; 613 614 if (unlikely(dm_stats_used(&md->stats))) 615 dm_stats_account_io(&md->stats, bio_data_dir(bio), 616 bio->bi_iter.bi_sector, bio_sectors(bio), 617 true, duration, stats_aux); 618 619 smp_wmb(); 620 621 bio_end_io_acct(bio, start_time); 622 623 /* nudge anyone waiting on suspend queue */ 624 if (unlikely(wq_has_sleeper(&md->wait))) 625 wake_up(&md->wait); 626} 627 628static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio) 629{ 630 struct dm_io *io; 631 struct dm_target_io *tio; 632 struct bio *clone; 633 634 clone = bio_alloc_bioset(GFP_NOIO, 0, &md->io_bs); 635 if (!clone) 636 return NULL; 637 638 tio = container_of(clone, struct dm_target_io, clone); 639 tio->inside_dm_io = true; 640 tio->io = NULL; 641 642 io = container_of(tio, struct dm_io, tio); 643 io->magic = DM_IO_MAGIC; 644 io->status = 0; 645 atomic_set(&io->io_count, 1); 646 io->orig_bio = bio; 647 io->md = md; 648 spin_lock_init(&io->endio_lock); 649 650 start_io_acct(io); 651 652 return io; 653} 654 655static void free_io(struct mapped_device *md, struct dm_io *io) 656{ 657 bio_put(&io->tio.clone); 658} 659 660static struct dm_target_io *alloc_tio(struct clone_info *ci, struct dm_target *ti, 661 unsigned target_bio_nr, gfp_t gfp_mask) 662{ 663 struct dm_target_io *tio; 664 665 if (!ci->io->tio.io) { 666 /* the dm_target_io embedded in ci->io is available */ 667 tio = &ci->io->tio; 668 } else { 669 struct bio *clone = bio_alloc_bioset(gfp_mask, 0, &ci->io->md->bs); 670 if (!clone) 671 return NULL; 672 673 tio = container_of(clone, struct dm_target_io, clone); 674 tio->inside_dm_io = false; 675 } 676 677 tio->magic = DM_TIO_MAGIC; 678 tio->io = ci->io; 679 tio->ti = ti; 680 tio->target_bio_nr = target_bio_nr; 681 682 return tio; 683} 684 685static void free_tio(struct dm_target_io *tio) 686{ 687 if (tio->inside_dm_io) 688 return; 689 bio_put(&tio->clone); 690} 691 692/* 693 * Add the bio to the list of deferred io. 694 */ 695static void queue_io(struct mapped_device *md, struct bio *bio) 696{ 697 unsigned long flags; 698 699 spin_lock_irqsave(&md->deferred_lock, flags); 700 bio_list_add(&md->deferred, bio); 701 spin_unlock_irqrestore(&md->deferred_lock, flags); 702 queue_work(md->wq, &md->work); 703} 704 705/* 706 * Everyone (including functions in this file), should use this 707 * function to access the md->map field, and make sure they call 708 * dm_put_live_table() when finished. 709 */ 710struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier) 711{ 712 *srcu_idx = srcu_read_lock(&md->io_barrier); 713 714 return srcu_dereference(md->map, &md->io_barrier); 715} 716 717void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier) 718{ 719 srcu_read_unlock(&md->io_barrier, srcu_idx); 720} 721 722void dm_sync_table(struct mapped_device *md) 723{ 724 synchronize_srcu(&md->io_barrier); 725 synchronize_rcu_expedited(); 726} 727 728/* 729 * A fast alternative to dm_get_live_table/dm_put_live_table. 730 * The caller must not block between these two functions. 731 */ 732static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU) 733{ 734 rcu_read_lock(); 735 return rcu_dereference(md->map); 736} 737 738static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU) 739{ 740 rcu_read_unlock(); 741} 742 743static char *_dm_claim_ptr = "I belong to device-mapper"; 744 745/* 746 * Open a table device so we can use it as a map destination. 747 */ 748static int open_table_device(struct table_device *td, dev_t dev, 749 struct mapped_device *md) 750{ 751 struct block_device *bdev; 752 753 int r; 754 755 BUG_ON(td->dm_dev.bdev); 756 757 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr); 758 if (IS_ERR(bdev)) 759 return PTR_ERR(bdev); 760 761 r = bd_link_disk_holder(bdev, dm_disk(md)); 762 if (r) { 763 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL); 764 return r; 765 } 766 767 td->dm_dev.bdev = bdev; 768 td->dm_dev.dax_dev = dax_get_by_host(bdev->bd_disk->disk_name); 769 return 0; 770} 771 772/* 773 * Close a table device that we've been using. 774 */ 775static void close_table_device(struct table_device *td, struct mapped_device *md) 776{ 777 if (!td->dm_dev.bdev) 778 return; 779 780 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md)); 781 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL); 782 put_dax(td->dm_dev.dax_dev); 783 td->dm_dev.bdev = NULL; 784 td->dm_dev.dax_dev = NULL; 785} 786 787static struct table_device *find_table_device(struct list_head *l, dev_t dev, 788 fmode_t mode) 789{ 790 struct table_device *td; 791 792 list_for_each_entry(td, l, list) 793 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode) 794 return td; 795 796 return NULL; 797} 798 799int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode, 800 struct dm_dev **result) 801{ 802 int r; 803 struct table_device *td; 804 805 mutex_lock(&md->table_devices_lock); 806 td = find_table_device(&md->table_devices, dev, mode); 807 if (!td) { 808 td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id); 809 if (!td) { 810 mutex_unlock(&md->table_devices_lock); 811 return -ENOMEM; 812 } 813 814 td->dm_dev.mode = mode; 815 td->dm_dev.bdev = NULL; 816 817 if ((r = open_table_device(td, dev, md))) { 818 mutex_unlock(&md->table_devices_lock); 819 kfree(td); 820 return r; 821 } 822 823 format_dev_t(td->dm_dev.name, dev); 824 825 refcount_set(&td->count, 1); 826 list_add(&td->list, &md->table_devices); 827 } else { 828 refcount_inc(&td->count); 829 } 830 mutex_unlock(&md->table_devices_lock); 831 832 *result = &td->dm_dev; 833 return 0; 834} 835EXPORT_SYMBOL_GPL(dm_get_table_device); 836 837void dm_put_table_device(struct mapped_device *md, struct dm_dev *d) 838{ 839 struct table_device *td = container_of(d, struct table_device, dm_dev); 840 841 mutex_lock(&md->table_devices_lock); 842 if (refcount_dec_and_test(&td->count)) { 843 close_table_device(td, md); 844 list_del(&td->list); 845 kfree(td); 846 } 847 mutex_unlock(&md->table_devices_lock); 848} 849EXPORT_SYMBOL(dm_put_table_device); 850 851static void free_table_devices(struct list_head *devices) 852{ 853 struct list_head *tmp, *next; 854 855 list_for_each_safe(tmp, next, devices) { 856 struct table_device *td = list_entry(tmp, struct table_device, list); 857 858 DMWARN("dm_destroy: %s still exists with %d references", 859 td->dm_dev.name, refcount_read(&td->count)); 860 kfree(td); 861 } 862} 863 864/* 865 * Get the geometry associated with a dm device 866 */ 867int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo) 868{ 869 *geo = md->geometry; 870 871 return 0; 872} 873 874/* 875 * Set the geometry of a device. 876 */ 877int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo) 878{ 879 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors; 880 881 if (geo->start > sz) { 882 DMWARN("Start sector is beyond the geometry limits."); 883 return -EINVAL; 884 } 885 886 md->geometry = *geo; 887 888 return 0; 889} 890 891static int __noflush_suspending(struct mapped_device *md) 892{ 893 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 894} 895 896/* 897 * Decrements the number of outstanding ios that a bio has been 898 * cloned into, completing the original io if necc. 899 */ 900static void dec_pending(struct dm_io *io, blk_status_t error) 901{ 902 unsigned long flags; 903 blk_status_t io_error; 904 struct bio *bio; 905 struct mapped_device *md = io->md; 906 unsigned long start_time = 0; 907 struct dm_stats_aux stats_aux; 908 909 /* Push-back supersedes any I/O errors */ 910 if (unlikely(error)) { 911 spin_lock_irqsave(&io->endio_lock, flags); 912 if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md))) 913 io->status = error; 914 spin_unlock_irqrestore(&io->endio_lock, flags); 915 } 916 917 if (atomic_dec_and_test(&io->io_count)) { 918 if (io->status == BLK_STS_DM_REQUEUE) { 919 /* 920 * Target requested pushing back the I/O. 921 */ 922 spin_lock_irqsave(&md->deferred_lock, flags); 923 if (__noflush_suspending(md)) 924 /* NOTE early return due to BLK_STS_DM_REQUEUE below */ 925 bio_list_add_head(&md->deferred, io->orig_bio); 926 else 927 /* noflush suspend was interrupted. */ 928 io->status = BLK_STS_IOERR; 929 spin_unlock_irqrestore(&md->deferred_lock, flags); 930 } 931 932 io_error = io->status; 933 bio = io->orig_bio; 934 start_time = io->start_time; 935 stats_aux = io->stats_aux; 936 free_io(md, io); 937 end_io_acct(md, bio, start_time, &stats_aux); 938 939 if (io_error == BLK_STS_DM_REQUEUE) 940 return; 941 942 if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) { 943 /* 944 * Preflush done for flush with data, reissue 945 * without REQ_PREFLUSH. 946 */ 947 bio->bi_opf &= ~REQ_PREFLUSH; 948 queue_io(md, bio); 949 } else { 950 /* done with normal IO or empty flush */ 951 if (io_error) 952 bio->bi_status = io_error; 953 bio_endio(bio); 954 } 955 } 956} 957 958void disable_discard(struct mapped_device *md) 959{ 960 struct queue_limits *limits = dm_get_queue_limits(md); 961 962 /* device doesn't really support DISCARD, disable it */ 963 limits->max_discard_sectors = 0; 964 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, md->queue); 965} 966 967void disable_write_same(struct mapped_device *md) 968{ 969 struct queue_limits *limits = dm_get_queue_limits(md); 970 971 /* device doesn't really support WRITE SAME, disable it */ 972 limits->max_write_same_sectors = 0; 973} 974 975void disable_write_zeroes(struct mapped_device *md) 976{ 977 struct queue_limits *limits = dm_get_queue_limits(md); 978 979 /* device doesn't really support WRITE ZEROES, disable it */ 980 limits->max_write_zeroes_sectors = 0; 981} 982 983static bool swap_bios_limit(struct dm_target *ti, struct bio *bio) 984{ 985 return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios); 986} 987 988static void clone_endio(struct bio *bio) 989{ 990 blk_status_t error = bio->bi_status; 991 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 992 struct dm_io *io = tio->io; 993 struct mapped_device *md = tio->io->md; 994 dm_endio_fn endio = tio->ti->type->end_io; 995 struct bio *orig_bio = io->orig_bio; 996 997 if (unlikely(error == BLK_STS_TARGET)) { 998 if (bio_op(bio) == REQ_OP_DISCARD && 999 !bio->bi_disk->queue->limits.max_discard_sectors) 1000 disable_discard(md); 1001 else if (bio_op(bio) == REQ_OP_WRITE_SAME && 1002 !bio->bi_disk->queue->limits.max_write_same_sectors) 1003 disable_write_same(md); 1004 else if (bio_op(bio) == REQ_OP_WRITE_ZEROES && 1005 !bio->bi_disk->queue->limits.max_write_zeroes_sectors) 1006 disable_write_zeroes(md); 1007 } 1008 1009 /* 1010 * For zone-append bios get offset in zone of the written 1011 * sector and add that to the original bio sector pos. 1012 */ 1013 if (bio_op(orig_bio) == REQ_OP_ZONE_APPEND) { 1014 sector_t written_sector = bio->bi_iter.bi_sector; 1015 struct request_queue *q = orig_bio->bi_disk->queue; 1016 u64 mask = (u64)blk_queue_zone_sectors(q) - 1; 1017 1018 orig_bio->bi_iter.bi_sector += written_sector & mask; 1019 } 1020 1021 if (endio) { 1022 int r = endio(tio->ti, bio, &error); 1023 switch (r) { 1024 case DM_ENDIO_REQUEUE: 1025 error = BLK_STS_DM_REQUEUE; 1026 fallthrough; 1027 case DM_ENDIO_DONE: 1028 break; 1029 case DM_ENDIO_INCOMPLETE: 1030 /* The target will handle the io */ 1031 return; 1032 default: 1033 DMWARN("unimplemented target endio return value: %d", r); 1034 BUG(); 1035 } 1036 } 1037 1038 if (unlikely(swap_bios_limit(tio->ti, bio))) { 1039 struct mapped_device *md = io->md; 1040 up(&md->swap_bios_semaphore); 1041 } 1042 1043 free_tio(tio); 1044 dec_pending(io, error); 1045} 1046 1047/* 1048 * Return maximum size of I/O possible at the supplied sector up to the current 1049 * target boundary. 1050 */ 1051static inline sector_t max_io_len_target_boundary(struct dm_target *ti, 1052 sector_t target_offset) 1053{ 1054 return ti->len - target_offset; 1055} 1056 1057static sector_t max_io_len(struct dm_target *ti, sector_t sector) 1058{ 1059 sector_t target_offset = dm_target_offset(ti, sector); 1060 sector_t len = max_io_len_target_boundary(ti, target_offset); 1061 sector_t max_len; 1062 1063 /* 1064 * Does the target need to split IO even further? 1065 * - varied (per target) IO splitting is a tenet of DM; this 1066 * explains why stacked chunk_sectors based splitting via 1067 * blk_max_size_offset() isn't possible here. So pass in 1068 * ti->max_io_len to override stacked chunk_sectors. 1069 */ 1070 if (ti->max_io_len) { 1071 max_len = blk_max_size_offset(ti->table->md->queue, 1072 target_offset, ti->max_io_len); 1073 if (len > max_len) 1074 len = max_len; 1075 } 1076 1077 return len; 1078} 1079 1080int dm_set_target_max_io_len(struct dm_target *ti, sector_t len) 1081{ 1082 if (len > UINT_MAX) { 1083 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)", 1084 (unsigned long long)len, UINT_MAX); 1085 ti->error = "Maximum size of target IO is too large"; 1086 return -EINVAL; 1087 } 1088 1089 ti->max_io_len = (uint32_t) len; 1090 1091 return 0; 1092} 1093EXPORT_SYMBOL_GPL(dm_set_target_max_io_len); 1094 1095static struct dm_target *dm_dax_get_live_target(struct mapped_device *md, 1096 sector_t sector, int *srcu_idx) 1097 __acquires(md->io_barrier) 1098{ 1099 struct dm_table *map; 1100 struct dm_target *ti; 1101 1102 map = dm_get_live_table(md, srcu_idx); 1103 if (!map) 1104 return NULL; 1105 1106 ti = dm_table_find_target(map, sector); 1107 if (!ti) 1108 return NULL; 1109 1110 return ti; 1111} 1112 1113static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, 1114 long nr_pages, void **kaddr, pfn_t *pfn) 1115{ 1116 struct mapped_device *md = dax_get_private(dax_dev); 1117 sector_t sector = pgoff * PAGE_SECTORS; 1118 struct dm_target *ti; 1119 long len, ret = -EIO; 1120 int srcu_idx; 1121 1122 ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1123 1124 if (!ti) 1125 goto out; 1126 if (!ti->type->direct_access) 1127 goto out; 1128 len = max_io_len(ti, sector) / PAGE_SECTORS; 1129 if (len < 1) 1130 goto out; 1131 nr_pages = min(len, nr_pages); 1132 ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn); 1133 1134 out: 1135 dm_put_live_table(md, srcu_idx); 1136 1137 return ret; 1138} 1139 1140static bool dm_dax_supported(struct dax_device *dax_dev, struct block_device *bdev, 1141 int blocksize, sector_t start, sector_t len) 1142{ 1143 struct mapped_device *md = dax_get_private(dax_dev); 1144 struct dm_table *map; 1145 bool ret = false; 1146 int srcu_idx; 1147 1148 map = dm_get_live_table(md, &srcu_idx); 1149 if (!map) 1150 goto out; 1151 1152 ret = dm_table_supports_dax(map, device_not_dax_capable, &blocksize); 1153 1154out: 1155 dm_put_live_table(md, srcu_idx); 1156 1157 return ret; 1158} 1159 1160static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, 1161 void *addr, size_t bytes, struct iov_iter *i) 1162{ 1163 struct mapped_device *md = dax_get_private(dax_dev); 1164 sector_t sector = pgoff * PAGE_SECTORS; 1165 struct dm_target *ti; 1166 long ret = 0; 1167 int srcu_idx; 1168 1169 ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1170 1171 if (!ti) 1172 goto out; 1173 if (!ti->type->dax_copy_from_iter) { 1174 ret = copy_from_iter(addr, bytes, i); 1175 goto out; 1176 } 1177 ret = ti->type->dax_copy_from_iter(ti, pgoff, addr, bytes, i); 1178 out: 1179 dm_put_live_table(md, srcu_idx); 1180 1181 return ret; 1182} 1183 1184static size_t dm_dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, 1185 void *addr, size_t bytes, struct iov_iter *i) 1186{ 1187 struct mapped_device *md = dax_get_private(dax_dev); 1188 sector_t sector = pgoff * PAGE_SECTORS; 1189 struct dm_target *ti; 1190 long ret = 0; 1191 int srcu_idx; 1192 1193 ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1194 1195 if (!ti) 1196 goto out; 1197 if (!ti->type->dax_copy_to_iter) { 1198 ret = copy_to_iter(addr, bytes, i); 1199 goto out; 1200 } 1201 ret = ti->type->dax_copy_to_iter(ti, pgoff, addr, bytes, i); 1202 out: 1203 dm_put_live_table(md, srcu_idx); 1204 1205 return ret; 1206} 1207 1208static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, 1209 size_t nr_pages) 1210{ 1211 struct mapped_device *md = dax_get_private(dax_dev); 1212 sector_t sector = pgoff * PAGE_SECTORS; 1213 struct dm_target *ti; 1214 int ret = -EIO; 1215 int srcu_idx; 1216 1217 ti = dm_dax_get_live_target(md, sector, &srcu_idx); 1218 1219 if (!ti) 1220 goto out; 1221 if (WARN_ON(!ti->type->dax_zero_page_range)) { 1222 /* 1223 * ->zero_page_range() is mandatory dax operation. If we are 1224 * here, something is wrong. 1225 */ 1226 goto out; 1227 } 1228 ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages); 1229 out: 1230 dm_put_live_table(md, srcu_idx); 1231 1232 return ret; 1233} 1234 1235/* 1236 * A target may call dm_accept_partial_bio only from the map routine. It is 1237 * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_* zone management 1238 * operations and REQ_OP_ZONE_APPEND (zone append writes). 1239 * 1240 * dm_accept_partial_bio informs the dm that the target only wants to process 1241 * additional n_sectors sectors of the bio and the rest of the data should be 1242 * sent in a next bio. 1243 * 1244 * A diagram that explains the arithmetics: 1245 * +--------------------+---------------+-------+ 1246 * | 1 | 2 | 3 | 1247 * +--------------------+---------------+-------+ 1248 * 1249 * <-------------- *tio->len_ptr ---------------> 1250 * <------- bi_size -------> 1251 * <-- n_sectors --> 1252 * 1253 * Region 1 was already iterated over with bio_advance or similar function. 1254 * (it may be empty if the target doesn't use bio_advance) 1255 * Region 2 is the remaining bio size that the target wants to process. 1256 * (it may be empty if region 1 is non-empty, although there is no reason 1257 * to make it empty) 1258 * The target requires that region 3 is to be sent in the next bio. 1259 * 1260 * If the target wants to receive multiple copies of the bio (via num_*bios, etc), 1261 * the partially processed part (the sum of regions 1+2) must be the same for all 1262 * copies of the bio. 1263 */ 1264void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors) 1265{ 1266 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone); 1267 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT; 1268 1269 BUG_ON(bio->bi_opf & REQ_PREFLUSH); 1270 BUG_ON(op_is_zone_mgmt(bio_op(bio))); 1271 BUG_ON(bio_op(bio) == REQ_OP_ZONE_APPEND); 1272 BUG_ON(bi_size > *tio->len_ptr); 1273 BUG_ON(n_sectors > bi_size); 1274 1275 *tio->len_ptr -= bi_size - n_sectors; 1276 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT; 1277} 1278EXPORT_SYMBOL_GPL(dm_accept_partial_bio); 1279 1280static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch) 1281{ 1282 mutex_lock(&md->swap_bios_lock); 1283 while (latch < md->swap_bios) { 1284 cond_resched(); 1285 down(&md->swap_bios_semaphore); 1286 md->swap_bios--; 1287 } 1288 while (latch > md->swap_bios) { 1289 cond_resched(); 1290 up(&md->swap_bios_semaphore); 1291 md->swap_bios++; 1292 } 1293 mutex_unlock(&md->swap_bios_lock); 1294} 1295 1296static blk_qc_t __map_bio(struct dm_target_io *tio) 1297{ 1298 int r; 1299 sector_t sector; 1300 struct bio *clone = &tio->clone; 1301 struct dm_io *io = tio->io; 1302 struct dm_target *ti = tio->ti; 1303 blk_qc_t ret = BLK_QC_T_NONE; 1304 1305 clone->bi_end_io = clone_endio; 1306 1307 /* 1308 * Map the clone. If r == 0 we don't need to do 1309 * anything, the target has assumed ownership of 1310 * this io. 1311 */ 1312 atomic_inc(&io->io_count); 1313 sector = clone->bi_iter.bi_sector; 1314 1315 if (unlikely(swap_bios_limit(ti, clone))) { 1316 struct mapped_device *md = io->md; 1317 int latch = get_swap_bios(); 1318 if (unlikely(latch != md->swap_bios)) 1319 __set_swap_bios_limit(md, latch); 1320 down(&md->swap_bios_semaphore); 1321 } 1322 1323 r = ti->type->map(ti, clone); 1324 switch (r) { 1325 case DM_MAPIO_SUBMITTED: 1326 break; 1327 case DM_MAPIO_REMAPPED: 1328 /* the bio has been remapped so dispatch it */ 1329 trace_block_bio_remap(clone->bi_disk->queue, clone, 1330 bio_dev(io->orig_bio), sector); 1331 ret = submit_bio_noacct(clone); 1332 break; 1333 case DM_MAPIO_KILL: 1334 if (unlikely(swap_bios_limit(ti, clone))) { 1335 struct mapped_device *md = io->md; 1336 up(&md->swap_bios_semaphore); 1337 } 1338 free_tio(tio); 1339 dec_pending(io, BLK_STS_IOERR); 1340 break; 1341 case DM_MAPIO_REQUEUE: 1342 if (unlikely(swap_bios_limit(ti, clone))) { 1343 struct mapped_device *md = io->md; 1344 up(&md->swap_bios_semaphore); 1345 } 1346 free_tio(tio); 1347 dec_pending(io, BLK_STS_DM_REQUEUE); 1348 break; 1349 default: 1350 DMWARN("unimplemented target map return value: %d", r); 1351 BUG(); 1352 } 1353 1354 return ret; 1355} 1356 1357static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len) 1358{ 1359 bio->bi_iter.bi_sector = sector; 1360 bio->bi_iter.bi_size = to_bytes(len); 1361} 1362 1363/* 1364 * Creates a bio that consists of range of complete bvecs. 1365 */ 1366static int clone_bio(struct dm_target_io *tio, struct bio *bio, 1367 sector_t sector, unsigned len) 1368{ 1369 struct bio *clone = &tio->clone; 1370 int r; 1371 1372 __bio_clone_fast(clone, bio); 1373 1374 r = bio_crypt_clone(clone, bio, GFP_NOIO); 1375 if (r < 0) 1376 return r; 1377 1378 if (bio_integrity(bio)) { 1379 if (unlikely(!dm_target_has_integrity(tio->ti->type) && 1380 !dm_target_passes_integrity(tio->ti->type))) { 1381 DMWARN("%s: the target %s doesn't support integrity data.", 1382 dm_device_name(tio->io->md), 1383 tio->ti->type->name); 1384 return -EIO; 1385 } 1386 1387 r = bio_integrity_clone(clone, bio, GFP_NOIO); 1388 if (r < 0) 1389 return r; 1390 } 1391 1392 bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector)); 1393 clone->bi_iter.bi_size = to_bytes(len); 1394 1395 if (bio_integrity(bio)) 1396 bio_integrity_trim(clone); 1397 1398 return 0; 1399} 1400 1401static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci, 1402 struct dm_target *ti, unsigned num_bios) 1403{ 1404 struct dm_target_io *tio; 1405 int try; 1406 1407 if (!num_bios) 1408 return; 1409 1410 if (num_bios == 1) { 1411 tio = alloc_tio(ci, ti, 0, GFP_NOIO); 1412 bio_list_add(blist, &tio->clone); 1413 return; 1414 } 1415 1416 for (try = 0; try < 2; try++) { 1417 int bio_nr; 1418 struct bio *bio; 1419 1420 if (try) 1421 mutex_lock(&ci->io->md->table_devices_lock); 1422 for (bio_nr = 0; bio_nr < num_bios; bio_nr++) { 1423 tio = alloc_tio(ci, ti, bio_nr, try ? GFP_NOIO : GFP_NOWAIT); 1424 if (!tio) 1425 break; 1426 1427 bio_list_add(blist, &tio->clone); 1428 } 1429 if (try) 1430 mutex_unlock(&ci->io->md->table_devices_lock); 1431 if (bio_nr == num_bios) 1432 return; 1433 1434 while ((bio = bio_list_pop(blist))) { 1435 tio = container_of(bio, struct dm_target_io, clone); 1436 free_tio(tio); 1437 } 1438 } 1439} 1440 1441static blk_qc_t __clone_and_map_simple_bio(struct clone_info *ci, 1442 struct dm_target_io *tio, unsigned *len) 1443{ 1444 struct bio *clone = &tio->clone; 1445 1446 tio->len_ptr = len; 1447 1448 __bio_clone_fast(clone, ci->bio); 1449 if (len) 1450 bio_setup_sector(clone, ci->sector, *len); 1451 1452 return __map_bio(tio); 1453} 1454 1455static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti, 1456 unsigned num_bios, unsigned *len) 1457{ 1458 struct bio_list blist = BIO_EMPTY_LIST; 1459 struct bio *bio; 1460 struct dm_target_io *tio; 1461 1462 alloc_multiple_bios(&blist, ci, ti, num_bios); 1463 1464 while ((bio = bio_list_pop(&blist))) { 1465 tio = container_of(bio, struct dm_target_io, clone); 1466 (void) __clone_and_map_simple_bio(ci, tio, len); 1467 } 1468} 1469 1470static int __send_empty_flush(struct clone_info *ci) 1471{ 1472 unsigned target_nr = 0; 1473 struct dm_target *ti; 1474 struct bio flush_bio; 1475 1476 /* 1477 * Use an on-stack bio for this, it's safe since we don't 1478 * need to reference it after submit. It's just used as 1479 * the basis for the clone(s). 1480 */ 1481 bio_init(&flush_bio, NULL, 0); 1482 flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC; 1483 ci->bio = &flush_bio; 1484 ci->sector_count = 0; 1485 1486 /* 1487 * Empty flush uses a statically initialized bio, as the base for 1488 * cloning. However, blkg association requires that a bdev is 1489 * associated with a gendisk, which doesn't happen until the bdev is 1490 * opened. So, blkg association is done at issue time of the flush 1491 * rather than when the device is created in alloc_dev(). 1492 */ 1493 bio_set_dev(ci->bio, ci->io->md->bdev); 1494 1495 BUG_ON(bio_has_data(ci->bio)); 1496 while ((ti = dm_table_get_target(ci->map, target_nr++))) 1497 __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL); 1498 1499 bio_uninit(ci->bio); 1500 return 0; 1501} 1502 1503static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti, 1504 sector_t sector, unsigned *len) 1505{ 1506 struct bio *bio = ci->bio; 1507 struct dm_target_io *tio; 1508 int r; 1509 1510 tio = alloc_tio(ci, ti, 0, GFP_NOIO); 1511 tio->len_ptr = len; 1512 r = clone_bio(tio, bio, sector, *len); 1513 if (r < 0) { 1514 free_tio(tio); 1515 return r; 1516 } 1517 (void) __map_bio(tio); 1518 1519 return 0; 1520} 1521 1522static int __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti, 1523 unsigned num_bios) 1524{ 1525 unsigned len; 1526 1527 /* 1528 * Even though the device advertised support for this type of 1529 * request, that does not mean every target supports it, and 1530 * reconfiguration might also have changed that since the 1531 * check was performed. 1532 */ 1533 if (!num_bios) 1534 return -EOPNOTSUPP; 1535 1536 len = min_t(sector_t, ci->sector_count, 1537 max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector))); 1538 1539 __send_duplicate_bios(ci, ti, num_bios, &len); 1540 1541 ci->sector += len; 1542 ci->sector_count -= len; 1543 1544 return 0; 1545} 1546 1547static bool is_abnormal_io(struct bio *bio) 1548{ 1549 bool r = false; 1550 1551 switch (bio_op(bio)) { 1552 case REQ_OP_DISCARD: 1553 case REQ_OP_SECURE_ERASE: 1554 case REQ_OP_WRITE_SAME: 1555 case REQ_OP_WRITE_ZEROES: 1556 r = true; 1557 break; 1558 } 1559 1560 return r; 1561} 1562 1563static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti, 1564 int *result) 1565{ 1566 struct bio *bio = ci->bio; 1567 unsigned num_bios = 0; 1568 1569 switch (bio_op(bio)) { 1570 case REQ_OP_DISCARD: 1571 num_bios = ti->num_discard_bios; 1572 break; 1573 case REQ_OP_SECURE_ERASE: 1574 num_bios = ti->num_secure_erase_bios; 1575 break; 1576 case REQ_OP_WRITE_SAME: 1577 num_bios = ti->num_write_same_bios; 1578 break; 1579 case REQ_OP_WRITE_ZEROES: 1580 num_bios = ti->num_write_zeroes_bios; 1581 break; 1582 default: 1583 return false; 1584 } 1585 1586 *result = __send_changing_extent_only(ci, ti, num_bios); 1587 return true; 1588} 1589 1590/* 1591 * Select the correct strategy for processing a non-flush bio. 1592 */ 1593static int __split_and_process_non_flush(struct clone_info *ci) 1594{ 1595 struct dm_target *ti; 1596 unsigned len; 1597 int r; 1598 1599 ti = dm_table_find_target(ci->map, ci->sector); 1600 if (!ti) 1601 return -EIO; 1602 1603 if (__process_abnormal_io(ci, ti, &r)) 1604 return r; 1605 1606 len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count); 1607 1608 r = __clone_and_map_data_bio(ci, ti, ci->sector, &len); 1609 if (r < 0) 1610 return r; 1611 1612 ci->sector += len; 1613 ci->sector_count -= len; 1614 1615 return 0; 1616} 1617 1618static void init_clone_info(struct clone_info *ci, struct mapped_device *md, 1619 struct dm_table *map, struct bio *bio) 1620{ 1621 ci->map = map; 1622 ci->io = alloc_io(md, bio); 1623 ci->sector = bio->bi_iter.bi_sector; 1624} 1625 1626#define __dm_part_stat_sub(part, field, subnd) \ 1627 (part_stat_get(part, field) -= (subnd)) 1628 1629/* 1630 * Entry point to split a bio into clones and submit them to the targets. 1631 */ 1632static blk_qc_t __split_and_process_bio(struct mapped_device *md, 1633 struct dm_table *map, struct bio *bio) 1634{ 1635 struct clone_info ci; 1636 blk_qc_t ret = BLK_QC_T_NONE; 1637 int error = 0; 1638 1639 init_clone_info(&ci, md, map, bio); 1640 1641 if (bio->bi_opf & REQ_PREFLUSH) { 1642 error = __send_empty_flush(&ci); 1643 /* dec_pending submits any data associated with flush */ 1644 } else if (op_is_zone_mgmt(bio_op(bio))) { 1645 ci.bio = bio; 1646 ci.sector_count = 0; 1647 error = __split_and_process_non_flush(&ci); 1648 } else { 1649 ci.bio = bio; 1650 ci.sector_count = bio_sectors(bio); 1651 while (ci.sector_count && !error) { 1652 error = __split_and_process_non_flush(&ci); 1653 if (current->bio_list && ci.sector_count && !error) { 1654 /* 1655 * Remainder must be passed to submit_bio_noacct() 1656 * so that it gets handled *after* bios already submitted 1657 * have been completely processed. 1658 * We take a clone of the original to store in 1659 * ci.io->orig_bio to be used by end_io_acct() and 1660 * for dec_pending to use for completion handling. 1661 */ 1662 struct bio *b = bio_split(bio, bio_sectors(bio) - ci.sector_count, 1663 GFP_NOIO, &md->queue->bio_split); 1664 ci.io->orig_bio = b; 1665 1666 /* 1667 * Adjust IO stats for each split, otherwise upon queue 1668 * reentry there will be redundant IO accounting. 1669 * NOTE: this is a stop-gap fix, a proper fix involves 1670 * significant refactoring of DM core's bio splitting 1671 * (by eliminating DM's splitting and just using bio_split) 1672 */ 1673 part_stat_lock(); 1674 __dm_part_stat_sub(&dm_disk(md)->part0, 1675 sectors[op_stat_group(bio_op(bio))], ci.sector_count); 1676 part_stat_unlock(); 1677 1678 bio_chain(b, bio); 1679 trace_block_split(md->queue, b, bio->bi_iter.bi_sector); 1680 ret = submit_bio_noacct(bio); 1681 break; 1682 } 1683 } 1684 } 1685 1686 /* drop the extra reference count */ 1687 dec_pending(ci.io, errno_to_blk_status(error)); 1688 return ret; 1689} 1690 1691static blk_qc_t dm_submit_bio(struct bio *bio) 1692{ 1693 struct mapped_device *md = bio->bi_disk->private_data; 1694 blk_qc_t ret = BLK_QC_T_NONE; 1695 int srcu_idx; 1696 struct dm_table *map; 1697 1698 map = dm_get_live_table(md, &srcu_idx); 1699 1700 /* If suspended, or map not yet available, queue this IO for later */ 1701 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) || 1702 unlikely(!map)) { 1703 if (bio->bi_opf & REQ_NOWAIT) 1704 bio_wouldblock_error(bio); 1705 else if (bio->bi_opf & REQ_RAHEAD) 1706 bio_io_error(bio); 1707 else 1708 queue_io(md, bio); 1709 goto out; 1710 } 1711 1712 /* 1713 * Use blk_queue_split() for abnormal IO (e.g. discard, writesame, etc) 1714 * otherwise associated queue_limits won't be imposed. 1715 */ 1716 if (is_abnormal_io(bio)) 1717 blk_queue_split(&bio); 1718 1719 ret = __split_and_process_bio(md, map, bio); 1720out: 1721 dm_put_live_table(md, srcu_idx); 1722 return ret; 1723} 1724 1725/*----------------------------------------------------------------- 1726 * An IDR is used to keep track of allocated minor numbers. 1727 *---------------------------------------------------------------*/ 1728static void free_minor(int minor) 1729{ 1730 spin_lock(&_minor_lock); 1731 idr_remove(&_minor_idr, minor); 1732 spin_unlock(&_minor_lock); 1733} 1734 1735/* 1736 * See if the device with a specific minor # is free. 1737 */ 1738static int specific_minor(int minor) 1739{ 1740 int r; 1741 1742 if (minor >= (1 << MINORBITS)) 1743 return -EINVAL; 1744 1745 idr_preload(GFP_KERNEL); 1746 spin_lock(&_minor_lock); 1747 1748 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT); 1749 1750 spin_unlock(&_minor_lock); 1751 idr_preload_end(); 1752 if (r < 0) 1753 return r == -ENOSPC ? -EBUSY : r; 1754 return 0; 1755} 1756 1757static int next_free_minor(int *minor) 1758{ 1759 int r; 1760 1761 idr_preload(GFP_KERNEL); 1762 spin_lock(&_minor_lock); 1763 1764 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT); 1765 1766 spin_unlock(&_minor_lock); 1767 idr_preload_end(); 1768 if (r < 0) 1769 return r; 1770 *minor = r; 1771 return 0; 1772} 1773 1774static const struct block_device_operations dm_blk_dops; 1775static const struct block_device_operations dm_rq_blk_dops; 1776static const struct dax_operations dm_dax_ops; 1777 1778static void dm_wq_work(struct work_struct *work); 1779 1780static void cleanup_mapped_device(struct mapped_device *md) 1781{ 1782 if (md->wq) 1783 destroy_workqueue(md->wq); 1784 bioset_exit(&md->bs); 1785 bioset_exit(&md->io_bs); 1786 1787 if (md->dax_dev) { 1788 kill_dax(md->dax_dev); 1789 put_dax(md->dax_dev); 1790 md->dax_dev = NULL; 1791 } 1792 1793 if (md->disk) { 1794 spin_lock(&_minor_lock); 1795 md->disk->private_data = NULL; 1796 spin_unlock(&_minor_lock); 1797 del_gendisk(md->disk); 1798 put_disk(md->disk); 1799 } 1800 1801 if (md->queue) 1802 blk_cleanup_queue(md->queue); 1803 1804 cleanup_srcu_struct(&md->io_barrier); 1805 1806 if (md->bdev) { 1807 bdput(md->bdev); 1808 md->bdev = NULL; 1809 } 1810 1811 mutex_destroy(&md->suspend_lock); 1812 mutex_destroy(&md->type_lock); 1813 mutex_destroy(&md->table_devices_lock); 1814 mutex_destroy(&md->swap_bios_lock); 1815 1816 dm_mq_cleanup_mapped_device(md); 1817} 1818 1819/* 1820 * Allocate and initialise a blank device with a given minor. 1821 */ 1822static struct mapped_device *alloc_dev(int minor) 1823{ 1824 int r, numa_node_id = dm_get_numa_node(); 1825 struct mapped_device *md; 1826 void *old_md; 1827 1828 md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id); 1829 if (!md) { 1830 DMWARN("unable to allocate device, out of memory."); 1831 return NULL; 1832 } 1833 1834 if (!try_module_get(THIS_MODULE)) 1835 goto bad_module_get; 1836 1837 /* get a minor number for the dev */ 1838 if (minor == DM_ANY_MINOR) 1839 r = next_free_minor(&minor); 1840 else 1841 r = specific_minor(minor); 1842 if (r < 0) 1843 goto bad_minor; 1844 1845 r = init_srcu_struct(&md->io_barrier); 1846 if (r < 0) 1847 goto bad_io_barrier; 1848 1849 md->numa_node_id = numa_node_id; 1850 md->init_tio_pdu = false; 1851 md->type = DM_TYPE_NONE; 1852 mutex_init(&md->suspend_lock); 1853 mutex_init(&md->type_lock); 1854 mutex_init(&md->table_devices_lock); 1855 spin_lock_init(&md->deferred_lock); 1856 atomic_set(&md->holders, 1); 1857 atomic_set(&md->open_count, 0); 1858 atomic_set(&md->event_nr, 0); 1859 atomic_set(&md->uevent_seq, 0); 1860 INIT_LIST_HEAD(&md->uevent_list); 1861 INIT_LIST_HEAD(&md->table_devices); 1862 spin_lock_init(&md->uevent_lock); 1863 1864 /* 1865 * default to bio-based until DM table is loaded and md->type 1866 * established. If request-based table is loaded: blk-mq will 1867 * override accordingly. 1868 */ 1869 md->queue = blk_alloc_queue(numa_node_id); 1870 if (!md->queue) 1871 goto bad; 1872 1873 md->disk = alloc_disk_node(1, md->numa_node_id); 1874 if (!md->disk) 1875 goto bad; 1876 1877 init_waitqueue_head(&md->wait); 1878 INIT_WORK(&md->work, dm_wq_work); 1879 init_waitqueue_head(&md->eventq); 1880 init_completion(&md->kobj_holder.completion); 1881 1882 md->swap_bios = get_swap_bios(); 1883 sema_init(&md->swap_bios_semaphore, md->swap_bios); 1884 mutex_init(&md->swap_bios_lock); 1885 1886 md->disk->major = _major; 1887 md->disk->first_minor = minor; 1888 md->disk->fops = &dm_blk_dops; 1889 md->disk->queue = md->queue; 1890 md->disk->private_data = md; 1891 sprintf(md->disk->disk_name, "dm-%d", minor); 1892 1893 if (IS_ENABLED(CONFIG_DAX_DRIVER)) { 1894 md->dax_dev = alloc_dax(md, md->disk->disk_name, 1895 &dm_dax_ops, 0); 1896 if (IS_ERR(md->dax_dev)) { 1897 md->dax_dev = NULL; 1898 goto bad; 1899 } 1900 } 1901 1902 add_disk_no_queue_reg(md->disk); 1903 format_dev_t(md->name, MKDEV(_major, minor)); 1904 1905 md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0); 1906 if (!md->wq) 1907 goto bad; 1908 1909 md->bdev = bdget_disk(md->disk, 0); 1910 if (!md->bdev) 1911 goto bad; 1912 1913 r = dm_stats_init(&md->stats); 1914 if (r < 0) 1915 goto bad; 1916 1917 /* Populate the mapping, nobody knows we exist yet */ 1918 spin_lock(&_minor_lock); 1919 old_md = idr_replace(&_minor_idr, md, minor); 1920 spin_unlock(&_minor_lock); 1921 1922 BUG_ON(old_md != MINOR_ALLOCED); 1923 1924 return md; 1925 1926bad: 1927 cleanup_mapped_device(md); 1928bad_io_barrier: 1929 free_minor(minor); 1930bad_minor: 1931 module_put(THIS_MODULE); 1932bad_module_get: 1933 kvfree(md); 1934 return NULL; 1935} 1936 1937static void unlock_fs(struct mapped_device *md); 1938 1939static void free_dev(struct mapped_device *md) 1940{ 1941 int minor = MINOR(disk_devt(md->disk)); 1942 1943 unlock_fs(md); 1944 1945 cleanup_mapped_device(md); 1946 1947 free_table_devices(&md->table_devices); 1948 dm_stats_cleanup(&md->stats); 1949 free_minor(minor); 1950 1951 module_put(THIS_MODULE); 1952 kvfree(md); 1953} 1954 1955static int __bind_mempools(struct mapped_device *md, struct dm_table *t) 1956{ 1957 struct dm_md_mempools *p = dm_table_get_md_mempools(t); 1958 int ret = 0; 1959 1960 if (dm_table_bio_based(t)) { 1961 /* 1962 * The md may already have mempools that need changing. 1963 * If so, reload bioset because front_pad may have changed 1964 * because a different table was loaded. 1965 */ 1966 bioset_exit(&md->bs); 1967 bioset_exit(&md->io_bs); 1968 1969 } else if (bioset_initialized(&md->bs)) { 1970 /* 1971 * There's no need to reload with request-based dm 1972 * because the size of front_pad doesn't change. 1973 * Note for future: If you are to reload bioset, 1974 * prep-ed requests in the queue may refer 1975 * to bio from the old bioset, so you must walk 1976 * through the queue to unprep. 1977 */ 1978 goto out; 1979 } 1980 1981 BUG_ON(!p || 1982 bioset_initialized(&md->bs) || 1983 bioset_initialized(&md->io_bs)); 1984 1985 ret = bioset_init_from_src(&md->bs, &p->bs); 1986 if (ret) 1987 goto out; 1988 ret = bioset_init_from_src(&md->io_bs, &p->io_bs); 1989 if (ret) 1990 bioset_exit(&md->bs); 1991out: 1992 /* mempool bind completed, no longer need any mempools in the table */ 1993 dm_table_free_md_mempools(t); 1994 return ret; 1995} 1996 1997/* 1998 * Bind a table to the device. 1999 */ 2000static void event_callback(void *context) 2001{ 2002 unsigned long flags; 2003 LIST_HEAD(uevents); 2004 struct mapped_device *md = (struct mapped_device *) context; 2005 2006 spin_lock_irqsave(&md->uevent_lock, flags); 2007 list_splice_init(&md->uevent_list, &uevents); 2008 spin_unlock_irqrestore(&md->uevent_lock, flags); 2009 2010 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj); 2011 2012 atomic_inc(&md->event_nr); 2013 wake_up(&md->eventq); 2014 dm_issue_global_event(); 2015} 2016 2017/* 2018 * Returns old map, which caller must destroy. 2019 */ 2020static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, 2021 struct queue_limits *limits) 2022{ 2023 struct dm_table *old_map; 2024 struct request_queue *q = md->queue; 2025 bool request_based = dm_table_request_based(t); 2026 sector_t size; 2027 int ret; 2028 2029 lockdep_assert_held(&md->suspend_lock); 2030 2031 size = dm_table_get_size(t); 2032 2033 /* 2034 * Wipe any geometry if the size of the table changed. 2035 */ 2036 if (size != dm_get_size(md)) 2037 memset(&md->geometry, 0, sizeof(md->geometry)); 2038 2039 set_capacity(md->disk, size); 2040 bd_set_nr_sectors(md->bdev, size); 2041 2042 dm_table_event_callback(t, event_callback, md); 2043 2044 if (request_based) { 2045 /* 2046 * Leverage the fact that request-based DM targets are 2047 * immutable singletons - used to optimize dm_mq_queue_rq. 2048 */ 2049 md->immutable_target = dm_table_get_immutable_target(t); 2050 } 2051 2052 ret = __bind_mempools(md, t); 2053 if (ret) { 2054 old_map = ERR_PTR(ret); 2055 goto out; 2056 } 2057 2058 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2059 rcu_assign_pointer(md->map, (void *)t); 2060 md->immutable_target_type = dm_table_get_immutable_target_type(t); 2061 2062 dm_table_set_restrictions(t, q, limits); 2063 if (old_map) 2064 dm_sync_table(md); 2065 2066out: 2067 return old_map; 2068} 2069 2070/* 2071 * Returns unbound table for the caller to free. 2072 */ 2073static struct dm_table *__unbind(struct mapped_device *md) 2074{ 2075 struct dm_table *map = rcu_dereference_protected(md->map, 1); 2076 2077 if (!map) 2078 return NULL; 2079 2080 dm_table_event_callback(map, NULL, NULL); 2081 RCU_INIT_POINTER(md->map, NULL); 2082 dm_sync_table(md); 2083 2084 return map; 2085} 2086 2087/* 2088 * Constructor for a new device. 2089 */ 2090int dm_create(int minor, struct mapped_device **result) 2091{ 2092 int r; 2093 struct mapped_device *md; 2094 2095 md = alloc_dev(minor); 2096 if (!md) 2097 return -ENXIO; 2098 2099 r = dm_sysfs_init(md); 2100 if (r) { 2101 free_dev(md); 2102 return r; 2103 } 2104 2105 *result = md; 2106 return 0; 2107} 2108 2109/* 2110 * Functions to manage md->type. 2111 * All are required to hold md->type_lock. 2112 */ 2113void dm_lock_md_type(struct mapped_device *md) 2114{ 2115 mutex_lock(&md->type_lock); 2116} 2117 2118void dm_unlock_md_type(struct mapped_device *md) 2119{ 2120 mutex_unlock(&md->type_lock); 2121} 2122 2123void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type) 2124{ 2125 BUG_ON(!mutex_is_locked(&md->type_lock)); 2126 md->type = type; 2127} 2128 2129enum dm_queue_mode dm_get_md_type(struct mapped_device *md) 2130{ 2131 return md->type; 2132} 2133 2134struct target_type *dm_get_immutable_target_type(struct mapped_device *md) 2135{ 2136 return md->immutable_target_type; 2137} 2138 2139/* 2140 * The queue_limits are only valid as long as you have a reference 2141 * count on 'md'. 2142 */ 2143struct queue_limits *dm_get_queue_limits(struct mapped_device *md) 2144{ 2145 BUG_ON(!atomic_read(&md->holders)); 2146 return &md->queue->limits; 2147} 2148EXPORT_SYMBOL_GPL(dm_get_queue_limits); 2149 2150/* 2151 * Setup the DM device's queue based on md's type 2152 */ 2153int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) 2154{ 2155 int r; 2156 struct queue_limits limits; 2157 enum dm_queue_mode type = dm_get_md_type(md); 2158 2159 switch (type) { 2160 case DM_TYPE_REQUEST_BASED: 2161 md->disk->fops = &dm_rq_blk_dops; 2162 r = dm_mq_init_request_queue(md, t); 2163 if (r) { 2164 DMERR("Cannot initialize queue for request-based dm mapped device"); 2165 return r; 2166 } 2167 break; 2168 case DM_TYPE_BIO_BASED: 2169 case DM_TYPE_DAX_BIO_BASED: 2170 break; 2171 case DM_TYPE_NONE: 2172 WARN_ON_ONCE(true); 2173 break; 2174 } 2175 2176 r = dm_calculate_queue_limits(t, &limits); 2177 if (r) { 2178 DMERR("Cannot calculate initial queue limits"); 2179 return r; 2180 } 2181 dm_table_set_restrictions(t, md->queue, &limits); 2182 blk_register_queue(md->disk); 2183 2184 return 0; 2185} 2186 2187struct mapped_device *dm_get_md(dev_t dev) 2188{ 2189 struct mapped_device *md; 2190 unsigned minor = MINOR(dev); 2191 2192 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS)) 2193 return NULL; 2194 2195 spin_lock(&_minor_lock); 2196 2197 md = idr_find(&_minor_idr, minor); 2198 if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) || 2199 test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { 2200 md = NULL; 2201 goto out; 2202 } 2203 dm_get(md); 2204out: 2205 spin_unlock(&_minor_lock); 2206 2207 return md; 2208} 2209EXPORT_SYMBOL_GPL(dm_get_md); 2210 2211void *dm_get_mdptr(struct mapped_device *md) 2212{ 2213 return md->interface_ptr; 2214} 2215 2216void dm_set_mdptr(struct mapped_device *md, void *ptr) 2217{ 2218 md->interface_ptr = ptr; 2219} 2220 2221void dm_get(struct mapped_device *md) 2222{ 2223 atomic_inc(&md->holders); 2224 BUG_ON(test_bit(DMF_FREEING, &md->flags)); 2225} 2226 2227int dm_hold(struct mapped_device *md) 2228{ 2229 spin_lock(&_minor_lock); 2230 if (test_bit(DMF_FREEING, &md->flags)) { 2231 spin_unlock(&_minor_lock); 2232 return -EBUSY; 2233 } 2234 dm_get(md); 2235 spin_unlock(&_minor_lock); 2236 return 0; 2237} 2238EXPORT_SYMBOL_GPL(dm_hold); 2239 2240const char *dm_device_name(struct mapped_device *md) 2241{ 2242 return md->name; 2243} 2244EXPORT_SYMBOL_GPL(dm_device_name); 2245 2246static void __dm_destroy(struct mapped_device *md, bool wait) 2247{ 2248 struct dm_table *map; 2249 int srcu_idx; 2250 2251 might_sleep(); 2252 2253 spin_lock(&_minor_lock); 2254 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md)))); 2255 set_bit(DMF_FREEING, &md->flags); 2256 spin_unlock(&_minor_lock); 2257 2258 blk_set_queue_dying(md->queue); 2259 2260 /* 2261 * Take suspend_lock so that presuspend and postsuspend methods 2262 * do not race with internal suspend. 2263 */ 2264 mutex_lock(&md->suspend_lock); 2265 map = dm_get_live_table(md, &srcu_idx); 2266 if (!dm_suspended_md(md)) { 2267 dm_table_presuspend_targets(map); 2268 set_bit(DMF_SUSPENDED, &md->flags); 2269 set_bit(DMF_POST_SUSPENDING, &md->flags); 2270 dm_table_postsuspend_targets(map); 2271 } 2272 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */ 2273 dm_put_live_table(md, srcu_idx); 2274 mutex_unlock(&md->suspend_lock); 2275 2276 /* 2277 * Rare, but there may be I/O requests still going to complete, 2278 * for example. Wait for all references to disappear. 2279 * No one should increment the reference count of the mapped_device, 2280 * after the mapped_device state becomes DMF_FREEING. 2281 */ 2282 if (wait) 2283 while (atomic_read(&md->holders)) 2284 msleep(1); 2285 else if (atomic_read(&md->holders)) 2286 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)", 2287 dm_device_name(md), atomic_read(&md->holders)); 2288 2289 dm_sysfs_exit(md); 2290 dm_table_destroy(__unbind(md)); 2291 free_dev(md); 2292} 2293 2294void dm_destroy(struct mapped_device *md) 2295{ 2296 __dm_destroy(md, true); 2297} 2298 2299void dm_destroy_immediate(struct mapped_device *md) 2300{ 2301 __dm_destroy(md, false); 2302} 2303 2304void dm_put(struct mapped_device *md) 2305{ 2306 atomic_dec(&md->holders); 2307} 2308EXPORT_SYMBOL_GPL(dm_put); 2309 2310static bool md_in_flight_bios(struct mapped_device *md) 2311{ 2312 int cpu; 2313 struct hd_struct *part = &dm_disk(md)->part0; 2314 long sum = 0; 2315 2316 for_each_possible_cpu(cpu) { 2317 sum += part_stat_local_read_cpu(part, in_flight[0], cpu); 2318 sum += part_stat_local_read_cpu(part, in_flight[1], cpu); 2319 } 2320 2321 return sum != 0; 2322} 2323 2324static int dm_wait_for_bios_completion(struct mapped_device *md, long task_state) 2325{ 2326 int r = 0; 2327 DEFINE_WAIT(wait); 2328 2329 while (true) { 2330 prepare_to_wait(&md->wait, &wait, task_state); 2331 2332 if (!md_in_flight_bios(md)) 2333 break; 2334 2335 if (signal_pending_state(task_state, current)) { 2336 r = -EINTR; 2337 break; 2338 } 2339 2340 io_schedule(); 2341 } 2342 finish_wait(&md->wait, &wait); 2343 2344 smp_rmb(); 2345 2346 return r; 2347} 2348 2349static int dm_wait_for_completion(struct mapped_device *md, long task_state) 2350{ 2351 int r = 0; 2352 2353 if (!queue_is_mq(md->queue)) 2354 return dm_wait_for_bios_completion(md, task_state); 2355 2356 while (true) { 2357 if (!blk_mq_queue_inflight(md->queue)) 2358 break; 2359 2360 if (signal_pending_state(task_state, current)) { 2361 r = -EINTR; 2362 break; 2363 } 2364 2365 msleep(5); 2366 } 2367 2368 return r; 2369} 2370 2371/* 2372 * Process the deferred bios 2373 */ 2374static void dm_wq_work(struct work_struct *work) 2375{ 2376 struct mapped_device *md = container_of(work, struct mapped_device, work); 2377 struct bio *bio; 2378 2379 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { 2380 spin_lock_irq(&md->deferred_lock); 2381 bio = bio_list_pop(&md->deferred); 2382 spin_unlock_irq(&md->deferred_lock); 2383 2384 if (!bio) 2385 break; 2386 2387 submit_bio_noacct(bio); 2388 cond_resched(); 2389 } 2390} 2391 2392static void dm_queue_flush(struct mapped_device *md) 2393{ 2394 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2395 smp_mb__after_atomic(); 2396 queue_work(md->wq, &md->work); 2397} 2398 2399/* 2400 * Swap in a new table, returning the old one for the caller to destroy. 2401 */ 2402struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table) 2403{ 2404 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL); 2405 struct queue_limits limits; 2406 int r; 2407 2408 mutex_lock(&md->suspend_lock); 2409 2410 /* device must be suspended */ 2411 if (!dm_suspended_md(md)) 2412 goto out; 2413 2414 /* 2415 * If the new table has no data devices, retain the existing limits. 2416 * This helps multipath with queue_if_no_path if all paths disappear, 2417 * then new I/O is queued based on these limits, and then some paths 2418 * reappear. 2419 */ 2420 if (dm_table_has_no_data_devices(table)) { 2421 live_map = dm_get_live_table_fast(md); 2422 if (live_map) 2423 limits = md->queue->limits; 2424 dm_put_live_table_fast(md); 2425 } 2426 2427 if (!live_map) { 2428 r = dm_calculate_queue_limits(table, &limits); 2429 if (r) { 2430 map = ERR_PTR(r); 2431 goto out; 2432 } 2433 } 2434 2435 map = __bind(md, table, &limits); 2436 dm_issue_global_event(); 2437 2438out: 2439 mutex_unlock(&md->suspend_lock); 2440 return map; 2441} 2442 2443/* 2444 * Functions to lock and unlock any filesystem running on the 2445 * device. 2446 */ 2447static int lock_fs(struct mapped_device *md) 2448{ 2449 int r; 2450 2451 WARN_ON(md->frozen_sb); 2452 2453 md->frozen_sb = freeze_bdev(md->bdev); 2454 if (IS_ERR(md->frozen_sb)) { 2455 r = PTR_ERR(md->frozen_sb); 2456 md->frozen_sb = NULL; 2457 return r; 2458 } 2459 2460 set_bit(DMF_FROZEN, &md->flags); 2461 2462 return 0; 2463} 2464 2465static void unlock_fs(struct mapped_device *md) 2466{ 2467 if (!test_bit(DMF_FROZEN, &md->flags)) 2468 return; 2469 2470 thaw_bdev(md->bdev, md->frozen_sb); 2471 md->frozen_sb = NULL; 2472 clear_bit(DMF_FROZEN, &md->flags); 2473} 2474 2475/* 2476 * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG 2477 * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE 2478 * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY 2479 * 2480 * If __dm_suspend returns 0, the device is completely quiescent 2481 * now. There is no request-processing activity. All new requests 2482 * are being added to md->deferred list. 2483 */ 2484static int __dm_suspend(struct mapped_device *md, struct dm_table *map, 2485 unsigned suspend_flags, long task_state, 2486 int dmf_suspended_flag) 2487{ 2488 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG; 2489 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG; 2490 int r; 2491 2492 lockdep_assert_held(&md->suspend_lock); 2493 2494 /* 2495 * DMF_NOFLUSH_SUSPENDING must be set before presuspend. 2496 * This flag is cleared before dm_suspend returns. 2497 */ 2498 if (noflush) 2499 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 2500 else 2501 DMDEBUG("%s: suspending with flush", dm_device_name(md)); 2502 2503 /* 2504 * This gets reverted if there's an error later and the targets 2505 * provide the .presuspend_undo hook. 2506 */ 2507 dm_table_presuspend_targets(map); 2508 2509 /* 2510 * Flush I/O to the device. 2511 * Any I/O submitted after lock_fs() may not be flushed. 2512 * noflush takes precedence over do_lockfs. 2513 * (lock_fs() flushes I/Os and waits for them to complete.) 2514 */ 2515 if (!noflush && do_lockfs) { 2516 r = lock_fs(md); 2517 if (r) { 2518 dm_table_presuspend_undo_targets(map); 2519 return r; 2520 } 2521 } 2522 2523 /* 2524 * Here we must make sure that no processes are submitting requests 2525 * to target drivers i.e. no one may be executing 2526 * __split_and_process_bio from dm_submit_bio. 2527 * 2528 * To get all processes out of __split_and_process_bio in dm_submit_bio, 2529 * we take the write lock. To prevent any process from reentering 2530 * __split_and_process_bio from dm_submit_bio and quiesce the thread 2531 * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call 2532 * flush_workqueue(md->wq). 2533 */ 2534 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2535 if (map) 2536 synchronize_srcu(&md->io_barrier); 2537 2538 /* 2539 * Stop md->queue before flushing md->wq in case request-based 2540 * dm defers requests to md->wq from md->queue. 2541 */ 2542 if (dm_request_based(md)) 2543 dm_stop_queue(md->queue); 2544 2545 flush_workqueue(md->wq); 2546 2547 /* 2548 * At this point no more requests are entering target request routines. 2549 * We call dm_wait_for_completion to wait for all existing requests 2550 * to finish. 2551 */ 2552 r = dm_wait_for_completion(md, task_state); 2553 if (!r) 2554 set_bit(dmf_suspended_flag, &md->flags); 2555 2556 if (noflush) 2557 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); 2558 if (map) 2559 synchronize_srcu(&md->io_barrier); 2560 2561 /* were we interrupted ? */ 2562 if (r < 0) { 2563 dm_queue_flush(md); 2564 2565 if (dm_request_based(md)) 2566 dm_start_queue(md->queue); 2567 2568 unlock_fs(md); 2569 dm_table_presuspend_undo_targets(map); 2570 /* pushback list is already flushed, so skip flush */ 2571 } 2572 2573 return r; 2574} 2575 2576/* 2577 * We need to be able to change a mapping table under a mounted 2578 * filesystem. For example we might want to move some data in 2579 * the background. Before the table can be swapped with 2580 * dm_bind_table, dm_suspend must be called to flush any in 2581 * flight bios and ensure that any further io gets deferred. 2582 */ 2583/* 2584 * Suspend mechanism in request-based dm. 2585 * 2586 * 1. Flush all I/Os by lock_fs() if needed. 2587 * 2. Stop dispatching any I/O by stopping the request_queue. 2588 * 3. Wait for all in-flight I/Os to be completed or requeued. 2589 * 2590 * To abort suspend, start the request_queue. 2591 */ 2592int dm_suspend(struct mapped_device *md, unsigned suspend_flags) 2593{ 2594 struct dm_table *map = NULL; 2595 int r = 0; 2596 2597retry: 2598 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 2599 2600 if (dm_suspended_md(md)) { 2601 r = -EINVAL; 2602 goto out_unlock; 2603 } 2604 2605 if (dm_suspended_internally_md(md)) { 2606 /* already internally suspended, wait for internal resume */ 2607 mutex_unlock(&md->suspend_lock); 2608 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 2609 if (r) 2610 return r; 2611 goto retry; 2612 } 2613 2614 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2615 2616 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED); 2617 if (r) 2618 goto out_unlock; 2619 2620 set_bit(DMF_POST_SUSPENDING, &md->flags); 2621 dm_table_postsuspend_targets(map); 2622 clear_bit(DMF_POST_SUSPENDING, &md->flags); 2623 2624out_unlock: 2625 mutex_unlock(&md->suspend_lock); 2626 return r; 2627} 2628 2629static int __dm_resume(struct mapped_device *md, struct dm_table *map) 2630{ 2631 if (map) { 2632 int r = dm_table_resume_targets(map); 2633 if (r) 2634 return r; 2635 } 2636 2637 dm_queue_flush(md); 2638 2639 /* 2640 * Flushing deferred I/Os must be done after targets are resumed 2641 * so that mapping of targets can work correctly. 2642 * Request-based dm is queueing the deferred I/Os in its request_queue. 2643 */ 2644 if (dm_request_based(md)) 2645 dm_start_queue(md->queue); 2646 2647 unlock_fs(md); 2648 2649 return 0; 2650} 2651 2652int dm_resume(struct mapped_device *md) 2653{ 2654 int r; 2655 struct dm_table *map = NULL; 2656 2657retry: 2658 r = -EINVAL; 2659 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); 2660 2661 if (!dm_suspended_md(md)) 2662 goto out; 2663 2664 if (dm_suspended_internally_md(md)) { 2665 /* already internally suspended, wait for internal resume */ 2666 mutex_unlock(&md->suspend_lock); 2667 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); 2668 if (r) 2669 return r; 2670 goto retry; 2671 } 2672 2673 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2674 if (!map || !dm_table_get_size(map)) 2675 goto out; 2676 2677 r = __dm_resume(md, map); 2678 if (r) 2679 goto out; 2680 2681 clear_bit(DMF_SUSPENDED, &md->flags); 2682out: 2683 mutex_unlock(&md->suspend_lock); 2684 2685 return r; 2686} 2687 2688/* 2689 * Internal suspend/resume works like userspace-driven suspend. It waits 2690 * until all bios finish and prevents issuing new bios to the target drivers. 2691 * It may be used only from the kernel. 2692 */ 2693 2694static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags) 2695{ 2696 struct dm_table *map = NULL; 2697 2698 lockdep_assert_held(&md->suspend_lock); 2699 2700 if (md->internal_suspend_count++) 2701 return; /* nested internal suspend */ 2702 2703 if (dm_suspended_md(md)) { 2704 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2705 return; /* nest suspend */ 2706 } 2707 2708 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); 2709 2710 /* 2711 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is 2712 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend 2713 * would require changing .presuspend to return an error -- avoid this 2714 * until there is a need for more elaborate variants of internal suspend. 2715 */ 2716 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE, 2717 DMF_SUSPENDED_INTERNALLY); 2718 2719 set_bit(DMF_POST_SUSPENDING, &md->flags); 2720 dm_table_postsuspend_targets(map); 2721 clear_bit(DMF_POST_SUSPENDING, &md->flags); 2722} 2723 2724static void __dm_internal_resume(struct mapped_device *md) 2725{ 2726 BUG_ON(!md->internal_suspend_count); 2727 2728 if (--md->internal_suspend_count) 2729 return; /* resume from nested internal suspend */ 2730 2731 if (dm_suspended_md(md)) 2732 goto done; /* resume from nested suspend */ 2733 2734 /* 2735 * NOTE: existing callers don't need to call dm_table_resume_targets 2736 * (which may fail -- so best to avoid it for now by passing NULL map) 2737 */ 2738 (void) __dm_resume(md, NULL); 2739 2740done: 2741 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2742 smp_mb__after_atomic(); 2743 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY); 2744} 2745 2746void dm_internal_suspend_noflush(struct mapped_device *md) 2747{ 2748 mutex_lock(&md->suspend_lock); 2749 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG); 2750 mutex_unlock(&md->suspend_lock); 2751} 2752EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush); 2753 2754void dm_internal_resume(struct mapped_device *md) 2755{ 2756 mutex_lock(&md->suspend_lock); 2757 __dm_internal_resume(md); 2758 mutex_unlock(&md->suspend_lock); 2759} 2760EXPORT_SYMBOL_GPL(dm_internal_resume); 2761 2762/* 2763 * Fast variants of internal suspend/resume hold md->suspend_lock, 2764 * which prevents interaction with userspace-driven suspend. 2765 */ 2766 2767void dm_internal_suspend_fast(struct mapped_device *md) 2768{ 2769 mutex_lock(&md->suspend_lock); 2770 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 2771 return; 2772 2773 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); 2774 synchronize_srcu(&md->io_barrier); 2775 flush_workqueue(md->wq); 2776 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE); 2777} 2778EXPORT_SYMBOL_GPL(dm_internal_suspend_fast); 2779 2780void dm_internal_resume_fast(struct mapped_device *md) 2781{ 2782 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) 2783 goto done; 2784 2785 dm_queue_flush(md); 2786 2787done: 2788 mutex_unlock(&md->suspend_lock); 2789} 2790EXPORT_SYMBOL_GPL(dm_internal_resume_fast); 2791 2792/*----------------------------------------------------------------- 2793 * Event notification. 2794 *---------------------------------------------------------------*/ 2795int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action, 2796 unsigned cookie) 2797{ 2798 int r; 2799 unsigned noio_flag; 2800 char udev_cookie[DM_COOKIE_LENGTH]; 2801 char *envp[] = { udev_cookie, NULL }; 2802 2803 noio_flag = memalloc_noio_save(); 2804 2805 if (!cookie) 2806 r = kobject_uevent(&disk_to_dev(md->disk)->kobj, action); 2807 else { 2808 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u", 2809 DM_COOKIE_ENV_VAR_NAME, cookie); 2810 r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj, 2811 action, envp); 2812 } 2813 2814 memalloc_noio_restore(noio_flag); 2815 2816 return r; 2817} 2818 2819uint32_t dm_next_uevent_seq(struct mapped_device *md) 2820{ 2821 return atomic_add_return(1, &md->uevent_seq); 2822} 2823 2824uint32_t dm_get_event_nr(struct mapped_device *md) 2825{ 2826 return atomic_read(&md->event_nr); 2827} 2828 2829int dm_wait_event(struct mapped_device *md, int event_nr) 2830{ 2831 return wait_event_interruptible(md->eventq, 2832 (event_nr != atomic_read(&md->event_nr))); 2833} 2834 2835void dm_uevent_add(struct mapped_device *md, struct list_head *elist) 2836{ 2837 unsigned long flags; 2838 2839 spin_lock_irqsave(&md->uevent_lock, flags); 2840 list_add(elist, &md->uevent_list); 2841 spin_unlock_irqrestore(&md->uevent_lock, flags); 2842} 2843 2844/* 2845 * The gendisk is only valid as long as you have a reference 2846 * count on 'md'. 2847 */ 2848struct gendisk *dm_disk(struct mapped_device *md) 2849{ 2850 return md->disk; 2851} 2852EXPORT_SYMBOL_GPL(dm_disk); 2853 2854struct kobject *dm_kobject(struct mapped_device *md) 2855{ 2856 return &md->kobj_holder.kobj; 2857} 2858 2859struct mapped_device *dm_get_from_kobject(struct kobject *kobj) 2860{ 2861 struct mapped_device *md; 2862 2863 md = container_of(kobj, struct mapped_device, kobj_holder.kobj); 2864 2865 spin_lock(&_minor_lock); 2866 if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { 2867 md = NULL; 2868 goto out; 2869 } 2870 dm_get(md); 2871out: 2872 spin_unlock(&_minor_lock); 2873 2874 return md; 2875} 2876 2877int dm_suspended_md(struct mapped_device *md) 2878{ 2879 return test_bit(DMF_SUSPENDED, &md->flags); 2880} 2881 2882static int dm_post_suspending_md(struct mapped_device *md) 2883{ 2884 return test_bit(DMF_POST_SUSPENDING, &md->flags); 2885} 2886 2887int dm_suspended_internally_md(struct mapped_device *md) 2888{ 2889 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); 2890} 2891 2892int dm_test_deferred_remove_flag(struct mapped_device *md) 2893{ 2894 return test_bit(DMF_DEFERRED_REMOVE, &md->flags); 2895} 2896 2897int dm_suspended(struct dm_target *ti) 2898{ 2899 return dm_suspended_md(ti->table->md); 2900} 2901EXPORT_SYMBOL_GPL(dm_suspended); 2902 2903int dm_post_suspending(struct dm_target *ti) 2904{ 2905 return dm_post_suspending_md(ti->table->md); 2906} 2907EXPORT_SYMBOL_GPL(dm_post_suspending); 2908 2909int dm_noflush_suspending(struct dm_target *ti) 2910{ 2911 return __noflush_suspending(ti->table->md); 2912} 2913EXPORT_SYMBOL_GPL(dm_noflush_suspending); 2914 2915struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type, 2916 unsigned integrity, unsigned per_io_data_size, 2917 unsigned min_pool_size) 2918{ 2919 struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id); 2920 unsigned int pool_size = 0; 2921 unsigned int front_pad, io_front_pad; 2922 int ret; 2923 2924 if (!pools) 2925 return NULL; 2926 2927 switch (type) { 2928 case DM_TYPE_BIO_BASED: 2929 case DM_TYPE_DAX_BIO_BASED: 2930 pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size); 2931 front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone); 2932 io_front_pad = roundup(front_pad, __alignof__(struct dm_io)) + offsetof(struct dm_io, tio); 2933 ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, 0); 2934 if (ret) 2935 goto out; 2936 if (integrity && bioset_integrity_create(&pools->io_bs, pool_size)) 2937 goto out; 2938 break; 2939 case DM_TYPE_REQUEST_BASED: 2940 pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size); 2941 front_pad = offsetof(struct dm_rq_clone_bio_info, clone); 2942 /* per_io_data_size is used for blk-mq pdu at queue allocation */ 2943 break; 2944 default: 2945 BUG(); 2946 } 2947 2948 ret = bioset_init(&pools->bs, pool_size, front_pad, 0); 2949 if (ret) 2950 goto out; 2951 2952 if (integrity && bioset_integrity_create(&pools->bs, pool_size)) 2953 goto out; 2954 2955 return pools; 2956 2957out: 2958 dm_free_md_mempools(pools); 2959 2960 return NULL; 2961} 2962 2963void dm_free_md_mempools(struct dm_md_mempools *pools) 2964{ 2965 if (!pools) 2966 return; 2967 2968 bioset_exit(&pools->bs); 2969 bioset_exit(&pools->io_bs); 2970 2971 kfree(pools); 2972} 2973 2974struct dm_pr { 2975 u64 old_key; 2976 u64 new_key; 2977 u32 flags; 2978 bool fail_early; 2979}; 2980 2981static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn, 2982 void *data) 2983{ 2984 struct mapped_device *md = bdev->bd_disk->private_data; 2985 struct dm_table *table; 2986 struct dm_target *ti; 2987 int ret = -ENOTTY, srcu_idx; 2988 2989 table = dm_get_live_table(md, &srcu_idx); 2990 if (!table || !dm_table_get_size(table)) 2991 goto out; 2992 2993 /* We only support devices that have a single target */ 2994 if (dm_table_get_num_targets(table) != 1) 2995 goto out; 2996 ti = dm_table_get_target(table, 0); 2997 2998 if (dm_suspended_md(md)) { 2999 ret = -EAGAIN; 3000 goto out; 3001 } 3002 3003 ret = -EINVAL; 3004 if (!ti->type->iterate_devices) 3005 goto out; 3006 3007 ret = ti->type->iterate_devices(ti, fn, data); 3008out: 3009 dm_put_live_table(md, srcu_idx); 3010 return ret; 3011} 3012 3013/* 3014 * For register / unregister we need to manually call out to every path. 3015 */ 3016static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev, 3017 sector_t start, sector_t len, void *data) 3018{ 3019 struct dm_pr *pr = data; 3020 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; 3021 3022 if (!ops || !ops->pr_register) 3023 return -EOPNOTSUPP; 3024 return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags); 3025} 3026 3027static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key, 3028 u32 flags) 3029{ 3030 struct dm_pr pr = { 3031 .old_key = old_key, 3032 .new_key = new_key, 3033 .flags = flags, 3034 .fail_early = true, 3035 }; 3036 int ret; 3037 3038 ret = dm_call_pr(bdev, __dm_pr_register, &pr); 3039 if (ret && new_key) { 3040 /* unregister all paths if we failed to register any path */ 3041 pr.old_key = new_key; 3042 pr.new_key = 0; 3043 pr.flags = 0; 3044 pr.fail_early = false; 3045 dm_call_pr(bdev, __dm_pr_register, &pr); 3046 } 3047 3048 return ret; 3049} 3050 3051static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type, 3052 u32 flags) 3053{ 3054 struct mapped_device *md = bdev->bd_disk->private_data; 3055 const struct pr_ops *ops; 3056 int r, srcu_idx; 3057 3058 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 3059 if (r < 0) 3060 goto out; 3061 3062 ops = bdev->bd_disk->fops->pr_ops; 3063 if (ops && ops->pr_reserve) 3064 r = ops->pr_reserve(bdev, key, type, flags); 3065 else 3066 r = -EOPNOTSUPP; 3067out: 3068 dm_unprepare_ioctl(md, srcu_idx); 3069 return r; 3070} 3071 3072static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type) 3073{ 3074 struct mapped_device *md = bdev->bd_disk->private_data; 3075 const struct pr_ops *ops; 3076 int r, srcu_idx; 3077 3078 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 3079 if (r < 0) 3080 goto out; 3081 3082 ops = bdev->bd_disk->fops->pr_ops; 3083 if (ops && ops->pr_release) 3084 r = ops->pr_release(bdev, key, type); 3085 else 3086 r = -EOPNOTSUPP; 3087out: 3088 dm_unprepare_ioctl(md, srcu_idx); 3089 return r; 3090} 3091 3092static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key, 3093 enum pr_type type, bool abort) 3094{ 3095 struct mapped_device *md = bdev->bd_disk->private_data; 3096 const struct pr_ops *ops; 3097 int r, srcu_idx; 3098 3099 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 3100 if (r < 0) 3101 goto out; 3102 3103 ops = bdev->bd_disk->fops->pr_ops; 3104 if (ops && ops->pr_preempt) 3105 r = ops->pr_preempt(bdev, old_key, new_key, type, abort); 3106 else 3107 r = -EOPNOTSUPP; 3108out: 3109 dm_unprepare_ioctl(md, srcu_idx); 3110 return r; 3111} 3112 3113static int dm_pr_clear(struct block_device *bdev, u64 key) 3114{ 3115 struct mapped_device *md = bdev->bd_disk->private_data; 3116 const struct pr_ops *ops; 3117 int r, srcu_idx; 3118 3119 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); 3120 if (r < 0) 3121 goto out; 3122 3123 ops = bdev->bd_disk->fops->pr_ops; 3124 if (ops && ops->pr_clear) 3125 r = ops->pr_clear(bdev, key); 3126 else 3127 r = -EOPNOTSUPP; 3128out: 3129 dm_unprepare_ioctl(md, srcu_idx); 3130 return r; 3131} 3132 3133static const struct pr_ops dm_pr_ops = { 3134 .pr_register = dm_pr_register, 3135 .pr_reserve = dm_pr_reserve, 3136 .pr_release = dm_pr_release, 3137 .pr_preempt = dm_pr_preempt, 3138 .pr_clear = dm_pr_clear, 3139}; 3140 3141static const struct block_device_operations dm_blk_dops = { 3142 .submit_bio = dm_submit_bio, 3143 .open = dm_blk_open, 3144 .release = dm_blk_close, 3145 .ioctl = dm_blk_ioctl, 3146 .getgeo = dm_blk_getgeo, 3147 .report_zones = dm_blk_report_zones, 3148 .pr_ops = &dm_pr_ops, 3149 .owner = THIS_MODULE 3150}; 3151 3152static const struct block_device_operations dm_rq_blk_dops = { 3153 .open = dm_blk_open, 3154 .release = dm_blk_close, 3155 .ioctl = dm_blk_ioctl, 3156 .getgeo = dm_blk_getgeo, 3157 .pr_ops = &dm_pr_ops, 3158 .owner = THIS_MODULE 3159}; 3160 3161static const struct dax_operations dm_dax_ops = { 3162 .direct_access = dm_dax_direct_access, 3163 .dax_supported = dm_dax_supported, 3164 .copy_from_iter = dm_dax_copy_from_iter, 3165 .copy_to_iter = dm_dax_copy_to_iter, 3166 .zero_page_range = dm_dax_zero_page_range, 3167}; 3168 3169/* 3170 * module hooks 3171 */ 3172module_init(dm_init); 3173module_exit(dm_exit); 3174 3175module_param(major, uint, 0); 3176MODULE_PARM_DESC(major, "The major number of the device mapper"); 3177 3178module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR); 3179MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools"); 3180 3181module_param(dm_numa_node, int, S_IRUGO | S_IWUSR); 3182MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations"); 3183 3184module_param(swap_bios, int, S_IRUGO | S_IWUSR); 3185MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs"); 3186 3187MODULE_DESCRIPTION(DM_NAME " driver"); 3188MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>"); 3189MODULE_LICENSE("GPL"); 3190