Lines Matching refs:md
97 struct mapped_device *md;
136 * Bits for the md->flags field.
337 int dm_deleting_md(struct mapped_device *md)
339 return test_bit(DMF_DELETING, &md->flags);
344 struct mapped_device *md;
348 md = bdev->bd_disk->private_data;
349 if (!md)
352 if (test_bit(DMF_FREEING, &md->flags) ||
353 dm_deleting_md(md)) {
354 md = NULL;
358 dm_get(md);
359 atomic_inc(&md->open_count);
363 return md ? 0 : -ENXIO;
368 struct mapped_device *md;
372 md = disk->private_data;
373 if (WARN_ON(!md))
376 if (atomic_dec_and_test(&md->open_count) &&
377 (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
380 dm_put(md);
385 int dm_open_count(struct mapped_device *md)
387 return atomic_read(&md->open_count);
393 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
399 if (dm_open_count(md)) {
402 set_bit(DMF_DEFERRED_REMOVE, &md->flags);
403 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
406 set_bit(DMF_DELETING, &md->flags);
413 int dm_cancel_deferred_remove(struct mapped_device *md)
419 if (test_bit(DMF_DELETING, &md->flags))
422 clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
436 struct mapped_device *md = bdev->bd_disk->private_data;
438 return dm_get_geometry(md, geo);
475 struct mapped_device *md = disk->private_data;
484 if (dm_suspended_md(md))
487 map = dm_get_live_table(md, &srcu_idx);
512 dm_put_live_table(md, srcu_idx);
519 static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx,
528 map = dm_get_live_table(md, srcu_idx);
540 if (dm_suspended_md(md))
545 dm_put_live_table(md, *srcu_idx);
553 static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx)
555 dm_put_live_table(md, srcu_idx);
561 struct mapped_device *md = bdev->bd_disk->private_data;
564 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
584 dm_unprepare_ioctl(md, srcu_idx);
599 struct mapped_device *md = io->md;
603 if (unlikely(dm_stats_used(&md->stats)))
604 dm_stats_account_io(&md->stats, bio_data_dir(bio),
609 static void end_io_acct(struct mapped_device *md, struct bio *bio,
614 if (unlikely(dm_stats_used(&md->stats)))
615 dm_stats_account_io(&md->stats, bio_data_dir(bio),
624 if (unlikely(wq_has_sleeper(&md->wait)))
625 wake_up(&md->wait);
628 static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
634 clone = bio_alloc_bioset(GFP_NOIO, 0, &md->io_bs);
647 io->md = md;
655 static void free_io(struct mapped_device *md, struct dm_io *io)
669 struct bio *clone = bio_alloc_bioset(gfp_mask, 0, &ci->io->md->bs);
695 static void queue_io(struct mapped_device *md, struct bio *bio)
699 spin_lock_irqsave(&md->deferred_lock, flags);
700 bio_list_add(&md->deferred, bio);
701 spin_unlock_irqrestore(&md->deferred_lock, flags);
702 queue_work(md->wq, &md->work);
707 * function to access the md->map field, and make sure they call
710 struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
712 *srcu_idx = srcu_read_lock(&md->io_barrier);
714 return srcu_dereference(md->map, &md->io_barrier);
717 void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
719 srcu_read_unlock(&md->io_barrier, srcu_idx);
722 void dm_sync_table(struct mapped_device *md)
724 synchronize_srcu(&md->io_barrier);
732 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
735 return rcu_dereference(md->map);
738 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
749 struct mapped_device *md)
761 r = bd_link_disk_holder(bdev, dm_disk(md));
775 static void close_table_device(struct table_device *td, struct mapped_device *md)
780 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
799 int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
805 mutex_lock(&md->table_devices_lock);
806 td = find_table_device(&md->table_devices, dev, mode);
808 td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
810 mutex_unlock(&md->table_devices_lock);
817 if ((r = open_table_device(td, dev, md))) {
818 mutex_unlock(&md->table_devices_lock);
826 list_add(&td->list, &md->table_devices);
830 mutex_unlock(&md->table_devices_lock);
837 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
841 mutex_lock(&md->table_devices_lock);
843 close_table_device(td, md);
847 mutex_unlock(&md->table_devices_lock);
867 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
869 *geo = md->geometry;
877 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
886 md->geometry = *geo;
891 static int __noflush_suspending(struct mapped_device *md)
893 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
905 struct mapped_device *md = io->md;
912 if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md)))
922 spin_lock_irqsave(&md->deferred_lock, flags);
923 if (__noflush_suspending(md))
925 bio_list_add_head(&md->deferred, io->orig_bio);
929 spin_unlock_irqrestore(&md->deferred_lock, flags);
936 free_io(md, io);
937 end_io_acct(md, bio, start_time, &stats_aux);
948 queue_io(md, bio);
958 void disable_discard(struct mapped_device *md)
960 struct queue_limits *limits = dm_get_queue_limits(md);
964 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, md->queue);
967 void disable_write_same(struct mapped_device *md)
969 struct queue_limits *limits = dm_get_queue_limits(md);
975 void disable_write_zeroes(struct mapped_device *md)
977 struct queue_limits *limits = dm_get_queue_limits(md);
993 struct mapped_device *md = tio->io->md;
1000 disable_discard(md);
1003 disable_write_same(md);
1006 disable_write_zeroes(md);
1039 struct mapped_device *md = io->md;
1040 up(&md->swap_bios_semaphore);
1071 max_len = blk_max_size_offset(ti->table->md->queue,
1095 static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,
1097 __acquires(md->io_barrier)
1102 map = dm_get_live_table(md, srcu_idx);
1116 struct mapped_device *md = dax_get_private(dax_dev);
1122 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1135 dm_put_live_table(md, srcu_idx);
1143 struct mapped_device *md = dax_get_private(dax_dev);
1148 map = dm_get_live_table(md, &srcu_idx);
1155 dm_put_live_table(md, srcu_idx);
1163 struct mapped_device *md = dax_get_private(dax_dev);
1169 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1179 dm_put_live_table(md, srcu_idx);
1187 struct mapped_device *md = dax_get_private(dax_dev);
1193 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1203 dm_put_live_table(md, srcu_idx);
1211 struct mapped_device *md = dax_get_private(dax_dev);
1217 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1230 dm_put_live_table(md, srcu_idx);
1280 static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch)
1282 mutex_lock(&md->swap_bios_lock);
1283 while (latch < md->swap_bios) {
1285 down(&md->swap_bios_semaphore);
1286 md->swap_bios--;
1288 while (latch > md->swap_bios) {
1290 up(&md->swap_bios_semaphore);
1291 md->swap_bios++;
1293 mutex_unlock(&md->swap_bios_lock);
1316 struct mapped_device *md = io->md;
1318 if (unlikely(latch != md->swap_bios))
1319 __set_swap_bios_limit(md, latch);
1320 down(&md->swap_bios_semaphore);
1335 struct mapped_device *md = io->md;
1336 up(&md->swap_bios_semaphore);
1343 struct mapped_device *md = io->md;
1344 up(&md->swap_bios_semaphore);
1382 dm_device_name(tio->io->md),
1421 mutex_lock(&ci->io->md->table_devices_lock);
1430 mutex_unlock(&ci->io->md->table_devices_lock);
1493 bio_set_dev(ci->bio, ci->io->md->bdev);
1618 static void init_clone_info(struct clone_info *ci, struct mapped_device *md,
1622 ci->io = alloc_io(md, bio);
1632 static blk_qc_t __split_and_process_bio(struct mapped_device *md,
1639 init_clone_info(&ci, md, map, bio);
1663 GFP_NOIO, &md->queue->bio_split);
1674 __dm_part_stat_sub(&dm_disk(md)->part0,
1679 trace_block_split(md->queue, b, bio->bi_iter.bi_sector);
1693 struct mapped_device *md = bio->bi_disk->private_data;
1698 map = dm_get_live_table(md, &srcu_idx);
1701 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) ||
1708 queue_io(md, bio);
1719 ret = __split_and_process_bio(md, map, bio);
1721 dm_put_live_table(md, srcu_idx);
1780 static void cleanup_mapped_device(struct mapped_device *md)
1782 if (md->wq)
1783 destroy_workqueue(md->wq);
1784 bioset_exit(&md->bs);
1785 bioset_exit(&md->io_bs);
1787 if (md->dax_dev) {
1788 kill_dax(md->dax_dev);
1789 put_dax(md->dax_dev);
1790 md->dax_dev = NULL;
1793 if (md->disk) {
1795 md->disk->private_data = NULL;
1797 del_gendisk(md->disk);
1798 put_disk(md->disk);
1801 if (md->queue)
1802 blk_cleanup_queue(md->queue);
1804 cleanup_srcu_struct(&md->io_barrier);
1806 if (md->bdev) {
1807 bdput(md->bdev);
1808 md->bdev = NULL;
1811 mutex_destroy(&md->suspend_lock);
1812 mutex_destroy(&md->type_lock);
1813 mutex_destroy(&md->table_devices_lock);
1814 mutex_destroy(&md->swap_bios_lock);
1816 dm_mq_cleanup_mapped_device(md);
1825 struct mapped_device *md;
1828 md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
1829 if (!md) {
1845 r = init_srcu_struct(&md->io_barrier);
1849 md->numa_node_id = numa_node_id;
1850 md->init_tio_pdu = false;
1851 md->type = DM_TYPE_NONE;
1852 mutex_init(&md->suspend_lock);
1853 mutex_init(&md->type_lock);
1854 mutex_init(&md->table_devices_lock);
1855 spin_lock_init(&md->deferred_lock);
1856 atomic_set(&md->holders, 1);
1857 atomic_set(&md->open_count, 0);
1858 atomic_set(&md->event_nr, 0);
1859 atomic_set(&md->uevent_seq, 0);
1860 INIT_LIST_HEAD(&md->uevent_list);
1861 INIT_LIST_HEAD(&md->table_devices);
1862 spin_lock_init(&md->uevent_lock);
1865 * default to bio-based until DM table is loaded and md->type
1869 md->queue = blk_alloc_queue(numa_node_id);
1870 if (!md->queue)
1873 md->disk = alloc_disk_node(1, md->numa_node_id);
1874 if (!md->disk)
1877 init_waitqueue_head(&md->wait);
1878 INIT_WORK(&md->work, dm_wq_work);
1879 init_waitqueue_head(&md->eventq);
1880 init_completion(&md->kobj_holder.completion);
1882 md->swap_bios = get_swap_bios();
1883 sema_init(&md->swap_bios_semaphore, md->swap_bios);
1884 mutex_init(&md->swap_bios_lock);
1886 md->disk->major = _major;
1887 md->disk->first_minor = minor;
1888 md->disk->fops = &dm_blk_dops;
1889 md->disk->queue = md->queue;
1890 md->disk->private_data = md;
1891 sprintf(md->disk->disk_name, "dm-%d", minor);
1894 md->dax_dev = alloc_dax(md, md->disk->disk_name,
1896 if (IS_ERR(md->dax_dev)) {
1897 md->dax_dev = NULL;
1902 add_disk_no_queue_reg(md->disk);
1903 format_dev_t(md->name, MKDEV(_major, minor));
1905 md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);
1906 if (!md->wq)
1909 md->bdev = bdget_disk(md->disk, 0);
1910 if (!md->bdev)
1913 r = dm_stats_init(&md->stats);
1919 old_md = idr_replace(&_minor_idr, md, minor);
1924 return md;
1927 cleanup_mapped_device(md);
1933 kvfree(md);
1937 static void unlock_fs(struct mapped_device *md);
1939 static void free_dev(struct mapped_device *md)
1941 int minor = MINOR(disk_devt(md->disk));
1943 unlock_fs(md);
1945 cleanup_mapped_device(md);
1947 free_table_devices(&md->table_devices);
1948 dm_stats_cleanup(&md->stats);
1952 kvfree(md);
1955 static int __bind_mempools(struct mapped_device *md, struct dm_table *t)
1962 * The md may already have mempools that need changing.
1966 bioset_exit(&md->bs);
1967 bioset_exit(&md->io_bs);
1969 } else if (bioset_initialized(&md->bs)) {
1982 bioset_initialized(&md->bs) ||
1983 bioset_initialized(&md->io_bs));
1985 ret = bioset_init_from_src(&md->bs, &p->bs);
1988 ret = bioset_init_from_src(&md->io_bs, &p->io_bs);
1990 bioset_exit(&md->bs);
2004 struct mapped_device *md = (struct mapped_device *) context;
2006 spin_lock_irqsave(&md->uevent_lock, flags);
2007 list_splice_init(&md->uevent_list, &uevents);
2008 spin_unlock_irqrestore(&md->uevent_lock, flags);
2010 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
2012 atomic_inc(&md->event_nr);
2013 wake_up(&md->eventq);
2020 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2024 struct request_queue *q = md->queue;
2029 lockdep_assert_held(&md->suspend_lock);
2036 if (size != dm_get_size(md))
2037 memset(&md->geometry, 0, sizeof(md->geometry));
2039 set_capacity(md->disk, size);
2040 bd_set_nr_sectors(md->bdev, size);
2042 dm_table_event_callback(t, event_callback, md);
2049 md->immutable_target = dm_table_get_immutable_target(t);
2052 ret = __bind_mempools(md, t);
2058 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2059 rcu_assign_pointer(md->map, (void *)t);
2060 md->immutable_target_type = dm_table_get_immutable_target_type(t);
2064 dm_sync_table(md);
2073 static struct dm_table *__unbind(struct mapped_device *md)
2075 struct dm_table *map = rcu_dereference_protected(md->map, 1);
2081 RCU_INIT_POINTER(md->map, NULL);
2082 dm_sync_table(md);
2093 struct mapped_device *md;
2095 md = alloc_dev(minor);
2096 if (!md)
2099 r = dm_sysfs_init(md);
2101 free_dev(md);
2105 *result = md;
2110 * Functions to manage md->type.
2111 * All are required to hold md->type_lock.
2113 void dm_lock_md_type(struct mapped_device *md)
2115 mutex_lock(&md->type_lock);
2118 void dm_unlock_md_type(struct mapped_device *md)
2120 mutex_unlock(&md->type_lock);
2123 void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type)
2125 BUG_ON(!mutex_is_locked(&md->type_lock));
2126 md->type = type;
2129 enum dm_queue_mode dm_get_md_type(struct mapped_device *md)
2131 return md->type;
2134 struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
2136 return md->immutable_target_type;
2141 * count on 'md'.
2143 struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
2145 BUG_ON(!atomic_read(&md->holders));
2146 return &md->queue->limits;
2151 * Setup the DM device's queue based on md's type
2153 int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
2157 enum dm_queue_mode type = dm_get_md_type(md);
2161 md->disk->fops = &dm_rq_blk_dops;
2162 r = dm_mq_init_request_queue(md, t);
2181 dm_table_set_restrictions(t, md->queue, &limits);
2182 blk_register_queue(md->disk);
2189 struct mapped_device *md;
2197 md = idr_find(&_minor_idr, minor);
2198 if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) ||
2199 test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2200 md = NULL;
2203 dm_get(md);
2207 return md;
2211 void *dm_get_mdptr(struct mapped_device *md)
2213 return md->interface_ptr;
2216 void dm_set_mdptr(struct mapped_device *md, void *ptr)
2218 md->interface_ptr = ptr;
2221 void dm_get(struct mapped_device *md)
2223 atomic_inc(&md->holders);
2224 BUG_ON(test_bit(DMF_FREEING, &md->flags));
2227 int dm_hold(struct mapped_device *md)
2230 if (test_bit(DMF_FREEING, &md->flags)) {
2234 dm_get(md);
2240 const char *dm_device_name(struct mapped_device *md)
2242 return md->name;
2246 static void __dm_destroy(struct mapped_device *md, bool wait)
2254 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2255 set_bit(DMF_FREEING, &md->flags);
2258 blk_set_queue_dying(md->queue);
2264 mutex_lock(&md->suspend_lock);
2265 map = dm_get_live_table(md, &srcu_idx);
2266 if (!dm_suspended_md(md)) {
2268 set_bit(DMF_SUSPENDED, &md->flags);
2269 set_bit(DMF_POST_SUSPENDING, &md->flags);
2273 dm_put_live_table(md, srcu_idx);
2274 mutex_unlock(&md->suspend_lock);
2283 while (atomic_read(&md->holders))
2285 else if (atomic_read(&md->holders))
2287 dm_device_name(md), atomic_read(&md->holders));
2289 dm_sysfs_exit(md);
2290 dm_table_destroy(__unbind(md));
2291 free_dev(md);
2294 void dm_destroy(struct mapped_device *md)
2296 __dm_destroy(md, true);
2299 void dm_destroy_immediate(struct mapped_device *md)
2301 __dm_destroy(md, false);
2304 void dm_put(struct mapped_device *md)
2306 atomic_dec(&md->holders);
2310 static bool md_in_flight_bios(struct mapped_device *md)
2313 struct hd_struct *part = &dm_disk(md)->part0;
2324 static int dm_wait_for_bios_completion(struct mapped_device *md, long task_state)
2330 prepare_to_wait(&md->wait, &wait, task_state);
2332 if (!md_in_flight_bios(md))
2342 finish_wait(&md->wait, &wait);
2349 static int dm_wait_for_completion(struct mapped_device *md, long task_state)
2353 if (!queue_is_mq(md->queue))
2354 return dm_wait_for_bios_completion(md, task_state);
2357 if (!blk_mq_queue_inflight(md->queue))
2376 struct mapped_device *md = container_of(work, struct mapped_device, work);
2379 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2380 spin_lock_irq(&md->deferred_lock);
2381 bio = bio_list_pop(&md->deferred);
2382 spin_unlock_irq(&md->deferred_lock);
2392 static void dm_queue_flush(struct mapped_device *md)
2394 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2396 queue_work(md->wq, &md->work);
2402 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
2408 mutex_lock(&md->suspend_lock);
2411 if (!dm_suspended_md(md))
2421 live_map = dm_get_live_table_fast(md);
2423 limits = md->queue->limits;
2424 dm_put_live_table_fast(md);
2435 map = __bind(md, table, &limits);
2439 mutex_unlock(&md->suspend_lock);
2447 static int lock_fs(struct mapped_device *md)
2451 WARN_ON(md->frozen_sb);
2453 md->frozen_sb = freeze_bdev(md->bdev);
2454 if (IS_ERR(md->frozen_sb)) {
2455 r = PTR_ERR(md->frozen_sb);
2456 md->frozen_sb = NULL;
2460 set_bit(DMF_FROZEN, &md->flags);
2465 static void unlock_fs(struct mapped_device *md)
2467 if (!test_bit(DMF_FROZEN, &md->flags))
2470 thaw_bdev(md->bdev, md->frozen_sb);
2471 md->frozen_sb = NULL;
2472 clear_bit(DMF_FROZEN, &md->flags);
2482 * are being added to md->deferred list.
2484 static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
2492 lockdep_assert_held(&md->suspend_lock);
2499 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2501 DMDEBUG("%s: suspending with flush", dm_device_name(md));
2516 r = lock_fs(md);
2532 * flush_workqueue(md->wq).
2534 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2536 synchronize_srcu(&md->io_barrier);
2539 * Stop md->queue before flushing md->wq in case request-based
2540 * dm defers requests to md->wq from md->queue.
2542 if (dm_request_based(md))
2543 dm_stop_queue(md->queue);
2545 flush_workqueue(md->wq);
2552 r = dm_wait_for_completion(md, task_state);
2554 set_bit(dmf_suspended_flag, &md->flags);
2557 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2559 synchronize_srcu(&md->io_barrier);
2563 dm_queue_flush(md);
2565 if (dm_request_based(md))
2566 dm_start_queue(md->queue);
2568 unlock_fs(md);
2592 int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2598 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2600 if (dm_suspended_md(md)) {
2605 if (dm_suspended_internally_md(md)) {
2607 mutex_unlock(&md->suspend_lock);
2608 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2614 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2616 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
2620 set_bit(DMF_POST_SUSPENDING, &md->flags);
2622 clear_bit(DMF_POST_SUSPENDING, &md->flags);
2625 mutex_unlock(&md->suspend_lock);
2629 static int __dm_resume(struct mapped_device *md, struct dm_table *map)
2637 dm_queue_flush(md);
2644 if (dm_request_based(md))
2645 dm_start_queue(md->queue);
2647 unlock_fs(md);
2652 int dm_resume(struct mapped_device *md)
2659 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2661 if (!dm_suspended_md(md))
2664 if (dm_suspended_internally_md(md)) {
2666 mutex_unlock(&md->suspend_lock);
2667 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2673 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2677 r = __dm_resume(md, map);
2681 clear_bit(DMF_SUSPENDED, &md->flags);
2683 mutex_unlock(&md->suspend_lock);
2694 static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
2698 lockdep_assert_held(&md->suspend_lock);
2700 if (md->internal_suspend_count++)
2703 if (dm_suspended_md(md)) {
2704 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2708 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2716 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2719 set_bit(DMF_POST_SUSPENDING, &md->flags);
2721 clear_bit(DMF_POST_SUSPENDING, &md->flags);
2724 static void __dm_internal_resume(struct mapped_device *md)
2726 BUG_ON(!md->internal_suspend_count);
2728 if (--md->internal_suspend_count)
2731 if (dm_suspended_md(md))
2738 (void) __dm_resume(md, NULL);
2741 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2743 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2746 void dm_internal_suspend_noflush(struct mapped_device *md)
2748 mutex_lock(&md->suspend_lock);
2749 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2750 mutex_unlock(&md->suspend_lock);
2754 void dm_internal_resume(struct mapped_device *md)
2756 mutex_lock(&md->suspend_lock);
2757 __dm_internal_resume(md);
2758 mutex_unlock(&md->suspend_lock);
2763 * Fast variants of internal suspend/resume hold md->suspend_lock,
2767 void dm_internal_suspend_fast(struct mapped_device *md)
2769 mutex_lock(&md->suspend_lock);
2770 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2773 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2774 synchronize_srcu(&md->io_barrier);
2775 flush_workqueue(md->wq);
2776 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2780 void dm_internal_resume_fast(struct mapped_device *md)
2782 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2785 dm_queue_flush(md);
2788 mutex_unlock(&md->suspend_lock);
2795 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
2806 r = kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
2810 r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2819 uint32_t dm_next_uevent_seq(struct mapped_device *md)
2821 return atomic_add_return(1, &md->uevent_seq);
2824 uint32_t dm_get_event_nr(struct mapped_device *md)
2826 return atomic_read(&md->event_nr);
2829 int dm_wait_event(struct mapped_device *md, int event_nr)
2831 return wait_event_interruptible(md->eventq,
2832 (event_nr != atomic_read(&md->event_nr)));
2835 void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2839 spin_lock_irqsave(&md->uevent_lock, flags);
2840 list_add(elist, &md->uevent_list);
2841 spin_unlock_irqrestore(&md->uevent_lock, flags);
2846 * count on 'md'.
2848 struct gendisk *dm_disk(struct mapped_device *md)
2850 return md->disk;
2854 struct kobject *dm_kobject(struct mapped_device *md)
2856 return &md->kobj_holder.kobj;
2861 struct mapped_device *md;
2863 md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
2866 if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2867 md = NULL;
2870 dm_get(md);
2874 return md;
2877 int dm_suspended_md(struct mapped_device *md)
2879 return test_bit(DMF_SUSPENDED, &md->flags);
2882 static int dm_post_suspending_md(struct mapped_device *md)
2884 return test_bit(DMF_POST_SUSPENDING, &md->flags);
2887 int dm_suspended_internally_md(struct mapped_device *md)
2889 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2892 int dm_test_deferred_remove_flag(struct mapped_device *md)
2894 return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
2899 return dm_suspended_md(ti->table->md);
2905 return dm_post_suspending_md(ti->table->md);
2911 return __noflush_suspending(ti->table->md);
2915 struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type,
2919 struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
2984 struct mapped_device *md = bdev->bd_disk->private_data;
2989 table = dm_get_live_table(md, &srcu_idx);
2998 if (dm_suspended_md(md)) {
3009 dm_put_live_table(md, srcu_idx);
3054 struct mapped_device *md = bdev->bd_disk->private_data;
3058 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3068 dm_unprepare_ioctl(md, srcu_idx);
3074 struct mapped_device *md = bdev->bd_disk->private_data;
3078 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3088 dm_unprepare_ioctl(md, srcu_idx);
3095 struct mapped_device *md = bdev->bd_disk->private_data;
3099 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3109 dm_unprepare_ioctl(md, srcu_idx);
3115 struct mapped_device *md = bdev->bd_disk->private_data;
3119 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3129 dm_unprepare_ioctl(md, srcu_idx);