Lines Matching refs:md
308 int dm_deleting_md(struct mapped_device *md)
310 return test_bit(DMF_DELETING, &md->flags);
315 struct mapped_device *md;
319 md = disk->private_data;
320 if (!md)
323 if (test_bit(DMF_FREEING, &md->flags) ||
324 dm_deleting_md(md)) {
325 md = NULL;
329 dm_get(md);
330 atomic_inc(&md->open_count);
334 return md ? 0 : -ENXIO;
339 struct mapped_device *md;
343 md = disk->private_data;
344 if (WARN_ON(!md))
347 if (atomic_dec_and_test(&md->open_count) &&
348 (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
351 dm_put(md);
356 int dm_open_count(struct mapped_device *md)
358 return atomic_read(&md->open_count);
364 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
370 if (dm_open_count(md)) {
373 set_bit(DMF_DEFERRED_REMOVE, &md->flags);
374 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
377 set_bit(DMF_DELETING, &md->flags);
384 int dm_cancel_deferred_remove(struct mapped_device *md)
390 if (test_bit(DMF_DELETING, &md->flags))
393 clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
407 struct mapped_device *md = bdev->bd_disk->private_data;
409 return dm_get_geometry(md, geo);
412 static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx,
421 map = dm_get_live_table(md, srcu_idx);
433 if (dm_suspended_md(md))
438 dm_put_live_table(md, *srcu_idx);
446 static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx)
448 dm_put_live_table(md, srcu_idx);
454 struct mapped_device *md = bdev->bd_disk->private_data;
457 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
480 dm_unprepare_ioctl(md, srcu_idx);
523 unlikely(dm_stats_used(&io->md->stats))) {
531 dm_stats_account_io(&io->md->stats, bio_data_dir(bio),
573 static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
579 clone = bio_alloc_clone(NULL, bio, GFP_NOIO, &md->mempools->io_bs);
591 this_cpu_inc(*md->pending_io);
593 io->md = md;
597 if (blk_queue_io_stat(md->queue))
601 unlikely(dm_stats_used(&md->stats)))
602 dm_stats_record_start(&md->stats, &io->stats_aux);
615 struct mapped_device *md = ci->io->md;
626 &md->mempools->bs);
645 clone->bi_bdev = md->disk->part0;
647 bio_set_dev(clone, md->disk->part0);
668 static void queue_io(struct mapped_device *md, struct bio *bio)
672 spin_lock_irqsave(&md->deferred_lock, flags);
673 bio_list_add(&md->deferred, bio);
674 spin_unlock_irqrestore(&md->deferred_lock, flags);
675 queue_work(md->wq, &md->work);
680 * function to access the md->map field, and make sure they call
683 struct dm_table *dm_get_live_table(struct mapped_device *md,
684 int *srcu_idx) __acquires(md->io_barrier)
686 *srcu_idx = srcu_read_lock(&md->io_barrier);
688 return srcu_dereference(md->map, &md->io_barrier);
691 void dm_put_live_table(struct mapped_device *md,
692 int srcu_idx) __releases(md->io_barrier)
694 srcu_read_unlock(&md->io_barrier, srcu_idx);
697 void dm_sync_table(struct mapped_device *md)
699 synchronize_srcu(&md->io_barrier);
707 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
710 return rcu_dereference(md->map);
713 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
723 static struct table_device *open_table_device(struct mapped_device *md,
731 td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
747 if (md->disk->slave_dir) {
748 r = bd_link_disk_holder(bdev, md->disk);
757 list_add(&td->list, &md->table_devices);
770 static void close_table_device(struct table_device *td, struct mapped_device *md)
772 if (md->disk->slave_dir)
773 bd_unlink_disk_holder(td->dm_dev.bdev, md->disk);
792 int dm_get_table_device(struct mapped_device *md, dev_t dev, blk_mode_t mode,
797 mutex_lock(&md->table_devices_lock);
798 td = find_table_device(&md->table_devices, dev, mode);
800 td = open_table_device(md, dev, mode);
802 mutex_unlock(&md->table_devices_lock);
808 mutex_unlock(&md->table_devices_lock);
814 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
818 mutex_lock(&md->table_devices_lock);
820 close_table_device(td, md);
821 mutex_unlock(&md->table_devices_lock);
827 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
829 *geo = md->geometry;
837 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
846 md->geometry = *geo;
851 static int __noflush_suspending(struct mapped_device *md)
853 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
858 struct mapped_device *md = io->md;
861 struct dm_io *next = md->requeue_list;
863 md->requeue_list = io;
866 bio_list_add_head(&md->deferred, io->orig_bio);
870 static void dm_kick_requeue(struct mapped_device *md, bool first_stage)
873 queue_work(md->wq, &md->requeue_work);
875 queue_work(md->wq, &md->work);
888 struct mapped_device *md = io->md;
907 spin_lock_irqsave(&md->deferred_lock, flags);
908 if ((__noflush_suspending(md) &&
909 !WARN_ON_ONCE(dm_is_zone_write(md, bio))) ||
920 spin_unlock_irqrestore(&md->deferred_lock, flags);
924 dm_kick_requeue(md, first_stage);
932 struct mapped_device *md = io->md;
953 this_cpu_dec(*md->pending_io);
956 if (unlikely(wq_has_sleeper(&md->wait)))
957 wake_up(&md->wait);
969 queue_io(md, bio);
980 struct mapped_device *md = container_of(work, struct mapped_device,
986 spin_lock_irqsave(&md->deferred_lock, flags);
987 io = md->requeue_list;
988 md->requeue_list = NULL;
989 spin_unlock_irqrestore(&md->deferred_lock, flags);
994 dm_io_rewind(io, &md->disk->bio_split);
1048 __noflush_suspending(io->md))) {
1064 * count on 'md'. But _not_ imposing verification to avoid atomic_read(),
1066 static inline struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
1068 return &md->queue->limits;
1071 void disable_discard(struct mapped_device *md)
1073 struct queue_limits *limits = dm_get_queue_limits(md);
1079 void disable_write_zeroes(struct mapped_device *md)
1081 struct queue_limits *limits = dm_get_queue_limits(md);
1099 struct mapped_device *md = io->md;
1104 disable_discard(md);
1107 disable_write_zeroes(md);
1125 if (WARN_ON_ONCE(dm_is_zone_write(md, bio)))
1145 up(&md->swap_bios_semaphore);
1177 min(max_sectors ? : queue_max_sectors(ti->table->md->queue),
1201 static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,
1203 __acquires(md->io_barrier)
1208 map = dm_get_live_table(md, srcu_idx);
1223 struct mapped_device *md = dax_get_private(dax_dev);
1229 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1242 dm_put_live_table(md, srcu_idx);
1250 struct mapped_device *md = dax_get_private(dax_dev);
1256 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1269 dm_put_live_table(md, srcu_idx);
1277 struct mapped_device *md = dax_get_private(dax_dev);
1283 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1289 dm_put_live_table(md, srcu_idx);
1372 trace_block_bio_remap(tgt_clone, disk_devt(io->md->disk),
1378 static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch)
1380 mutex_lock(&md->swap_bios_lock);
1381 while (latch < md->swap_bios) {
1383 down(&md->swap_bios_semaphore);
1384 md->swap_bios--;
1386 while (latch > md->swap_bios) {
1388 up(&md->swap_bios_semaphore);
1389 md->swap_bios++;
1391 mutex_unlock(&md->swap_bios_lock);
1399 struct mapped_device *md = io->md;
1413 if (unlikely(latch != md->swap_bios))
1414 __set_swap_bios_limit(md, latch);
1415 down(&md->swap_bios_semaphore);
1424 if (unlikely(dm_emulate_zone_append(md)))
1444 up(&md->swap_bios_semaphore);
1483 mutex_lock(&ci->io->md->table_devices_lock);
1493 mutex_unlock(&ci->io->md->table_devices_lock);
1545 bio_init(&flush_bio, ci->io->md->disk->part0, NULL, 0,
1616 struct queue_limits *limits = dm_get_queue_limits(ti->table->md);
1734 static void init_clone_info(struct clone_info *ci, struct mapped_device *md,
1738 ci->io = alloc_io(md, bio);
1754 static void dm_split_and_process_bio(struct mapped_device *md,
1773 init_clone_info(&ci, md, map, bio, is_abnormal);
1815 struct mapped_device *md = bio->bi_bdev->bd_disk->private_data;
1819 map = dm_get_live_table(md, &srcu_idx);
1822 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) ||
1829 queue_io(md, bio);
1833 dm_split_and_process_bio(md, map, bio);
1835 dm_put_live_table(md, srcu_idx);
1970 static void cleanup_mapped_device(struct mapped_device *md)
1972 if (md->wq)
1973 destroy_workqueue(md->wq);
1974 dm_free_md_mempools(md->mempools);
1976 if (md->dax_dev) {
1977 dax_remove_host(md->disk);
1978 kill_dax(md->dax_dev);
1979 put_dax(md->dax_dev);
1980 md->dax_dev = NULL;
1983 dm_cleanup_zoned_dev(md);
1984 if (md->disk) {
1986 md->disk->private_data = NULL;
1988 if (dm_get_md_type(md) != DM_TYPE_NONE) {
1991 dm_sysfs_exit(md);
1992 list_for_each_entry(td, &md->table_devices, list) {
1994 md->disk);
2001 mutex_lock(&md->table_devices_lock);
2002 del_gendisk(md->disk);
2003 mutex_unlock(&md->table_devices_lock);
2005 dm_queue_destroy_crypto_profile(md->queue);
2006 put_disk(md->disk);
2009 if (md->pending_io) {
2010 free_percpu(md->pending_io);
2011 md->pending_io = NULL;
2014 cleanup_srcu_struct(&md->io_barrier);
2016 mutex_destroy(&md->suspend_lock);
2017 mutex_destroy(&md->type_lock);
2018 mutex_destroy(&md->table_devices_lock);
2019 mutex_destroy(&md->swap_bios_lock);
2021 dm_mq_cleanup_mapped_device(md);
2030 struct mapped_device *md;
2033 md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
2034 if (!md) {
2050 r = init_srcu_struct(&md->io_barrier);
2054 md->numa_node_id = numa_node_id;
2055 md->init_tio_pdu = false;
2056 md->type = DM_TYPE_NONE;
2057 mutex_init(&md->suspend_lock);
2058 mutex_init(&md->type_lock);
2059 mutex_init(&md->table_devices_lock);
2060 spin_lock_init(&md->deferred_lock);
2061 atomic_set(&md->holders, 1);
2062 atomic_set(&md->open_count, 0);
2063 atomic_set(&md->event_nr, 0);
2064 atomic_set(&md->uevent_seq, 0);
2065 INIT_LIST_HEAD(&md->uevent_list);
2066 INIT_LIST_HEAD(&md->table_devices);
2067 spin_lock_init(&md->uevent_lock);
2070 * default to bio-based until DM table is loaded and md->type
2074 md->disk = blk_alloc_disk(md->numa_node_id);
2075 if (!md->disk)
2077 md->queue = md->disk->queue;
2079 init_waitqueue_head(&md->wait);
2080 INIT_WORK(&md->work, dm_wq_work);
2081 INIT_WORK(&md->requeue_work, dm_wq_requeue_work);
2082 init_waitqueue_head(&md->eventq);
2083 init_completion(&md->kobj_holder.completion);
2085 md->requeue_list = NULL;
2086 md->swap_bios = get_swap_bios();
2087 sema_init(&md->swap_bios_semaphore, md->swap_bios);
2088 mutex_init(&md->swap_bios_lock);
2090 md->disk->major = _major;
2091 md->disk->first_minor = minor;
2092 md->disk->minors = 1;
2093 md->disk->flags |= GENHD_FL_NO_PART;
2094 md->disk->fops = &dm_blk_dops;
2095 md->disk->private_data = md;
2096 sprintf(md->disk->disk_name, "dm-%d", minor);
2099 md->dax_dev = alloc_dax(md, &dm_dax_ops);
2100 if (IS_ERR(md->dax_dev)) {
2101 md->dax_dev = NULL;
2104 set_dax_nocache(md->dax_dev);
2105 set_dax_nomc(md->dax_dev);
2106 if (dax_add_host(md->dax_dev, md->disk))
2110 format_dev_t(md->name, MKDEV(_major, minor));
2112 md->wq = alloc_workqueue("kdmflush/%s", WQ_MEM_RECLAIM, 0, md->name);
2113 if (!md->wq)
2116 md->pending_io = alloc_percpu(unsigned long);
2117 if (!md->pending_io)
2120 r = dm_stats_init(&md->stats);
2126 old_md = idr_replace(&_minor_idr, md, minor);
2131 return md;
2134 cleanup_mapped_device(md);
2140 kvfree(md);
2144 static void unlock_fs(struct mapped_device *md);
2146 static void free_dev(struct mapped_device *md)
2148 int minor = MINOR(disk_devt(md->disk));
2150 unlock_fs(md);
2152 cleanup_mapped_device(md);
2154 WARN_ON_ONCE(!list_empty(&md->table_devices));
2155 dm_stats_cleanup(&md->stats);
2159 kvfree(md);
2169 struct mapped_device *md = context;
2171 spin_lock_irqsave(&md->uevent_lock, flags);
2172 list_splice_init(&md->uevent_list, &uevents);
2173 spin_unlock_irqrestore(&md->uevent_lock, flags);
2175 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
2177 atomic_inc(&md->event_nr);
2178 wake_up(&md->eventq);
2185 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
2192 lockdep_assert_held(&md->suspend_lock);
2199 if (size != dm_get_size(md))
2200 memset(&md->geometry, 0, sizeof(md->geometry));
2202 set_capacity(md->disk, size);
2204 dm_table_event_callback(t, event_callback, md);
2211 md->immutable_target = dm_table_get_immutable_target(t);
2221 if (!md->mempools) {
2222 md->mempools = t->mempools;
2227 * The md may already have mempools that need changing.
2231 dm_free_md_mempools(md->mempools);
2232 md->mempools = t->mempools;
2236 ret = dm_table_set_restrictions(t, md->queue, limits);
2242 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2243 rcu_assign_pointer(md->map, (void *)t);
2244 md->immutable_target_type = dm_table_get_immutable_target_type(t);
2247 dm_sync_table(md);
2255 static struct dm_table *__unbind(struct mapped_device *md)
2257 struct dm_table *map = rcu_dereference_protected(md->map, 1);
2263 RCU_INIT_POINTER(md->map, NULL);
2264 dm_sync_table(md);
2274 struct mapped_device *md;
2276 md = alloc_dev(minor);
2277 if (!md)
2280 dm_ima_reset_data(md);
2282 *result = md;
2287 * Functions to manage md->type.
2288 * All are required to hold md->type_lock.
2290 void dm_lock_md_type(struct mapped_device *md)
2292 mutex_lock(&md->type_lock);
2295 void dm_unlock_md_type(struct mapped_device *md)
2297 mutex_unlock(&md->type_lock);
2300 void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type)
2302 BUG_ON(!mutex_is_locked(&md->type_lock));
2303 md->type = type;
2306 enum dm_queue_mode dm_get_md_type(struct mapped_device *md)
2308 return md->type;
2311 struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
2313 return md->immutable_target_type;
2317 * Setup the DM device's queue based on md's type
2319 int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
2328 md->disk->fops = &dm_rq_blk_dops;
2329 r = dm_mq_init_request_queue(md, t);
2337 blk_queue_flag_set(QUEUE_FLAG_IO_STAT, md->queue);
2349 r = dm_table_set_restrictions(t, md->queue, &limits);
2357 mutex_lock(&md->table_devices_lock);
2358 r = add_disk(md->disk);
2359 mutex_unlock(&md->table_devices_lock);
2367 list_for_each_entry(td, &md->table_devices, list) {
2368 r = bd_link_disk_holder(td->dm_dev.bdev, md->disk);
2373 r = dm_sysfs_init(md);
2377 md->type = type;
2381 list_for_each_entry_continue_reverse(td, &md->table_devices, list)
2382 bd_unlink_disk_holder(td->dm_dev.bdev, md->disk);
2383 mutex_lock(&md->table_devices_lock);
2384 del_gendisk(md->disk);
2385 mutex_unlock(&md->table_devices_lock);
2391 struct mapped_device *md;
2399 md = idr_find(&_minor_idr, minor);
2400 if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) ||
2401 test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2402 md = NULL;
2405 dm_get(md);
2409 return md;
2413 void *dm_get_mdptr(struct mapped_device *md)
2415 return md->interface_ptr;
2418 void dm_set_mdptr(struct mapped_device *md, void *ptr)
2420 md->interface_ptr = ptr;
2423 void dm_get(struct mapped_device *md)
2425 atomic_inc(&md->holders);
2426 BUG_ON(test_bit(DMF_FREEING, &md->flags));
2429 int dm_hold(struct mapped_device *md)
2432 if (test_bit(DMF_FREEING, &md->flags)) {
2436 dm_get(md);
2442 const char *dm_device_name(struct mapped_device *md)
2444 return md->name;
2448 static void __dm_destroy(struct mapped_device *md, bool wait)
2456 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2457 set_bit(DMF_FREEING, &md->flags);
2460 blk_mark_disk_dead(md->disk);
2466 mutex_lock(&md->suspend_lock);
2467 map = dm_get_live_table(md, &srcu_idx);
2468 if (!dm_suspended_md(md)) {
2470 set_bit(DMF_SUSPENDED, &md->flags);
2471 set_bit(DMF_POST_SUSPENDING, &md->flags);
2475 dm_put_live_table(md, srcu_idx);
2476 mutex_unlock(&md->suspend_lock);
2485 while (atomic_read(&md->holders))
2487 else if (atomic_read(&md->holders))
2489 dm_device_name(md), atomic_read(&md->holders));
2491 dm_table_destroy(__unbind(md));
2492 free_dev(md);
2495 void dm_destroy(struct mapped_device *md)
2497 __dm_destroy(md, true);
2500 void dm_destroy_immediate(struct mapped_device *md)
2502 __dm_destroy(md, false);
2505 void dm_put(struct mapped_device *md)
2507 atomic_dec(&md->holders);
2511 static bool dm_in_flight_bios(struct mapped_device *md)
2517 sum += *per_cpu_ptr(md->pending_io, cpu);
2522 static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int task_state)
2528 prepare_to_wait(&md->wait, &wait, task_state);
2530 if (!dm_in_flight_bios(md))
2540 finish_wait(&md->wait, &wait);
2547 static int dm_wait_for_completion(struct mapped_device *md, unsigned int task_state)
2551 if (!queue_is_mq(md->queue))
2552 return dm_wait_for_bios_completion(md, task_state);
2555 if (!blk_mq_queue_inflight(md->queue))
2574 struct mapped_device *md = container_of(work, struct mapped_device, work);
2577 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2578 spin_lock_irq(&md->deferred_lock);
2579 bio = bio_list_pop(&md->deferred);
2580 spin_unlock_irq(&md->deferred_lock);
2590 static void dm_queue_flush(struct mapped_device *md)
2592 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2594 queue_work(md->wq, &md->work);
2600 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
2606 mutex_lock(&md->suspend_lock);
2609 if (!dm_suspended_md(md))
2619 live_map = dm_get_live_table_fast(md);
2621 limits = md->queue->limits;
2622 dm_put_live_table_fast(md);
2633 map = __bind(md, table, &limits);
2637 mutex_unlock(&md->suspend_lock);
2645 static int lock_fs(struct mapped_device *md)
2649 WARN_ON(test_bit(DMF_FROZEN, &md->flags));
2651 r = freeze_bdev(md->disk->part0);
2653 set_bit(DMF_FROZEN, &md->flags);
2657 static void unlock_fs(struct mapped_device *md)
2659 if (!test_bit(DMF_FROZEN, &md->flags))
2661 thaw_bdev(md->disk->part0);
2662 clear_bit(DMF_FROZEN, &md->flags);
2672 * are being added to md->deferred list.
2674 static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
2682 lockdep_assert_held(&md->suspend_lock);
2689 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2691 DMDEBUG("%s: suspending with flush", dm_device_name(md));
2706 r = lock_fs(md);
2722 * flush_workqueue(md->wq).
2724 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2726 synchronize_srcu(&md->io_barrier);
2729 * Stop md->queue before flushing md->wq in case request-based
2730 * dm defers requests to md->wq from md->queue.
2732 if (dm_request_based(md))
2733 dm_stop_queue(md->queue);
2735 flush_workqueue(md->wq);
2742 r = dm_wait_for_completion(md, task_state);
2744 set_bit(dmf_suspended_flag, &md->flags);
2747 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2749 synchronize_srcu(&md->io_barrier);
2753 dm_queue_flush(md);
2755 if (dm_request_based(md))
2756 dm_start_queue(md->queue);
2758 unlock_fs(md);
2782 int dm_suspend(struct mapped_device *md, unsigned int suspend_flags)
2788 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2790 if (dm_suspended_md(md)) {
2795 if (dm_suspended_internally_md(md)) {
2797 mutex_unlock(&md->suspend_lock);
2798 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2804 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2810 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
2814 set_bit(DMF_POST_SUSPENDING, &md->flags);
2816 clear_bit(DMF_POST_SUSPENDING, &md->flags);
2819 mutex_unlock(&md->suspend_lock);
2823 static int __dm_resume(struct mapped_device *md, struct dm_table *map)
2832 dm_queue_flush(md);
2839 if (dm_request_based(md))
2840 dm_start_queue(md->queue);
2842 unlock_fs(md);
2847 int dm_resume(struct mapped_device *md)
2854 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2856 if (!dm_suspended_md(md))
2859 if (dm_suspended_internally_md(md)) {
2861 mutex_unlock(&md->suspend_lock);
2862 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2868 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2872 r = __dm_resume(md, map);
2876 clear_bit(DMF_SUSPENDED, &md->flags);
2878 mutex_unlock(&md->suspend_lock);
2889 static void __dm_internal_suspend(struct mapped_device *md, unsigned int suspend_flags)
2893 lockdep_assert_held(&md->suspend_lock);
2895 if (md->internal_suspend_count++)
2898 if (dm_suspended_md(md)) {
2899 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2903 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2911 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2914 set_bit(DMF_POST_SUSPENDING, &md->flags);
2916 clear_bit(DMF_POST_SUSPENDING, &md->flags);
2919 static void __dm_internal_resume(struct mapped_device *md)
2924 BUG_ON(!md->internal_suspend_count);
2926 if (--md->internal_suspend_count)
2929 if (dm_suspended_md(md))
2932 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2933 r = __dm_resume(md, map);
2947 set_bit(DMF_SUSPENDED, &md->flags);
2950 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2952 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2955 void dm_internal_suspend_noflush(struct mapped_device *md)
2957 mutex_lock(&md->suspend_lock);
2958 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2959 mutex_unlock(&md->suspend_lock);
2963 void dm_internal_resume(struct mapped_device *md)
2965 mutex_lock(&md->suspend_lock);
2966 __dm_internal_resume(md);
2967 mutex_unlock(&md->suspend_lock);
2972 * Fast variants of internal suspend/resume hold md->suspend_lock,
2976 void dm_internal_suspend_fast(struct mapped_device *md)
2978 mutex_lock(&md->suspend_lock);
2979 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2982 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2983 synchronize_srcu(&md->io_barrier);
2984 flush_workqueue(md->wq);
2985 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2989 void dm_internal_resume_fast(struct mapped_device *md)
2991 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2994 dm_queue_flush(md);
2997 mutex_unlock(&md->suspend_lock);
3006 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
3025 r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj, action, envp);
3032 uint32_t dm_next_uevent_seq(struct mapped_device *md)
3034 return atomic_add_return(1, &md->uevent_seq);
3037 uint32_t dm_get_event_nr(struct mapped_device *md)
3039 return atomic_read(&md->event_nr);
3042 int dm_wait_event(struct mapped_device *md, int event_nr)
3044 return wait_event_interruptible(md->eventq,
3045 (event_nr != atomic_read(&md->event_nr)));
3048 void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
3052 spin_lock_irqsave(&md->uevent_lock, flags);
3053 list_add(elist, &md->uevent_list);
3054 spin_unlock_irqrestore(&md->uevent_lock, flags);
3059 * count on 'md'.
3061 struct gendisk *dm_disk(struct mapped_device *md)
3063 return md->disk;
3067 struct kobject *dm_kobject(struct mapped_device *md)
3069 return &md->kobj_holder.kobj;
3074 struct mapped_device *md;
3076 md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
3079 if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
3080 md = NULL;
3083 dm_get(md);
3087 return md;
3090 int dm_suspended_md(struct mapped_device *md)
3092 return test_bit(DMF_SUSPENDED, &md->flags);
3095 static int dm_post_suspending_md(struct mapped_device *md)
3097 return test_bit(DMF_POST_SUSPENDING, &md->flags);
3100 int dm_suspended_internally_md(struct mapped_device *md)
3102 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
3105 int dm_test_deferred_remove_flag(struct mapped_device *md)
3107 return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
3112 return dm_suspended_md(ti->table->md);
3118 return dm_post_suspending_md(ti->table->md);
3124 return __noflush_suspending(ti->table->md);
3154 struct mapped_device *md = bdev->bd_disk->private_data;
3159 table = dm_get_live_table(md, &srcu_idx);
3168 if (dm_suspended_md(md)) {
3180 dm_put_live_table(md, srcu_idx);
3363 struct mapped_device *md = bdev->bd_disk->private_data;
3367 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3377 dm_unprepare_ioctl(md, srcu_idx);