Lines Matching full:md
309 int dm_deleting_md(struct mapped_device *md) in dm_deleting_md() argument
311 return test_bit(DMF_DELETING, &md->flags); in dm_deleting_md()
316 struct mapped_device *md; in dm_blk_open() local
320 md = disk->private_data; in dm_blk_open()
321 if (!md) in dm_blk_open()
324 if (test_bit(DMF_FREEING, &md->flags) || in dm_blk_open()
325 dm_deleting_md(md)) { in dm_blk_open()
326 md = NULL; in dm_blk_open()
330 dm_get(md); in dm_blk_open()
331 atomic_inc(&md->open_count); in dm_blk_open()
335 return md ? 0 : -ENXIO; in dm_blk_open()
340 struct mapped_device *md; in dm_blk_close() local
344 md = disk->private_data; in dm_blk_close()
345 if (WARN_ON(!md)) in dm_blk_close()
348 if (atomic_dec_and_test(&md->open_count) && in dm_blk_close()
349 (test_bit(DMF_DEFERRED_REMOVE, &md->flags))) in dm_blk_close()
352 dm_put(md); in dm_blk_close()
357 int dm_open_count(struct mapped_device *md) in dm_open_count() argument
359 return atomic_read(&md->open_count); in dm_open_count()
365 int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred) in dm_lock_for_deletion() argument
371 if (dm_open_count(md)) { in dm_lock_for_deletion()
374 set_bit(DMF_DEFERRED_REMOVE, &md->flags); in dm_lock_for_deletion()
375 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags)) in dm_lock_for_deletion()
378 set_bit(DMF_DELETING, &md->flags); in dm_lock_for_deletion()
385 int dm_cancel_deferred_remove(struct mapped_device *md) in dm_cancel_deferred_remove() argument
391 if (test_bit(DMF_DELETING, &md->flags)) in dm_cancel_deferred_remove()
394 clear_bit(DMF_DEFERRED_REMOVE, &md->flags); in dm_cancel_deferred_remove()
408 struct mapped_device *md = bdev->bd_disk->private_data; in dm_blk_getgeo() local
410 return dm_get_geometry(md, geo); in dm_blk_getgeo()
413 static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx, in dm_prepare_ioctl() argument
422 map = dm_get_live_table(md, srcu_idx); in dm_prepare_ioctl()
434 if (dm_suspended_md(md)) in dm_prepare_ioctl()
439 dm_put_live_table(md, *srcu_idx); in dm_prepare_ioctl()
447 static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx) in dm_unprepare_ioctl() argument
449 dm_put_live_table(md, srcu_idx); in dm_unprepare_ioctl()
455 struct mapped_device *md = bdev->bd_disk->private_data; in dm_blk_ioctl() local
458 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); in dm_blk_ioctl()
481 dm_unprepare_ioctl(md, srcu_idx); in dm_blk_ioctl()
524 unlikely(dm_stats_used(&io->md->stats))) { in dm_io_acct()
532 dm_stats_account_io(&io->md->stats, bio_data_dir(bio), in dm_io_acct()
574 static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio, gfp_t gfp_mask) in alloc_io() argument
580 clone = bio_alloc_clone(NULL, bio, gfp_mask, &md->mempools->io_bs); in alloc_io()
594 this_cpu_inc(*md->pending_io); in alloc_io()
596 io->md = md; in alloc_io()
600 if (blk_queue_io_stat(md->queue)) in alloc_io()
604 unlikely(dm_stats_used(&md->stats))) in alloc_io()
605 dm_stats_record_start(&md->stats, &io->stats_aux); in alloc_io()
618 struct mapped_device *md = ci->io->md; in alloc_tio() local
629 &md->mempools->bs); in alloc_tio()
648 clone->bi_bdev = md->disk->part0; in alloc_tio()
650 bio_set_dev(clone, md->disk->part0); in alloc_tio()
671 static void queue_io(struct mapped_device *md, struct bio *bio) in queue_io() argument
675 spin_lock_irqsave(&md->deferred_lock, flags); in queue_io()
676 bio_list_add(&md->deferred, bio); in queue_io()
677 spin_unlock_irqrestore(&md->deferred_lock, flags); in queue_io()
678 queue_work(md->wq, &md->work); in queue_io()
683 * function to access the md->map field, and make sure they call
686 struct dm_table *dm_get_live_table(struct mapped_device *md, in dm_get_live_table() argument
687 int *srcu_idx) __acquires(md->io_barrier) in dm_get_live_table()
689 *srcu_idx = srcu_read_lock(&md->io_barrier); in dm_get_live_table()
691 return srcu_dereference(md->map, &md->io_barrier); in dm_get_live_table()
694 void dm_put_live_table(struct mapped_device *md, in dm_put_live_table() argument
695 int srcu_idx) __releases(md->io_barrier) in dm_put_live_table()
697 srcu_read_unlock(&md->io_barrier, srcu_idx); in dm_put_live_table()
700 void dm_sync_table(struct mapped_device *md) in dm_sync_table() argument
702 synchronize_srcu(&md->io_barrier); in dm_sync_table()
710 static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU) in dm_get_live_table_fast() argument
713 return rcu_dereference(md->map); in dm_get_live_table_fast()
716 static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU) in dm_put_live_table_fast() argument
726 static struct table_device *open_table_device(struct mapped_device *md, in open_table_device() argument
735 td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id); in open_table_device()
753 if (md->disk->slave_dir) { in open_table_device()
754 r = bd_link_disk_holder(bdev, md->disk); in open_table_device()
765 list_add(&td->list, &md->table_devices); in open_table_device()
778 static void close_table_device(struct table_device *td, struct mapped_device *md) in close_table_device() argument
780 if (md->disk->slave_dir) in close_table_device()
781 bd_unlink_disk_holder(td->dm_dev.bdev, md->disk); in close_table_device()
784 if (unlikely(test_bit(DMF_DEFERRED_REMOVE, &md->flags))) in close_table_device()
806 int dm_get_table_device(struct mapped_device *md, dev_t dev, blk_mode_t mode, in dm_get_table_device() argument
811 mutex_lock(&md->table_devices_lock); in dm_get_table_device()
812 td = find_table_device(&md->table_devices, dev, mode); in dm_get_table_device()
814 td = open_table_device(md, dev, mode); in dm_get_table_device()
816 mutex_unlock(&md->table_devices_lock); in dm_get_table_device()
822 mutex_unlock(&md->table_devices_lock); in dm_get_table_device()
828 void dm_put_table_device(struct mapped_device *md, struct dm_dev *d) in dm_put_table_device() argument
832 mutex_lock(&md->table_devices_lock); in dm_put_table_device()
834 close_table_device(td, md); in dm_put_table_device()
835 mutex_unlock(&md->table_devices_lock); in dm_put_table_device()
841 int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo) in dm_get_geometry() argument
843 *geo = md->geometry; in dm_get_geometry()
851 int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo) in dm_set_geometry() argument
860 md->geometry = *geo; in dm_set_geometry()
865 static int __noflush_suspending(struct mapped_device *md) in __noflush_suspending() argument
867 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); in __noflush_suspending()
872 struct mapped_device *md = io->md; in dm_requeue_add_io() local
875 struct dm_io *next = md->requeue_list; in dm_requeue_add_io()
877 md->requeue_list = io; in dm_requeue_add_io()
880 bio_list_add_head(&md->deferred, io->orig_bio); in dm_requeue_add_io()
884 static void dm_kick_requeue(struct mapped_device *md, bool first_stage) in dm_kick_requeue() argument
887 queue_work(md->wq, &md->requeue_work); in dm_kick_requeue()
889 queue_work(md->wq, &md->work); in dm_kick_requeue()
902 struct mapped_device *md = io->md; in dm_handle_requeue() local
921 spin_lock_irqsave(&md->deferred_lock, flags); in dm_handle_requeue()
922 if ((__noflush_suspending(md) && in dm_handle_requeue()
923 !WARN_ON_ONCE(dm_is_zone_write(md, bio))) || in dm_handle_requeue()
934 spin_unlock_irqrestore(&md->deferred_lock, flags); in dm_handle_requeue()
938 dm_kick_requeue(md, first_stage); in dm_handle_requeue()
946 struct mapped_device *md = io->md; in __dm_io_complete() local
967 this_cpu_dec(*md->pending_io); in __dm_io_complete()
970 if (unlikely(wq_has_sleeper(&md->wait))) in __dm_io_complete()
971 wake_up(&md->wait); in __dm_io_complete()
983 queue_io(md, bio); in __dm_io_complete()
994 struct mapped_device *md = container_of(work, struct mapped_device, in dm_wq_requeue_work() local
1000 spin_lock_irqsave(&md->deferred_lock, flags); in dm_wq_requeue_work()
1001 io = md->requeue_list; in dm_wq_requeue_work()
1002 md->requeue_list = NULL; in dm_wq_requeue_work()
1003 spin_unlock_irqrestore(&md->deferred_lock, flags); in dm_wq_requeue_work()
1008 dm_io_rewind(io, &md->disk->bio_split); in dm_wq_requeue_work()
1062 __noflush_suspending(io->md))) { in dm_io_set_error()
1078 * count on 'md'. But _not_ imposing verification to avoid atomic_read(),
1080 static inline struct queue_limits *dm_get_queue_limits(struct mapped_device *md) in dm_get_queue_limits() argument
1082 return &md->queue->limits; in dm_get_queue_limits()
1085 void disable_discard(struct mapped_device *md) in disable_discard() argument
1087 struct queue_limits *limits = dm_get_queue_limits(md); in disable_discard()
1093 void disable_write_zeroes(struct mapped_device *md) in disable_write_zeroes() argument
1095 struct queue_limits *limits = dm_get_queue_limits(md); in disable_write_zeroes()
1113 struct mapped_device *md = io->md; in clone_endio() local
1118 disable_discard(md); in clone_endio()
1121 disable_write_zeroes(md); in clone_endio()
1139 if (WARN_ON_ONCE(dm_is_zone_write(md, bio))) in clone_endio()
1159 up(&md->swap_bios_semaphore); in clone_endio()
1191 min(max_sectors ? : queue_max_sectors(ti->table->md->queue), in __max_io_len()
1215 static struct dm_target *dm_dax_get_live_target(struct mapped_device *md, in dm_dax_get_live_target() argument
1217 __acquires(md->io_barrier) in dm_dax_get_live_target()
1222 map = dm_get_live_table(md, srcu_idx); in dm_dax_get_live_target()
1237 struct mapped_device *md = dax_get_private(dax_dev); in dm_dax_direct_access() local
1243 ti = dm_dax_get_live_target(md, sector, &srcu_idx); in dm_dax_direct_access()
1256 dm_put_live_table(md, srcu_idx); in dm_dax_direct_access()
1264 struct mapped_device *md = dax_get_private(dax_dev); in dm_dax_zero_page_range() local
1270 ti = dm_dax_get_live_target(md, sector, &srcu_idx); in dm_dax_zero_page_range()
1283 dm_put_live_table(md, srcu_idx); in dm_dax_zero_page_range()
1291 struct mapped_device *md = dax_get_private(dax_dev); in dm_dax_recovery_write() local
1297 ti = dm_dax_get_live_target(md, sector, &srcu_idx); in dm_dax_recovery_write()
1303 dm_put_live_table(md, srcu_idx); in dm_dax_recovery_write()
1386 trace_block_bio_remap(tgt_clone, disk_devt(io->md->disk), in dm_submit_bio_remap()
1392 static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch) in __set_swap_bios_limit() argument
1394 mutex_lock(&md->swap_bios_lock); in __set_swap_bios_limit()
1395 while (latch < md->swap_bios) { in __set_swap_bios_limit()
1397 down(&md->swap_bios_semaphore); in __set_swap_bios_limit()
1398 md->swap_bios--; in __set_swap_bios_limit()
1400 while (latch > md->swap_bios) { in __set_swap_bios_limit()
1402 up(&md->swap_bios_semaphore); in __set_swap_bios_limit()
1403 md->swap_bios++; in __set_swap_bios_limit()
1405 mutex_unlock(&md->swap_bios_lock); in __set_swap_bios_limit()
1413 struct mapped_device *md = io->md; in __map_bio() local
1427 if (unlikely(latch != md->swap_bios)) in __map_bio()
1428 __set_swap_bios_limit(md, latch); in __map_bio()
1429 down(&md->swap_bios_semaphore); in __map_bio()
1452 up(&md->swap_bios_semaphore); in __map_bio()
1491 mutex_lock(&ci->io->md->table_devices_lock); in alloc_multiple_bios()
1501 mutex_unlock(&ci->io->md->table_devices_lock); in alloc_multiple_bios()
1549 bio_init(&flush_bio, ci->io->md->disk->part0, NULL, 0, in __send_empty_flush()
1648 struct queue_limits *limits = dm_get_queue_limits(ti->table->md); in __process_abnormal_io()
1789 static inline bool dm_zone_bio_needs_split(struct mapped_device *md, in dm_zone_bio_needs_split() argument
1796 return dm_emulate_zone_append(md) && bio_straddles_zones(bio) && in dm_zone_bio_needs_split()
1799 static inline bool dm_zone_plug_bio(struct mapped_device *md, struct bio *bio) in dm_zone_plug_bio() argument
1801 return dm_emulate_zone_append(md) && blk_zone_plug_bio(bio, 0); in dm_zone_plug_bio()
1808 struct mapped_device *md = ci->io->md; in __send_zone_reset_all_emulated() local
1809 unsigned int zone_sectors = md->disk->queue->limits.chunk_sectors; in __send_zone_reset_all_emulated()
1823 ret = dm_zone_get_reset_bitmap(md, ci->map, ti->begin, in __send_zone_reset_all_emulated()
1917 static inline bool dm_zone_bio_needs_split(struct mapped_device *md, in dm_zone_bio_needs_split() argument
1922 static inline bool dm_zone_plug_bio(struct mapped_device *md, struct bio *bio) in dm_zone_plug_bio() argument
1935 static void dm_split_and_process_bio(struct mapped_device *md, in dm_split_and_process_bio() argument
1947 (is_abnormal || dm_zone_bio_needs_split(md, bio)); in dm_split_and_process_bio()
1969 if (static_branch_unlikely(&zoned_enabled) && dm_zone_plug_bio(md, bio)) in dm_split_and_process_bio()
1983 io = alloc_io(md, bio, GFP_NOWAIT); in dm_split_and_process_bio()
1990 io = alloc_io(md, bio, GFP_NOIO); in dm_split_and_process_bio()
2039 struct mapped_device *md = bio->bi_bdev->bd_disk->private_data; in dm_submit_bio() local
2043 map = dm_get_live_table(md, &srcu_idx); in dm_submit_bio()
2046 dm_device_name(md)); in dm_submit_bio()
2052 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) { in dm_submit_bio()
2058 queue_io(md, bio); in dm_submit_bio()
2062 dm_split_and_process_bio(md, map, bio); in dm_submit_bio()
2064 dm_put_live_table(md, srcu_idx); in dm_submit_bio()
2199 static void cleanup_mapped_device(struct mapped_device *md) in cleanup_mapped_device() argument
2201 if (md->wq) in cleanup_mapped_device()
2202 destroy_workqueue(md->wq); in cleanup_mapped_device()
2203 dm_free_md_mempools(md->mempools); in cleanup_mapped_device()
2205 if (md->dax_dev) { in cleanup_mapped_device()
2206 dax_remove_host(md->disk); in cleanup_mapped_device()
2207 kill_dax(md->dax_dev); in cleanup_mapped_device()
2208 put_dax(md->dax_dev); in cleanup_mapped_device()
2209 md->dax_dev = NULL; in cleanup_mapped_device()
2212 if (md->disk) { in cleanup_mapped_device()
2214 md->disk->private_data = NULL; in cleanup_mapped_device()
2216 if (dm_get_md_type(md) != DM_TYPE_NONE) { in cleanup_mapped_device()
2219 dm_sysfs_exit(md); in cleanup_mapped_device()
2220 list_for_each_entry(td, &md->table_devices, list) { in cleanup_mapped_device()
2222 md->disk); in cleanup_mapped_device()
2229 mutex_lock(&md->table_devices_lock); in cleanup_mapped_device()
2230 del_gendisk(md->disk); in cleanup_mapped_device()
2231 mutex_unlock(&md->table_devices_lock); in cleanup_mapped_device()
2233 dm_queue_destroy_crypto_profile(md->queue); in cleanup_mapped_device()
2234 put_disk(md->disk); in cleanup_mapped_device()
2237 if (md->pending_io) { in cleanup_mapped_device()
2238 free_percpu(md->pending_io); in cleanup_mapped_device()
2239 md->pending_io = NULL; in cleanup_mapped_device()
2242 cleanup_srcu_struct(&md->io_barrier); in cleanup_mapped_device()
2244 mutex_destroy(&md->suspend_lock); in cleanup_mapped_device()
2245 mutex_destroy(&md->type_lock); in cleanup_mapped_device()
2246 mutex_destroy(&md->table_devices_lock); in cleanup_mapped_device()
2247 mutex_destroy(&md->swap_bios_lock); in cleanup_mapped_device()
2249 dm_mq_cleanup_mapped_device(md); in cleanup_mapped_device()
2259 struct mapped_device *md; in alloc_dev() local
2262 md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id); in alloc_dev()
2263 if (!md) { in alloc_dev()
2279 r = init_srcu_struct(&md->io_barrier); in alloc_dev()
2283 md->numa_node_id = numa_node_id; in alloc_dev()
2284 md->init_tio_pdu = false; in alloc_dev()
2285 md->type = DM_TYPE_NONE; in alloc_dev()
2286 mutex_init(&md->suspend_lock); in alloc_dev()
2287 mutex_init(&md->type_lock); in alloc_dev()
2288 mutex_init(&md->table_devices_lock); in alloc_dev()
2289 spin_lock_init(&md->deferred_lock); in alloc_dev()
2290 atomic_set(&md->holders, 1); in alloc_dev()
2291 atomic_set(&md->open_count, 0); in alloc_dev()
2292 atomic_set(&md->event_nr, 0); in alloc_dev()
2293 atomic_set(&md->uevent_seq, 0); in alloc_dev()
2294 INIT_LIST_HEAD(&md->uevent_list); in alloc_dev()
2295 INIT_LIST_HEAD(&md->table_devices); in alloc_dev()
2296 spin_lock_init(&md->uevent_lock); in alloc_dev()
2299 * default to bio-based until DM table is loaded and md->type in alloc_dev()
2303 md->disk = blk_alloc_disk(NULL, md->numa_node_id); in alloc_dev()
2304 if (IS_ERR(md->disk)) { in alloc_dev()
2305 md->disk = NULL; in alloc_dev()
2308 md->queue = md->disk->queue; in alloc_dev()
2310 init_waitqueue_head(&md->wait); in alloc_dev()
2311 INIT_WORK(&md->work, dm_wq_work); in alloc_dev()
2312 INIT_WORK(&md->requeue_work, dm_wq_requeue_work); in alloc_dev()
2313 init_waitqueue_head(&md->eventq); in alloc_dev()
2314 init_completion(&md->kobj_holder.completion); in alloc_dev()
2316 md->requeue_list = NULL; in alloc_dev()
2317 md->swap_bios = get_swap_bios(); in alloc_dev()
2318 sema_init(&md->swap_bios_semaphore, md->swap_bios); in alloc_dev()
2319 mutex_init(&md->swap_bios_lock); in alloc_dev()
2321 md->disk->major = _major; in alloc_dev()
2322 md->disk->first_minor = minor; in alloc_dev()
2323 md->disk->minors = 1; in alloc_dev()
2324 md->disk->flags |= GENHD_FL_NO_PART; in alloc_dev()
2325 md->disk->fops = &dm_blk_dops; in alloc_dev()
2326 md->disk->private_data = md; in alloc_dev()
2327 sprintf(md->disk->disk_name, "dm-%d", minor); in alloc_dev()
2329 dax_dev = alloc_dax(md, &dm_dax_ops); in alloc_dev()
2336 md->dax_dev = dax_dev; in alloc_dev()
2337 if (dax_add_host(dax_dev, md->disk)) in alloc_dev()
2341 format_dev_t(md->name, MKDEV(_major, minor)); in alloc_dev()
2343 md->wq = alloc_workqueue("kdmflush/%s", WQ_MEM_RECLAIM, 0, md->name); in alloc_dev()
2344 if (!md->wq) in alloc_dev()
2347 md->pending_io = alloc_percpu(unsigned long); in alloc_dev()
2348 if (!md->pending_io) in alloc_dev()
2351 r = dm_stats_init(&md->stats); in alloc_dev()
2357 old_md = idr_replace(&_minor_idr, md, minor); in alloc_dev()
2362 return md; in alloc_dev()
2365 cleanup_mapped_device(md); in alloc_dev()
2371 kvfree(md); in alloc_dev()
2375 static void unlock_fs(struct mapped_device *md);
2377 static void free_dev(struct mapped_device *md) in free_dev() argument
2379 int minor = MINOR(disk_devt(md->disk)); in free_dev()
2381 unlock_fs(md); in free_dev()
2383 cleanup_mapped_device(md); in free_dev()
2385 WARN_ON_ONCE(!list_empty(&md->table_devices)); in free_dev()
2386 dm_stats_cleanup(&md->stats); in free_dev()
2390 kvfree(md); in free_dev()
2400 struct mapped_device *md = context; in event_callback() local
2402 spin_lock_irqsave(&md->uevent_lock, flags); in event_callback()
2403 list_splice_init(&md->uevent_list, &uevents); in event_callback()
2404 spin_unlock_irqrestore(&md->uevent_lock, flags); in event_callback()
2406 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj); in event_callback()
2408 atomic_inc(&md->event_nr); in event_callback()
2409 wake_up(&md->eventq); in event_callback()
2416 static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, in __bind() argument
2423 lockdep_assert_held(&md->suspend_lock); in __bind()
2430 if (size != dm_get_size(md)) in __bind()
2431 memset(&md->geometry, 0, sizeof(md->geometry)); in __bind()
2433 set_capacity(md->disk, size); in __bind()
2435 dm_table_event_callback(t, event_callback, md); in __bind()
2442 md->immutable_target = dm_table_get_immutable_target(t); in __bind()
2452 if (!md->mempools) { in __bind()
2453 md->mempools = t->mempools; in __bind()
2458 * The md may already have mempools that need changing. in __bind()
2462 dm_free_md_mempools(md->mempools); in __bind()
2463 md->mempools = t->mempools; in __bind()
2467 ret = dm_table_set_restrictions(t, md->queue, limits); in __bind()
2473 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); in __bind()
2474 rcu_assign_pointer(md->map, (void *)t); in __bind()
2475 md->immutable_target_type = dm_table_get_immutable_target_type(t); in __bind()
2478 dm_sync_table(md); in __bind()
2486 static struct dm_table *__unbind(struct mapped_device *md) in __unbind() argument
2488 struct dm_table *map = rcu_dereference_protected(md->map, 1); in __unbind()
2494 RCU_INIT_POINTER(md->map, NULL); in __unbind()
2495 dm_sync_table(md); in __unbind()
2505 struct mapped_device *md; in dm_create() local
2507 md = alloc_dev(minor); in dm_create()
2508 if (!md) in dm_create()
2511 dm_ima_reset_data(md); in dm_create()
2513 *result = md; in dm_create()
2518 * Functions to manage md->type.
2519 * All are required to hold md->type_lock.
2521 void dm_lock_md_type(struct mapped_device *md) in dm_lock_md_type() argument
2523 mutex_lock(&md->type_lock); in dm_lock_md_type()
2526 void dm_unlock_md_type(struct mapped_device *md) in dm_unlock_md_type() argument
2528 mutex_unlock(&md->type_lock); in dm_unlock_md_type()
2531 enum dm_queue_mode dm_get_md_type(struct mapped_device *md) in dm_get_md_type() argument
2533 return md->type; in dm_get_md_type()
2536 struct target_type *dm_get_immutable_target_type(struct mapped_device *md) in dm_get_immutable_target_type() argument
2538 return md->immutable_target_type; in dm_get_immutable_target_type()
2542 * Setup the DM device's queue based on md's type
2544 int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) in dm_setup_md_queue() argument
2554 md->disk->fops = &dm_rq_blk_dops; in dm_setup_md_queue()
2555 r = dm_mq_init_request_queue(md, t); in dm_setup_md_queue()
2567 r = dm_table_set_restrictions(t, md->queue, &limits); in dm_setup_md_queue()
2575 mutex_lock(&md->table_devices_lock); in dm_setup_md_queue()
2576 r = add_disk(md->disk); in dm_setup_md_queue()
2577 mutex_unlock(&md->table_devices_lock); in dm_setup_md_queue()
2585 list_for_each_entry(td, &md->table_devices, list) { in dm_setup_md_queue()
2586 r = bd_link_disk_holder(td->dm_dev.bdev, md->disk); in dm_setup_md_queue()
2591 r = dm_sysfs_init(md); in dm_setup_md_queue()
2595 md->type = type; in dm_setup_md_queue()
2599 list_for_each_entry_continue_reverse(td, &md->table_devices, list) in dm_setup_md_queue()
2600 bd_unlink_disk_holder(td->dm_dev.bdev, md->disk); in dm_setup_md_queue()
2601 mutex_lock(&md->table_devices_lock); in dm_setup_md_queue()
2602 del_gendisk(md->disk); in dm_setup_md_queue()
2603 mutex_unlock(&md->table_devices_lock); in dm_setup_md_queue()
2609 struct mapped_device *md; in dm_get_md() local
2617 md = idr_find(&_minor_idr, minor); in dm_get_md()
2618 if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) || in dm_get_md()
2619 test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { in dm_get_md()
2620 md = NULL; in dm_get_md()
2623 dm_get(md); in dm_get_md()
2627 return md; in dm_get_md()
2631 void *dm_get_mdptr(struct mapped_device *md) in dm_get_mdptr() argument
2633 return md->interface_ptr; in dm_get_mdptr()
2636 void dm_set_mdptr(struct mapped_device *md, void *ptr) in dm_set_mdptr() argument
2638 md->interface_ptr = ptr; in dm_set_mdptr()
2641 void dm_get(struct mapped_device *md) in dm_get() argument
2643 atomic_inc(&md->holders); in dm_get()
2644 BUG_ON(test_bit(DMF_FREEING, &md->flags)); in dm_get()
2647 int dm_hold(struct mapped_device *md) in dm_hold() argument
2650 if (test_bit(DMF_FREEING, &md->flags)) { in dm_hold()
2654 dm_get(md); in dm_hold()
2660 const char *dm_device_name(struct mapped_device *md) in dm_device_name() argument
2662 return md->name; in dm_device_name()
2666 static void __dm_destroy(struct mapped_device *md, bool wait) in __dm_destroy() argument
2674 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md)))); in __dm_destroy()
2675 set_bit(DMF_FREEING, &md->flags); in __dm_destroy()
2678 blk_mark_disk_dead(md->disk); in __dm_destroy()
2684 mutex_lock(&md->suspend_lock); in __dm_destroy()
2685 map = dm_get_live_table(md, &srcu_idx); in __dm_destroy()
2686 if (!dm_suspended_md(md)) { in __dm_destroy()
2688 set_bit(DMF_SUSPENDED, &md->flags); in __dm_destroy()
2689 set_bit(DMF_POST_SUSPENDING, &md->flags); in __dm_destroy()
2693 dm_put_live_table(md, srcu_idx); in __dm_destroy()
2694 mutex_unlock(&md->suspend_lock); in __dm_destroy()
2703 while (atomic_read(&md->holders)) in __dm_destroy()
2705 else if (atomic_read(&md->holders)) in __dm_destroy()
2707 dm_device_name(md), atomic_read(&md->holders)); in __dm_destroy()
2709 dm_table_destroy(__unbind(md)); in __dm_destroy()
2710 free_dev(md); in __dm_destroy()
2713 void dm_destroy(struct mapped_device *md) in dm_destroy() argument
2715 __dm_destroy(md, true); in dm_destroy()
2718 void dm_destroy_immediate(struct mapped_device *md) in dm_destroy_immediate() argument
2720 __dm_destroy(md, false); in dm_destroy_immediate()
2723 void dm_put(struct mapped_device *md) in dm_put() argument
2725 atomic_dec(&md->holders); in dm_put()
2729 static bool dm_in_flight_bios(struct mapped_device *md) in dm_in_flight_bios() argument
2735 sum += *per_cpu_ptr(md->pending_io, cpu); in dm_in_flight_bios()
2740 static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int task_state) in dm_wait_for_bios_completion() argument
2746 prepare_to_wait(&md->wait, &wait, task_state); in dm_wait_for_bios_completion()
2748 if (!dm_in_flight_bios(md)) in dm_wait_for_bios_completion()
2758 finish_wait(&md->wait, &wait); in dm_wait_for_bios_completion()
2765 static int dm_wait_for_completion(struct mapped_device *md, unsigned int task_state) in dm_wait_for_completion() argument
2769 if (!queue_is_mq(md->queue)) in dm_wait_for_completion()
2770 return dm_wait_for_bios_completion(md, task_state); in dm_wait_for_completion()
2773 if (!blk_mq_queue_inflight(md->queue)) in dm_wait_for_completion()
2792 struct mapped_device *md = container_of(work, struct mapped_device, work); in dm_wq_work() local
2795 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { in dm_wq_work()
2796 spin_lock_irq(&md->deferred_lock); in dm_wq_work()
2797 bio = bio_list_pop(&md->deferred); in dm_wq_work()
2798 spin_unlock_irq(&md->deferred_lock); in dm_wq_work()
2808 static void dm_queue_flush(struct mapped_device *md) in dm_queue_flush() argument
2810 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); in dm_queue_flush()
2812 queue_work(md->wq, &md->work); in dm_queue_flush()
2818 struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table) in dm_swap_table() argument
2824 mutex_lock(&md->suspend_lock); in dm_swap_table()
2827 if (!dm_suspended_md(md)) in dm_swap_table()
2837 live_map = dm_get_live_table_fast(md); in dm_swap_table()
2839 limits = md->queue->limits; in dm_swap_table()
2840 dm_put_live_table_fast(md); in dm_swap_table()
2851 map = __bind(md, table, &limits); in dm_swap_table()
2855 mutex_unlock(&md->suspend_lock); in dm_swap_table()
2863 static int lock_fs(struct mapped_device *md) in lock_fs() argument
2867 WARN_ON(test_bit(DMF_FROZEN, &md->flags)); in lock_fs()
2869 r = bdev_freeze(md->disk->part0); in lock_fs()
2871 set_bit(DMF_FROZEN, &md->flags); in lock_fs()
2875 static void unlock_fs(struct mapped_device *md) in unlock_fs() argument
2877 if (!test_bit(DMF_FROZEN, &md->flags)) in unlock_fs()
2879 bdev_thaw(md->disk->part0); in unlock_fs()
2880 clear_bit(DMF_FROZEN, &md->flags); in unlock_fs()
2890 * are being added to md->deferred list.
2892 static int __dm_suspend(struct mapped_device *md, struct dm_table *map, in __dm_suspend() argument
2900 lockdep_assert_held(&md->suspend_lock); in __dm_suspend()
2907 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); in __dm_suspend()
2909 DMDEBUG("%s: suspending with flush", dm_device_name(md)); in __dm_suspend()
2924 r = lock_fs(md); in __dm_suspend()
2940 * flush_workqueue(md->wq). in __dm_suspend()
2942 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); in __dm_suspend()
2944 synchronize_srcu(&md->io_barrier); in __dm_suspend()
2947 * Stop md->queue before flushing md->wq in case request-based in __dm_suspend()
2948 * dm defers requests to md->wq from md->queue. in __dm_suspend()
2950 if (dm_request_based(md)) in __dm_suspend()
2951 dm_stop_queue(md->queue); in __dm_suspend()
2953 flush_workqueue(md->wq); in __dm_suspend()
2960 r = dm_wait_for_completion(md, task_state); in __dm_suspend()
2962 set_bit(dmf_suspended_flag, &md->flags); in __dm_suspend()
2965 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); in __dm_suspend()
2967 synchronize_srcu(&md->io_barrier); in __dm_suspend()
2971 dm_queue_flush(md); in __dm_suspend()
2973 if (dm_request_based(md)) in __dm_suspend()
2974 dm_start_queue(md->queue); in __dm_suspend()
2976 unlock_fs(md); in __dm_suspend()
3000 int dm_suspend(struct mapped_device *md, unsigned int suspend_flags) in dm_suspend() argument
3006 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); in dm_suspend()
3008 if (dm_suspended_md(md)) { in dm_suspend()
3013 if (dm_suspended_internally_md(md)) { in dm_suspend()
3015 mutex_unlock(&md->suspend_lock); in dm_suspend()
3016 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); in dm_suspend()
3022 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); in dm_suspend()
3028 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED); in dm_suspend()
3032 set_bit(DMF_POST_SUSPENDING, &md->flags); in dm_suspend()
3034 clear_bit(DMF_POST_SUSPENDING, &md->flags); in dm_suspend()
3037 mutex_unlock(&md->suspend_lock); in dm_suspend()
3041 static int __dm_resume(struct mapped_device *md, struct dm_table *map) in __dm_resume() argument
3050 dm_queue_flush(md); in __dm_resume()
3057 if (dm_request_based(md)) in __dm_resume()
3058 dm_start_queue(md->queue); in __dm_resume()
3060 unlock_fs(md); in __dm_resume()
3065 int dm_resume(struct mapped_device *md) in dm_resume() argument
3072 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); in dm_resume()
3074 if (!dm_suspended_md(md)) in dm_resume()
3077 if (dm_suspended_internally_md(md)) { in dm_resume()
3079 mutex_unlock(&md->suspend_lock); in dm_resume()
3080 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); in dm_resume()
3086 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); in dm_resume()
3090 r = __dm_resume(md, map); in dm_resume()
3094 clear_bit(DMF_SUSPENDED, &md->flags); in dm_resume()
3096 mutex_unlock(&md->suspend_lock); in dm_resume()
3107 static void __dm_internal_suspend(struct mapped_device *md, unsigned int suspend_flags) in __dm_internal_suspend() argument
3111 lockdep_assert_held(&md->suspend_lock); in __dm_internal_suspend()
3113 if (md->internal_suspend_count++) in __dm_internal_suspend()
3116 if (dm_suspended_md(md)) { in __dm_internal_suspend()
3117 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); in __dm_internal_suspend()
3121 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); in __dm_internal_suspend()
3129 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE, in __dm_internal_suspend()
3132 set_bit(DMF_POST_SUSPENDING, &md->flags); in __dm_internal_suspend()
3134 clear_bit(DMF_POST_SUSPENDING, &md->flags); in __dm_internal_suspend()
3137 static void __dm_internal_resume(struct mapped_device *md) in __dm_internal_resume() argument
3142 BUG_ON(!md->internal_suspend_count); in __dm_internal_resume()
3144 if (--md->internal_suspend_count) in __dm_internal_resume()
3147 if (dm_suspended_md(md)) in __dm_internal_resume()
3150 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); in __dm_internal_resume()
3151 r = __dm_resume(md, map); in __dm_internal_resume()
3165 set_bit(DMF_SUSPENDED, &md->flags); in __dm_internal_resume()
3168 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); in __dm_internal_resume()
3170 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY); in __dm_internal_resume()
3173 void dm_internal_suspend_noflush(struct mapped_device *md) in dm_internal_suspend_noflush() argument
3175 mutex_lock(&md->suspend_lock); in dm_internal_suspend_noflush()
3176 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG); in dm_internal_suspend_noflush()
3177 mutex_unlock(&md->suspend_lock); in dm_internal_suspend_noflush()
3181 void dm_internal_resume(struct mapped_device *md) in dm_internal_resume() argument
3183 mutex_lock(&md->suspend_lock); in dm_internal_resume()
3184 __dm_internal_resume(md); in dm_internal_resume()
3185 mutex_unlock(&md->suspend_lock); in dm_internal_resume()
3190 * Fast variants of internal suspend/resume hold md->suspend_lock,
3194 void dm_internal_suspend_fast(struct mapped_device *md) in dm_internal_suspend_fast() argument
3196 mutex_lock(&md->suspend_lock); in dm_internal_suspend_fast()
3197 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) in dm_internal_suspend_fast()
3200 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); in dm_internal_suspend_fast()
3201 synchronize_srcu(&md->io_barrier); in dm_internal_suspend_fast()
3202 flush_workqueue(md->wq); in dm_internal_suspend_fast()
3203 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE); in dm_internal_suspend_fast()
3207 void dm_internal_resume_fast(struct mapped_device *md) in dm_internal_resume_fast() argument
3209 if (dm_suspended_md(md) || dm_suspended_internally_md(md)) in dm_internal_resume_fast()
3212 dm_queue_flush(md); in dm_internal_resume_fast()
3215 mutex_unlock(&md->suspend_lock); in dm_internal_resume_fast()
3224 int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action, in dm_kobject_uevent() argument
3243 r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj, action, envp); in dm_kobject_uevent()
3250 uint32_t dm_next_uevent_seq(struct mapped_device *md) in dm_next_uevent_seq() argument
3252 return atomic_add_return(1, &md->uevent_seq); in dm_next_uevent_seq()
3255 uint32_t dm_get_event_nr(struct mapped_device *md) in dm_get_event_nr() argument
3257 return atomic_read(&md->event_nr); in dm_get_event_nr()
3260 int dm_wait_event(struct mapped_device *md, int event_nr) in dm_wait_event() argument
3262 return wait_event_interruptible(md->eventq, in dm_wait_event()
3263 (event_nr != atomic_read(&md->event_nr))); in dm_wait_event()
3266 void dm_uevent_add(struct mapped_device *md, struct list_head *elist) in dm_uevent_add() argument
3270 spin_lock_irqsave(&md->uevent_lock, flags); in dm_uevent_add()
3271 list_add(elist, &md->uevent_list); in dm_uevent_add()
3272 spin_unlock_irqrestore(&md->uevent_lock, flags); in dm_uevent_add()
3277 * count on 'md'.
3279 struct gendisk *dm_disk(struct mapped_device *md) in dm_disk() argument
3281 return md->disk; in dm_disk()
3285 struct kobject *dm_kobject(struct mapped_device *md) in dm_kobject() argument
3287 return &md->kobj_holder.kobj; in dm_kobject()
3292 struct mapped_device *md; in dm_get_from_kobject() local
3294 md = container_of(kobj, struct mapped_device, kobj_holder.kobj); in dm_get_from_kobject()
3297 if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { in dm_get_from_kobject()
3298 md = NULL; in dm_get_from_kobject()
3301 dm_get(md); in dm_get_from_kobject()
3305 return md; in dm_get_from_kobject()
3308 int dm_suspended_md(struct mapped_device *md) in dm_suspended_md() argument
3310 return test_bit(DMF_SUSPENDED, &md->flags); in dm_suspended_md()
3313 static int dm_post_suspending_md(struct mapped_device *md) in dm_post_suspending_md() argument
3315 return test_bit(DMF_POST_SUSPENDING, &md->flags); in dm_post_suspending_md()
3318 int dm_suspended_internally_md(struct mapped_device *md) in dm_suspended_internally_md() argument
3320 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); in dm_suspended_internally_md()
3323 int dm_test_deferred_remove_flag(struct mapped_device *md) in dm_test_deferred_remove_flag() argument
3325 return test_bit(DMF_DEFERRED_REMOVE, &md->flags); in dm_test_deferred_remove_flag()
3330 return dm_suspended_md(ti->table->md); in dm_suspended()
3336 return dm_post_suspending_md(ti->table->md); in dm_post_suspending()
3342 return __noflush_suspending(ti->table->md); in dm_noflush_suspending()
3382 struct mapped_device *md = disk->private_data; in dm_blk_get_unique_id() local
3392 table = dm_get_live_table(md, &srcu_idx); in dm_blk_get_unique_id()
3406 dm_put_live_table(md, srcu_idx); in dm_blk_get_unique_id()
3425 struct mapped_device *md = bdev->bd_disk->private_data; in dm_call_pr() local
3430 table = dm_get_live_table(md, &srcu_idx); in dm_call_pr()
3439 if (dm_suspended_md(md)) { in dm_call_pr()
3451 dm_put_live_table(md, srcu_idx); in dm_call_pr()
3634 struct mapped_device *md = bdev->bd_disk->private_data; in dm_pr_clear() local
3638 r = dm_prepare_ioctl(md, &srcu_idx, &bdev); in dm_pr_clear()
3648 dm_unprepare_ioctl(md, srcu_idx); in dm_pr_clear()