41  *       always at least 1 when the plug is hashed in the disk plug hash table.
57  * @disk: The gendisk the plug belongs to.
69 	struct gendisk		*disk;  member
80  *    from the disk hash table and that the initial reference to the zone
111 	struct gendisk	*disk;  member
116 static void disk_zone_wplug_sync_wp_offset(struct gendisk *disk,
123 	struct gendisk *disk = args->disk;  in disk_report_zones_cb()  local
125 	if (disk->zone_wplugs_hash)  in disk_report_zones_cb()
126 		disk_zone_wplug_sync_wp_offset(disk, zone);  in disk_report_zones_cb()
156 	struct gendisk *disk = bdev->bd_disk;  in blkdev_report_zones()  local
157 	sector_t capacity = get_capacity(disk);  in blkdev_report_zones()
159 		.disk = disk,  in blkdev_report_zones()
164 	if (!bdev_is_zoned(bdev) || WARN_ON_ONCE(!disk->fops->report_zones))  in blkdev_report_zones()
170 	return disk->fops->report_zones(disk, sector, nr_zones,  in blkdev_report_zones()
193  *    @sector..@sector+@nr_sectors. Specifying the entire disk sector range
373 static bool disk_zone_is_last(struct gendisk *disk, struct blk_zone *zone)  in disk_zone_is_last()  argument
375 	return zone->start + zone->len >= get_capacity(disk);  in disk_zone_is_last()
378 static bool disk_zone_is_full(struct gendisk *disk,  in disk_zone_is_full()  argument
381 	if (zno < disk->nr_zones - 1)  in disk_zone_is_full()
382 		return offset_in_zone >= disk->zone_capacity;  in disk_zone_is_full()
383 	return offset_in_zone >= disk->last_zone_capacity;  in disk_zone_is_full()
386 static bool disk_zone_wplug_is_full(struct gendisk *disk,  in disk_zone_wplug_is_full()  argument
389 	return disk_zone_is_full(disk, zwplug->zone_no, zwplug->wp_offset);  in disk_zone_wplug_is_full()
392 static bool disk_insert_zone_wplug(struct gendisk *disk,  in disk_insert_zone_wplug()  argument
398 		hash_32(zwplug->zone_no, disk->zone_wplugs_hash_bits);  in disk_insert_zone_wplug()
405 	spin_lock_irqsave(&disk->zone_wplugs_lock, flags);  in disk_insert_zone_wplug()
406 	hlist_for_each_entry_rcu(zwplg, &disk->zone_wplugs_hash[idx], node) {  in disk_insert_zone_wplug()
408 			spin_unlock_irqrestore(&disk->zone_wplugs_lock, flags);  in disk_insert_zone_wplug()
412 	hlist_add_head_rcu(&zwplug->node, &disk->zone_wplugs_hash[idx]);  in disk_insert_zone_wplug()
413 	atomic_inc(&disk->nr_zone_wplugs);  in disk_insert_zone_wplug()
414 	spin_unlock_irqrestore(&disk->zone_wplugs_lock, flags);  in disk_insert_zone_wplug()
419 static struct blk_zone_wplug *disk_get_hashed_zone_wplug(struct gendisk *disk,  in disk_get_hashed_zone_wplug()  argument
422 	unsigned int zno = disk_zone_no(disk, sector);  in disk_get_hashed_zone_wplug()
423 	unsigned int idx = hash_32(zno, disk->zone_wplugs_hash_bits);  in disk_get_hashed_zone_wplug()
428 	hlist_for_each_entry_rcu(zwplug, &disk->zone_wplugs_hash[idx], node) {  in disk_get_hashed_zone_wplug()
441 static inline struct blk_zone_wplug *disk_get_zone_wplug(struct gendisk *disk,  in disk_get_zone_wplug()  argument
444 	if (!atomic_read(&disk->nr_zone_wplugs))  in disk_get_zone_wplug()
447 	return disk_get_hashed_zone_wplug(disk, sector);  in disk_get_zone_wplug()
455 	mempool_free(zwplug, zwplug->disk->zone_wplugs_pool);  in disk_free_zone_wplug_rcu()
469 static inline bool disk_should_remove_zone_wplug(struct gendisk *disk,  in disk_should_remove_zone_wplug()  argument
497 	return !zwplug->wp_offset || disk_zone_wplug_is_full(disk, zwplug);  in disk_should_remove_zone_wplug()
500 static void disk_remove_zone_wplug(struct gendisk *disk,  in disk_remove_zone_wplug()  argument
514 	spin_lock_irqsave(&disk->zone_wplugs_lock, flags);  in disk_remove_zone_wplug()
516 	atomic_dec(&disk->nr_zone_wplugs);  in disk_remove_zone_wplug()
517 	spin_unlock_irqrestore(&disk->zone_wplugs_lock, flags);  in disk_remove_zone_wplug()
528 static struct blk_zone_wplug *disk_get_and_lock_zone_wplug(struct gendisk *disk,  in disk_get_and_lock_zone_wplug()  argument
532 	unsigned int zno = disk_zone_no(disk, sector);  in disk_get_and_lock_zone_wplug()
536 	zwplug = disk_get_zone_wplug(disk, sector);  in disk_get_and_lock_zone_wplug()
558 	zwplug = mempool_alloc(disk->zone_wplugs_pool, gfp_mask);  in disk_get_and_lock_zone_wplug()
567 	zwplug->wp_offset = bdev_offset_from_zone_start(disk->part0, sector);  in disk_get_and_lock_zone_wplug()
570 	zwplug->disk = disk;  in disk_get_and_lock_zone_wplug()
579 	if (!disk_insert_zone_wplug(disk, zwplug)) {  in disk_get_and_lock_zone_wplug()
581 		mempool_free(zwplug, disk->zone_wplugs_pool);  in disk_get_and_lock_zone_wplug()
591 	struct request_queue *q = zwplug->disk->queue;  in blk_zone_wplug_bio_io_error()
611 			    zwplug->disk->disk_name, zwplug->zone_no);  in disk_zone_wplug_abort()
622 static void disk_zone_wplug_set_wp_offset(struct gendisk *disk,  in disk_zone_wplug_set_wp_offset()  argument
638 	if (disk_should_remove_zone_wplug(disk, zwplug))  in disk_zone_wplug_set_wp_offset()
639 		disk_remove_zone_wplug(disk, zwplug);  in disk_zone_wplug_set_wp_offset()
665 static void disk_zone_wplug_sync_wp_offset(struct gendisk *disk,  in disk_zone_wplug_sync_wp_offset()  argument
671 	zwplug = disk_get_zone_wplug(disk, zone->start);  in disk_zone_wplug_sync_wp_offset()
677 		disk_zone_wplug_set_wp_offset(disk, zwplug,  in disk_zone_wplug_sync_wp_offset()
684 static int disk_zone_sync_wp_offset(struct gendisk *disk, sector_t sector)  in disk_zone_sync_wp_offset()  argument
687 		.disk = disk,  in disk_zone_sync_wp_offset()
690 	return disk->fops->report_zones(disk, sector, 1,  in disk_zone_sync_wp_offset()
697 	struct gendisk *disk = bio->bi_bdev->bd_disk;  in blk_zone_wplug_handle_reset_or_finish()  local
724 	zwplug = disk_get_zone_wplug(disk, sector);  in blk_zone_wplug_handle_reset_or_finish()
727 		disk_zone_wplug_set_wp_offset(disk, zwplug, wp_offset);  in blk_zone_wplug_handle_reset_or_finish()
737 	struct gendisk *disk = bio->bi_bdev->bd_disk;  in blk_zone_wplug_handle_reset_all()  local
747 	for (sector = 0; sector < get_capacity(disk);  in blk_zone_wplug_handle_reset_all()
748 	     sector += disk->queue->limits.chunk_sectors) {  in blk_zone_wplug_handle_reset_all()
749 		zwplug = disk_get_zone_wplug(disk, sector);  in blk_zone_wplug_handle_reset_all()
752 			disk_zone_wplug_set_wp_offset(disk, zwplug, 0);  in blk_zone_wplug_handle_reset_all()
761 static void disk_zone_wplug_schedule_bio_work(struct gendisk *disk,  in disk_zone_wplug_schedule_bio_work()  argument
771 	queue_work(disk->zone_wplugs_wq, &zwplug->bio_work);  in disk_zone_wplug_schedule_bio_work()
774 static inline void disk_zone_wplug_add_bio(struct gendisk *disk,  in disk_zone_wplug_add_bio()  argument
822 		disk_zone_wplug_schedule_bio_work(disk, zwplug);  in disk_zone_wplug_add_bio()
870 	struct gendisk *disk = q->disk;  in blk_zone_write_plug_init_request()  local
872 		disk_get_zone_wplug(disk, blk_rq_pos(req));  in blk_zone_write_plug_init_request()
894 	while (!disk_zone_wplug_is_full(disk, zwplug)) {  in blk_zone_write_plug_init_request()
930 	struct gendisk *disk = bio->bi_bdev->bd_disk;  in blk_zone_wplug_prepare_bio()  local
949 	if (disk_zone_wplug_is_full(disk, zwplug))  in blk_zone_wplug_prepare_bio()
985 	struct gendisk *disk = bio->bi_bdev->bd_disk;  in blk_zone_wplug_handle_write()  local
1017 	zwplug = disk_get_and_lock_zone_wplug(disk, sector, gfp_mask, &flags);  in blk_zone_wplug_handle_write()
1052 	disk_zone_wplug_add_bio(disk, zwplug, bio, nr_segs);  in blk_zone_wplug_handle_write()
1061 	struct gendisk *disk = bio->bi_bdev->bd_disk;  in blk_zone_wplug_handle_native_zone_append()  local
1070 	 * the plug in the disk hash table if the zone is fully written using  in blk_zone_wplug_handle_native_zone_append()
1073 	zwplug = disk_get_zone_wplug(disk, bio->bi_iter.bi_sector);  in blk_zone_wplug_handle_native_zone_append()
1091 				    disk->disk_name, zwplug->zone_no);  in blk_zone_wplug_handle_native_zone_append()
1094 	disk_remove_zone_wplug(disk, zwplug);  in blk_zone_wplug_handle_native_zone_append()
1179 static void disk_zone_wplug_unplug_bio(struct gendisk *disk,  in disk_zone_wplug_unplug_bio()  argument
1188 		disk_zone_wplug_schedule_bio_work(disk, zwplug);  in disk_zone_wplug_unplug_bio()
1199 	if (disk_should_remove_zone_wplug(disk, zwplug))  in disk_zone_wplug_unplug_bio()
1200 		disk_remove_zone_wplug(disk, zwplug);  in disk_zone_wplug_unplug_bio()
1207 	struct gendisk *disk = bio->bi_bdev->bd_disk;  in blk_zone_write_plug_bio_endio()  local
1209 		disk_get_zone_wplug(disk, bio->bi_iter.bi_sector);  in blk_zone_write_plug_bio_endio()
1247 		disk_zone_wplug_unplug_bio(disk, zwplug);  in blk_zone_write_plug_bio_endio()
1255 	struct gendisk *disk = req->q->disk;  in blk_zone_write_plug_finish_request()  local
1258 	zwplug = disk_get_zone_wplug(disk, req->__sector);  in blk_zone_write_plug_finish_request()
1270 	disk_zone_wplug_unplug_bio(disk, zwplug);  in blk_zone_write_plug_finish_request()
1322 static inline unsigned int disk_zone_wplugs_hash_size(struct gendisk *disk)  in disk_zone_wplugs_hash_size()  argument
1324 	return 1U << disk->zone_wplugs_hash_bits;  in disk_zone_wplugs_hash_size()
1327 void disk_init_zone_resources(struct gendisk *disk)  in disk_init_zone_resources()  argument
1329 	spin_lock_init(&disk->zone_wplugs_lock);  in disk_init_zone_resources()
1333  * For the size of a disk zone write plug hash table, use the size of the
1334  * zone write plug mempool, which is the maximum of the disk open zones and
1336  * 9 bits. For a disk that has no limits, mempool size defaults to 128.
1341 static int disk_alloc_zone_resources(struct gendisk *disk,  in disk_alloc_zone_resources()  argument
1346 	atomic_set(&disk->nr_zone_wplugs, 0);  in disk_alloc_zone_resources()
1347 	disk->zone_wplugs_hash_bits =  in disk_alloc_zone_resources()
1350 	disk->zone_wplugs_hash =  in disk_alloc_zone_resources()
1351 		kcalloc(disk_zone_wplugs_hash_size(disk),  in disk_alloc_zone_resources()
1353 	if (!disk->zone_wplugs_hash)  in disk_alloc_zone_resources()
1356 	for (i = 0; i < disk_zone_wplugs_hash_size(disk); i++)  in disk_alloc_zone_resources()
1357 		INIT_HLIST_HEAD(&disk->zone_wplugs_hash[i]);  in disk_alloc_zone_resources()
1359 	disk->zone_wplugs_pool = mempool_create_kmalloc_pool(pool_size,  in disk_alloc_zone_resources()
1361 	if (!disk->zone_wplugs_pool)  in disk_alloc_zone_resources()
1364 	disk->zone_wplugs_wq =  in disk_alloc_zone_resources()
1366 				pool_size, disk->disk_name);  in disk_alloc_zone_resources()
1367 	if (!disk->zone_wplugs_wq)  in disk_alloc_zone_resources()
1373 	mempool_destroy(disk->zone_wplugs_pool);  in disk_alloc_zone_resources()
1374 	disk->zone_wplugs_pool = NULL;  in disk_alloc_zone_resources()
1376 	kfree(disk->zone_wplugs_hash);  in disk_alloc_zone_resources()
1377 	disk->zone_wplugs_hash = NULL;  in disk_alloc_zone_resources()
1378 	disk->zone_wplugs_hash_bits = 0;  in disk_alloc_zone_resources()
1382 static void disk_destroy_zone_wplugs_hash_table(struct gendisk *disk)  in disk_destroy_zone_wplugs_hash_table()  argument
1387 	if (!disk->zone_wplugs_hash)  in disk_destroy_zone_wplugs_hash_table()
1391 	for (i = 0; i < disk_zone_wplugs_hash_size(disk); i++) {  in disk_destroy_zone_wplugs_hash_table()
1392 		while (!hlist_empty(&disk->zone_wplugs_hash[i])) {  in disk_destroy_zone_wplugs_hash_table()
1393 			zwplug = hlist_entry(disk->zone_wplugs_hash[i].first,  in disk_destroy_zone_wplugs_hash_table()
1396 			disk_remove_zone_wplug(disk, zwplug);  in disk_destroy_zone_wplugs_hash_table()
1401 	WARN_ON_ONCE(atomic_read(&disk->nr_zone_wplugs));  in disk_destroy_zone_wplugs_hash_table()
1402 	kfree(disk->zone_wplugs_hash);  in disk_destroy_zone_wplugs_hash_table()
1403 	disk->zone_wplugs_hash = NULL;  in disk_destroy_zone_wplugs_hash_table()
1404 	disk->zone_wplugs_hash_bits = 0;  in disk_destroy_zone_wplugs_hash_table()
1407 static unsigned int disk_set_conv_zones_bitmap(struct gendisk *disk,  in disk_set_conv_zones_bitmap()  argument
1413 	spin_lock_irqsave(&disk->zone_wplugs_lock, flags);  in disk_set_conv_zones_bitmap()
1415 		nr_conv_zones = bitmap_weight(bitmap, disk->nr_zones);  in disk_set_conv_zones_bitmap()
1416 	bitmap = rcu_replace_pointer(disk->conv_zones_bitmap, bitmap,  in disk_set_conv_zones_bitmap()
1417 				     lockdep_is_held(&disk->zone_wplugs_lock));  in disk_set_conv_zones_bitmap()
1418 	spin_unlock_irqrestore(&disk->zone_wplugs_lock, flags);  in disk_set_conv_zones_bitmap()
1425 void disk_free_zone_resources(struct gendisk *disk)  in disk_free_zone_resources()  argument
1427 	if (!disk->zone_wplugs_pool)  in disk_free_zone_resources()
1430 	if (disk->zone_wplugs_wq) {  in disk_free_zone_resources()
1431 		destroy_workqueue(disk->zone_wplugs_wq);  in disk_free_zone_resources()
1432 		disk->zone_wplugs_wq = NULL;  in disk_free_zone_resources()
1435 	disk_destroy_zone_wplugs_hash_table(disk);  in disk_free_zone_resources()
1443 	mempool_destroy(disk->zone_wplugs_pool);  in disk_free_zone_resources()
1444 	disk->zone_wplugs_pool = NULL;  in disk_free_zone_resources()
1446 	disk_set_conv_zones_bitmap(disk, NULL);  in disk_free_zone_resources()
1447 	disk->zone_capacity = 0;  in disk_free_zone_resources()
1448 	disk->last_zone_capacity = 0;  in disk_free_zone_resources()
1449 	disk->nr_zones = 0;  in disk_free_zone_resources()
1452 static inline bool disk_need_zone_resources(struct gendisk *disk)  in disk_need_zone_resources()  argument
1461 	return queue_is_mq(disk->queue) ||  in disk_need_zone_resources()
1462 		queue_emulates_zone_append(disk->queue);  in disk_need_zone_resources()
1465 static int disk_revalidate_zone_resources(struct gendisk *disk,  in disk_revalidate_zone_resources()  argument
1468 	struct queue_limits *lim = &disk->queue->limits;  in disk_revalidate_zone_resources()
1471 	if (!disk_need_zone_resources(disk))  in disk_revalidate_zone_resources()
1482 	if (!disk->zone_wplugs_hash)  in disk_revalidate_zone_resources()
1483 		return disk_alloc_zone_resources(disk, pool_size);  in disk_revalidate_zone_resources()
1489 	struct gendisk	*disk;  member
1498  * Update the disk zone resources information and device queue limits.
1499  * The disk queue is frozen when this is executed.
1501 static int disk_update_zone_resources(struct gendisk *disk,  in disk_update_zone_resources()  argument
1504 	struct request_queue *q = disk->queue;  in disk_update_zone_resources()
1509 	disk->nr_zones = args->nr_zones;  in disk_update_zone_resources()
1510 	disk->zone_capacity = args->zone_capacity;  in disk_update_zone_resources()
1511 	disk->last_zone_capacity = args->last_zone_capacity;  in disk_update_zone_resources()
1513 		disk_set_conv_zones_bitmap(disk, args->conv_zones_bitmap);  in disk_update_zone_resources()
1514 	if (nr_conv_zones >= disk->nr_zones) {  in disk_update_zone_resources()
1516 			disk->disk_name, nr_conv_zones, disk->nr_zones);  in disk_update_zone_resources()
1528 	nr_seq_zones = disk->nr_zones - nr_conv_zones;  in disk_update_zone_resources()
1534 	if (!disk->zone_wplugs_pool)  in disk_update_zone_resources()
1548 	mempool_resize(disk->zone_wplugs_pool, pool_size);  in disk_update_zone_resources()
1564 	struct gendisk *disk = args->disk;  in blk_revalidate_conv_zone()  local
1568 			disk->disk_name);  in blk_revalidate_conv_zone()
1572 	if (disk_zone_is_last(disk, zone))  in blk_revalidate_conv_zone()
1575 	if (!disk_need_zone_resources(disk))  in blk_revalidate_conv_zone()
1593 	struct gendisk *disk = args->disk;  in blk_revalidate_seq_zone()  local
1605 	if (disk_zone_is_last(disk, zone)) {  in blk_revalidate_seq_zone()
1609 			disk->disk_name);  in blk_revalidate_seq_zone()
1619 	if (!queue_emulates_zone_append(disk->queue) || !disk->zone_wplugs_hash)  in blk_revalidate_seq_zone()
1622 	disk_zone_wplug_sync_wp_offset(disk, zone);  in blk_revalidate_seq_zone()
1628 	zwplug = disk_get_and_lock_zone_wplug(disk, zone->wp, GFP_NOIO, &flags);  in blk_revalidate_seq_zone()
1644 	struct gendisk *disk = args->disk;  in blk_revalidate_zone_cb()  local
1645 	sector_t zone_sectors = disk->queue->limits.chunk_sectors;  in blk_revalidate_zone_cb()
1651 			disk->disk_name, args->sector, zone->start);  in blk_revalidate_zone_cb()
1655 	if (zone->start >= get_capacity(disk) || !zone->len) {  in blk_revalidate_zone_cb()
1657 			disk->disk_name, zone->start, zone->len);  in blk_revalidate_zone_cb()
1665 	if (!disk_zone_is_last(disk, zone)) {  in blk_revalidate_zone_cb()
1668 				disk->disk_name);  in blk_revalidate_zone_cb()
1673 			disk->disk_name);  in blk_revalidate_zone_cb()
1679 			disk->disk_name);  in blk_revalidate_zone_cb()
1694 			disk->disk_name, (int)zone->type, zone->start);  in blk_revalidate_zone_cb()
1706  * @disk:	Target disk
1717 int blk_revalidate_disk_zones(struct gendisk *disk)  in blk_revalidate_disk_zones()  argument
1719 	struct request_queue *q = disk->queue;  in blk_revalidate_disk_zones()
1721 	sector_t capacity = get_capacity(disk);  in blk_revalidate_disk_zones()
1738 			disk->disk_name, zone_sectors);  in blk_revalidate_disk_zones()
1746 	args.disk = disk;  in blk_revalidate_disk_zones()
1749 	ret = disk_revalidate_zone_resources(disk, args.nr_zones);  in blk_revalidate_disk_zones()
1755 	ret = disk->fops->report_zones(disk, 0, UINT_MAX,  in blk_revalidate_disk_zones()
1758 		pr_warn("%s: No zones reported\n", disk->disk_name);  in blk_revalidate_disk_zones()
1764 	 * If zones where reported, make sure that the entire disk capacity  in blk_revalidate_disk_zones()
1769 			disk->disk_name, args.sector);  in blk_revalidate_disk_zones()
1774 	 * Set the new disk zone parameters only once the queue is frozen and  in blk_revalidate_disk_zones()
1778 		ret = disk_update_zone_resources(disk, &args);  in blk_revalidate_disk_zones()
1780 		pr_warn("%s: failed to revalidate zones\n", disk->disk_name);  in blk_revalidate_disk_zones()
1784 		disk_free_zone_resources(disk);  in blk_revalidate_disk_zones()
1858 	struct gendisk *disk = q->disk;  in queue_zone_wplugs_show()  local
1862 	if (!disk->zone_wplugs_hash)  in queue_zone_wplugs_show()
1866 	for (i = 0; i < disk_zone_wplugs_hash_size(disk); i++)  in queue_zone_wplugs_show()
1867 		hlist_for_each_entry_rcu(zwplug, &disk->zone_wplugs_hash[i],  in queue_zone_wplugs_show()