22  * conf->seq_write is the number of the last batch successfully written.
23  * conf->seq_flush is the number of the last batch that was closed to
74 static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)  in stripe_hash()  argument
76 	int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & HASH_MASK;  in stripe_hash()
77 	return &conf->stripe_hashtbl[hash];  in stripe_hash()
80 static inline int stripe_hash_locks_hash(struct r5conf *conf, sector_t sect)  in stripe_hash_locks_hash()  argument
82 	return (sect >> RAID5_STRIPE_SHIFT(conf)) & STRIPE_HASH_LOCKS_MASK;  in stripe_hash_locks_hash()
85 static inline void lock_device_hash_lock(struct r5conf *conf, int hash)  in lock_device_hash_lock()  argument
86 	__acquires(&conf->device_lock)  in lock_device_hash_lock()
88 	spin_lock_irq(conf->hash_locks + hash);  in lock_device_hash_lock()
89 	spin_lock(&conf->device_lock);  in lock_device_hash_lock()
92 static inline void unlock_device_hash_lock(struct r5conf *conf, int hash)  in unlock_device_hash_lock()  argument
93 	__releases(&conf->device_lock)  in unlock_device_hash_lock()
95 	spin_unlock(&conf->device_lock);  in unlock_device_hash_lock()
96 	spin_unlock_irq(conf->hash_locks + hash);  in unlock_device_hash_lock()
99 static inline void lock_all_device_hash_locks_irq(struct r5conf *conf)  in lock_all_device_hash_locks_irq()  argument
100 	__acquires(&conf->device_lock)  in lock_all_device_hash_locks_irq()
103 	spin_lock_irq(conf->hash_locks);  in lock_all_device_hash_locks_irq()
105 		spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks);  in lock_all_device_hash_locks_irq()
106 	spin_lock(&conf->device_lock);  in lock_all_device_hash_locks_irq()
109 static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf)  in unlock_all_device_hash_locks_irq()  argument
110 	__releases(&conf->device_lock)  in unlock_all_device_hash_locks_irq()
113 	spin_unlock(&conf->device_lock);  in unlock_all_device_hash_locks_irq()
115 		spin_unlock(conf->hash_locks + i);  in unlock_all_device_hash_locks_irq()
116 	spin_unlock_irq(conf->hash_locks);  in unlock_all_device_hash_locks_irq()
158 static void print_raid5_conf(struct r5conf *conf);
177 	struct r5conf *conf = sh->raid_conf;  in raid5_wakeup_stripe_thread()  local
189 		group = conf->worker_groups + cpu_to_group(cpu);  in raid5_wakeup_stripe_thread()
198 	if (conf->worker_cnt_per_group == 0) {  in raid5_wakeup_stripe_thread()
199 		md_wakeup_thread(conf->mddev->thread);  in raid5_wakeup_stripe_thread()
203 	group = conf->worker_groups + cpu_to_group(sh->cpu);  in raid5_wakeup_stripe_thread()
211 	for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) {  in raid5_wakeup_stripe_thread()
221 static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,  in do_release_stripe()  argument
223 	__must_hold(&conf->device_lock)  in do_release_stripe()
229 	BUG_ON(atomic_read(&conf->active_stripes)==0);  in do_release_stripe()
231 	if (r5c_is_writeback(conf->log))  in do_release_stripe()
243 	    (conf->quiesce && r5c_is_writeback(conf->log) &&  in do_release_stripe()
253 			list_add_tail(&sh->lru, &conf->delayed_list);  in do_release_stripe()
255 			   sh->bm_seq - conf->seq_write > 0)  in do_release_stripe()
256 			list_add_tail(&sh->lru, &conf->bitmap_list);  in do_release_stripe()
260 			if (conf->worker_cnt_per_group == 0) {  in do_release_stripe()
263 							&conf->loprio_list);  in do_release_stripe()
266 							&conf->handle_list);  in do_release_stripe()
272 		md_wakeup_thread(conf->mddev->thread);  in do_release_stripe()
276 			if (atomic_dec_return(&conf->preread_active_stripes)  in do_release_stripe()
278 				md_wakeup_thread(conf->mddev->thread);  in do_release_stripe()
279 		atomic_dec(&conf->active_stripes);  in do_release_stripe()
281 			if (!r5c_is_writeback(conf->log))  in do_release_stripe()
287 				else if (injournal == conf->raid_disks - conf->max_degraded) {  in do_release_stripe()
290 						atomic_inc(&conf->r5c_cached_full_stripes);  in do_release_stripe()
292 						atomic_dec(&conf->r5c_cached_partial_stripes);  in do_release_stripe()
293 					list_add_tail(&sh->lru, &conf->r5c_full_stripe_list);  in do_release_stripe()
294 					r5c_check_cached_full_stripe(conf);  in do_release_stripe()
301 					list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list);  in do_release_stripe()
307 static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,  in __release_stripe()  argument
309 	__must_hold(&conf->device_lock)  in __release_stripe()
312 		do_release_stripe(conf, sh, temp_inactive_list);  in __release_stripe()
322 static void release_inactive_stripe_list(struct r5conf *conf,  in release_inactive_stripe_list()  argument
343 			spin_lock_irqsave(conf->hash_locks + hash, flags);  in release_inactive_stripe_list()
344 			if (list_empty(conf->inactive_list + hash) &&  in release_inactive_stripe_list()
346 				atomic_dec(&conf->empty_inactive_list_nr);  in release_inactive_stripe_list()
347 			list_splice_tail_init(list, conf->inactive_list + hash);  in release_inactive_stripe_list()
349 			spin_unlock_irqrestore(conf->hash_locks + hash, flags);  in release_inactive_stripe_list()
356 		wake_up(&conf->wait_for_stripe);  in release_inactive_stripe_list()
357 		if (atomic_read(&conf->active_stripes) == 0)  in release_inactive_stripe_list()
358 			wake_up(&conf->wait_for_quiescent);  in release_inactive_stripe_list()
359 		if (conf->retry_read_aligned)  in release_inactive_stripe_list()
360 			md_wakeup_thread(conf->mddev->thread);  in release_inactive_stripe_list()
364 static int release_stripe_list(struct r5conf *conf,  in release_stripe_list()  argument
366 	__must_hold(&conf->device_lock)  in release_stripe_list()
372 	head = llist_del_all(&conf->released_stripes);  in release_stripe_list()
386 		__release_stripe(conf, sh, &temp_inactive_list[hash]);  in release_stripe_list()
395 	struct r5conf *conf = sh->raid_conf;  in raid5_release_stripe()  local
406 	if (unlikely(!conf->mddev->thread) ||  in raid5_release_stripe()
409 	wakeup = llist_add(&sh->release_list, &conf->released_stripes);  in raid5_release_stripe()
411 		md_wakeup_thread(conf->mddev->thread);  in raid5_release_stripe()
415 	if (atomic_dec_and_lock_irqsave(&sh->count, &conf->device_lock, flags)) {  in raid5_release_stripe()
418 		do_release_stripe(conf, sh, &list);  in raid5_release_stripe()
419 		spin_unlock_irqrestore(&conf->device_lock, flags);  in raid5_release_stripe()
420 		release_inactive_stripe_list(conf, &list, hash);  in raid5_release_stripe()
432 static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)  in insert_hash()  argument
434 	struct hlist_head *hp = stripe_hash(conf, sh->sector);  in insert_hash()
443 static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash)  in get_free_stripe()  argument
448 	if (list_empty(conf->inactive_list + hash))  in get_free_stripe()
450 	first = (conf->inactive_list + hash)->next;  in get_free_stripe()
454 	atomic_inc(&conf->active_stripes);  in get_free_stripe()
456 	if (list_empty(conf->inactive_list + hash))  in get_free_stripe()
457 		atomic_inc(&conf->empty_inactive_list_nr);  in get_free_stripe()
501 init_stripe_shared_pages(struct stripe_head *sh, struct r5conf *conf, int disks)  in init_stripe_shared_pages()  argument
508 	/* Each of the sh->dev[i] need one conf->stripe_size */  in init_stripe_shared_pages()
509 	cnt = PAGE_SIZE / conf->stripe_size;  in init_stripe_shared_pages()
573 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
578 	struct r5conf *conf = sh->raid_conf;  in init_stripe()  local
589 	seq = read_seqcount_begin(&conf->gen_lock);  in init_stripe()
590 	sh->generation = conf->generation - previous;  in init_stripe()
591 	sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;  in init_stripe()
593 	stripe_set_idx(sector, conf, previous, sh);  in init_stripe()
610 	if (read_seqcount_retry(&conf->gen_lock, seq))  in init_stripe()
613 	insert_hash(conf, sh);  in init_stripe()
618 static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,  in __find_stripe()  argument
624 	hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)  in __find_stripe()
631 static struct stripe_head *find_get_stripe(struct r5conf *conf,  in find_get_stripe()  argument
637 	sh = __find_stripe(conf, sector, generation);  in find_get_stripe()
650 	spin_lock(&conf->device_lock);  in find_get_stripe()
653 			atomic_inc(&conf->active_stripes);  in find_get_stripe()
657 		if (!list_empty(conf->inactive_list + hash))  in find_get_stripe()
660 		if (list_empty(conf->inactive_list + hash) &&  in find_get_stripe()
662 			atomic_inc(&conf->empty_inactive_list_nr);  in find_get_stripe()
669 	spin_unlock(&conf->device_lock);  in find_get_stripe()
687  * Most calls to this function hold &conf->device_lock. Calls
691 int raid5_calc_degraded(struct r5conf *conf)  in raid5_calc_degraded()  argument
697 	for (i = 0; i < conf->previous_raid_disks; i++) {  in raid5_calc_degraded()
698 		struct md_rdev *rdev = READ_ONCE(conf->disks[i].rdev);  in raid5_calc_degraded()
701 			rdev = READ_ONCE(conf->disks[i].replacement);  in raid5_calc_degraded()
716 			if (conf->raid_disks >= conf->previous_raid_disks)  in raid5_calc_degraded()
719 	if (conf->raid_disks == conf->previous_raid_disks)  in raid5_calc_degraded()
722 	for (i = 0; i < conf->raid_disks; i++) {  in raid5_calc_degraded()
723 		struct md_rdev *rdev = READ_ONCE(conf->disks[i].rdev);  in raid5_calc_degraded()
726 			rdev = READ_ONCE(conf->disks[i].replacement);  in raid5_calc_degraded()
737 			if (conf->raid_disks <= conf->previous_raid_disks)  in raid5_calc_degraded()
745 static bool has_failed(struct r5conf *conf)  in has_failed()  argument
747 	int degraded = conf->mddev->degraded;  in has_failed()
749 	if (test_bit(MD_BROKEN, &conf->mddev->flags))  in has_failed()
752 	if (conf->mddev->reshape_position != MaxSector)  in has_failed()
753 		degraded = raid5_calc_degraded(conf);  in has_failed()
755 	return degraded > conf->max_degraded;  in has_failed()
791 static bool is_inactive_blocked(struct r5conf *conf, int hash)  in is_inactive_blocked()  argument
793 	if (list_empty(conf->inactive_list + hash))  in is_inactive_blocked()
796 	if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state))  in is_inactive_blocked()
799 	return (atomic_read(&conf->active_stripes) <  in is_inactive_blocked()
800 		(conf->max_nr_stripes * 3 / 4));  in is_inactive_blocked()
803 struct stripe_head *raid5_get_active_stripe(struct r5conf *conf,  in raid5_get_active_stripe()  argument
808 	int hash = stripe_hash_locks_hash(conf, sector);  in raid5_get_active_stripe()
813 	spin_lock_irq(conf->hash_locks + hash);  in raid5_get_active_stripe()
816 		if (!(flags & R5_GAS_NOQUIESCE) && conf->quiesce) {  in raid5_get_active_stripe()
829 			wait_event_lock_irq(conf->wait_for_quiescent,  in raid5_get_active_stripe()
830 					    !conf->quiesce,  in raid5_get_active_stripe()
831 					    *(conf->hash_locks + hash));  in raid5_get_active_stripe()
834 		sh = find_get_stripe(conf, sector, conf->generation - previous,  in raid5_get_active_stripe()
839 		if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) {  in raid5_get_active_stripe()
840 			sh = get_free_stripe(conf, hash);  in raid5_get_active_stripe()
842 				r5c_check_stripe_cache_usage(conf);  in raid5_get_active_stripe()
848 			if (!test_bit(R5_DID_ALLOC, &conf->cache_state))  in raid5_get_active_stripe()
849 				set_bit(R5_ALLOC_MORE, &conf->cache_state);  in raid5_get_active_stripe()
855 		set_bit(R5_INACTIVE_BLOCKED, &conf->cache_state);  in raid5_get_active_stripe()
856 		r5l_wake_reclaim(conf->log, 0);  in raid5_get_active_stripe()
864 		wait_event_lock_irq(conf->wait_for_stripe,  in raid5_get_active_stripe()
865 				    is_inactive_blocked(conf, hash),  in raid5_get_active_stripe()
866 				    *(conf->hash_locks + hash));  in raid5_get_active_stripe()
867 		clear_bit(R5_INACTIVE_BLOCKED, &conf->cache_state);  in raid5_get_active_stripe()
870 	spin_unlock_irq(conf->hash_locks + hash);  in raid5_get_active_stripe()
904 	struct r5conf *conf = sh->raid_conf;  in stripe_can_batch()  local
906 	if (raid5_has_log(conf) || raid5_has_ppl(conf))  in stripe_can_batch()
913 static void stripe_add_to_batch_list(struct r5conf *conf,  in stripe_add_to_batch_list()  argument
923 	if (!sector_div(tmp_sec, conf->chunk_sectors))  in stripe_add_to_batch_list()
925 	head_sector = sh->sector - RAID5_STRIPE_SECTORS(conf);  in stripe_add_to_batch_list()
931 		hash = stripe_hash_locks_hash(conf, head_sector);  in stripe_add_to_batch_list()
932 		spin_lock_irq(conf->hash_locks + hash);  in stripe_add_to_batch_list()
933 		head = find_get_stripe(conf, head_sector, conf->generation,  in stripe_add_to_batch_list()
935 		spin_unlock_irq(conf->hash_locks + hash);  in stripe_add_to_batch_list()
988 		if (atomic_dec_return(&conf->preread_active_stripes)  in stripe_add_to_batch_list()
990 			md_wakeup_thread(conf->mddev->thread);  in stripe_add_to_batch_list()
1011 static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)  in use_new_offset()  argument
1013 	sector_t progress = conf->reshape_progress;  in use_new_offset()
1015 	 * of conf->generation, or ->data_offset that was set before  in use_new_offset()
1021 	if (sh->generation == conf->generation - 1)  in use_new_offset()
1051 static void dispatch_defer_bios(struct r5conf *conf, int target,  in dispatch_defer_bios()  argument
1058 	if (conf->pending_data_cnt == 0)  in dispatch_defer_bios()
1061 	list_sort(NULL, &conf->pending_list, cmp_stripe);  in dispatch_defer_bios()
1063 	first = conf->pending_list.next;  in dispatch_defer_bios()
1066 	if (conf->next_pending_data)  in dispatch_defer_bios()
1067 		list_move_tail(&conf->pending_list,  in dispatch_defer_bios()
1068 				&conf->next_pending_data->sibling);  in dispatch_defer_bios()
1070 	while (!list_empty(&conf->pending_list)) {  in dispatch_defer_bios()
1071 		data = list_first_entry(&conf->pending_list,  in dispatch_defer_bios()
1078 		list_move(&data->sibling, &conf->free_list);  in dispatch_defer_bios()
1083 	conf->pending_data_cnt -= cnt;  in dispatch_defer_bios()
1084 	BUG_ON(conf->pending_data_cnt < 0 || cnt < target);  in dispatch_defer_bios()
1086 	if (next != &conf->pending_list)  in dispatch_defer_bios()
1087 		conf->next_pending_data = list_entry(next,  in dispatch_defer_bios()
1090 		conf->next_pending_data = NULL;  in dispatch_defer_bios()
1092 	if (first != &conf->pending_list)  in dispatch_defer_bios()
1093 		list_move_tail(&conf->pending_list, first);  in dispatch_defer_bios()
1096 static void flush_deferred_bios(struct r5conf *conf)  in flush_deferred_bios()  argument
1100 	if (conf->pending_data_cnt == 0)  in flush_deferred_bios()
1103 	spin_lock(&conf->pending_bios_lock);  in flush_deferred_bios()
1104 	dispatch_defer_bios(conf, conf->pending_data_cnt, &tmp);  in flush_deferred_bios()
1105 	BUG_ON(conf->pending_data_cnt != 0);  in flush_deferred_bios()
1106 	spin_unlock(&conf->pending_bios_lock);  in flush_deferred_bios()
1111 static void defer_issue_bios(struct r5conf *conf, sector_t sector,  in defer_issue_bios()  argument
1117 	spin_lock(&conf->pending_bios_lock);  in defer_issue_bios()
1118 	ent = list_first_entry(&conf->free_list, struct r5pending_data,  in defer_issue_bios()
1120 	list_move_tail(&ent->sibling, &conf->pending_list);  in defer_issue_bios()
1124 	conf->pending_data_cnt++;  in defer_issue_bios()
1125 	if (conf->pending_data_cnt >= PENDING_IO_MAX)  in defer_issue_bios()
1126 		dispatch_defer_bios(conf, PENDING_IO_ONE_FLUSH, &tmp);  in defer_issue_bios()
1128 	spin_unlock(&conf->pending_bios_lock);  in defer_issue_bios()
1140 	struct r5conf *conf = sh->raid_conf;  in ops_run_io()  local
1152 	should_defer = conf->batch_bio_dispatch && conf->group_cnt;  in ops_run_io()
1184 		rdev = conf->disks[i].rdev;  in ops_run_io()
1185 		rrdev = conf->disks[i].replacement;  in ops_run_io()
1214 						    RAID5_STRIPE_SECTORS(conf));  in ops_run_io()
1220 				if (!conf->mddev->external &&  in ops_run_io()
1221 				    conf->mddev->sb_flags) {  in ops_run_io()
1226 					md_check_recovery(conf->mddev);  in ops_run_io()
1234 				md_wait_for_blocked_rdev(rdev, conf->mddev);  in ops_run_io()
1237 				rdev_dec_pending(rdev, conf->mddev);  in ops_run_io()
1245 				md_sync_acct(rdev->bdev, RAID5_STRIPE_SECTORS(conf));  in ops_run_io()
1261 			if (use_new_offset(conf, sh))  in ops_run_io()
1284 			bi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);  in ops_run_io()
1286 			bi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);  in ops_run_io()
1296 			mddev_trace_remap(conf->mddev, bi, sh->dev[i].sector);  in ops_run_io()
1305 				md_sync_acct(rrdev->bdev, RAID5_STRIPE_SECTORS(conf));  in ops_run_io()
1321 			if (use_new_offset(conf, sh))  in ops_run_io()
1331 			rbi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);  in ops_run_io()
1333 			rbi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);  in ops_run_io()
1340 			mddev_trace_remap(conf->mddev, rbi, sh->dev[i].sector);  in ops_run_io()
1362 		defer_issue_bios(conf, head_sh->sector, &pending_bios);  in ops_run_io()
1376 	struct r5conf *conf = sh->raid_conf;  in async_copy_data()  local
1398 		if (len > 0 && page_offset + len > RAID5_STRIPE_SIZE(conf))  in async_copy_data()
1399 			clen = RAID5_STRIPE_SIZE(conf) - page_offset;  in async_copy_data()
1407 				if (conf->skip_copy &&  in async_copy_data()
1409 				    clen == RAID5_STRIPE_SIZE(conf) &&  in async_copy_data()
1434 	struct r5conf *conf = sh->raid_conf;  in ops_complete_biofill()  local
1455 				dev->sector + RAID5_STRIPE_SECTORS(conf)) {  in ops_complete_biofill()
1456 				rbi2 = r5_next_bio(conf, rbi, dev->sector);  in ops_complete_biofill()
1473 	struct r5conf *conf = sh->raid_conf;  in ops_run_biofill()  local
1488 				dev->sector + RAID5_STRIPE_SECTORS(conf)) {  in ops_run_biofill()
1492 				rbi = r5_next_bio(conf, rbi, dev->sector);  in ops_run_biofill()
1918 	struct r5conf *conf = sh->raid_conf;  in ops_run_biodrain()  local
1951 				dev->sector + RAID5_STRIPE_SECTORS(conf)) {  in ops_run_biodrain()
1962 							     r5c_is_writeback(conf->log));  in ops_run_biodrain()
1964 					    !r5c_is_writeback(conf->log)) {  in ops_run_biodrain()
1970 				wbi = r5_next_bio(conf, wbi, dev->sector);  in ops_run_biodrain()
2275 	struct r5conf *conf = sh->raid_conf;  in raid_run_ops()  local
2276 	int level = conf->level;  in raid_run_ops()
2279 	local_lock(&conf->percpu->lock);  in raid_run_ops()
2280 	percpu = this_cpu_ptr(conf->percpu);  in raid_run_ops()
2340 	local_unlock(&conf->percpu->lock);  in raid_run_ops()
2354 	int disks, struct r5conf *conf)  in alloc_stripe()  argument
2367 		sh->raid_conf = conf;  in alloc_stripe()
2370 		if (raid5_has_ppl(conf)) {  in alloc_stripe()
2378 		if (init_stripe_shared_pages(sh, conf, disks)) {  in alloc_stripe()
2386 static int grow_one_stripe(struct r5conf *conf, gfp_t gfp)  in grow_one_stripe()  argument
2390 	sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size, conf);  in grow_one_stripe()
2396 		free_stripe(conf->slab_cache, sh);  in grow_one_stripe()
2400 		conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;  in grow_one_stripe()
2402 	atomic_inc(&conf->active_stripes);  in grow_one_stripe()
2405 	WRITE_ONCE(conf->max_nr_stripes, conf->max_nr_stripes + 1);  in grow_one_stripe()
2409 static int grow_stripes(struct r5conf *conf, int num)  in grow_stripes()  argument
2412 	size_t namelen = sizeof(conf->cache_name[0]);  in grow_stripes()
2413 	int devs = max(conf->raid_disks, conf->previous_raid_disks);  in grow_stripes()
2415 	if (mddev_is_dm(conf->mddev))  in grow_stripes()
2416 		snprintf(conf->cache_name[0], namelen,  in grow_stripes()
2417 			"raid%d-%p", conf->level, conf->mddev);  in grow_stripes()
2419 		snprintf(conf->cache_name[0], namelen,  in grow_stripes()
2420 			"raid%d-%s", conf->level, mdname(conf->mddev));  in grow_stripes()
2421 	snprintf(conf->cache_name[1], namelen, "%.27s-alt", conf->cache_name[0]);  in grow_stripes()
2423 	conf->active_name = 0;  in grow_stripes()
2424 	sc = kmem_cache_create(conf->cache_name[conf->active_name],  in grow_stripes()
2429 	conf->slab_cache = sc;  in grow_stripes()
2430 	conf->pool_size = devs;  in grow_stripes()
2432 		if (!grow_one_stripe(conf, GFP_KERNEL))  in grow_stripes()
2479 static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)  in resize_chunks()  argument
2485 	if (conf->scribble_disks >= new_disks &&  in resize_chunks()
2486 	    conf->scribble_sectors >= new_sectors)  in resize_chunks()
2489 	raid5_quiesce(conf->mddev, true);  in resize_chunks()
2495 		percpu = per_cpu_ptr(conf->percpu, cpu);  in resize_chunks()
2497 				     new_sectors / RAID5_STRIPE_SECTORS(conf));  in resize_chunks()
2503 	raid5_quiesce(conf->mddev, false);  in resize_chunks()
2506 		conf->scribble_disks = new_disks;  in resize_chunks()
2507 		conf->scribble_sectors = new_sectors;  in resize_chunks()
2512 static int resize_stripes(struct r5conf *conf, int newsize)  in resize_stripes()  argument
2526 	 * 3/ reallocate conf->disks to be suitable bigger.  If this fails,  in resize_stripes()
2545 	md_allow_write(conf->mddev);  in resize_stripes()
2548 	sc = kmem_cache_create(conf->cache_name[1-conf->active_name],  in resize_stripes()
2555 	mutex_lock(&conf->cache_size_mutex);  in resize_stripes()
2557 	for (i = conf->max_nr_stripes; i; i--) {  in resize_stripes()
2558 		nsh = alloc_stripe(sc, GFP_KERNEL, newsize, conf);  in resize_stripes()
2572 		mutex_unlock(&conf->cache_size_mutex);  in resize_stripes()
2582 		lock_device_hash_lock(conf, hash);  in resize_stripes()
2583 		wait_event_cmd(conf->wait_for_stripe,  in resize_stripes()
2584 				    !list_empty(conf->inactive_list + hash),  in resize_stripes()
2585 				    unlock_device_hash_lock(conf, hash),  in resize_stripes()
2586 				    lock_device_hash_lock(conf, hash));  in resize_stripes()
2587 		osh = get_free_stripe(conf, hash);  in resize_stripes()
2588 		unlock_device_hash_lock(conf, hash);  in resize_stripes()
2596 		for(i=0; i<conf->pool_size; i++) {  in resize_stripes()
2602 		free_stripe(conf->slab_cache, osh);  in resize_stripes()
2604 		if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS +  in resize_stripes()
2605 		    !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) {  in resize_stripes()
2610 	kmem_cache_destroy(conf->slab_cache);  in resize_stripes()
2615 	 * conf->disks and the scribble region  in resize_stripes()
2619 		for (i = 0; i < conf->pool_size; i++)  in resize_stripes()
2620 			ndisks[i] = conf->disks[i];  in resize_stripes()
2622 		for (i = conf->pool_size; i < newsize; i++) {  in resize_stripes()
2629 			for (i = conf->pool_size; i < newsize; i++)  in resize_stripes()
2634 			kfree(conf->disks);  in resize_stripes()
2635 			conf->disks = ndisks;  in resize_stripes()
2640 	conf->slab_cache = sc;  in resize_stripes()
2641 	conf->active_name = 1-conf->active_name;  in resize_stripes()
2657 		for (i = conf->raid_disks; i < newsize; i++) {  in resize_stripes()
2665 		for (i=conf->raid_disks; i < newsize; i++)  in resize_stripes()
2680 		conf->pool_size = newsize;  in resize_stripes()
2681 	mutex_unlock(&conf->cache_size_mutex);  in resize_stripes()
2686 static int drop_one_stripe(struct r5conf *conf)  in drop_one_stripe()  argument
2689 	int hash = (conf->max_nr_stripes - 1) & STRIPE_HASH_LOCKS_MASK;  in drop_one_stripe()
2691 	spin_lock_irq(conf->hash_locks + hash);  in drop_one_stripe()
2692 	sh = get_free_stripe(conf, hash);  in drop_one_stripe()
2693 	spin_unlock_irq(conf->hash_locks + hash);  in drop_one_stripe()
2698 	free_stripe(conf->slab_cache, sh);  in drop_one_stripe()
2699 	atomic_dec(&conf->active_stripes);  in drop_one_stripe()
2700 	WRITE_ONCE(conf->max_nr_stripes, conf->max_nr_stripes - 1);  in drop_one_stripe()
2704 static void shrink_stripes(struct r5conf *conf)  in shrink_stripes()  argument
2706 	while (conf->max_nr_stripes &&  in shrink_stripes()
2707 	       drop_one_stripe(conf))  in shrink_stripes()
2710 	kmem_cache_destroy(conf->slab_cache);  in shrink_stripes()
2711 	conf->slab_cache = NULL;  in shrink_stripes()
2717 	struct r5conf *conf = sh->raid_conf;  in raid5_end_read_request()  local
2739 		rdev = conf->disks[i].replacement;  in raid5_end_read_request()
2741 		rdev = conf->disks[i].rdev;  in raid5_end_read_request()
2743 	if (use_new_offset(conf, sh))  in raid5_end_read_request()
2756 				mdname(conf->mddev), RAID5_STRIPE_SECTORS(conf),  in raid5_end_read_request()
2759 			atomic_add(RAID5_STRIPE_SECTORS(conf), &rdev->corrected_errors);  in raid5_end_read_request()
2784 				mdname(conf->mddev),  in raid5_end_read_request()
2787 		else if (conf->mddev->degraded >= conf->max_degraded) {  in raid5_end_read_request()
2791 				mdname(conf->mddev),  in raid5_end_read_request()
2799 				mdname(conf->mddev),  in raid5_end_read_request()
2803 			 > conf->max_nr_stripes) {  in raid5_end_read_request()
2806 				    mdname(conf->mddev),  in raid5_end_read_request()
2808 				    conf->max_nr_stripes);  in raid5_end_read_request()
2810 				    mdname(conf->mddev), rdev->bdev);  in raid5_end_read_request()
2831 				      rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), 0)))  in raid5_end_read_request()
2832 				md_error(conf->mddev, rdev);  in raid5_end_read_request()
2835 	rdev_dec_pending(rdev, conf->mddev);  in raid5_end_read_request()
2845 	struct r5conf *conf = sh->raid_conf;  in raid5_end_write_request()  local
2852 			rdev = conf->disks[i].rdev;  in raid5_end_write_request()
2856 			rdev = conf->disks[i].replacement;  in raid5_end_write_request()
2864 				rdev = conf->disks[i].rdev;  in raid5_end_write_request()
2878 			md_error(conf->mddev, rdev);  in raid5_end_write_request()
2880 					   RAID5_STRIPE_SECTORS(conf)))  in raid5_end_write_request()
2890 					     RAID5_STRIPE_SECTORS(conf))) {  in raid5_end_write_request()
2900 	rdev_dec_pending(rdev, conf->mddev);  in raid5_end_write_request()
2917 	struct r5conf *conf = mddev->private;  in raid5_error()  local
2924 	spin_lock_irqsave(&conf->device_lock, flags);  in raid5_error()
2927 	mddev->degraded = raid5_calc_degraded(conf);  in raid5_error()
2929 	if (has_failed(conf)) {  in raid5_error()
2930 		set_bit(MD_BROKEN, &conf->mddev->flags);  in raid5_error()
2931 		conf->recovery_disabled = mddev->recovery_disabled;  in raid5_error()
2934 			mdname(mddev), mddev->degraded, conf->raid_disks);  in raid5_error()
2937 			mdname(mddev), conf->raid_disks - mddev->degraded);  in raid5_error()
2940 	spin_unlock_irqrestore(&conf->device_lock, flags);  in raid5_error()
2953 sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,  in raid5_compute_sector()  argument
2963 	int algorithm = previous ? conf->prev_algo  in raid5_compute_sector()
2964 				 : conf->algorithm;  in raid5_compute_sector()
2965 	int sectors_per_chunk = previous ? conf->prev_chunk_sectors  in raid5_compute_sector()
2966 					 : conf->chunk_sectors;  in raid5_compute_sector()
2967 	int raid_disks = previous ? conf->previous_raid_disks  in raid5_compute_sector()
2968 				  : conf->raid_disks;  in raid5_compute_sector()
2969 	int data_disks = raid_disks - conf->max_degraded;  in raid5_compute_sector()
2989 	switch(conf->level) {  in raid5_compute_sector()
3157 	struct r5conf *conf = sh->raid_conf;  in raid5_compute_blocknr()  local
3159 	int data_disks = raid_disks - conf->max_degraded;  in raid5_compute_blocknr()
3161 	int sectors_per_chunk = previous ? conf->prev_chunk_sectors  in raid5_compute_blocknr()
3162 					 : conf->chunk_sectors;  in raid5_compute_blocknr()
3163 	int algorithm = previous ? conf->prev_algo  in raid5_compute_blocknr()
3164 				 : conf->algorithm;  in raid5_compute_blocknr()
3177 	switch(conf->level) {  in raid5_compute_blocknr()
3264 	check = raid5_compute_sector(conf, r_sector,  in raid5_compute_blocknr()
3269 			mdname(conf->mddev));  in raid5_compute_blocknr()
3296  *      stripe, we need to reserve (conf->raid_disk + 1) pages per stripe
3298  *      operation, we only need (conf->max_degraded + 1) pages per stripe.
3313 static inline bool delay_towrite(struct r5conf *conf,  in delay_towrite()  argument
3322 	if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&  in delay_towrite()
3336 	struct r5conf *conf = sh->raid_conf;  in schedule_reconstruction()  local
3337 	int level = conf->level;  in schedule_reconstruction()
3351 			if (dev->towrite && !delay_towrite(conf, dev, s)) {  in schedule_reconstruction()
3377 		if (s->locked + conf->max_degraded == disks)  in schedule_reconstruction()
3379 				atomic_inc(&conf->pending_full_writes);  in schedule_reconstruction()
3443 	struct r5conf *conf = sh->raid_conf;  in stripe_bio_overlaps()  local
3467 	if (forwrite && raid5_has_ppl(conf)) {  in stripe_bio_overlaps()
3493 		if (first + conf->chunk_sectors * (count - 1) != last)  in stripe_bio_overlaps()
3503 	struct r5conf *conf = sh->raid_conf;  in __add_stripe_bio()  local
3526 	md_write_inc(conf->mddev, bi);  in __add_stripe_bio()
3532 		     sector < sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf) &&  in __add_stripe_bio()
3534 		     bi = r5_next_bio(conf, bi, sh->dev[dd_idx].sector)) {  in __add_stripe_bio()
3538 		if (sector >= sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf))  in __add_stripe_bio()
3547 	if (conf->mddev->bitmap && firstwrite && !sh->batch_head) {  in __add_stripe_bio()
3548 		sh->bm_seq = conf->seq_flush+1;  in __add_stripe_bio()
3574 static void end_reshape(struct r5conf *conf);
3576 static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,  in stripe_set_idx()  argument
3580 		previous ? conf->prev_chunk_sectors : conf->chunk_sectors;  in stripe_set_idx()
3583 	int disks = previous ? conf->previous_raid_disks : conf->raid_disks;  in stripe_set_idx()
3585 	raid5_compute_sector(conf,  in stripe_set_idx()
3586 			     stripe * (disks - conf->max_degraded)  in stripe_set_idx()
3593 handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,  in handle_failed_stripe()  argument
3602 			struct md_rdev *rdev = conf->disks[i].rdev;  in handle_failed_stripe()
3613 					    RAID5_STRIPE_SECTORS(conf), 0))  in handle_failed_stripe()
3614 					md_error(conf->mddev, rdev);  in handle_failed_stripe()
3615 				rdev_dec_pending(rdev, conf->mddev);  in handle_failed_stripe()
3631 			sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {  in handle_failed_stripe()
3632 			struct bio *nextbi = r5_next_bio(conf, bi, sh->dev[i].sector);  in handle_failed_stripe()
3634 			md_write_end(conf->mddev);  in handle_failed_stripe()
3647 		       sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {  in handle_failed_stripe()
3648 			struct bio *bi2 = r5_next_bio(conf, bi, sh->dev[i].sector);  in handle_failed_stripe()
3650 			md_write_end(conf->mddev);  in handle_failed_stripe()
3659 		    s->failed > conf->max_degraded &&  in handle_failed_stripe()
3671 			       sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {  in handle_failed_stripe()
3673 					r5_next_bio(conf, bi, sh->dev[i].sector);  in handle_failed_stripe()
3688 		if (atomic_dec_and_test(&conf->pending_full_writes))  in handle_failed_stripe()
3689 			md_wakeup_thread(conf->mddev->thread);  in handle_failed_stripe()
3693 handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,  in handle_failed_sync()  argument
3712 	if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) {  in handle_failed_sync()
3716 		for (i = 0; i < conf->raid_disks; i++) {  in handle_failed_sync()
3717 			struct md_rdev *rdev = conf->disks[i].rdev;  in handle_failed_sync()
3723 						   RAID5_STRIPE_SECTORS(conf), 0))  in handle_failed_sync()
3725 			rdev = conf->disks[i].replacement;  in handle_failed_sync()
3731 						   RAID5_STRIPE_SECTORS(conf), 0))  in handle_failed_sync()
3735 			conf->recovery_disabled =  in handle_failed_sync()
3736 				conf->mddev->recovery_disabled;  in handle_failed_sync()
3738 	md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), !abort);  in handle_failed_sync()
3990 static void handle_stripe_clean_event(struct r5conf *conf,  in handle_stripe_clean_event()  argument
4021 					dev->sector + RAID5_STRIPE_SECTORS(conf)) {  in handle_stripe_clean_event()
4022 					wbi2 = r5_next_bio(conf, wbi, dev->sector);  in handle_stripe_clean_event()
4023 					md_write_end(conf->mddev);  in handle_stripe_clean_event()
4063 		spin_lock_irq(conf->hash_locks + hash);  in handle_stripe_clean_event()
4065 		spin_unlock_irq(conf->hash_locks + hash);  in handle_stripe_clean_event()
4080 		if (atomic_dec_and_test(&conf->pending_full_writes))  in handle_stripe_clean_event()
4081 			md_wakeup_thread(conf->mddev->thread);  in handle_stripe_clean_event()
4102 static int handle_stripe_dirtying(struct r5conf *conf,  in handle_stripe_dirtying()  argument
4108 	sector_t recovery_cp = conf->mddev->recovery_cp;  in handle_stripe_dirtying()
4117 	if (conf->rmw_level == PARITY_DISABLE_RMW ||  in handle_stripe_dirtying()
4125 			 conf->rmw_level, (unsigned long long)recovery_cp,  in handle_stripe_dirtying()
4130 		if (((dev->towrite && !delay_towrite(conf, dev, s)) ||  in handle_stripe_dirtying()
4157 	if ((rmw < rcw || (rmw == rcw && conf->rmw_level == PARITY_PREFER_RMW)) && rmw > 0) {  in handle_stripe_dirtying()
4159 		mddev_add_trace_msg(conf->mddev, "raid5 rmw %llu %d",  in handle_stripe_dirtying()
4180 						      &conf->cache_state)) {  in handle_stripe_dirtying()
4194 			if (((dev->towrite && !delay_towrite(conf, dev, s)) ||  in handle_stripe_dirtying()
4213 	if ((rcw < rmw || (rcw == rmw && conf->rmw_level != PARITY_PREFER_RMW)) && rcw > 0) {  in handle_stripe_dirtying()
4238 		if (rcw && !mddev_is_dm(conf->mddev))  in handle_stripe_dirtying()
4239 			blk_add_trace_msg(conf->mddev->gendisk->queue,  in handle_stripe_dirtying()
4266 static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,  in handle_parity_checks5()  argument
4327 			atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);  in handle_parity_checks5()
4328 			if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {  in handle_parity_checks5()
4332 						    "%llu-%llu\n", mdname(conf->mddev),  in handle_parity_checks5()
4335 						    RAID5_STRIPE_SECTORS(conf));  in handle_parity_checks5()
4358 static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,  in handle_parity_checks6()  argument
4454 			      mdname(conf->mddev),  in handle_parity_checks6()
4491 			atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);  in handle_parity_checks6()
4492 			if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {  in handle_parity_checks6()
4496 						    "%llu-%llu\n", mdname(conf->mddev),  in handle_parity_checks6()
4499 						    RAID5_STRIPE_SECTORS(conf));  in handle_parity_checks6()
4534 static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)  in handle_stripe_expansion()  argument
4551 			sector_t s = raid5_compute_sector(conf, bn, 0,  in handle_stripe_expansion()
4553 			sh2 = raid5_get_active_stripe(conf, NULL, s,  in handle_stripe_expansion()
4572 					  sh->dev[i].offset, RAID5_STRIPE_SIZE(conf),  in handle_stripe_expansion()
4577 			for (j = 0; j < conf->raid_disks; j++)  in handle_stripe_expansion()
4582 			if (j == conf->raid_disks) {  in handle_stripe_expansion()
4609 	struct r5conf *conf = sh->raid_conf;  in analyse_stripe()  local
4621 	s->log_failed = r5l_log_disk_error(conf);  in analyse_stripe()
4666 		rdev = conf->disks[i].replacement;  in analyse_stripe()
4668 		    rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) &&  in analyse_stripe()
4670 				       RAID5_STRIPE_SECTORS(conf)))  in analyse_stripe()
4677 			rdev = conf->disks[i].rdev;  in analyse_stripe()
4684 						   RAID5_STRIPE_SECTORS(conf));  in analyse_stripe()
4709 		else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= rdev->recovery_offset)  in analyse_stripe()
4723 			struct md_rdev *rdev2 = conf->disks[i].rdev;  in analyse_stripe()
4736 			struct md_rdev *rdev2 = conf->disks[i].rdev;  in analyse_stripe()
4745 			struct md_rdev *rdev2 = conf->disks[i].replacement;  in analyse_stripe()
4767 				rdev = conf->disks[i].replacement;  in analyse_stripe()
4788 		    sh->sector >= conf->mddev->recovery_cp ||  in analyse_stripe()
4789 		    test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery)))  in analyse_stripe()
4892 	struct r5conf *conf = sh->raid_conf;  in handle_stripe()  local
4949 	    test_bit(MD_SB_CHANGE_PENDING, &conf->mddev->sb_flags)) {  in handle_stripe()
4961 		rdev_dec_pending(s.blocked_rdev, conf->mddev);  in handle_stripe()
4981 	if (s.failed > conf->max_degraded ||  in handle_stripe()
4987 			handle_failed_stripe(conf, sh, &s, disks);  in handle_stripe()
4989 			handle_failed_sync(conf, sh, &s);  in handle_stripe()
5042 		|| conf->level < 6;  in handle_stripe()
5053 		handle_stripe_clean_event(conf, sh, disks);  in handle_stripe()
5056 		r5c_handle_cached_data_endio(conf, sh, disks);  in handle_stripe()
5075 	r5c_finish_stripe_write_out(conf, sh, &s);  in handle_stripe()
5087 		if (!r5c_is_writeback(conf->log)) {  in handle_stripe()
5089 				handle_stripe_dirtying(conf, sh, &s, disks);  in handle_stripe()
5095 				ret = r5c_try_caching_write(conf, sh, &s,  in handle_stripe()
5108 				ret = handle_stripe_dirtying(conf, sh, &s,  in handle_stripe()
5125 		if (conf->level == 6)  in handle_stripe()
5126 			handle_parity_checks6(conf, sh, &s, disks);  in handle_stripe()
5128 			handle_parity_checks5(conf, sh, &s, disks);  in handle_stripe()
5135 		for (i = 0; i < conf->raid_disks; i++)  in handle_stripe()
5149 		md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);  in handle_stripe()
5158 	if (s.failed <= conf->max_degraded && !conf->mddev->ro)  in handle_stripe()
5179 			= raid5_get_active_stripe(conf, NULL, sh->sector,  in handle_stripe()
5190 				atomic_inc(&conf->preread_active_stripes);  in handle_stripe()
5199 		for (i = conf->raid_disks; i--; ) {  in handle_stripe()
5209 		sh->disks = conf->raid_disks;  in handle_stripe()
5210 		stripe_set_idx(sh->sector, conf, 0, sh);  in handle_stripe()
5214 		atomic_dec(&conf->reshape_stripes);  in handle_stripe()
5215 		wake_up(&conf->wait_for_reshape);  in handle_stripe()
5216 		md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);  in handle_stripe()
5221 		handle_stripe_expansion(conf, sh);  in handle_stripe()
5226 		if (conf->mddev->external)  in handle_stripe()
5228 						 conf->mddev);  in handle_stripe()
5235 					 conf->mddev);  in handle_stripe()
5244 				rdev = conf->disks[i].rdev;  in handle_stripe()
5246 							RAID5_STRIPE_SECTORS(conf), 0))  in handle_stripe()
5247 					md_error(conf->mddev, rdev);  in handle_stripe()
5248 				rdev_dec_pending(rdev, conf->mddev);  in handle_stripe()
5251 				rdev = conf->disks[i].rdev;  in handle_stripe()
5253 						     RAID5_STRIPE_SECTORS(conf), 0);  in handle_stripe()
5254 				rdev_dec_pending(rdev, conf->mddev);  in handle_stripe()
5257 				rdev = conf->disks[i].replacement;  in handle_stripe()
5260 					rdev = conf->disks[i].rdev;  in handle_stripe()
5262 						     RAID5_STRIPE_SECTORS(conf), 0);  in handle_stripe()
5263 				rdev_dec_pending(rdev, conf->mddev);  in handle_stripe()
5277 		atomic_dec(&conf->preread_active_stripes);  in handle_stripe()
5278 		if (atomic_read(&conf->preread_active_stripes) <  in handle_stripe()
5280 			md_wakeup_thread(conf->mddev->thread);  in handle_stripe()
5286 static void raid5_activate_delayed(struct r5conf *conf)  in raid5_activate_delayed()  argument
5287 	__must_hold(&conf->device_lock)  in raid5_activate_delayed()
5289 	if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {  in raid5_activate_delayed()
5290 		while (!list_empty(&conf->delayed_list)) {  in raid5_activate_delayed()
5291 			struct list_head *l = conf->delayed_list.next;  in raid5_activate_delayed()
5297 				atomic_inc(&conf->preread_active_stripes);  in raid5_activate_delayed()
5298 			list_add_tail(&sh->lru, &conf->hold_list);  in raid5_activate_delayed()
5304 static void activate_bit_delay(struct r5conf *conf,  in activate_bit_delay()  argument
5306 	__must_hold(&conf->device_lock)  in activate_bit_delay()
5309 	list_add(&head, &conf->bitmap_list);  in activate_bit_delay()
5310 	list_del_init(&conf->bitmap_list);  in activate_bit_delay()
5317 		__release_stripe(conf, sh, &temp_inactive_list[hash]);  in activate_bit_delay()
5323 	struct r5conf *conf = mddev->private;  in in_chunk_boundary()  local
5328 	chunk_sectors = min(conf->chunk_sectors, conf->prev_chunk_sectors);  in in_chunk_boundary()
5337 static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)  in add_bio_to_retry()  argument
5341 	spin_lock_irqsave(&conf->device_lock, flags);  in add_bio_to_retry()
5343 	bi->bi_next = conf->retry_read_aligned_list;  in add_bio_to_retry()
5344 	conf->retry_read_aligned_list = bi;  in add_bio_to_retry()
5346 	spin_unlock_irqrestore(&conf->device_lock, flags);  in add_bio_to_retry()
5347 	md_wakeup_thread(conf->mddev->thread);  in add_bio_to_retry()
5350 static struct bio *remove_bio_from_retry(struct r5conf *conf,  in remove_bio_from_retry()  argument
5355 	bi = conf->retry_read_aligned;  in remove_bio_from_retry()
5357 		*offset = conf->retry_read_offset;  in remove_bio_from_retry()
5358 		conf->retry_read_aligned = NULL;  in remove_bio_from_retry()
5361 	bi = conf->retry_read_aligned_list;  in remove_bio_from_retry()
5363 		conf->retry_read_aligned_list = bi->bi_next;  in remove_bio_from_retry()
5382 	struct r5conf *conf = mddev->private;  in raid5_align_endio()  local
5387 	rdev_dec_pending(rdev, conf->mddev);  in raid5_align_endio()
5391 		if (atomic_dec_and_test(&conf->active_aligned_reads))  in raid5_align_endio()
5392 			wake_up(&conf->wait_for_quiescent);  in raid5_align_endio()
5398 	add_bio_to_retry(raid_bi, conf);  in raid5_align_endio()
5403 	struct r5conf *conf = mddev->private;  in raid5_read_one_chunk()  local
5415 	sector = raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector, 0,  in raid5_read_one_chunk()
5419 	if (r5c_big_stripe_cached(conf, sector))  in raid5_read_one_chunk()
5422 	rdev = conf->disks[dd_idx].replacement;  in raid5_read_one_chunk()
5425 		rdev = conf->disks[dd_idx].rdev;  in raid5_read_one_chunk()
5454 	if (conf->quiesce == 0) {  in raid5_read_one_chunk()
5455 		atomic_inc(&conf->active_aligned_reads);  in raid5_read_one_chunk()
5459 	if (!did_inc || smp_load_acquire(&conf->quiesce) != 0) {  in raid5_read_one_chunk()
5463 		if (did_inc && atomic_dec_and_test(&conf->active_aligned_reads))  in raid5_read_one_chunk()
5464 			wake_up(&conf->wait_for_quiescent);  in raid5_read_one_chunk()
5465 		spin_lock_irq(&conf->device_lock);  in raid5_read_one_chunk()
5466 		wait_event_lock_irq(conf->wait_for_quiescent, conf->quiesce == 0,  in raid5_read_one_chunk()
5467 				    conf->device_lock);  in raid5_read_one_chunk()
5468 		atomic_inc(&conf->active_aligned_reads);  in raid5_read_one_chunk()
5469 		spin_unlock_irq(&conf->device_lock);  in raid5_read_one_chunk()
5485 		struct r5conf *conf = mddev->private;  in chunk_aligned_read()  local
5486 		split = bio_split(raid_bio, sectors, GFP_NOIO, &conf->bio_split);  in chunk_aligned_read()
5508 static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group)  in __get_priority_stripe()  argument
5509 	__must_hold(&conf->device_lock)  in __get_priority_stripe()
5514 	bool second_try = !r5c_is_writeback(conf->log) &&  in __get_priority_stripe()
5515 		!r5l_log_disk_error(conf);  in __get_priority_stripe()
5516 	bool try_loprio = test_bit(R5C_LOG_TIGHT, &conf->cache_state) ||  in __get_priority_stripe()
5517 		r5l_log_disk_error(conf);  in __get_priority_stripe()
5522 	if (conf->worker_cnt_per_group == 0) {  in __get_priority_stripe()
5523 		handle_list = try_loprio ? &conf->loprio_list :  in __get_priority_stripe()
5524 					&conf->handle_list;  in __get_priority_stripe()
5526 		handle_list = try_loprio ? &conf->worker_groups[group].loprio_list :  in __get_priority_stripe()
5527 				&conf->worker_groups[group].handle_list;  in __get_priority_stripe()
5528 		wg = &conf->worker_groups[group];  in __get_priority_stripe()
5531 		for (i = 0; i < conf->group_cnt; i++) {  in __get_priority_stripe()
5532 			handle_list = try_loprio ? &conf->worker_groups[i].loprio_list :  in __get_priority_stripe()
5533 				&conf->worker_groups[i].handle_list;  in __get_priority_stripe()
5534 			wg = &conf->worker_groups[i];  in __get_priority_stripe()
5543 		  list_empty(&conf->hold_list) ? "empty" : "busy",  in __get_priority_stripe()
5544 		  atomic_read(&conf->pending_full_writes), conf->bypass_count);  in __get_priority_stripe()
5549 		if (list_empty(&conf->hold_list))  in __get_priority_stripe()
5550 			conf->bypass_count = 0;  in __get_priority_stripe()
5552 			if (conf->hold_list.next == conf->last_hold)  in __get_priority_stripe()
5553 				conf->bypass_count++;  in __get_priority_stripe()
5555 				conf->last_hold = conf->hold_list.next;  in __get_priority_stripe()
5556 				conf->bypass_count -= conf->bypass_threshold;  in __get_priority_stripe()
5557 				if (conf->bypass_count < 0)  in __get_priority_stripe()
5558 					conf->bypass_count = 0;  in __get_priority_stripe()
5561 	} else if (!list_empty(&conf->hold_list) &&  in __get_priority_stripe()
5562 		   ((conf->bypass_threshold &&  in __get_priority_stripe()
5563 		     conf->bypass_count > conf->bypass_threshold) ||  in __get_priority_stripe()
5564 		    atomic_read(&conf->pending_full_writes) == 0)) {  in __get_priority_stripe()
5566 		list_for_each_entry(tmp, &conf->hold_list,  lru) {  in __get_priority_stripe()
5567 			if (conf->worker_cnt_per_group == 0 ||  in __get_priority_stripe()
5577 			conf->bypass_count -= conf->bypass_threshold;  in __get_priority_stripe()
5578 			if (conf->bypass_count < 0)  in __get_priority_stripe()
5579 				conf->bypass_count = 0;  in __get_priority_stripe()
5613 	struct r5conf *conf = mddev->private;  in raid5_unplug()  local
5618 		spin_lock_irq(&conf->device_lock);  in raid5_unplug()
5634 			__release_stripe(conf, sh, &cb->temp_inactive_list[hash]);  in raid5_unplug()
5637 		spin_unlock_irq(&conf->device_lock);  in raid5_unplug()
5639 	release_inactive_stripe_list(conf, cb->temp_inactive_list,  in raid5_unplug()
5676 	struct r5conf *conf = mddev->private;  in make_discard_request()  local
5689 	logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);  in make_discard_request()
5694 	stripe_sectors = conf->chunk_sectors *  in make_discard_request()
5695 		(conf->raid_disks - conf->max_degraded);  in make_discard_request()
5700 	logical_sector *= conf->chunk_sectors;  in make_discard_request()
5701 	last_sector *= conf->chunk_sectors;  in make_discard_request()
5704 	     logical_sector += RAID5_STRIPE_SECTORS(conf)) {  in make_discard_request()
5708 		sh = raid5_get_active_stripe(conf, NULL, logical_sector, 0);  in make_discard_request()
5718 		for (d = 0; d < conf->raid_disks; d++) {  in make_discard_request()
5732 		for (d = 0; d < conf->raid_disks; d++) {  in make_discard_request()
5742 		if (conf->mddev->bitmap) {  in make_discard_request()
5743 			sh->bm_seq = conf->seq_flush + 1;  in make_discard_request()
5750 			atomic_inc(&conf->preread_active_stripes);  in make_discard_request()
5771 static bool stripe_ahead_of_reshape(struct mddev *mddev, struct r5conf *conf,  in stripe_ahead_of_reshape()  argument
5786 	spin_lock_irq(&conf->device_lock);  in stripe_ahead_of_reshape()
5789 				     conf->reshape_progress))  in stripe_ahead_of_reshape()
5793 	spin_unlock_irq(&conf->device_lock);  in stripe_ahead_of_reshape()
5798 static int add_all_stripe_bios(struct r5conf *conf,  in add_all_stripe_bios()  argument
5825 			md_wakeup_thread(conf->mddev->thread);  in add_all_stripe_bios()
5843 			  RAID5_STRIPE_SHIFT(conf), ctx->sectors_to_do);  in add_all_stripe_bios()
5858 		struct r5conf *conf, sector_t logical_sector)  in get_reshape_loc()  argument
5870 	spin_lock_irq(&conf->device_lock);  in get_reshape_loc()
5871 	reshape_progress = conf->reshape_progress;  in get_reshape_loc()
5872 	reshape_safe = conf->reshape_safe;  in get_reshape_loc()
5873 	spin_unlock_irq(&conf->device_lock);  in get_reshape_loc()
5886 	struct r5conf *conf = mddev->private;  in raid5_bitmap_sector()  local
5895 	sectors_per_chunk = conf->chunk_sectors *  in raid5_bitmap_sector()
5896 		(conf->raid_disks - conf->max_degraded);  in raid5_bitmap_sector()
5900 	start = raid5_compute_sector(conf, start, 0, &dd_idx, NULL);  in raid5_bitmap_sector()
5901 	end = raid5_compute_sector(conf, end, 0, &dd_idx, NULL);  in raid5_bitmap_sector()
5907 	loc = get_reshape_loc(mddev, conf, prev_start);  in raid5_bitmap_sector()
5914 	sectors_per_chunk = conf->prev_chunk_sectors *  in raid5_bitmap_sector()
5915 		(conf->previous_raid_disks - conf->max_degraded);  in raid5_bitmap_sector()
5919 	prev_start = raid5_compute_sector(conf, prev_start, 1, &dd_idx, NULL);  in raid5_bitmap_sector()
5920 	prev_end = raid5_compute_sector(conf, prev_end, 1, &dd_idx, NULL);  in raid5_bitmap_sector()
5932 		struct r5conf *conf, struct stripe_request_ctx *ctx,  in make_stripe_request()  argument
5942 	seq = read_seqcount_begin(&conf->gen_lock);  in make_stripe_request()
5944 	if (unlikely(conf->reshape_progress != MaxSector)) {  in make_stripe_request()
5945 		enum reshape_loc loc = get_reshape_loc(mddev, conf,  in make_stripe_request()
5955 	new_sector = raid5_compute_sector(conf, logical_sector, previous,  in make_stripe_request()
5964 	sh = raid5_get_active_stripe(conf, ctx, new_sector, flags);  in make_stripe_request()
5972 	    stripe_ahead_of_reshape(mddev, conf, sh)) {  in make_stripe_request()
5985 	if (read_seqcount_retry(&conf->gen_lock, seq)) {  in make_stripe_request()
5997 	if (!add_all_stripe_bios(conf, ctx, sh, bi, rw, previous)) {  in make_stripe_request()
6003 		stripe_add_to_batch_list(conf, sh, ctx->batch_last);  in make_stripe_request()
6021 		atomic_inc(&conf->preread_active_stripes);  in make_stripe_request()
6041 static sector_t raid5_bio_lowest_chunk_sector(struct r5conf *conf,  in raid5_bio_lowest_chunk_sector()  argument
6044 	int sectors_per_chunk = conf->chunk_sectors;  in raid5_bio_lowest_chunk_sector()
6045 	int raid_disks = conf->raid_disks;  in raid5_bio_lowest_chunk_sector()
6049 	sector_t r_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);  in raid5_bio_lowest_chunk_sector()
6053 	sector = raid5_compute_sector(conf, r_sector, 0, &dd_idx, &sh);  in raid5_bio_lowest_chunk_sector()
6073 	struct r5conf *conf = mddev->private;  in raid5_make_request()  local
6081 		int ret = log_handle_flush_request(conf, bi);  in raid5_make_request()
6116 	logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);  in raid5_make_request()
6122 					   RAID5_STRIPE_SECTORS(conf));  in raid5_make_request()
6130 	    (conf->reshape_progress != MaxSector) &&  in raid5_make_request()
6131 	    get_reshape_loc(mddev, conf, logical_sector) == LOC_INSIDE_RESHAPE) {  in raid5_make_request()
6146 	if (likely(conf->reshape_progress == MaxSector)) {  in raid5_make_request()
6147 		logical_sector = raid5_bio_lowest_chunk_sector(conf, bi);  in raid5_make_request()
6150 		add_wait_queue(&conf->wait_for_reshape, &wait);  in raid5_make_request()
6153 	s = (logical_sector - ctx.first_sector) >> RAID5_STRIPE_SHIFT(conf);  in raid5_make_request()
6156 		res = make_stripe_request(mddev, conf, &ctx, logical_sector,  in raid5_make_request()
6188 			(s << RAID5_STRIPE_SHIFT(conf));  in raid5_make_request()
6191 		remove_wait_queue(&conf->wait_for_reshape, &wait);  in raid5_make_request()
6220 	struct r5conf *conf = mddev->private;  in reshape_request()  local
6224 	int raid_disks = conf->previous_raid_disks;  in reshape_request()
6225 	int data_disks = raid_disks - conf->max_degraded;  in reshape_request()
6226 	int new_data_disks = conf->raid_disks - conf->max_degraded;  in reshape_request()
6238 		    conf->reshape_progress < raid5_size(mddev, 0, 0)) {  in reshape_request()
6240 				- conf->reshape_progress;  in reshape_request()
6242 			   conf->reshape_progress == MaxSector) {  in reshape_request()
6246 			   conf->reshape_progress > 0)  in reshape_request()
6247 			sector_nr = conf->reshape_progress;  in reshape_request()
6263 	reshape_sectors = max(conf->chunk_sectors, conf->prev_chunk_sectors);  in reshape_request()
6271 	writepos = conf->reshape_progress;  in reshape_request()
6273 	readpos = conf->reshape_progress;  in reshape_request()
6275 	safepos = conf->reshape_safe;  in reshape_request()
6298 		if (WARN_ON(conf->reshape_progress == 0))  in reshape_request()
6333 	if (conf->min_offset_diff < 0) {  in reshape_request()
6334 		safepos += -conf->min_offset_diff;  in reshape_request()
6335 		readpos += -conf->min_offset_diff;  in reshape_request()
6337 		writepos += conf->min_offset_diff;  in reshape_request()
6342 	    time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {  in reshape_request()
6344 		wait_event(conf->wait_for_reshape,  in reshape_request()
6345 			   atomic_read(&conf->reshape_stripes)==0  in reshape_request()
6347 		if (atomic_read(&conf->reshape_stripes) != 0)  in reshape_request()
6349 		mddev->reshape_position = conf->reshape_progress;  in reshape_request()
6360 		conf->reshape_checkpoint = jiffies;  in reshape_request()
6367 		spin_lock_irq(&conf->device_lock);  in reshape_request()
6368 		conf->reshape_safe = mddev->reshape_position;  in reshape_request()
6369 		spin_unlock_irq(&conf->device_lock);  in reshape_request()
6370 		wake_up(&conf->wait_for_reshape);  in reshape_request()
6375 	for (i = 0; i < reshape_sectors; i += RAID5_STRIPE_SECTORS(conf)) {  in reshape_request()
6378 		sh = raid5_get_active_stripe(conf, NULL, stripe_addr+i,  in reshape_request()
6381 		atomic_inc(&conf->reshape_stripes);  in reshape_request()
6389 			if (conf->level == 6 &&  in reshape_request()
6397 			memset(page_address(sh->dev[j].page), 0, RAID5_STRIPE_SIZE(conf));  in reshape_request()
6407 	spin_lock_irq(&conf->device_lock);  in reshape_request()
6409 		conf->reshape_progress -= reshape_sectors * new_data_disks;  in reshape_request()
6411 		conf->reshape_progress += reshape_sectors * new_data_disks;  in reshape_request()
6412 	spin_unlock_irq(&conf->device_lock);  in reshape_request()
6419 		raid5_compute_sector(conf, stripe_addr*(new_data_disks),  in reshape_request()
6422 		raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)  in reshape_request()
6428 		sh = raid5_get_active_stripe(conf, NULL, first_sector,  in reshape_request()
6433 		first_sector += RAID5_STRIPE_SECTORS(conf);  in reshape_request()
6453 		wait_event(conf->wait_for_reshape,  in reshape_request()
6454 			   atomic_read(&conf->reshape_stripes) == 0  in reshape_request()
6456 		if (atomic_read(&conf->reshape_stripes) != 0)  in reshape_request()
6458 		mddev->reshape_position = conf->reshape_progress;  in reshape_request()
6468 		conf->reshape_checkpoint = jiffies;  in reshape_request()
6476 		spin_lock_irq(&conf->device_lock);  in reshape_request()
6477 		conf->reshape_safe = mddev->reshape_position;  in reshape_request()
6478 		spin_unlock_irq(&conf->device_lock);  in reshape_request()
6479 		wake_up(&conf->wait_for_reshape);  in reshape_request()
6489 	struct r5conf *conf = mddev->private;  in raid5_sync_request()  local
6499 			end_reshape(conf);  in raid5_sync_request()
6507 			conf->fullsync = 0;  in raid5_sync_request()
6514 	wait_event(conf->wait_for_reshape, conf->quiesce != 2);  in raid5_sync_request()
6529 	if (mddev->degraded >= conf->max_degraded &&  in raid5_sync_request()
6536 	    !conf->fullsync &&  in raid5_sync_request()
6539 	    sync_blocks >= RAID5_STRIPE_SECTORS(conf)) {  in raid5_sync_request()
6541 		do_div(sync_blocks, RAID5_STRIPE_SECTORS(conf));  in raid5_sync_request()
6544 		return sync_blocks * RAID5_STRIPE_SECTORS(conf);  in raid5_sync_request()
6549 	sh = raid5_get_active_stripe(conf, NULL, sector_nr,  in raid5_sync_request()
6552 		sh = raid5_get_active_stripe(conf, NULL, sector_nr, 0);  in raid5_sync_request()
6562 	for (i = 0; i < conf->raid_disks; i++) {  in raid5_sync_request()
6563 		struct md_rdev *rdev = conf->disks[i].rdev;  in raid5_sync_request()
6577 	return RAID5_STRIPE_SECTORS(conf);  in raid5_sync_request()
6580 static int  retry_aligned_read(struct r5conf *conf, struct bio *raid_bio,  in retry_aligned_read()  argument
6600 		~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);  in retry_aligned_read()
6601 	sector = raid5_compute_sector(conf, logical_sector,  in retry_aligned_read()
6606 	     logical_sector += RAID5_STRIPE_SECTORS(conf),  in retry_aligned_read()
6607 		     sector += RAID5_STRIPE_SECTORS(conf),  in retry_aligned_read()
6614 		sh = raid5_get_active_stripe(conf, NULL, sector,  in retry_aligned_read()
6618 			conf->retry_read_aligned = raid_bio;  in retry_aligned_read()
6619 			conf->retry_read_offset = scnt;  in retry_aligned_read()
6625 			conf->retry_read_aligned = raid_bio;  in retry_aligned_read()
6626 			conf->retry_read_offset = scnt;  in retry_aligned_read()
6638 	if (atomic_dec_and_test(&conf->active_aligned_reads))  in retry_aligned_read()
6639 		wake_up(&conf->wait_for_quiescent);  in retry_aligned_read()
6643 static int handle_active_stripes(struct r5conf *conf, int group,  in handle_active_stripes()  argument
6646 		__must_hold(&conf->device_lock)  in handle_active_stripes()
6653 			(sh = __get_priority_stripe(conf, group)) != NULL)  in handle_active_stripes()
6661 			spin_unlock_irq(&conf->device_lock);  in handle_active_stripes()
6662 			log_flush_stripe_to_raid(conf);  in handle_active_stripes()
6663 			spin_lock_irq(&conf->device_lock);  in handle_active_stripes()
6668 	spin_unlock_irq(&conf->device_lock);  in handle_active_stripes()
6670 	release_inactive_stripe_list(conf, temp_inactive_list,  in handle_active_stripes()
6673 	r5l_flush_stripe_to_raid(conf->log);  in handle_active_stripes()
6675 		spin_lock_irq(&conf->device_lock);  in handle_active_stripes()
6681 	log_write_stripe_run(conf);  in handle_active_stripes()
6685 	spin_lock_irq(&conf->device_lock);  in handle_active_stripes()
6688 		__release_stripe(conf, batch[i], &temp_inactive_list[hash]);  in handle_active_stripes()
6697 	struct r5conf *conf = group->conf;  in raid5_do_work()  local
6698 	struct mddev *mddev = conf->mddev;  in raid5_do_work()
6699 	int group_id = group - conf->worker_groups;  in raid5_do_work()
6707 	spin_lock_irq(&conf->device_lock);  in raid5_do_work()
6711 		released = release_stripe_list(conf, worker->temp_inactive_list);  in raid5_do_work()
6713 		batch_size = handle_active_stripes(conf, group_id, worker,  in raid5_do_work()
6721 			conf->device_lock);  in raid5_do_work()
6725 	spin_unlock_irq(&conf->device_lock);  in raid5_do_work()
6727 	flush_deferred_bios(conf);  in raid5_do_work()
6729 	r5l_flush_stripe_to_raid(conf->log);  in raid5_do_work()
6747 	struct r5conf *conf = mddev->private;  in raid5d()  local
6757 	spin_lock_irq(&conf->device_lock);  in raid5d()
6766 		released = release_stripe_list(conf, conf->temp_inactive_list);  in raid5d()
6768 			clear_bit(R5_DID_ALLOC, &conf->cache_state);  in raid5d()
6771 		    !list_empty(&conf->bitmap_list)) {  in raid5d()
6773 			conf->seq_flush++;  in raid5d()
6774 			spin_unlock_irq(&conf->device_lock);  in raid5d()
6776 			spin_lock_irq(&conf->device_lock);  in raid5d()
6777 			conf->seq_write = conf->seq_flush;  in raid5d()
6778 			activate_bit_delay(conf, conf->temp_inactive_list);  in raid5d()
6780 		raid5_activate_delayed(conf);  in raid5d()
6782 		while ((bio = remove_bio_from_retry(conf, &offset))) {  in raid5d()
6784 			spin_unlock_irq(&conf->device_lock);  in raid5d()
6785 			ok = retry_aligned_read(conf, bio, offset);  in raid5d()
6786 			spin_lock_irq(&conf->device_lock);  in raid5d()
6792 		batch_size = handle_active_stripes(conf, ANY_GROUP, NULL,  in raid5d()
6793 						   conf->temp_inactive_list);  in raid5d()
6799 			spin_unlock_irq(&conf->device_lock);  in raid5d()
6801 			spin_lock_irq(&conf->device_lock);  in raid5d()
6806 	spin_unlock_irq(&conf->device_lock);  in raid5d()
6807 	if (test_and_clear_bit(R5_ALLOC_MORE, &conf->cache_state) &&  in raid5d()
6808 	    mutex_trylock(&conf->cache_size_mutex)) {  in raid5d()
6809 		grow_one_stripe(conf, __GFP_NOWARN);  in raid5d()
6813 		set_bit(R5_DID_ALLOC, &conf->cache_state);  in raid5d()
6814 		mutex_unlock(&conf->cache_size_mutex);  in raid5d()
6817 	flush_deferred_bios(conf);  in raid5d()
6819 	r5l_flush_stripe_to_raid(conf->log);  in raid5d()
6830 	struct r5conf *conf;  in raid5_show_stripe_cache_size()  local
6833 	conf = mddev->private;  in raid5_show_stripe_cache_size()
6834 	if (conf)  in raid5_show_stripe_cache_size()
6835 		ret = sprintf(page, "%d\n", conf->min_nr_stripes);  in raid5_show_stripe_cache_size()
6844 	struct r5conf *conf = mddev->private;  in raid5_set_cache_size()  local
6849 	WRITE_ONCE(conf->min_nr_stripes, size);  in raid5_set_cache_size()
6850 	mutex_lock(&conf->cache_size_mutex);  in raid5_set_cache_size()
6851 	while (size < conf->max_nr_stripes &&  in raid5_set_cache_size()
6852 	       drop_one_stripe(conf))  in raid5_set_cache_size()
6854 	mutex_unlock(&conf->cache_size_mutex);  in raid5_set_cache_size()
6858 	mutex_lock(&conf->cache_size_mutex);  in raid5_set_cache_size()
6859 	while (size > conf->max_nr_stripes)  in raid5_set_cache_size()
6860 		if (!grow_one_stripe(conf, GFP_KERNEL)) {  in raid5_set_cache_size()
6861 			WRITE_ONCE(conf->min_nr_stripes, conf->max_nr_stripes);  in raid5_set_cache_size()
6865 	mutex_unlock(&conf->cache_size_mutex);  in raid5_set_cache_size()
6874 	struct r5conf *conf;  in raid5_store_stripe_cache_size()  local
6885 	conf = mddev->private;  in raid5_store_stripe_cache_size()
6886 	if (!conf)  in raid5_store_stripe_cache_size()
6903 	struct r5conf *conf = mddev->private;  in raid5_show_rmw_level()  local
6904 	if (conf)  in raid5_show_rmw_level()
6905 		return sprintf(page, "%d\n", conf->rmw_level);  in raid5_show_rmw_level()
6913 	struct r5conf *conf = mddev->private;  in raid5_store_rmw_level()  local
6916 	if (!conf)  in raid5_store_rmw_level()
6933 	conf->rmw_level = new;  in raid5_store_rmw_level()
6945 	struct r5conf *conf;  in raid5_show_stripe_size()  local
6949 	conf = mddev->private;  in raid5_show_stripe_size()
6950 	if (conf)  in raid5_show_stripe_size()
6951 		ret = sprintf(page, "%lu\n", RAID5_STRIPE_SIZE(conf));  in raid5_show_stripe_size()
6960 	struct r5conf *conf;  in raid5_store_stripe_size()  local
6984 	conf = mddev->private;  in raid5_store_stripe_size()
6985 	if (!conf) {  in raid5_store_stripe_size()
6990 	if (new == conf->stripe_size)  in raid5_store_stripe_size()
6994 			conf->stripe_size, new);  in raid5_store_stripe_size()
7002 	mutex_lock(&conf->cache_size_mutex);  in raid5_store_stripe_size()
7003 	size = conf->max_nr_stripes;  in raid5_store_stripe_size()
7005 	shrink_stripes(conf);  in raid5_store_stripe_size()
7007 	conf->stripe_size = new;  in raid5_store_stripe_size()
7008 	conf->stripe_shift = ilog2(new) - 9;  in raid5_store_stripe_size()
7009 	conf->stripe_sectors = new >> 9;  in raid5_store_stripe_size()
7010 	if (grow_stripes(conf, size)) {  in raid5_store_stripe_size()
7015 	mutex_unlock(&conf->cache_size_mutex);  in raid5_store_stripe_size()
7036 	struct r5conf *conf;  in raid5_show_preread_threshold()  local
7039 	conf = mddev->private;  in raid5_show_preread_threshold()
7040 	if (conf)  in raid5_show_preread_threshold()
7041 		ret = sprintf(page, "%d\n", conf->bypass_threshold);  in raid5_show_preread_threshold()
7049 	struct r5conf *conf;  in raid5_store_preread_threshold()  local
7061 	conf = mddev->private;  in raid5_store_preread_threshold()
7062 	if (!conf)  in raid5_store_preread_threshold()
7064 	else if (new > conf->min_nr_stripes)  in raid5_store_preread_threshold()
7067 		conf->bypass_threshold = new;  in raid5_store_preread_threshold()
7081 	struct r5conf *conf;  in raid5_show_skip_copy()  local
7084 	conf = mddev->private;  in raid5_show_skip_copy()
7085 	if (conf)  in raid5_show_skip_copy()
7086 		ret = sprintf(page, "%d\n", conf->skip_copy);  in raid5_show_skip_copy()
7094 	struct r5conf *conf;  in raid5_store_skip_copy()  local
7107 	conf = mddev->private;  in raid5_store_skip_copy()
7108 	if (!conf)  in raid5_store_skip_copy()
7110 	else if (new != conf->skip_copy) {  in raid5_store_skip_copy()
7114 		conf->skip_copy = new;  in raid5_store_skip_copy()
7133 	struct r5conf *conf = mddev->private;  in stripe_cache_active_show()  local
7134 	if (conf)  in stripe_cache_active_show()
7135 		return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));  in stripe_cache_active_show()
7146 	struct r5conf *conf;  in raid5_show_group_thread_cnt()  local
7149 	conf = mddev->private;  in raid5_show_group_thread_cnt()
7150 	if (conf)  in raid5_show_group_thread_cnt()
7151 		ret = sprintf(page, "%d\n", conf->worker_cnt_per_group);  in raid5_show_group_thread_cnt()
7156 static int alloc_thread_groups(struct r5conf *conf, int cnt,
7162 	struct r5conf *conf;  in raid5_store_group_thread_cnt()  local
7181 	conf = mddev->private;  in raid5_store_group_thread_cnt()
7182 	if (!conf)  in raid5_store_group_thread_cnt()
7184 	else if (new != conf->worker_cnt_per_group) {  in raid5_store_group_thread_cnt()
7185 		old_groups = conf->worker_groups;  in raid5_store_group_thread_cnt()
7189 		err = alloc_thread_groups(conf, new, &group_cnt, &new_groups);  in raid5_store_group_thread_cnt()
7191 			spin_lock_irq(&conf->device_lock);  in raid5_store_group_thread_cnt()
7192 			conf->group_cnt = group_cnt;  in raid5_store_group_thread_cnt()
7193 			conf->worker_cnt_per_group = new;  in raid5_store_group_thread_cnt()
7194 			conf->worker_groups = new_groups;  in raid5_store_group_thread_cnt()
7195 			spin_unlock_irq(&conf->device_lock);  in raid5_store_group_thread_cnt()
7231 static int alloc_thread_groups(struct r5conf *conf, int cnt, int *group_cnt,  in alloc_thread_groups()  argument
7260 		group->conf = conf;  in alloc_thread_groups()
7276 static void free_thread_groups(struct r5conf *conf)  in free_thread_groups()  argument
7278 	if (conf->worker_groups)  in free_thread_groups()
7279 		kfree(conf->worker_groups[0].workers);  in free_thread_groups()
7280 	kfree(conf->worker_groups);  in free_thread_groups()
7281 	conf->worker_groups = NULL;  in free_thread_groups()
7287 	struct r5conf *conf = mddev->private;  in raid5_size()  local
7293 		raid_disks = min(conf->raid_disks, conf->previous_raid_disks);  in raid5_size()
7295 	sectors &= ~((sector_t)conf->chunk_sectors - 1);  in raid5_size()
7296 	sectors &= ~((sector_t)conf->prev_chunk_sectors - 1);  in raid5_size()
7297 	return sectors * (raid_disks - conf->max_degraded);  in raid5_size()
7300 static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)  in free_scratch_buffer()  argument
7308 static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)  in alloc_scratch_buffer()  argument
7310 	if (conf->level == 6 && !percpu->spare_page) {  in alloc_scratch_buffer()
7317 			   max(conf->raid_disks,  in alloc_scratch_buffer()
7318 			       conf->previous_raid_disks),  in alloc_scratch_buffer()
7319 			   max(conf->chunk_sectors,  in alloc_scratch_buffer()
7320 			       conf->prev_chunk_sectors)  in alloc_scratch_buffer()
7321 			   / RAID5_STRIPE_SECTORS(conf))) {  in alloc_scratch_buffer()
7322 		free_scratch_buffer(conf, percpu);  in alloc_scratch_buffer()
7332 	struct r5conf *conf = hlist_entry_safe(node, struct r5conf, node);  in raid456_cpu_dead()  local
7334 	free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));  in raid456_cpu_dead()
7338 static void raid5_free_percpu(struct r5conf *conf)  in raid5_free_percpu()  argument
7340 	if (!conf->percpu)  in raid5_free_percpu()
7343 	cpuhp_state_remove_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);  in raid5_free_percpu()
7344 	free_percpu(conf->percpu);  in raid5_free_percpu()
7347 static void free_conf(struct r5conf *conf)  in free_conf()  argument
7351 	log_exit(conf);  in free_conf()
7353 	shrinker_free(conf->shrinker);  in free_conf()
7354 	free_thread_groups(conf);  in free_conf()
7355 	shrink_stripes(conf);  in free_conf()
7356 	raid5_free_percpu(conf);  in free_conf()
7357 	for (i = 0; i < conf->pool_size; i++)  in free_conf()
7358 		if (conf->disks[i].extra_page)  in free_conf()
7359 			put_page(conf->disks[i].extra_page);  in free_conf()
7360 	kfree(conf->disks);  in free_conf()
7361 	bioset_exit(&conf->bio_split);  in free_conf()
7362 	kfree(conf->stripe_hashtbl);  in free_conf()
7363 	kfree(conf->pending_data);  in free_conf()
7364 	kfree(conf);  in free_conf()
7369 	struct r5conf *conf = hlist_entry_safe(node, struct r5conf, node);  in raid456_cpu_up_prepare()  local
7370 	struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);  in raid456_cpu_up_prepare()
7372 	if (alloc_scratch_buffer(conf, percpu)) {  in raid456_cpu_up_prepare()
7380 static int raid5_alloc_percpu(struct r5conf *conf)  in raid5_alloc_percpu()  argument
7384 	conf->percpu = alloc_percpu(struct raid5_percpu);  in raid5_alloc_percpu()
7385 	if (!conf->percpu)  in raid5_alloc_percpu()
7388 	err = cpuhp_state_add_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);  in raid5_alloc_percpu()
7390 		conf->scribble_disks = max(conf->raid_disks,  in raid5_alloc_percpu()
7391 			conf->previous_raid_disks);  in raid5_alloc_percpu()
7392 		conf->scribble_sectors = max(conf->chunk_sectors,  in raid5_alloc_percpu()
7393 			conf->prev_chunk_sectors);  in raid5_alloc_percpu()
7401 	struct r5conf *conf = shrink->private_data;  in raid5_cache_scan()  local
7404 	if (mutex_trylock(&conf->cache_size_mutex)) {  in raid5_cache_scan()
7407 		       conf->max_nr_stripes > conf->min_nr_stripes) {  in raid5_cache_scan()
7408 			if (drop_one_stripe(conf) == 0) {  in raid5_cache_scan()
7414 		mutex_unlock(&conf->cache_size_mutex);  in raid5_cache_scan()
7422 	struct r5conf *conf = shrink->private_data;  in raid5_cache_count()  local
7423 	int max_stripes = READ_ONCE(conf->max_nr_stripes);  in raid5_cache_count()
7424 	int min_stripes = READ_ONCE(conf->min_nr_stripes);  in raid5_cache_count()
7434 	struct r5conf *conf;  in setup_conf()  local
7473 	conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);  in setup_conf()
7474 	if (conf == NULL)  in setup_conf()
7478 	conf->stripe_size = DEFAULT_STRIPE_SIZE;  in setup_conf()
7479 	conf->stripe_shift = ilog2(DEFAULT_STRIPE_SIZE) - 9;  in setup_conf()
7480 	conf->stripe_sectors = DEFAULT_STRIPE_SIZE >> 9;  in setup_conf()
7482 	INIT_LIST_HEAD(&conf->free_list);  in setup_conf()
7483 	INIT_LIST_HEAD(&conf->pending_list);  in setup_conf()
7484 	conf->pending_data = kcalloc(PENDING_IO_MAX,  in setup_conf()
7487 	if (!conf->pending_data)  in setup_conf()
7490 		list_add(&conf->pending_data[i].sibling, &conf->free_list);  in setup_conf()
7492 	if (!alloc_thread_groups(conf, 0, &group_cnt, &new_group)) {  in setup_conf()
7493 		conf->group_cnt = group_cnt;  in setup_conf()
7494 		conf->worker_cnt_per_group = 0;  in setup_conf()
7495 		conf->worker_groups = new_group;  in setup_conf()
7498 	spin_lock_init(&conf->device_lock);  in setup_conf()
7499 	seqcount_spinlock_init(&conf->gen_lock, &conf->device_lock);  in setup_conf()
7500 	mutex_init(&conf->cache_size_mutex);  in setup_conf()
7502 	init_waitqueue_head(&conf->wait_for_quiescent);  in setup_conf()
7503 	init_waitqueue_head(&conf->wait_for_stripe);  in setup_conf()
7504 	init_waitqueue_head(&conf->wait_for_reshape);  in setup_conf()
7505 	INIT_LIST_HEAD(&conf->handle_list);  in setup_conf()
7506 	INIT_LIST_HEAD(&conf->loprio_list);  in setup_conf()
7507 	INIT_LIST_HEAD(&conf->hold_list);  in setup_conf()
7508 	INIT_LIST_HEAD(&conf->delayed_list);  in setup_conf()
7509 	INIT_LIST_HEAD(&conf->bitmap_list);  in setup_conf()
7510 	init_llist_head(&conf->released_stripes);  in setup_conf()
7511 	atomic_set(&conf->active_stripes, 0);  in setup_conf()
7512 	atomic_set(&conf->preread_active_stripes, 0);  in setup_conf()
7513 	atomic_set(&conf->active_aligned_reads, 0);  in setup_conf()
7514 	spin_lock_init(&conf->pending_bios_lock);  in setup_conf()
7515 	conf->batch_bio_dispatch = true;  in setup_conf()
7520 			conf->batch_bio_dispatch = false;  in setup_conf()
7525 	conf->bypass_threshold = BYPASS_THRESHOLD;  in setup_conf()
7526 	conf->recovery_disabled = mddev->recovery_disabled - 1;  in setup_conf()
7528 	conf->raid_disks = mddev->raid_disks;  in setup_conf()
7530 		conf->previous_raid_disks = mddev->raid_disks;  in setup_conf()
7532 		conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;  in setup_conf()
7533 	max_disks = max(conf->raid_disks, conf->previous_raid_disks);  in setup_conf()
7535 	conf->disks = kcalloc(max_disks, sizeof(struct disk_info),  in setup_conf()
7538 	if (!conf->disks)  in setup_conf()
7542 		conf->disks[i].extra_page = alloc_page(GFP_KERNEL);  in setup_conf()
7543 		if (!conf->disks[i].extra_page)  in setup_conf()
7547 	ret = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0);  in setup_conf()
7550 	conf->mddev = mddev;  in setup_conf()
7553 	conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL);  in setup_conf()
7554 	if (!conf->stripe_hashtbl)  in setup_conf()
7562 	spin_lock_init(conf->hash_locks);  in setup_conf()
7564 		spin_lock_init(conf->hash_locks + i);  in setup_conf()
7567 		INIT_LIST_HEAD(conf->inactive_list + i);  in setup_conf()
7570 		INIT_LIST_HEAD(conf->temp_inactive_list + i);  in setup_conf()
7572 	atomic_set(&conf->r5c_cached_full_stripes, 0);  in setup_conf()
7573 	INIT_LIST_HEAD(&conf->r5c_full_stripe_list);  in setup_conf()
7574 	atomic_set(&conf->r5c_cached_partial_stripes, 0);  in setup_conf()
7575 	INIT_LIST_HEAD(&conf->r5c_partial_stripe_list);  in setup_conf()
7576 	atomic_set(&conf->r5c_flushing_full_stripes, 0);  in setup_conf()
7577 	atomic_set(&conf->r5c_flushing_partial_stripes, 0);  in setup_conf()
7579 	conf->level = mddev->new_level;  in setup_conf()
7580 	conf->chunk_sectors = mddev->new_chunk_sectors;  in setup_conf()
7581 	ret = raid5_alloc_percpu(conf);  in setup_conf()
7593 		disk = conf->disks + raid_disk;  in setup_conf()
7610 			conf->fullsync = 1;  in setup_conf()
7613 	conf->level = mddev->new_level;  in setup_conf()
7614 	if (conf->level == 6) {  in setup_conf()
7615 		conf->max_degraded = 2;  in setup_conf()
7617 			conf->rmw_level = PARITY_ENABLE_RMW;  in setup_conf()
7619 			conf->rmw_level = PARITY_DISABLE_RMW;  in setup_conf()
7621 		conf->max_degraded = 1;  in setup_conf()
7622 		conf->rmw_level = PARITY_ENABLE_RMW;  in setup_conf()
7624 	conf->algorithm = mddev->new_layout;  in setup_conf()
7625 	conf->reshape_progress = mddev->reshape_position;  in setup_conf()
7626 	if (conf->reshape_progress != MaxSector) {  in setup_conf()
7627 		conf->prev_chunk_sectors = mddev->chunk_sectors;  in setup_conf()
7628 		conf->prev_algo = mddev->layout;  in setup_conf()
7630 		conf->prev_chunk_sectors = conf->chunk_sectors;  in setup_conf()
7631 		conf->prev_algo = conf->algorithm;  in setup_conf()
7634 	conf->min_nr_stripes = NR_STRIPES;  in setup_conf()
7637 			((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4,  in setup_conf()
7638 			((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4);  in setup_conf()
7639 		conf->min_nr_stripes = max(NR_STRIPES, stripes);  in setup_conf()
7640 		if (conf->min_nr_stripes != NR_STRIPES)  in setup_conf()
7642 				mdname(mddev), conf->min_nr_stripes);  in setup_conf()
7644 	memory = conf->min_nr_stripes * (sizeof(struct stripe_head) +  in setup_conf()
7646 	atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS);  in setup_conf()
7647 	if (grow_stripes(conf, conf->min_nr_stripes)) {  in setup_conf()
7659 	conf->shrinker = shrinker_alloc(0, "md-raid5:%s", mdname(mddev));  in setup_conf()
7660 	if (!conf->shrinker) {  in setup_conf()
7667 	conf->shrinker->seeks = DEFAULT_SEEKS * conf->raid_disks * 4;  in setup_conf()
7668 	conf->shrinker->scan_objects = raid5_cache_scan;  in setup_conf()
7669 	conf->shrinker->count_objects = raid5_cache_count;  in setup_conf()
7670 	conf->shrinker->batch = 128;  in setup_conf()
7671 	conf->shrinker->private_data = conf;  in setup_conf()
7673 	shrinker_register(conf->shrinker);  in setup_conf()
7676 	rcu_assign_pointer(conf->thread,  in setup_conf()
7678 	if (!conf->thread) {  in setup_conf()
7685 	return conf;  in setup_conf()
7688 	if (conf)  in setup_conf()
7689 		free_conf(conf);  in setup_conf()
7721 	struct r5conf *conf = mddev->private;  in raid5_set_limits()  local
7730 	data_disks = conf->previous_raid_disks - conf->max_degraded;  in raid5_set_limits()
7740 	lim.io_opt = lim.io_min * (conf->raid_disks - conf->max_degraded);  in raid5_set_limits()
7772 	lim.max_hw_sectors = RAID5_MAX_REQ_STRIPES << RAID5_STRIPE_SHIFT(conf);  in raid5_set_limits()
7782 	struct r5conf *conf;  in raid5_run()  local
7920 		conf = setup_conf(mddev);  in raid5_run()
7922 		conf = mddev->private;  in raid5_run()
7924 	if (IS_ERR(conf))  in raid5_run()
7925 		return PTR_ERR(conf);  in raid5_run()
7937 	conf->min_offset_diff = min_offset_diff;  in raid5_run()
7938 	rcu_assign_pointer(mddev->thread, conf->thread);  in raid5_run()
7939 	rcu_assign_pointer(conf->thread, NULL);  in raid5_run()
7940 	mddev->private = conf;  in raid5_run()
7942 	for (i = 0; i < conf->raid_disks && conf->previous_raid_disks;  in raid5_run()
7944 		rdev = conf->disks[i].rdev;  in raid5_run()
7947 		if (conf->disks[i].replacement &&  in raid5_run()
7948 		    conf->reshape_progress != MaxSector) {  in raid5_run()
7971 					 conf->algorithm,  in raid5_run()
7972 					 conf->raid_disks,  in raid5_run()
7973 					 conf->max_degraded))  in raid5_run()
7977 				 conf->prev_algo,  in raid5_run()
7978 				 conf->previous_raid_disks,  in raid5_run()
7979 				 conf->max_degraded))  in raid5_run()
7987 	mddev->degraded = raid5_calc_degraded(conf);  in raid5_run()
7989 	if (has_failed(conf)) {  in raid5_run()
7991 			mdname(mddev), mddev->degraded, conf->raid_disks);  in raid5_run()
8015 		mdname(mddev), conf->level,  in raid5_run()
8019 	print_raid5_conf(conf);  in raid5_run()
8021 	if (conf->reshape_progress != MaxSector) {  in raid5_run()
8022 		conf->reshape_safe = conf->reshape_progress;  in raid5_run()
8023 		atomic_set(&conf->reshape_stripes, 0);  in raid5_run()
8045 	if (log_init(conf, journal_dev, raid5_has_ppl(conf)))  in raid5_run()
8051 	print_raid5_conf(conf);  in raid5_run()
8052 	free_conf(conf);  in raid5_run()
8060 	struct r5conf *conf = priv;  in raid5_free()  local
8062 	free_conf(conf);  in raid5_free()
8068 	struct r5conf *conf = mddev->private;  in raid5_status()  local
8074 		conf->chunk_sectors / 2, mddev->layout);  in raid5_status()
8075 	seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);  in raid5_status()
8076 	for (i = 0; i < conf->raid_disks; i++) {  in raid5_status()
8077 		struct md_rdev *rdev = READ_ONCE(conf->disks[i].rdev);  in raid5_status()
8084 static void print_raid5_conf(struct r5conf *conf)  in print_raid5_conf()  argument
8089 	pr_debug("RAID conf printout:\n");  in print_raid5_conf()
8090 	if (!conf) {  in print_raid5_conf()
8091 		pr_debug("(conf==NULL)\n");  in print_raid5_conf()
8094 	pr_debug(" --- level:%d rd:%d wd:%d\n", conf->level,  in print_raid5_conf()
8095 	       conf->raid_disks,  in print_raid5_conf()
8096 	       conf->raid_disks - conf->mddev->degraded);  in print_raid5_conf()
8098 	for (i = 0; i < conf->raid_disks; i++) {  in print_raid5_conf()
8099 		rdev = conf->disks[i].rdev;  in print_raid5_conf()
8110 	struct r5conf *conf = mddev->private;  in raid5_spare_active()  local
8115 	for (i = 0; i < conf->raid_disks; i++) {  in raid5_spare_active()
8116 		rdev = conf->disks[i].rdev;  in raid5_spare_active()
8117 		replacement = conf->disks[i].replacement;  in raid5_spare_active()
8144 	spin_lock_irqsave(&conf->device_lock, flags);  in raid5_spare_active()
8145 	mddev->degraded = raid5_calc_degraded(conf);  in raid5_spare_active()
8146 	spin_unlock_irqrestore(&conf->device_lock, flags);  in raid5_spare_active()
8147 	print_raid5_conf(conf);  in raid5_spare_active()
8153 	struct r5conf *conf = mddev->private;  in raid5_remove_disk()  local
8160 	print_raid5_conf(conf);  in raid5_remove_disk()
8161 	if (test_bit(Journal, &rdev->flags) && conf->log) {  in raid5_remove_disk()
8168 		if (atomic_read(&conf->active_stripes) ||  in raid5_remove_disk()
8169 		    atomic_read(&conf->r5c_cached_full_stripes) ||  in raid5_remove_disk()
8170 		    atomic_read(&conf->r5c_cached_partial_stripes)) {  in raid5_remove_disk()
8173 		log_exit(conf);  in raid5_remove_disk()
8176 	if (unlikely(number >= conf->pool_size))  in raid5_remove_disk()
8178 	p = conf->disks + number;  in raid5_remove_disk()
8186 	if (number >= conf->raid_disks &&  in raid5_remove_disk()
8187 	    conf->reshape_progress == MaxSector)  in raid5_remove_disk()
8199 	    mddev->recovery_disabled != conf->recovery_disabled &&  in raid5_remove_disk()
8200 	    !has_failed(conf) &&  in raid5_remove_disk()
8202 	    number < conf->raid_disks) {  in raid5_remove_disk()
8208 		err = log_modify(conf, rdev, false);  in raid5_remove_disk()
8221 			err = log_modify(conf, tmp, true);  in raid5_remove_disk()
8227 	print_raid5_conf(conf);  in raid5_remove_disk()
8233 	struct r5conf *conf = mddev->private;  in raid5_add_disk()  local
8239 	int last = conf->raid_disks - 1;  in raid5_add_disk()
8242 		if (conf->log)  in raid5_add_disk()
8250 		ret = log_init(conf, rdev, false);  in raid5_add_disk()
8254 		ret = r5l_start(conf->log);  in raid5_add_disk()
8260 	if (mddev->recovery_disabled == conf->recovery_disabled)  in raid5_add_disk()
8263 	if (rdev->saved_raid_disk < 0 && has_failed(conf))  in raid5_add_disk()
8276 	    conf->disks[rdev->saved_raid_disk].rdev == NULL)  in raid5_add_disk()
8280 		p = conf->disks + disk;  in raid5_add_disk()
8285 				conf->fullsync = 1;  in raid5_add_disk()
8288 			err = log_modify(conf, rdev, true);  in raid5_add_disk()
8294 		p = conf->disks + disk;  in raid5_add_disk()
8303 			conf->fullsync = 1;  in raid5_add_disk()
8309 	print_raid5_conf(conf);  in raid5_add_disk()
8323 	struct r5conf *conf = mddev->private;  in raid5_resize()  local
8326 	if (raid5_has_log(conf) || raid5_has_ppl(conf))  in raid5_resize()
8328 	sectors &= ~((sector_t)conf->chunk_sectors - 1);  in raid5_resize()
8359 	struct r5conf *conf = mddev->private;  in check_stripe_cache()  local
8360 	if (((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4  in check_stripe_cache()
8361 	    > conf->min_nr_stripes ||  in check_stripe_cache()
8362 	    ((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4  in check_stripe_cache()
8363 	    > conf->min_nr_stripes) {  in check_stripe_cache()
8367 			 / RAID5_STRIPE_SIZE(conf))*4);  in check_stripe_cache()
8375 	struct r5conf *conf = mddev->private;  in check_reshape()  local
8377 	if (raid5_has_log(conf) || raid5_has_ppl(conf))  in check_reshape()
8383 	if (has_failed(conf))  in check_reshape()
8403 		if (resize_chunks(conf,  in check_reshape()
8404 				  conf->previous_raid_disks  in check_reshape()
8411 	if (conf->previous_raid_disks + mddev->delta_disks <= conf->pool_size)  in check_reshape()
8413 	return resize_stripes(conf, (conf->previous_raid_disks  in check_reshape()
8419 	struct r5conf *conf = mddev->private;  in raid5_start_reshape()  local
8431 	if (has_failed(conf))  in raid5_start_reshape()
8437 	for (i = 0; i < conf->raid_disks; i++)  in raid5_start_reshape()
8438 		if (conf->disks[i].replacement)  in raid5_start_reshape()
8447 	if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)  in raid5_start_reshape()
8457 	if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)  in raid5_start_reshape()
8464 	atomic_set(&conf->reshape_stripes, 0);  in raid5_start_reshape()
8465 	spin_lock_irq(&conf->device_lock);  in raid5_start_reshape()
8466 	write_seqcount_begin(&conf->gen_lock);  in raid5_start_reshape()
8467 	conf->previous_raid_disks = conf->raid_disks;  in raid5_start_reshape()
8468 	conf->raid_disks += mddev->delta_disks;  in raid5_start_reshape()
8469 	conf->prev_chunk_sectors = conf->chunk_sectors;  in raid5_start_reshape()
8470 	conf->chunk_sectors = mddev->new_chunk_sectors;  in raid5_start_reshape()
8471 	conf->prev_algo = conf->algorithm;  in raid5_start_reshape()
8472 	conf->algorithm = mddev->new_layout;  in raid5_start_reshape()
8473 	conf->generation++;  in raid5_start_reshape()
8479 		conf->reshape_progress = raid5_size(mddev, 0, 0);  in raid5_start_reshape()
8481 		conf->reshape_progress = 0;  in raid5_start_reshape()
8482 	conf->reshape_safe = conf->reshape_progress;  in raid5_start_reshape()
8483 	write_seqcount_end(&conf->gen_lock);  in raid5_start_reshape()
8484 	spin_unlock_irq(&conf->device_lock);  in raid5_start_reshape()
8506 					    >= conf->previous_raid_disks)  in raid5_start_reshape()
8514 			} else if (rdev->raid_disk >= conf->previous_raid_disks  in raid5_start_reshape()
8524 		spin_lock_irqsave(&conf->device_lock, flags);  in raid5_start_reshape()
8525 		mddev->degraded = raid5_calc_degraded(conf);  in raid5_start_reshape()
8526 		spin_unlock_irqrestore(&conf->device_lock, flags);  in raid5_start_reshape()
8528 	mddev->raid_disks = conf->raid_disks;  in raid5_start_reshape()
8529 	mddev->reshape_position = conf->reshape_progress;  in raid5_start_reshape()
8537 	conf->reshape_checkpoint = jiffies;  in raid5_start_reshape()
8543  * changes needed in 'conf'
8545 static void end_reshape(struct r5conf *conf)  in end_reshape()  argument
8548 	if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {  in end_reshape()
8551 		spin_lock_irq(&conf->device_lock);  in end_reshape()
8552 		conf->previous_raid_disks = conf->raid_disks;  in end_reshape()
8553 		md_finish_reshape(conf->mddev);  in end_reshape()
8555 		conf->reshape_progress = MaxSector;  in end_reshape()
8556 		conf->mddev->reshape_position = MaxSector;  in end_reshape()
8557 		rdev_for_each(rdev, conf->mddev)  in end_reshape()
8562 		spin_unlock_irq(&conf->device_lock);  in end_reshape()
8563 		wake_up(&conf->wait_for_reshape);  in end_reshape()
8565 		mddev_update_io_opt(conf->mddev,  in end_reshape()
8566 			conf->raid_disks - conf->max_degraded);  in end_reshape()
8575 	struct r5conf *conf = mddev->private;  in raid5_finish_reshape()  local
8582 			spin_lock_irq(&conf->device_lock);  in raid5_finish_reshape()
8583 			mddev->degraded = raid5_calc_degraded(conf);  in raid5_finish_reshape()
8584 			spin_unlock_irq(&conf->device_lock);  in raid5_finish_reshape()
8585 			for (d = conf->raid_disks ;  in raid5_finish_reshape()
8586 			     d < conf->raid_disks - mddev->delta_disks;  in raid5_finish_reshape()
8588 				rdev = conf->disks[d].rdev;  in raid5_finish_reshape()
8591 				rdev = conf->disks[d].replacement;  in raid5_finish_reshape()
8596 		mddev->layout = conf->algorithm;  in raid5_finish_reshape()
8597 		mddev->chunk_sectors = conf->chunk_sectors;  in raid5_finish_reshape()
8606 	struct r5conf *conf = mddev->private;  in raid5_quiesce()  local
8610 		lock_all_device_hash_locks_irq(conf);  in raid5_quiesce()
8614 		r5c_flush_cache(conf, INT_MAX);  in raid5_quiesce()
8618 		smp_store_release(&conf->quiesce, 2);  in raid5_quiesce()
8619 		wait_event_cmd(conf->wait_for_quiescent,  in raid5_quiesce()
8620 				    atomic_read(&conf->active_stripes) == 0 &&  in raid5_quiesce()
8621 				    atomic_read(&conf->active_aligned_reads) == 0,  in raid5_quiesce()
8622 				    unlock_all_device_hash_locks_irq(conf),  in raid5_quiesce()
8623 				    lock_all_device_hash_locks_irq(conf));  in raid5_quiesce()
8624 		conf->quiesce = 1;  in raid5_quiesce()
8625 		unlock_all_device_hash_locks_irq(conf);  in raid5_quiesce()
8627 		wake_up(&conf->wait_for_reshape);  in raid5_quiesce()
8630 		lock_all_device_hash_locks_irq(conf);  in raid5_quiesce()
8631 		conf->quiesce = 0;  in raid5_quiesce()
8632 		wake_up(&conf->wait_for_quiescent);  in raid5_quiesce()
8633 		wake_up(&conf->wait_for_reshape);  in raid5_quiesce()
8634 		unlock_all_device_hash_locks_irq(conf);  in raid5_quiesce()
8636 	log_quiesce(conf, quiesce);  in raid5_quiesce()
8737 	struct r5conf *conf = mddev->private;  in raid5_check_reshape()  local
8757 			conf->algorithm = mddev->new_layout;  in raid5_check_reshape()
8761 			conf->chunk_sectors = new_chunk ;  in raid5_check_reshape()
8880 	struct r5conf *conf;  in raid5_change_consistency_policy()  local
8886 	conf = mddev->private;  in raid5_change_consistency_policy()
8887 	if (!conf) {  in raid5_change_consistency_policy()
8894 		if (!raid5_has_ppl(conf) && conf->level == 5) {  in raid5_change_consistency_policy()
8895 			err = log_init(conf, NULL, true);  in raid5_change_consistency_policy()
8897 				err = resize_stripes(conf, conf->pool_size);  in raid5_change_consistency_policy()
8899 					log_exit(conf);  in raid5_change_consistency_policy()
8904 		if (raid5_has_ppl(conf)) {  in raid5_change_consistency_policy()
8905 			log_exit(conf);  in raid5_change_consistency_policy()
8906 			err = resize_stripes(conf, conf->pool_size);  in raid5_change_consistency_policy()
8907 		} else if (test_bit(MD_HAS_JOURNAL, &conf->mddev->flags) &&  in raid5_change_consistency_policy()
8908 			   r5l_log_disk_error(conf)) {  in raid5_change_consistency_policy()
8938 	struct r5conf *conf = mddev->private;  in raid5_start()  local
8940 	return r5l_start(conf->log);  in raid5_start()
8950 	struct r5conf *conf = mddev->private;  in raid5_prepare_suspend()  local
8952 	wake_up(&conf->wait_for_reshape);  in raid5_prepare_suspend()