2519 	struct slab_scrubber *scrubber = &slab->allocator->scrubber;  in register_slab_for_scrubbing()  local
2529 		WRITE_ONCE(scrubber->slab_count, scrubber->slab_count + 1);  in register_slab_for_scrubbing()
2535 		list_add_tail(&slab->allocq_entry, &scrubber->high_priority_slabs);  in register_slab_for_scrubbing()
2539 	list_add_tail(&slab->allocq_entry, &scrubber->slabs);  in register_slab_for_scrubbing()
2626  * @scrubber: The slab scrubber.
2630 static struct vdo_slab *get_next_slab(struct slab_scrubber *scrubber)  in get_next_slab()  argument
2634 	slab = list_first_entry_or_null(&scrubber->high_priority_slabs,  in get_next_slab()
2639 	return list_first_entry_or_null(&scrubber->slabs, struct vdo_slab,  in get_next_slab()
2644  * has_slabs_to_scrub() - Check whether a scrubber has slabs to scrub.
2645  * @scrubber: The scrubber to check.
2647  * Return: true if the scrubber has slabs to scrub.
2649 static inline bool __must_check has_slabs_to_scrub(struct slab_scrubber *scrubber)  in has_slabs_to_scrub()  argument
2651 	return (get_next_slab(scrubber) != NULL);  in has_slabs_to_scrub()
2656  * @scrubber: The scrubber.
2658 static void uninitialize_scrubber_vio(struct slab_scrubber *scrubber)  in uninitialize_scrubber_vio()  argument
2660 	vdo_free(vdo_forget(scrubber->vio.data));  in uninitialize_scrubber_vio()
2661 	free_vio_components(&scrubber->vio);  in uninitialize_scrubber_vio()
2667  * @scrubber: The scrubber.
2669 static void finish_scrubbing(struct slab_scrubber *scrubber, int result)  in finish_scrubbing()  argument
2671 	bool notify = vdo_waitq_has_waiters(&scrubber->waiters);  in finish_scrubbing()
2672 	bool done = !has_slabs_to_scrub(scrubber);  in finish_scrubbing()
2674 		container_of(scrubber, struct block_allocator, scrubber);  in finish_scrubbing()
2677 		uninitialize_scrubber_vio(scrubber);  in finish_scrubbing()
2679 	if (scrubber->high_priority_only) {  in finish_scrubbing()
2680 		scrubber->high_priority_only = false;  in finish_scrubbing()
2681 		vdo_fail_completion(vdo_forget(scrubber->vio.completion.parent), result);  in finish_scrubbing()
2706 	 * Note that the scrubber has stopped, and inform anyone who might be waiting for that to  in finish_scrubbing()
2709 	if (!vdo_finish_draining(&scrubber->admin_state))  in finish_scrubbing()
2710 		WRITE_ONCE(scrubber->admin_state.current_state,  in finish_scrubbing()
2718 		vdo_waitq_notify_all_waiters(&scrubber->waiters, NULL, NULL);  in finish_scrubbing()
2721 static void scrub_next_slab(struct slab_scrubber *scrubber);
2724  * slab_scrubbed() - Notify the scrubber that a slab has been scrubbed.
2731 	struct slab_scrubber *scrubber =  in slab_scrubbed()  local
2733 	struct vdo_slab *slab = scrubber->slab;  in slab_scrubbed()
2738 	WRITE_ONCE(scrubber->slab_count, scrubber->slab_count - 1);  in slab_scrubbed()
2739 	scrub_next_slab(scrubber);  in slab_scrubbed()
2744  * @scrubber: The slab scrubber.
2747 static void abort_scrubbing(struct slab_scrubber *scrubber, int result)  in abort_scrubbing()  argument
2749 	vdo_enter_read_only_mode(scrubber->vio.completion.vdo, result);  in abort_scrubbing()
2750 	finish_scrubbing(scrubber, result);  in abort_scrubbing()
2824 	struct slab_scrubber *scrubber =  in apply_journal_entries()  local
2826 	struct vdo_slab *slab = scrubber->slab;  in apply_journal_entries()
2832 	char *end_data = scrubber->vio.data + (end_index * VDO_BLOCK_SIZE);  in apply_journal_entries()
2845 		char *block_data = scrubber->vio.data + (index * VDO_BLOCK_SIZE);  in apply_journal_entries()
2861 			abort_scrubbing(scrubber, VDO_CORRUPT_JOURNAL);  in apply_journal_entries()
2867 			abort_scrubbing(scrubber, result);  in apply_journal_entries()
2886 		abort_scrubbing(scrubber, result);  in apply_journal_entries()
2901 	struct slab_scrubber *scrubber = container_of(vio, struct slab_scrubber, vio);  in read_slab_journal_endio()  local
2904 			      scrubber->slab->allocator->thread_id);  in read_slab_journal_endio()
2909  * @completion: The scrubber's vio completion.
2915 	struct slab_scrubber *scrubber =  in start_scrubbing()  local
2917 	struct vdo_slab *slab = scrubber->slab;  in start_scrubbing()
2924 	vdo_submit_metadata_vio(&scrubber->vio, slab->journal_origin,  in start_scrubbing()
2931  * @scrubber: The scrubber.
2933 static void scrub_next_slab(struct slab_scrubber *scrubber)  in scrub_next_slab()  argument
2935 	struct vdo_completion *completion = &scrubber->vio.completion;  in scrub_next_slab()
2942 	vdo_waitq_notify_all_waiters(&scrubber->waiters, NULL, NULL);  in scrub_next_slab()
2945 		finish_scrubbing(scrubber, VDO_READ_ONLY);  in scrub_next_slab()
2949 	slab = get_next_slab(scrubber);  in scrub_next_slab()
2951 	    (scrubber->high_priority_only && list_empty(&scrubber->high_priority_slabs))) {  in scrub_next_slab()
2952 		finish_scrubbing(scrubber, VDO_SUCCESS);  in scrub_next_slab()
2956 	if (vdo_finish_draining(&scrubber->admin_state))  in scrub_next_slab()
2960 	scrubber->slab = slab;  in scrub_next_slab()
2974 	struct slab_scrubber *scrubber = &allocator->scrubber;  in scrub_slabs()  local
2976 	scrubber->vio.completion.parent = parent;  in scrub_slabs()
2977 	scrubber->high_priority_only = (parent != NULL);  in scrub_slabs()
2978 	if (!has_slabs_to_scrub(scrubber)) {  in scrub_slabs()
2979 		finish_scrubbing(scrubber, VDO_SUCCESS);  in scrub_slabs()
2983 	if (scrubber->high_priority_only &&  in scrub_slabs()
2985 	    list_empty(&scrubber->high_priority_slabs))  in scrub_slabs()
2986 		register_slab_for_scrubbing(get_next_slab(scrubber), true);  in scrub_slabs()
2988 	vdo_resume_if_quiescent(&scrubber->admin_state);  in scrub_slabs()
2989 	scrub_next_slab(scrubber);  in scrub_slabs()
3224 	if (vdo_is_state_quiescent(&allocator->scrubber.admin_state))  in vdo_enqueue_clean_slab_waiter()
3227 	vdo_waitq_enqueue_waiter(&allocator->scrubber.waiters, waiter);  in vdo_enqueue_clean_slab_waiter()
3571 	const struct slab_scrubber *scrubber = &allocator->scrubber;  in vdo_dump_block_allocator()  local
3629 		     READ_ONCE(scrubber->slab_count),  in vdo_dump_block_allocator()
3630 		     vdo_waitq_num_waiters(&scrubber->waiters),  in vdo_dump_block_allocator()
3631 		     vdo_get_admin_state_code(&scrubber->admin_state)->name,  in vdo_dump_block_allocator()
3632 		     scrubber->high_priority_only ? ", high_priority_only " : "");  in vdo_dump_block_allocator()
3914  * initialize_slab_scrubber() - Initialize an allocator's slab scrubber.
3921 	struct slab_scrubber *scrubber = &allocator->scrubber;  in initialize_slab_scrubber()  local
3936 					 journal_data, &scrubber->vio);  in initialize_slab_scrubber()
3942 	INIT_LIST_HEAD(&scrubber->high_priority_slabs);  in initialize_slab_scrubber()
3943 	INIT_LIST_HEAD(&scrubber->slabs);  in initialize_slab_scrubber()
3944 	vdo_set_admin_state_code(&scrubber->admin_state, VDO_ADMIN_STATE_SUSPENDED);  in initialize_slab_scrubber()
4224 		uninitialize_scrubber_vio(&allocator->scrubber);  in vdo_free_slab_depot()
4741  * stop_scrubbing() - Tell the scrubber to stop scrubbing after it finishes the slab it is
4743  * @allocator: The block allocator owning the scrubber to stop.
4747 	struct slab_scrubber *scrubber = &allocator->scrubber;  in stop_scrubbing()  local
4749 	if (vdo_is_state_quiescent(&scrubber->admin_state)) {  in stop_scrubbing()
4752 		vdo_start_draining(&scrubber->admin_state,  in stop_scrubbing()
4842  * resume_scrubbing() - Tell the scrubber to resume scrubbing if it has been stopped.
4848 	struct slab_scrubber *scrubber = &allocator->scrubber;  in resume_scrubbing()  local
4850 	if (!has_slabs_to_scrub(scrubber)) {  in resume_scrubbing()
4855 	result = vdo_resume_if_quiescent(&scrubber->admin_state);  in resume_scrubbing()
4861 	scrub_next_slab(scrubber);  in resume_scrubbing()
5066 		unrecovered += READ_ONCE(depot->allocators[zone].scrubber.slab_count);  in vdo_get_slab_depot_statistics()