18 #include "completion.h"
101 	struct vdo_completion *completion;  member
165 	struct vdo_page_completion *completion;  in page_completion_from_waiter()  local
170 	completion = container_of(waiter, struct vdo_page_completion, waiter);  in page_completion_from_waiter()
171 	vdo_assert_completion_type(&completion->completion, VDO_PAGE_COMPLETION);  in page_completion_from_waiter()
172 	return completion;  in page_completion_from_waiter()
199 		info->vio->completion.callback_thread_id = cache->zone->thread_id;  in initialize_info()
474  * complete_with_page() - Helper to complete the VDO Page Completion request successfully.
476  * @vdo_page_comp: The VDO page completion to complete.
489 		vdo_fail_completion(&vdo_page_comp->completion, VDO_BAD_PAGE);  in complete_with_page()
495 	vdo_finish_completion(&vdo_page_comp->completion);  in complete_with_page()
499  * complete_waiter_with_error() - Complete a page completion with an error code.
500  * @waiter: The page completion, as a waiter.
509 	vdo_fail_completion(&page_completion_from_waiter(waiter)->completion, *result);  in complete_waiter_with_error()
513  * complete_waiter_with_page() - Complete a page completion with a page.
514  * @waiter: The page completion, as a waiter.
527  * Upon completion the waitq will be empty.
540 	 * Increment the busy count once for each pending completion so that this page does not  in distribute_page_over_waitq()
582  * validate_completed_page() - Check that a page completion which is being freed to the cache
588 static int __must_check validate_completed_page(struct vdo_page_completion *completion,  in validate_completed_page()  argument
593 	result = VDO_ASSERT(completion->ready, "VDO Page completion not ready");  in validate_completed_page()
597 	result = VDO_ASSERT(completion->info != NULL,  in validate_completed_page()
598 			    "VDO Page Completion must be complete");  in validate_completed_page()
602 	result = VDO_ASSERT(completion->info->pbn == completion->pbn,  in validate_completed_page()
603 			    "VDO Page Completion pbn must be consistent");  in validate_completed_page()
607 	result = VDO_ASSERT(is_valid(completion->info),  in validate_completed_page()
608 			    "VDO Page Completion page must be valid");  in validate_completed_page()
613 		result = VDO_ASSERT(completion->writable,  in validate_completed_page()
614 				    "VDO Page Completion must be writable");  in validate_completed_page()
649 validate_completed_page_or_enter_read_only_mode(struct vdo_page_completion *completion,  in validate_completed_page_or_enter_read_only_mode()  argument
652 	int result = validate_completed_page(completion, writable);  in validate_completed_page_or_enter_read_only_mode()
657 	enter_zone_read_only_mode(completion->info->cache->zone, result);  in validate_completed_page_or_enter_read_only_mode()
663  * @completion: The page read vio.
665 static void handle_load_error(struct vdo_completion *completion)  in handle_load_error()  argument
667 	int result = completion->result;  in handle_load_error()
668 	struct page_info *info = completion->parent;  in handle_load_error()
672 	vio_record_metadata_io_error(as_vio(completion));  in handle_load_error()
689  * @completion: The vio which has loaded the page. Its parent is the page_info.
691 static void page_is_loaded(struct vdo_completion *completion)  in page_is_loaded()  argument
693 	struct page_info *info = completion->parent;  in page_is_loaded()
710 		vdo_continue_completion(completion, result);  in page_is_loaded()
731  * @completion: The page load completion.
733 static void handle_rebuild_read_error(struct vdo_completion *completion)  in handle_rebuild_read_error()  argument
735 	struct page_info *info = completion->parent;  in handle_rebuild_read_error()
744 	vio_record_metadata_io_error(as_vio(completion));  in handle_rebuild_read_error()
747 	vdo_reset_completion(completion);  in handle_rebuild_read_error()
748 	page_is_loaded(completion);  in handle_rebuild_read_error()
754 	struct page_info *info = vio->completion.parent;  in load_cache_page_endio()
790 static void write_pages(struct vdo_completion *completion);
793 static void handle_flush_error(struct vdo_completion *completion)  in handle_flush_error()  argument
795 	struct page_info *info = completion->parent;  in handle_flush_error()
797 	vio_record_metadata_io_error(as_vio(completion));  in handle_flush_error()
798 	set_persistent_error(info->cache, "flush failed", completion->result);  in handle_flush_error()
799 	write_pages(completion);  in handle_flush_error()
805 	struct page_info *info = vio->completion.parent;  in flush_endio()
871  * Return: true if the page completion is for the desired page number.
881  * allocate_free_page() - Allocate a free page to the first completion in the waiting queue, and
933  * If the selected page is not dirty, immediately allocates the page to the oldest completion
959  * discard_page_for_completion() - Helper used to trigger a discard so that the completion can get
1001  * @completion: The page write vio.
1003 static void handle_page_write_error(struct vdo_completion *completion)  in handle_page_write_error()  argument
1005 	int result = completion->result;  in handle_page_write_error()
1006 	struct page_info *info = completion->parent;  in handle_page_write_error()
1009 	vio_record_metadata_io_error(as_vio(completion));  in handle_page_write_error()
1028 static void page_is_written_out(struct vdo_completion *completion);
1033 	struct page_info *info = vio->completion.parent;  in write_cache_page_endio()
1040  * @completion: The vio which wrote the page. Its parent is a page_info.
1042 static void page_is_written_out(struct vdo_completion *completion)  in page_is_written_out()  argument
1046 	struct page_info *info = completion->parent;  in page_is_written_out()
1111 			struct vdo_completion *completion = &info->vio->completion;  in write_pages()  local
1113 			vdo_reset_completion(completion);  in write_pages()
1114 			completion->callback = page_is_written_out;  in write_pages()
1115 			completion->error_handler = handle_page_write_error;  in write_pages()
1116 			vdo_fail_completion(completion, VDO_READ_ONLY);  in write_pages()
1134  * vdo_release_page_completion() - Release a VDO Page Completion.
1136  * The page referenced by this completion (if any) will no longer be held busy by this completion.
1140 void vdo_release_page_completion(struct vdo_completion *completion)  in vdo_release_page_completion()  argument
1143 	struct vdo_page_completion *page_completion = as_vdo_page_completion(completion);  in vdo_release_page_completion()
1146 	if (completion->result == VDO_SUCCESS) {  in vdo_release_page_completion()
1177  *                              Completion.
1193  * vdo_get_page() - Initialize a page completion and get a block map page.
1204  * by the completion to be loaded from disk. When the callback is invoked, the page will be
1214 	struct vdo_completion *completion = &page_completion->completion;  in vdo_get_page()  local
1219 			    "New page completion was not already on a wait queue");  in vdo_get_page()
1227 	vdo_initialize_completion(completion, cache->vdo, VDO_PAGE_COMPLETION);  in vdo_get_page()
1228 	vdo_prepare_completion(completion, callback, error_handler,  in vdo_get_page()
1230 	completion->requeue = requeue;  in vdo_get_page()
1233 		vdo_fail_completion(completion, VDO_READ_ONLY);  in vdo_get_page()
1284  * @completion: The vdo_page_completion containing the page.
1286 void vdo_request_page_write(struct vdo_completion *completion)  in vdo_request_page_write()  argument
1289 	struct vdo_page_completion *vdo_page_comp = as_vdo_page_completion(completion);  in vdo_request_page_write()
1300  * vdo_get_cached_page() - Get the block map page from a page completion.
1301  * @completion: A vdo page completion whose callback has been called.
1306 int vdo_get_cached_page(struct vdo_completion *completion,  in vdo_get_cached_page()  argument
1312 	vpc = as_vdo_page_completion(completion);  in vdo_get_cached_page()
1552 static void finish_page_write(struct vdo_completion *completion)  in finish_page_write()  argument
1555 	struct vio *vio = as_vio(completion);  in finish_page_write()
1557 	struct tree_page *page = completion->parent;  in finish_page_write()
1598 static void handle_write_error(struct vdo_completion *completion)  in handle_write_error()  argument
1600 	int result = completion->result;  in handle_write_error()
1601 	struct vio *vio = as_vio(completion);  in handle_write_error()
1612 static void write_initialized_page(struct vdo_completion *completion)  in write_initialized_page()  argument
1614 	struct vio *vio = as_vio(completion);  in write_initialized_page()
1617 	struct tree_page *tree_page = completion->parent;  in write_initialized_page()
1649 	struct vdo_completion *completion = &vio->vio.completion;  in write_page()  local
1664 	completion->parent = tree_page;  in write_page()
1666 	completion->callback_thread_id = zone->thread_id;  in write_page()
1682 		write_initialized_page(completion);  in write_page()
1718 	data_vio->vio.completion.error_handler = handle_data_vio_error;  in finish_lookup()
1820 static void finish_block_map_page_load(struct vdo_completion *completion)  in finish_block_map_page_load()  argument
1826 	struct vio *vio = as_vio(completion);  in finish_block_map_page_load()
1828 	struct data_vio *data_vio = completion->parent;  in finish_block_map_page_load()
1848 static void handle_io_error(struct vdo_completion *completion)  in handle_io_error()  argument
1850 	int result = completion->result;  in handle_io_error()
1851 	struct vio *vio = as_vio(completion);  in handle_io_error()
1853 	struct data_vio *data_vio = completion->parent;  in handle_io_error()
1864 	struct data_vio *data_vio = vio->completion.parent;  in load_page_endio()
1877 	pooled->vio.completion.parent = data_vio;  in load_page()
1936 static void allocation_failure(struct vdo_completion *completion)  in allocation_failure()  argument
1938 	struct data_vio *data_vio = as_data_vio(completion);  in allocation_failure()
1940 	if (vdo_requeue_completion_if_needed(completion,  in allocation_failure()
1944 	abort_lookup(data_vio, completion->result, "allocation");  in allocation_failure()
2065 static void finish_block_map_allocation(struct vdo_completion *completion)  in finish_block_map_allocation()  argument
2071 	struct data_vio *data_vio = as_data_vio(completion);  in finish_block_map_allocation()
2126 static void release_block_map_write_lock(struct vdo_completion *completion)  in release_block_map_write_lock()  argument
2128 	struct data_vio *data_vio = as_data_vio(completion);  in release_block_map_write_lock()
2141 static void set_block_map_page_reference_count(struct vdo_completion *completion)  in set_block_map_page_reference_count()  argument
2143 	struct data_vio *data_vio = as_data_vio(completion);  in set_block_map_page_reference_count()
2147 	completion->callback = release_block_map_write_lock;  in set_block_map_page_reference_count()
2148 	vdo_modify_reference_count(completion, &data_vio->increment_updater);  in set_block_map_page_reference_count()
2151 static void journal_block_map_allocation(struct vdo_completion *completion)  in journal_block_map_allocation()  argument
2153 	struct data_vio *data_vio = as_data_vio(completion);  in journal_block_map_allocation()
2159 	vdo_add_recovery_journal_entry(completion->vdo->recovery_journal, data_vio);  in journal_block_map_allocation()
2162 static void allocate_block(struct vdo_completion *completion)  in allocate_block()  argument
2164 	struct data_vio *data_vio = as_data_vio(completion);  in allocate_block()
2500 	struct vdo_completion *completion = cursors->completion;  in finish_cursor()  local
2508 	vdo_finish_completion(completion);  in finish_cursor()
2515  * @completion: The VIO doing a read or write.
2517 static void continue_traversal(struct vdo_completion *completion)  in continue_traversal()  argument
2519 	vio_record_metadata_io_error(as_vio(completion));  in continue_traversal()
2520 	traverse(completion->parent);  in continue_traversal()
2525  * @completion: The VIO doing the read.
2527 static void finish_traversal_load(struct vdo_completion *completion)  in finish_traversal_load()  argument
2529 	struct cursor *cursor = completion->parent;  in finish_traversal_load()
2545 	struct cursor *cursor = vio->completion.parent;  in traversal_endio()
2594 									    cursor->parent->completion);  in traverse()
2633 	pooled->vio.completion.parent = cursor;  in launch_cursor()
2634 	pooled->vio.completion.callback_thread_id = cursor->parent->zone->thread_id;  in launch_cursor()
2673  * @completion: The completion to notify on each traversed PBN, and when traversal completes.
2676 			 struct vdo_completion *completion)  in vdo_traverse_forest()  argument
2685 		vdo_fail_completion(completion, result);  in vdo_traverse_forest()
2692 	cursors->completion = completion;  in vdo_traverse_forest()
3038 static void grow_forest(void *context, struct vdo_completion *completion)  in grow_forest()  argument
3041 	vdo_finish_completion(completion);  in grow_forest()
3062 /* Release the page completion and then continue the requester. */
3063 static inline void finish_processing_page(struct vdo_completion *completion, int result)  in finish_processing_page()  argument
3065 	struct vdo_completion *parent = completion->parent;  in finish_processing_page()
3067 	vdo_release_page_completion(completion);  in finish_processing_page()
3071 static void handle_page_error(struct vdo_completion *completion)  in handle_page_error()  argument
3073 	finish_processing_page(completion, completion->result);  in handle_page_error()
3089 		     modifiable, &data_vio->vio.completion,  in fetch_mapping_page()
3161 static void get_mapping_from_fetched_page(struct vdo_completion *completion)  in get_mapping_from_fetched_page()  argument
3164 	struct vdo_page_completion *vpc = as_vdo_page_completion(completion);  in get_mapping_from_fetched_page()
3167 	struct data_vio *data_vio = as_data_vio(completion->parent);  in get_mapping_from_fetched_page()
3170 	if (completion->result != VDO_SUCCESS) {  in get_mapping_from_fetched_page()
3171 		finish_processing_page(completion, completion->result);  in get_mapping_from_fetched_page()
3177 		finish_processing_page(completion, result);  in get_mapping_from_fetched_page()
3186 	finish_processing_page(completion, result);  in get_mapping_from_fetched_page()
3230 static void put_mapping_in_fetched_page(struct vdo_completion *completion)  in put_mapping_in_fetched_page()  argument
3232 	struct data_vio *data_vio = as_data_vio(completion->parent);  in put_mapping_in_fetched_page()
3238 	if (completion->result != VDO_SUCCESS) {  in put_mapping_in_fetched_page()
3239 		finish_processing_page(completion, completion->result);  in put_mapping_in_fetched_page()
3243 	vpc = as_vdo_page_completion(completion);  in put_mapping_in_fetched_page()
3246 		finish_processing_page(completion, result);  in put_mapping_in_fetched_page()
3258 	finish_processing_page(completion, VDO_SUCCESS);  in put_mapping_in_fetched_page()