16 #include "completion.h"
57 	/* The completion header */
58 	struct vdo_completion completion;  member
191  * as_repair_completion() - Convert a generic completion to a repair_completion.
192  * @completion: The completion to convert.
197 as_repair_completion(struct vdo_completion *completion)  in as_repair_completion()  argument
199 	vdo_assert_completion_type(completion, VDO_REPAIR_COMPLETION);  in as_repair_completion()
200 	return container_of(completion, struct repair_completion, completion);  in as_repair_completion()
206 	struct vdo_completion *completion = &repair->completion;  in prepare_repair_completion()  local
207 	const struct thread_config *thread_config = &completion->vdo->thread_config;  in prepare_repair_completion()
214 	vdo_reset_completion(completion);  in prepare_repair_completion()
215 	vdo_set_completion_callback(completion, callback, thread_id);  in prepare_repair_completion()
222 	vdo_launch_completion(&repair->completion);  in launch_repair_completion()
242 	repair->completion.vdo->block_map->zones[0].page_cache.rebuilding = false;  in free_repair_completion()
250 static void finish_repair(struct vdo_completion *completion)  in finish_repair()  argument
252 	struct vdo_completion *parent = completion->parent;  in finish_repair()
253 	struct vdo *vdo = completion->vdo;  in finish_repair()
254 	struct repair_completion *repair = as_repair_completion(completion);  in finish_repair()
278 	 * Now that we've freed the repair completion and its vast array of journal entries, we  in finish_repair()
286  * @completion: The repair completion.
288 static void abort_repair(struct vdo_completion *completion)  in abort_repair()  argument
290 	struct vdo_completion *parent = completion->parent;  in abort_repair()
291 	int result = completion->result;  in abort_repair()
292 	struct repair_completion *repair = as_repair_completion(completion);  in abort_repair()
294 	if (vdo_state_requires_read_only_rebuild(completion->vdo->load_state))  in abort_repair()
306  * @repair: The repair completion.
315 	vdo_fail_completion(&repair->completion, result);  in abort_on_error()
322  * @completion: The repair completion.
324 static void drain_slab_depot(struct vdo_completion *completion)  in drain_slab_depot()  argument
326 	struct vdo *vdo = completion->vdo;  in drain_slab_depot()
327 	struct repair_completion *repair = as_repair_completion(completion);  in drain_slab_depot()
342 	vdo_drain_slab_depot(vdo->depot, operation, completion);  in drain_slab_depot()
347  * @completion: The repair completion.
351 static void flush_block_map_updates(struct vdo_completion *completion)  in flush_block_map_updates()  argument
353 	vdo_assert_on_admin_thread(completion->vdo, __func__);  in flush_block_map_updates()
356 	prepare_repair_completion(as_repair_completion(completion), drain_slab_depot,  in flush_block_map_updates()
358 	vdo_drain_block_map(completion->vdo->block_map, VDO_ADMIN_STATE_RECOVERING,  in flush_block_map_updates()
359 			    completion);  in flush_block_map_updates()
363 		       struct vdo_completion *completion);
367  * @completion: The vdo_page_completion.
369 static void handle_page_load_error(struct vdo_completion *completion)  in handle_page_load_error()  argument
371 	struct repair_completion *repair = completion->parent;  in handle_page_load_error()
374 	vdo_set_completion_result(&repair->completion, completion->result);  in handle_page_load_error()
375 	vdo_release_page_completion(completion);  in handle_page_load_error()
376 	fetch_page(repair, completion);  in handle_page_load_error()
382  * @completion: The page_completion for writing the page
385 static void unmap_entry(struct block_map_page *page, struct vdo_completion *completion,  in unmap_entry()  argument
389 	vdo_request_page_write(completion);  in unmap_entry()
395  * @completion: The page_completion for writing the page
399 					 struct vdo_completion *completion,  in remove_out_of_bounds_entries()  argument
408 			unmap_entry(page, completion, slot);  in remove_out_of_bounds_entries()
415  * @completion: The page_completion for writing the page
420 static bool process_slot(struct block_map_page *page, struct vdo_completion *completion,  in process_slot()  argument
423 	struct slab_depot *depot = completion->vdo->depot;  in process_slot()
429 		unmap_entry(page, completion, slot);  in process_slot()
445 		unmap_entry(page, completion, slot);  in process_slot()
458 	unmap_entry(page, completion, slot);  in process_slot()
464  * @repair: The repair completion.
465  * @completion: The page completion holding the page.
468 					       struct vdo_completion *completion)  in rebuild_reference_counts_from_page()  argument
474 	result = vdo_get_cached_page(completion, &page);  in rebuild_reference_counts_from_page()
476 		vdo_set_completion_result(&repair->completion, result);  in rebuild_reference_counts_from_page()
486 		remove_out_of_bounds_entries(page, completion, last_slot);  in rebuild_reference_counts_from_page()
493 		if (process_slot(page, completion, slot))  in rebuild_reference_counts_from_page()
500  * @completion: The vdo_page_completion for the fetched page.
504 static void page_loaded(struct vdo_completion *completion)  in page_loaded()  argument
506 	struct repair_completion *repair = completion->parent;  in page_loaded()
509 	rebuild_reference_counts_from_page(repair, completion);  in page_loaded()
510 	vdo_release_page_completion(completion);  in page_loaded()
513 	fetch_page(repair, completion);  in page_loaded()
521 	if (repair->completion.result != VDO_SUCCESS)  in get_pbn_to_fetch()
527 	if (vdo_is_physical_data_block(repair->completion.vdo->depot, pbn))  in get_pbn_to_fetch()
530 	vdo_set_completion_result(&repair->completion, VDO_BAD_MAPPING);  in get_pbn_to_fetch()
537  * @completion: The page completion to use.
542 		       struct vdo_completion *completion)  in fetch_page()  argument
544 	struct vdo_page_completion *page_completion = (struct vdo_page_completion *) completion;  in fetch_page()
545 	struct block_map *block_map = repair->completion.vdo->block_map;  in fetch_page()
567  * @completion: The repair completion.
573 static void rebuild_from_leaves(struct vdo_completion *completion)  in rebuild_from_leaves()  argument
576 	struct repair_completion *repair = as_repair_completion(completion);  in rebuild_from_leaves()
577 	struct block_map *map = completion->vdo->block_map;  in rebuild_from_leaves()
594 		if (fetch_page(repair, &repair->page_completions[i].completion)) {  in rebuild_from_leaves()
607  * @completion: The parent completion of the traversal.
613 static int process_entry(physical_block_number_t pbn, struct vdo_completion *completion)  in process_entry()  argument
615 	struct repair_completion *repair = as_repair_completion(completion);  in process_entry()
616 	struct slab_depot *depot = completion->vdo->depot;  in process_entry()
637 static void rebuild_reference_counts(struct vdo_completion *completion)  in rebuild_reference_counts()  argument
639 	struct repair_completion *repair = as_repair_completion(completion);  in rebuild_reference_counts()
640 	struct vdo *vdo = completion->vdo;  in rebuild_reference_counts()
648 	 * Completion chaining from page cache hits can lead to stack overflow during the rebuild,  in rebuild_reference_counts()
655 	vdo_traverse_forest(vdo->block_map, process_entry, completion);  in rebuild_reference_counts()
728  * @repair: The repair completion.
738 	sector = get_sector(repair->completion.vdo->recovery_journal,  in get_entry()
789  * @completion: The allocator completion.
793 static void add_slab_journal_entries(struct vdo_completion *completion)  in add_slab_journal_entries()  argument
796 	struct repair_completion *repair = completion->parent;  in add_slab_journal_entries()
797 	struct vdo *vdo = completion->vdo;  in add_slab_journal_entries()
799 	struct block_allocator *allocator = vdo_as_block_allocator(completion);  in add_slab_journal_entries()
802 	vdo_prepare_completion(completion, add_slab_journal_entries,  in add_slab_journal_entries()
804 			       completion->callback_thread_id, repair);  in add_slab_journal_entries()
818 				vdo_fail_completion(completion, result);  in add_slab_journal_entries()
836 						  completion))  in add_slab_journal_entries()
842 	vdo_notify_slab_journals_are_recovered(completion);  in add_slab_journal_entries()
853 	struct vdo_completion *completion = &allocator->completion;  in vdo_replay_into_slab_journals()  local
855 	struct vdo *vdo = completion->vdo;  in vdo_replay_into_slab_journals()
862 		vdo_notify_slab_journals_are_recovered(completion);  in vdo_replay_into_slab_journals()
879 	completion->parent = repair;  in vdo_replay_into_slab_journals()
880 	add_slab_journal_entries(completion);  in vdo_replay_into_slab_journals()
883 static void load_slab_depot(struct vdo_completion *completion)  in load_slab_depot()  argument
885 	struct repair_completion *repair = as_repair_completion(completion);  in load_slab_depot()
888 	vdo_assert_on_admin_thread(completion->vdo, __func__);  in load_slab_depot()
890 	if (vdo_state_requires_read_only_rebuild(completion->vdo->load_state)) {  in load_slab_depot()
899 	vdo_load_slab_depot(completion->vdo->depot, operation, completion, repair);  in load_slab_depot()
902 static void flush_block_map(struct vdo_completion *completion)  in flush_block_map()  argument
904 	struct repair_completion *repair = as_repair_completion(completion);  in flush_block_map()
907 	vdo_assert_on_admin_thread(completion->vdo, __func__);  in flush_block_map()
911 	operation = (vdo_state_requires_read_only_rebuild(completion->vdo->load_state) ?  in flush_block_map()
914 	vdo_drain_block_map(completion->vdo->block_map, operation, completion);  in flush_block_map()
923 	if (repair->completion.result != VDO_SUCCESS) {  in finish_if_done()
931 				vdo_release_page_completion(&page_completion->completion);  in finish_if_done()
934 		vdo_launch_completion(&repair->completion);  in finish_if_done()
947 	vdo_set_completion_result(&repair->completion, result);  in abort_block_map_recovery()
954  * @repair: The repair completion.
1006 				struct vdo_completion *completion);
1008 static void block_map_page_loaded(struct vdo_completion *completion)  in block_map_page_loaded()  argument
1010 	struct repair_completion *repair = as_repair_completion(completion->parent);  in block_map_page_loaded()
1014 		recover_ready_pages(repair, completion);  in block_map_page_loaded()
1017 static void handle_block_map_page_load_error(struct vdo_completion *completion)  in handle_block_map_page_load_error()  argument
1019 	struct repair_completion *repair = as_repair_completion(completion->parent);  in handle_block_map_page_load_error()
1022 	abort_block_map_recovery(repair, completion->result);  in handle_block_map_page_load_error()
1026 				 struct vdo_completion *completion)  in fetch_block_map_page()  argument
1040 	vdo_get_page(((struct vdo_page_completion *) completion),  in fetch_block_map_page()
1041 		     &repair->completion.vdo->block_map->zones[0], pbn, true,  in fetch_block_map_page()
1042 		     &repair->completion, block_map_page_loaded,  in fetch_block_map_page()
1047 							    struct vdo_page_completion *completion)  in get_next_page_completion()  argument
1049 	completion++;  in get_next_page_completion()
1050 	if (completion == (&repair->page_completions[repair->page_count]))  in get_next_page_completion()
1051 		completion = &repair->page_completions[0];  in get_next_page_completion()
1052 	return completion;  in get_next_page_completion()
1056 				struct vdo_completion *completion)  in recover_ready_pages()  argument
1058 	struct vdo_page_completion *page_completion = (struct vdo_page_completion *) completion;  in recover_ready_pages()
1071 		result = vdo_get_cached_page(completion, &page);  in recover_ready_pages()
1083 		vdo_request_page_write(completion);  in recover_ready_pages()
1084 		vdo_release_page_completion(completion);  in recover_ready_pages()
1090 		fetch_block_map_page(repair, completion);  in recover_ready_pages()
1092 		completion = &page_completion->completion;  in recover_ready_pages()
1096 static void recover_block_map(struct vdo_completion *completion)  in recover_block_map()  argument
1098 	struct repair_completion *repair = as_repair_completion(completion);  in recover_block_map()
1099 	struct vdo *vdo = completion->vdo;  in recover_block_map()
1143 		fetch_block_map_page(repair, &repair->page_completions[i].completion);  in recover_block_map()
1148 	recover_ready_pages(repair, &repair->page_completions[0].completion);  in recover_block_map()
1218  * @repair: The repair completion.
1224 	struct recovery_journal *journal = repair->completion.vdo->recovery_journal;  in find_recovery_journal_head_and_tail()
1314  *                           sector to the array of numbered mappings in the repair completion,
1316  * @repair: The repair completion.
1326 	struct vdo *vdo = repair->completion.vdo;  in append_sector_entries()
1406 	struct vdo *vdo = repair->completion.vdo;  in parse_journal_for_rebuild()
1448  * @repair: The repair completion.
1456 	struct vdo *vdo = repair->completion.vdo;  in extract_new_mappings()
1503  * @repair: The repair completion.
1517 	struct vdo *vdo = repair->completion.vdo;  in compute_usages()
1557 	struct recovery_journal *journal = repair->completion.vdo->recovery_journal;  in parse_journal_for_recovery()
1583 			vdo_enter_read_only_mode(repair->completion.vdo, VDO_CORRUPT_JOURNAL);  in parse_journal_for_recovery()
1653 	return (vdo_state_requires_read_only_rebuild(repair->completion.vdo->load_state) ?  in parse_journal()
1658 static void finish_journal_load(struct vdo_completion *completion)  in finish_journal_load()  argument
1660 	struct repair_completion *repair = completion->parent;  in finish_journal_load()
1668 	vdo_continue_completion(&repair->completion, parse_journal(repair));  in finish_journal_load()
1671 static void handle_journal_load_error(struct vdo_completion *completion)  in handle_journal_load_error()  argument
1673 	struct repair_completion *repair = completion->parent;  in handle_journal_load_error()
1676 	vdo_set_completion_result(&repair->completion, completion->result);  in handle_journal_load_error()
1677 	vio_record_metadata_io_error(as_vio(completion));  in handle_journal_load_error()
1678 	completion->callback(completion);  in handle_journal_load_error()
1684 	struct vdo *vdo = vio->completion.vdo;  in read_journal_endio()
1691  * @parent: The completion to notify when the operation is complete
1726 	vdo_initialize_completion(&repair->completion, vdo, VDO_REPAIR_COMPLETION);  in vdo_repair()
1727 	repair->completion.error_handler = abort_repair;  in vdo_repair()
1728 	repair->completion.parent = parent;  in vdo_repair()