58  *   completion is required for further work to be done by the issuer.
101  * completion will be enqueued on a cpu queue. This obviates the need for the releasing threads to
105  * Whenever the pool's completion is run on a cpu thread, it calls process_release_callback() which
159 	/* Completion for scheduling releases */
160 	struct vdo_completion completion;  member
215 as_data_vio_pool(struct vdo_completion *completion)  in as_data_vio_pool()  argument
217 	vdo_assert_completion_type(completion, VDO_DATA_VIO_POOL_COMPLETION);  in as_data_vio_pool()
218 	return container_of(completion, struct data_vio_pool, completion);  in as_data_vio_pool()
275 	int error = vdo_status_to_errno(data_vio->vio.completion.result);  in acknowledge_data_vio()
415  * @completion: The data_vio for an external data request as a completion.
419 static void attempt_logical_block_lock(struct vdo_completion *completion)  in attempt_logical_block_lock()  argument
421 	struct data_vio *data_vio = as_data_vio(completion);  in attempt_logical_block_lock()
464 		complete_data_vio(completion);  in attempt_logical_block_lock()
488 	struct vdo_completion *completion = &data_vio->vio.completion;  in launch_data_vio()  local
506 	vdo_reset_completion(completion);  in launch_data_vio()
507 	completion->error_handler = handle_data_vio_error;  in launch_data_vio()
509 	vdo_enqueue_completion(completion, VDO_DEFAULT_Q_MAP_BIO_PRIORITY);  in launch_data_vio()
590 	launch_bio(limiter->pool->completion.vdo, data_vio, bio);  in assign_data_vio()
666 	pool->completion.requeue = true;  in schedule_releases()
667 	vdo_launch_completion_with_priority(&pool->completion,  in schedule_releases()
696  * @completion: The pool with data_vios to release.
698 static void process_release_callback(struct vdo_completion *completion)  in process_release_callback()  argument
700 	struct data_vio_pool *pool = as_data_vio_pool(completion);  in process_release_callback()
864 	vdo_initialize_completion(&pool->completion, vdo, VDO_DATA_VIO_POOL_COMPLETION);  in make_data_vio_pool()
865 	vdo_prepare_completion(&pool->completion, process_release_callback,  in make_data_vio_pool()
983 	launch_bio(pool->completion.vdo, data_vio, bio);  in vdo_launch_bio()
1008  * @completion: The completion to notify when the pool has drained.
1010 void drain_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion)  in drain_data_vio_pool()  argument
1012 	assert_on_vdo_cpu_thread(completion->vdo, __func__);  in drain_data_vio_pool()
1013 	vdo_start_draining(&pool->state, VDO_ADMIN_STATE_SUSPENDING, completion,  in drain_data_vio_pool()
1019  * @completion: The completion to notify when the pool has resumed.
1021 void resume_data_vio_pool(struct data_vio_pool *pool, struct vdo_completion *completion)  in resume_data_vio_pool()  argument
1023 	assert_on_vdo_cpu_thread(completion->vdo, __func__);  in resume_data_vio_pool()
1024 	vdo_continue_completion(completion, vdo_resume_if_quiescent(&pool->state));  in resume_data_vio_pool()
1129 static void release_allocated_lock(struct vdo_completion *completion)  in release_allocated_lock()  argument
1131 	struct data_vio *data_vio = as_data_vio(completion);  in release_allocated_lock()
1201 	next_lock_holder->vio.completion.requeue = true;  in transfer_lock()
1209 static void release_logical_lock(struct vdo_completion *completion)  in release_logical_lock()  argument
1211 	struct data_vio *data_vio = as_data_vio(completion);  in release_logical_lock()
1226 static void clean_hash_lock(struct vdo_completion *completion)  in clean_hash_lock()  argument
1228 	struct data_vio *data_vio = as_data_vio(completion);  in clean_hash_lock()
1231 	if (completion->result != VDO_SUCCESS) {  in clean_hash_lock()
1248 	struct vdo_completion *completion = &data_vio->vio.completion;  in finish_cleanup()  local
1257 	    (completion->result != VDO_SUCCESS)) {  in finish_cleanup()
1258 		struct data_vio_pool *pool = completion->vdo->data_vio_pool;  in finish_cleanup()
1260 		vdo_funnel_queue_put(pool->queue, &completion->work_queue_entry_link);  in finish_cleanup()
1269 	completion->requeue = true;  in finish_cleanup()
1299 		    (data_vio->vio.completion.result != VDO_READ_ONLY))  in perform_cleanup_stage()
1312 void complete_data_vio(struct vdo_completion *completion)  in complete_data_vio()  argument
1314 	struct data_vio *data_vio = as_data_vio(completion);  in complete_data_vio()
1316 	completion->error_handler = NULL;  in complete_data_vio()
1322 static void enter_read_only_mode(struct vdo_completion *completion)  in enter_read_only_mode()  argument
1324 	if (vdo_is_read_only(completion->vdo))  in enter_read_only_mode()
1327 	if (completion->result != VDO_READ_ONLY) {  in enter_read_only_mode()
1328 		struct data_vio *data_vio = as_data_vio(completion);  in enter_read_only_mode()
1330 		vdo_log_error_strerror(completion->result,  in enter_read_only_mode()
1339 	vdo_enter_read_only_mode(completion->vdo, completion->result);  in enter_read_only_mode()
1342 void handle_data_vio_error(struct vdo_completion *completion)  in handle_data_vio_error()  argument
1344 	struct data_vio *data_vio = as_data_vio(completion);  in handle_data_vio_error()
1346 	if ((completion->result == VDO_READ_ONLY) || (data_vio->user_bio == NULL))  in handle_data_vio_error()
1347 		enter_read_only_mode(completion);  in handle_data_vio_error()
1350 	complete_data_vio(completion);  in handle_data_vio_error()
1387 	data_vio->vio.completion.error_handler = error_handler;  in data_vio_allocate_data_block()
1442  * @completion: The data_vio which has just finished its read.
1446 static void modify_for_partial_write(struct vdo_completion *completion)  in modify_for_partial_write()  argument
1448 	struct data_vio *data_vio = as_data_vio(completion);  in modify_for_partial_write()
1468 static void complete_read(struct vdo_completion *completion)  in complete_read()  argument
1470 	struct data_vio *data_vio = as_data_vio(completion);  in complete_read()
1486 		modify_for_partial_write(completion);  in complete_read()
1494 	complete_data_vio(completion);  in complete_read()
1512 static void complete_zero_read(struct vdo_completion *completion)  in complete_zero_read()  argument
1514 	struct data_vio *data_vio = as_data_vio(completion);  in complete_zero_read()
1521 			modify_for_partial_write(completion);  in complete_zero_read()
1528 	complete_read(completion);  in complete_zero_read()
1536 static void read_block(struct vdo_completion *completion)  in read_block()  argument
1538 	struct data_vio *data_vio = as_data_vio(completion);  in read_block()
1539 	struct vio *vio = as_vio(completion);  in read_block()
1579 reference_count_update_completion_as_data_vio(struct vdo_completion *completion)  in reference_count_update_completion_as_data_vio()  argument
1581 	if (completion->type == VIO_COMPLETION)  in reference_count_update_completion_as_data_vio()
1582 		return as_data_vio(completion);  in reference_count_update_completion_as_data_vio()
1584 	return container_of(completion, struct data_vio, decrement_completion);  in reference_count_update_completion_as_data_vio()
1591  * @completion: The completion of the write in progress.
1593 static void update_block_map(struct vdo_completion *completion)  in update_block_map()  argument
1595 	struct data_vio *data_vio = reference_count_update_completion_as_data_vio(completion);  in update_block_map()
1605 	completion = &data_vio->vio.completion;  in update_block_map()
1606 	vdo_set_completion_result(completion, data_vio->decrement_completion.result);  in update_block_map()
1607 	if (completion->result != VDO_SUCCESS) {  in update_block_map()
1608 		handle_data_vio_error(completion);  in update_block_map()
1612 	completion->error_handler = handle_data_vio_error;  in update_block_map()
1616 		completion->callback = complete_data_vio;  in update_block_map()
1622 static void decrement_reference_count(struct vdo_completion *completion)  in decrement_reference_count()  argument
1624 	struct data_vio *data_vio = container_of(completion, struct data_vio,  in decrement_reference_count()
1629 	vdo_set_completion_callback(completion, update_block_map,  in decrement_reference_count()
1631 	completion->error_handler = update_block_map;  in decrement_reference_count()
1632 	vdo_modify_reference_count(completion, &data_vio->decrement_updater);  in decrement_reference_count()
1635 static void increment_reference_count(struct vdo_completion *completion)  in increment_reference_count()  argument
1637 	struct data_vio *data_vio = as_data_vio(completion);  in increment_reference_count()
1653 	completion->error_handler = update_block_map;  in increment_reference_count()
1654 	vdo_modify_reference_count(completion, &data_vio->increment_updater);  in increment_reference_count()
1658 static void journal_remapping(struct vdo_completion *completion)  in journal_remapping()  argument
1660 	struct data_vio *data_vio = as_data_vio(completion);  in journal_remapping()
1684 	vdo_add_recovery_journal_entry(completion->vdo->recovery_journal, data_vio);  in journal_remapping()
1693 static void read_old_block_mapping(struct vdo_completion *completion)  in read_old_block_mapping()  argument
1695 	struct data_vio *data_vio = as_data_vio(completion);  in read_old_block_mapping()
1721 static void pack_compressed_data(struct vdo_completion *completion)  in pack_compressed_data()  argument
1723 	struct data_vio *data_vio = as_data_vio(completion);  in pack_compressed_data()
1742 static void compress_data_vio(struct vdo_completion *completion)  in compress_data_vio()  argument
1744 	struct data_vio *data_vio = as_data_vio(completion);  in compress_data_vio()
1813 static void hash_data_vio(struct vdo_completion *completion)  in hash_data_vio()  argument
1815 	struct data_vio *data_vio = as_data_vio(completion);  in hash_data_vio()
1852 	vdo_set_completion_result(&data_vio->vio.completion,  in write_bio_finished()
1901 static void acknowledge_write_callback(struct vdo_completion *completion)  in acknowledge_write_callback()  argument
1903 	struct data_vio *data_vio = as_data_vio(completion);  in acknowledge_write_callback()
1904 	struct vdo *vdo = completion->vdo;  in acknowledge_write_callback()
1926 static void allocate_block(struct vdo_completion *completion)  in allocate_block()  argument
1928 	struct data_vio *data_vio = as_data_vio(completion);  in allocate_block()
1935 	completion->error_handler = handle_data_vio_error;  in allocate_block()
1958 static void handle_allocation_error(struct vdo_completion *completion)  in handle_allocation_error()  argument
1960 	struct data_vio *data_vio = as_data_vio(completion);  in handle_allocation_error()
1962 	if (completion->result == VDO_NO_SPACE) {  in handle_allocation_error()
1964 		vdo_reset_completion(completion);  in handle_allocation_error()
1965 		completion->error_handler = handle_data_vio_error;  in handle_allocation_error()
1971 	handle_data_vio_error(completion);  in handle_allocation_error()
1987 void continue_data_vio_with_block_map_slot(struct vdo_completion *completion)  in continue_data_vio_with_block_map_slot()  argument
1989 	struct data_vio *data_vio = as_data_vio(completion);  in continue_data_vio_with_block_map_slot()
2006 		completion->callback = complete_data_vio;  in continue_data_vio_with_block_map_slot()