25 #include "data-vio.h"
35 #include "vio.h"
276  * @completion: The write vio.
281 		container_of(as_vio(completion), struct slab_summary_block, vio);  in finish_update()
289  * @completion: The write VIO.
294 		container_of(as_vio(completion), struct slab_summary_block, vio);  in handle_write_error()
303 	struct vio *vio = bio->bi_private;  in write_slab_summary_endio()  local
305 		container_of(vio, struct slab_summary_block, vio);  in write_slab_summary_endio()
307 	continue_vio_after_io(vio, finish_update, block->allocator->thread_id);  in write_slab_summary_endio()
346 	vdo_submit_metadata_vio(&block->vio, pbn, write_slab_summary_endio,  in launch_write()
370 	if (vdo_is_read_only(block->vio.completion.vdo)) {  in update_slab_summary_entry()
410  *                      reaping again in case we deferred reaping due to an outstanding vio.
411  * @completion: The flush vio.
425  * @completion: The flush vio.
436 	struct vio *vio = bio->bi_private;  in flush_endio()  local
437 	struct slab_journal *journal = vio->completion.parent;  in flush_endio()
439 	continue_vio_after_io(vio, complete_reaping,  in flush_endio()
444  * flush_for_reaping() - A waiter callback for getting a vio with which to flush the lower layer
447  * @context: The newly acquired flush vio.
454 	struct vio *vio = &pooled->vio;  in flush_for_reaping()  local
456 	vio->completion.parent = journal;  in flush_for_reaping()
457 	vdo_submit_flush_vio(vio, flush_endio, handle_flush_error);  in flush_for_reaping()
679 static sequence_number_t get_committing_sequence_number(const struct pooled_vio *vio)  in get_committing_sequence_number()  argument
682 		(const struct packed_slab_journal_block *) vio->vio.data;  in get_committing_sequence_number()
689  * @completion: The write vio as a completion.
729 	struct vio *vio = bio->bi_private;  in write_slab_journal_endio()  local
730 	struct slab_journal *journal = vio->completion.parent;  in write_slab_journal_endio()
732 	continue_vio_after_io(vio, complete_write, journal->slab->allocator->thread_id);  in write_slab_journal_endio()
737  * @waiter: The vio pool waiter which was just notified.
738  * @context: The vio pool entry for the write.
745 	struct vio *vio = &pooled->vio;  in write_slab_journal_block()  local
757 	/* Copy the tail block into the vio. */  in write_slab_journal_block()
758 	memcpy(pooled->vio.data, journal->block, VDO_BLOCK_SIZE);  in write_slab_journal_block()
773 	vio->completion.parent = journal;  in write_slab_journal_block()
780 	vdo_submit_metadata_vio(vdo_forget(vio), block_number, write_slab_journal_endio,  in write_slab_journal_block()
1034  *                                  a VIO for it from the pool.
1063  *                                  locks, and return its VIO to the pool.
1064  * @completion: The VIO that just finished writing.
1068 	struct vio *vio = as_vio(completion);  in finish_reference_block_write()  local
1069 	struct pooled_vio *pooled = vio_as_pooled_vio(vio);  in finish_reference_block_write()
1155 	struct vio *vio = bio->bi_private;  in write_reference_block_endio()  local
1156 	struct reference_block *block = vio->completion.parent;  in write_reference_block_endio()
1159 	continue_vio_after_io(vio, finish_reference_block_write, thread_id);  in write_reference_block_endio()
1164  * @completion: The VIO doing the I/O as a completion.
1169 	struct vio *vio = as_vio(completion);  in handle_io_error()  local
1172 	vio_record_metadata_io_error(vio);  in handle_io_error()
1173 	return_vio_to_pool(slab->allocator->vio_pool, vio_as_pooled_vio(vio));  in handle_io_error()
1180  * write_reference_block() - After a dirty block waiter has gotten a VIO from the VIO pool, copy
1181  *                           its counters and associated data into the VIO, and launch the write.
1183  * @context: The VIO returned by the pool.
1190 	struct vdo_completion *completion = &pooled->vio.completion;  in write_reference_block()
1194 	pack_reference_block(block, pooled->vio.data);  in write_reference_block()
1202 	 * moment. As long as VIO order is preserved, two VIOs updating this block at once will not  in write_reference_block()
1216 	vdo_submit_metadata_vio(&pooled->vio, pbn, write_reference_block_endio,  in write_reference_block()
1682  * @waiter: The vio which should make an entry now.
1810 		/* If the slab is over the blocking threshold, make the vio wait. */  in add_entries()
2235  * @completion: The VIO that just finished reading.
2239 	struct vio *vio = as_vio(completion);  in finish_reference_block_load()  local
2240 	struct pooled_vio *pooled = vio_as_pooled_vio(vio);  in finish_reference_block_load()
2244 	unpack_reference_block((struct packed_reference_block *) vio->data, block);  in finish_reference_block_load()
2255 	struct vio *vio = bio->bi_private;  in load_reference_block_endio()  local
2256 	struct reference_block *block = vio->completion.parent;  in load_reference_block_endio()
2258 	continue_vio_after_io(vio, finish_reference_block_load,  in load_reference_block_endio()
2263  * load_reference_block() - After a block waiter has gotten a VIO from the VIO pool, load the
2266  * @context: The VIO returned by the pool.
2271 	struct vio *vio = &pooled->vio;  in load_reference_block()  local
2276 	vio->completion.parent = block;  in load_reference_block()
2277 	vdo_submit_metadata_vio(vio, block->slab->ref_counts_origin + block_offset,  in load_reference_block()
2409 	struct vio *vio = as_vio(completion);  in finish_loading_journal()  local
2412 	struct packed_slab_journal_block *block = (struct packed_slab_journal_block *) vio->data;  in finish_loading_journal()
2432 	return_vio_to_pool(slab->allocator->vio_pool, vio_as_pooled_vio(vio));  in finish_loading_journal()
2438 	struct vio *vio = bio->bi_private;  in read_slab_journal_tail_endio()  local
2439 	struct slab_journal *journal = vio->completion.parent;  in read_slab_journal_tail_endio()
2441 	continue_vio_after_io(vio, finish_loading_journal,  in read_slab_journal_tail_endio()
2449 	struct vio *vio = as_vio(completion);  in handle_load_error()  local
2451 	vio_record_metadata_io_error(vio);  in handle_load_error()
2452 	return_vio_to_pool(journal->slab->allocator->vio_pool, vio_as_pooled_vio(vio));  in handle_load_error()
2457  * read_slab_journal_tail() - Read the slab journal tail block by using a vio acquired from the vio
2459  * @waiter: The vio pool waiter which has just been notified.
2460  * @context: The vio pool entry given to the waiter.
2470 	struct vio *vio = &pooled->vio;  in read_slab_journal_tail()  local
2482 	vio->completion.parent = journal;  in read_slab_journal_tail()
2483 	vio->completion.callback_thread_id = slab->allocator->thread_id;  in read_slab_journal_tail()
2484 	vdo_submit_metadata_vio(vio, slab->journal_origin + tail_block,  in read_slab_journal_tail()
2655  * uninitialize_scrubber_vio() - Clean up the slab_scrubber's vio.
2660 	vdo_free(vdo_forget(scrubber->vio.data));  in uninitialize_scrubber_vio()
2661 	free_vio_components(&scrubber->vio);  in uninitialize_scrubber_vio()
2681 		vdo_fail_completion(vdo_forget(scrubber->vio.completion.parent), result);  in finish_scrubbing()
2732 		container_of(as_vio(completion), struct slab_scrubber, vio);  in slab_scrubbed()
2749 	vdo_enter_read_only_mode(scrubber->vio.completion.vdo, result);  in abort_scrubbing()
2759 	struct vio *vio = as_vio(completion);  in handle_scrubber_error()  local
2761 	vio_record_metadata_io_error(vio);  in handle_scrubber_error()
2762 	abort_scrubbing(container_of(vio, struct slab_scrubber, vio),  in handle_scrubber_error()
2816  * apply_journal_entries() - Find the relevant vio of the slab journal and apply all valid entries.
2817  * @completion: The metadata read vio completion.
2825 		container_of(as_vio(completion), struct slab_scrubber, vio);  in apply_journal_entries()
2832 	char *end_data = scrubber->vio.data + (end_index * VDO_BLOCK_SIZE);  in apply_journal_entries()
2845 		char *block_data = scrubber->vio.data + (index * VDO_BLOCK_SIZE);  in apply_journal_entries()
2900 	struct vio *vio = bio->bi_private;  in read_slab_journal_endio()  local
2901 	struct slab_scrubber *scrubber = container_of(vio, struct slab_scrubber, vio);  in read_slab_journal_endio()
2909  * @completion: The scrubber's vio completion.
2916 		container_of(as_vio(completion), struct slab_scrubber, vio);  in start_scrubbing()
2924 	vdo_submit_metadata_vio(&scrubber->vio, slab->journal_origin,  in start_scrubbing()
2935 	struct vdo_completion *completion = &scrubber->vio.completion;  in scrub_next_slab()
2976 	scrubber->vio.completion.parent = parent;  in scrub_slabs()
3936 					 journal_data, &scrubber->vio);  in initialize_slab_scrubber()
3967 					 block->outgoing_entries, &block->vio);  in initialize_slab_summary_block()
4197 		free_vio_components(&allocator->summary_blocks[i].vio);  in uninitialize_allocator_summary()
4433  * @completion: The vio which was used to write the summary data.
4452 	struct vio *vio = bio->bi_private;  in write_summary_endio()  local
4453 	struct vdo *vdo = vio->completion.vdo;  in write_summary_endio()
4455 	continue_vio_after_io(vio, finish_combining_zones,  in write_summary_endio()
4498  * @completion: The vio which was used to read the summary data.
4519 	struct vio *vio = bio->bi_private;  in load_summary_endio()  local
4520 	struct vdo *vdo = vio->completion.vdo;  in load_summary_endio()
4522 	continue_vio_after_io(vio, finish_loading_summary,  in load_summary_endio()
4534 	struct vio *vio;  in load_slab_summary()  local
4542 						 (char *) depot->summary_entries, &vio);  in load_slab_summary()
4550 		finish_loading_summary(&vio->completion);  in load_slab_summary()
4554 	vdo_submit_metadata_vio(vio, depot->summary_origin, load_summary_endio,  in load_slab_summary()
4789 				    "vio pool not busy");  in do_drain_step()