1 // SPDX-License-Identifier: GPL-2.0-only
12 #include "memory-alloc.h"
14 #include "string-utils.h"
16 #include "admin-state.h"
19 #include "data-vio.h"
22 #include "io-submitter.h"
23 #include "physical-zone.h"
24 #include "status-codes.h"
36  * vdo_get_compressed_block_fragment() - Get a reference to a compressed fragment from a compressed
41  * @fragment_size [out] The size of the fragment.
53 	u8 slot;  in vdo_get_compressed_block_fragment()  local
59 	version = vdo_unpack_version_number(block->header.version);  in vdo_get_compressed_block_fragment()
63 	slot = mapping_state - VDO_MAPPING_STATE_COMPRESSED_BASE;  in vdo_get_compressed_block_fragment()
64 	if (slot >= VDO_MAX_COMPRESSION_SLOTS)  in vdo_get_compressed_block_fragment()
67 	compressed_size = __le16_to_cpu(block->header.sizes[slot]);  in vdo_get_compressed_block_fragment()
68 	for (i = 0; i < slot; i++) {  in vdo_get_compressed_block_fragment()
69 		offset += __le16_to_cpu(block->header.sizes[i]);  in vdo_get_compressed_block_fragment()
83  * assert_on_packer_thread() - Check that we are on the packer thread.
89 	VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == packer->thread_id),  in assert_on_packer_thread()
94  * insert_in_sorted_list() - Insert a bin to the list.
105 	list_for_each_entry(active_bin, &packer->bins, list)  in insert_in_sorted_list()
106 		if (active_bin->free_space > bin->free_space) {  in insert_in_sorted_list()
107 			list_move_tail(&bin->list, &active_bin->list);  in insert_in_sorted_list()
111 	list_move_tail(&bin->list, &packer->bins);  in insert_in_sorted_list()
115  * make_bin() - Allocate a bin and put it into the packer's list.
128 	bin->free_space = VDO_COMPRESSED_BLOCK_DATA_SIZE;  in make_bin()
129 	INIT_LIST_HEAD(&bin->list);  in make_bin()
130 	list_add_tail(&bin->list, &packer->bins);  in make_bin()
135  * vdo_make_packer() - Make a new block packer.
153 	packer->thread_id = vdo->thread_config.packer_thread;  in vdo_make_packer()
154 	packer->size = bin_count;  in vdo_make_packer()
155 	INIT_LIST_HEAD(&packer->bins);  in vdo_make_packer()
156 	vdo_set_admin_state_code(&packer->state, VDO_ADMIN_STATE_NORMAL_OPERATION);  in vdo_make_packer()
172 				       struct vio *, __func__, &packer->canceled_bin);  in vdo_make_packer()
178 	result = vdo_make_default_thread(vdo, packer->thread_id);  in vdo_make_packer()
189  * vdo_free_packer() - Free a block packer.
199 	list_for_each_entry_safe(bin, tmp, &packer->bins, list) {  in vdo_free_packer()
200 		list_del_init(&bin->list);  in vdo_free_packer()
204 	vdo_free(vdo_forget(packer->canceled_bin));  in vdo_free_packer()
209  * get_packer_from_data_vio() - Get the packer from a data_vio.
216 	return vdo_from_data_vio(data_vio)->packer;  in get_packer_from_data_vio()
220  * vdo_get_packer_statistics() - Get the current statistics from the packer.
227 	const struct packer_statistics *stats = &packer->statistics;  in vdo_get_packer_statistics()
230 		.compressed_fragments_written = READ_ONCE(stats->compressed_fragments_written),  in vdo_get_packer_statistics()
231 		.compressed_blocks_written = READ_ONCE(stats->compressed_blocks_written),  in vdo_get_packer_statistics()
232 		.compressed_fragments_in_packer = READ_ONCE(stats->compressed_fragments_in_packer),  in vdo_get_packer_statistics()
237  * abort_packing() - Abort packing a data_vio.
244 	WRITE_ONCE(packer->statistics.compressed_fragments_in_packer,  in abort_packing()
245 		   packer->statistics.compressed_fragments_in_packer - 1);  in abort_packing()
251  * release_compressed_write_waiter() - Update a data_vio for which a successful compressed write
259 	data_vio->new_mapped = (struct zoned_pbn) {  in release_compressed_write_waiter()
260 		.pbn = allocation->pbn,  in release_compressed_write_waiter()
261 		.zone = allocation->zone,  in release_compressed_write_waiter()
262 		.state = data_vio->compression.slot + VDO_MAPPING_STATE_COMPRESSED_BASE,  in release_compressed_write_waiter()
265 	vdo_share_compressed_write_lock(data_vio, allocation->lock);  in release_compressed_write_waiter()
266 	update_metadata_for_data_vio_write(data_vio, allocation->lock);  in release_compressed_write_waiter()
270  * finish_compressed_write() - Finish a compressed block write.
283 	 * Process all the non-agent waiters first to ensure that the pbn lock can not be released  in finish_compressed_write()
286 	for (client = agent->compression.next_in_batch; client != NULL; client = next) {  in finish_compressed_write()
287 		next = client->compression.next_in_batch;  in finish_compressed_write()
288 		release_compressed_write_waiter(client, &agent->allocation);  in finish_compressed_write()
291 	completion->error_handler = handle_data_vio_error;  in finish_compressed_write()
292 	release_compressed_write_waiter(agent, &agent->allocation);  in finish_compressed_write()
298 	struct allocation *allocation = &agent->allocation;  in handle_compressed_write_error()
301 	if (vdo_requeue_completion_if_needed(completion, allocation->zone->thread_id))  in handle_compressed_write_error()
306 			       (unsigned long long) allocation->pbn);  in handle_compressed_write_error()
308 	for (client = agent->compression.next_in_batch; client != NULL; client = next) {  in handle_compressed_write_error()
309 		next = client->compression.next_in_batch;  in handle_compressed_write_error()
315 	completion->error_handler = handle_data_vio_error;  in handle_compressed_write_error()
320  * add_to_bin() - Put a data_vio in a specific packer_bin in which it will definitely fit.
326 	data_vio->compression.bin = bin;  in add_to_bin()
327 	data_vio->compression.slot = bin->slots_used;  in add_to_bin()
328 	bin->incoming[bin->slots_used++] = data_vio;  in add_to_bin()
332  * remove_from_bin() - Get the next data_vio whose compression has not been canceled from a bin.
341 	while (bin->slots_used > 0) {  in remove_from_bin()
342 		struct data_vio *data_vio = bin->incoming[--bin->slots_used];  in remove_from_bin()
345 			data_vio->compression.bin = NULL;  in remove_from_bin()
349 		add_to_bin(packer->canceled_bin, data_vio);  in remove_from_bin()
353 	bin->free_space = VDO_COMPRESSED_BLOCK_DATA_SIZE;  in remove_from_bin()
358  * initialize_compressed_block() - Initialize a compressed block.
360  * @size: The size of the agent's fragment.
364  * data field, it needn't be copied. So all we need do is initialize the header and set the size of
367 static void initialize_compressed_block(struct compressed_block *block, u16 size)  in initialize_compressed_block()  argument
375 	block->header.version = vdo_pack_version_number(COMPRESSED_BLOCK_1_0);  in initialize_compressed_block()
376 	block->header.sizes[0] = __cpu_to_le16(size);  in initialize_compressed_block()
380  * pack_fragment() - Pack a data_vio's fragment into the compressed block in which it is already
391 					       block_size_t offset, slot_number_t slot,  in pack_fragment()  argument
394 	struct compression_state *to_pack = &data_vio->compression;  in pack_fragment()
395 	char *fragment = to_pack->block->data;  in pack_fragment()
397 	to_pack->next_in_batch = compression->next_in_batch;  in pack_fragment()
398 	compression->next_in_batch = data_vio;  in pack_fragment()
399 	to_pack->slot = slot;  in pack_fragment()
400 	block->header.sizes[slot] = __cpu_to_le16(to_pack->size);  in pack_fragment()
401 	memcpy(&block->data[offset], fragment, to_pack->size);  in pack_fragment()
402 	return (offset + to_pack->size);  in pack_fragment()
406  * compressed_write_end_io() - The bio_end_io for a compressed block write.
411 	struct data_vio *data_vio = vio_as_data_vio(bio->bi_private);  in compressed_write_end_io()
415 	continue_data_vio_with_error(data_vio, blk_status_to_errno(bio->bi_status));  in compressed_write_end_io()
419  * write_bin() - Write out a bin.
427 	slot_number_t slot = 1;  in write_bin()  local
437 	compression = &agent->compression;  in write_bin()
438 	compression->slot = 0;  in write_bin()
439 	block = compression->block;  in write_bin()
440 	initialize_compressed_block(block, compression->size);  in write_bin()
441 	offset = compression->size;  in write_bin()
444 		offset = pack_fragment(compression, client, offset, slot++, block);  in write_bin()
450 	if (slot == 1) {  in write_bin()
455 	if (slot < VDO_MAX_COMPRESSION_SLOTS) {  in write_bin()
457 		memset(&block->header.sizes[slot], 0,  in write_bin()
458 		       (VDO_MAX_COMPRESSION_SLOTS - slot) * sizeof(__le16));  in write_bin()
461 	agent->vio.completion.error_handler = handle_compressed_write_error;  in write_bin()
467 	result = vio_reset_bio(&agent->vio, (char *) block, compressed_write_end_io,  in write_bin()
468 			       REQ_OP_WRITE, agent->allocation.pbn);  in write_bin()
478 	stats = &packer->statistics;  in write_bin()
479 	WRITE_ONCE(stats->compressed_fragments_in_packer,  in write_bin()
480 		   (stats->compressed_fragments_in_packer - slot));  in write_bin()
481 	WRITE_ONCE(stats->compressed_fragments_written,  in write_bin()
482 		   (stats->compressed_fragments_written + slot));  in write_bin()
483 	WRITE_ONCE(stats->compressed_blocks_written,  in write_bin()
484 		   stats->compressed_blocks_written + 1);  in write_bin()
490  * add_data_vio_to_packer_bin() - Add a data_vio to a bin's incoming queue
502 	if (bin->free_space < data_vio->compression.size)  in add_data_vio_to_packer_bin()
506 	bin->free_space -= data_vio->compression.size;  in add_data_vio_to_packer_bin()
509 	if ((bin->slots_used == VDO_MAX_COMPRESSION_SLOTS) ||  in add_data_vio_to_packer_bin()
510 	    (bin->free_space == 0))  in add_data_vio_to_packer_bin()
518  * select_bin() - Select the bin that should be used to pack the compressed data in a data_vio with
532 	list_for_each_entry(bin, &packer->bins, list) {  in select_bin()
533 		if (bin->free_space >= data_vio->compression.size)  in select_bin()
542 	 * size of the incoming block, it seems wrong to force that bin to write when giving up on  in select_bin()
545 	fullest_bin = list_first_entry(&packer->bins, struct packer_bin, list);  in select_bin()
546 	if (data_vio->compression.size >=  in select_bin()
547 	    (VDO_COMPRESSED_BLOCK_DATA_SIZE - fullest_bin->free_space))  in select_bin()
558  * vdo_attempt_packing() - Attempt to rewrite the data in this data_vio as part of a compressed
581 	WRITE_ONCE(packer->statistics.compressed_fragments_in_packer,  in vdo_attempt_packing()
582 		   packer->statistics.compressed_fragments_in_packer + 1);  in vdo_attempt_packing()
588 	if (!vdo_is_state_normal(&packer->state) ||  in vdo_attempt_packing()
589 	    (data_vio->flush_generation < packer->flush_generation)) {  in vdo_attempt_packing()
614  * check_for_drain_complete() - Check whether the packer has drained.
619 	if (vdo_is_state_draining(&packer->state) && (packer->canceled_bin->slots_used == 0))  in check_for_drain_complete()
620 		vdo_finish_draining(&packer->state);  in check_for_drain_complete()
624  * write_all_non_empty_bins() - Write out all non-empty bins on behalf of a flush or suspend.
631 	list_for_each_entry(bin, &packer->bins, list)  in write_all_non_empty_bins()
634 		 * We don't need to re-sort the bin here since this loop will make every bin have  in write_all_non_empty_bins()
642  * vdo_flush_packer() - Request that the packer flush asynchronously.
652 	if (vdo_is_state_normal(&packer->state))  in vdo_flush_packer()
657  * vdo_remove_lock_holder_from_packer() - Remove a lock holder from the packer.
667 	slot_number_t slot;  in vdo_remove_lock_holder_from_packer()  local
671 	lock_holder = vdo_forget(data_vio->compression.lock_holder);  in vdo_remove_lock_holder_from_packer()
672 	bin = lock_holder->compression.bin;  in vdo_remove_lock_holder_from_packer()
675 	slot = lock_holder->compression.slot;  in vdo_remove_lock_holder_from_packer()
676 	bin->slots_used--;  in vdo_remove_lock_holder_from_packer()
677 	if (slot < bin->slots_used) {  in vdo_remove_lock_holder_from_packer()
678 		bin->incoming[slot] = bin->incoming[bin->slots_used];  in vdo_remove_lock_holder_from_packer()
679 		bin->incoming[slot]->compression.slot = slot;  in vdo_remove_lock_holder_from_packer()
682 	lock_holder->compression.bin = NULL;  in vdo_remove_lock_holder_from_packer()
683 	lock_holder->compression.slot = 0;  in vdo_remove_lock_holder_from_packer()
685 	if (bin != packer->canceled_bin) {  in vdo_remove_lock_holder_from_packer()
686 		bin->free_space += lock_holder->compression.size;  in vdo_remove_lock_holder_from_packer()
695  * vdo_increment_packer_flush_generation() - Increment the flush generation in the packer.
704 	packer->flush_generation++;  in vdo_increment_packer_flush_generation()
709  * initiate_drain() - Initiate a drain.
721  * vdo_drain_packer() - Drain the packer by preventing any more VIOs from entering the packer and
729 	vdo_start_draining(&packer->state, VDO_ADMIN_STATE_SUSPENDING, completion,  in vdo_drain_packer()
734  * vdo_resume_packer() - Resume a packer which has been suspended.
741 	vdo_continue_completion(parent, vdo_resume_if_quiescent(&packer->state));  in vdo_resume_packer()
746 	if (bin->slots_used == 0)  in dump_packer_bin()
751 		     (canceled ? "Canceled" : ""), bin->slots_used, bin->free_space);  in dump_packer_bin()
754 	 * FIXME: dump vios in bin->incoming? The vios should have been dumped from the vio pool.  in dump_packer_bin()
760  * vdo_dump_packer() - Dump the packer.
763  * Context: dumps in a thread-unsafe fashion.
771 		     (unsigned long long) packer->flush_generation,  in vdo_dump_packer()
772 		     vdo_get_admin_state_code(&packer->state)->name,  in vdo_dump_packer()
773 		     (unsigned long long) packer->size);  in vdo_dump_packer()
775 	list_for_each_entry(bin, &packer->bins, list)  in vdo_dump_packer()
778 	dump_packer_bin(packer->canceled_bin, true);  in vdo_dump_packer()