15 #include "block-map.h"
31  * block write from overwriting a block which appears to still be a valid head block of the
129  *                                                  block.
131  * @sequence_number: The journal sequence number of the referenced block.
195  * pop_free_list() - Get a block from the end of the free list.
198  * Return: The block or NULL if the list is empty.
202 	struct recovery_journal_block *block;  in pop_free_list()  local
207 	block = list_last_entry(&journal->free_tail_blocks,  in pop_free_list()
209 	list_del_init(&block->list_node);  in pop_free_list()
210 	return block;  in pop_free_list()
214  * is_block_dirty() - Check whether a recovery block is dirty.
215  * @block: The block to check.
220  * Return: true if the block has any uncommitted entries.
222 static inline bool __must_check is_block_dirty(const struct recovery_journal_block *block)  in is_block_dirty()  argument
224 	return (block->uncommitted_entry_count > 0);  in is_block_dirty()
228  * is_block_empty() - Check whether a journal block is empty.
229  * @block: The block to check.
231  * Return: true if the block has no entries.
233 static inline bool __must_check is_block_empty(const struct recovery_journal_block *block)  in is_block_empty()  argument
235 	return (block->entry_count == 0);  in is_block_empty()
239  * is_block_full() - Check whether a journal block is full.
240  * @block: The block to check.
242  * Return: true if the block is full.
244 static inline bool __must_check is_block_full(const struct recovery_journal_block *block)  in is_block_full()  argument
246 	return ((block == NULL) || (block->journal->entries_per_block == block->entry_count));  in is_block_full()
276  * Return: true if any block has a waiter.
280 	struct recovery_journal_block *block = get_journal_block(&journal->active_tail_blocks);  in has_block_waiters()  local
283 	 * Either the first active tail block (if it exists) has waiters, or no active tail block  in has_block_waiters()
286 	return ((block != NULL) &&  in has_block_waiters()
287 		(vdo_waitq_has_waiters(&block->entry_waiters) ||  in has_block_waiters()
288 		 vdo_waitq_has_waiters(&block->commit_waiters)));  in has_block_waiters()
292 static void recycle_journal_block(struct recovery_journal_block *block);
358 					    "journal being saved has clean active block");  in check_for_drain_complete()
404  * Exposed only so the block map can be initialized therefrom.
406  * Return: The sequence number of the tail block.
417  * The head is the lowest sequence number of the block map head and the slab journal head.
439  *                                         so, force the oldest slab journal tail block to commit.
514  *                              current active block.
515  * @journal: The journal to be reset based on its active block.
553  * Attempts to reap the journal now that all the locks on some journal block have been released.
649  * initialize_recovery_block() - Initialize a journal block.
651  * @journal: The journal to which the block will belong.
652  * @block: The block to initialize.
657 				     struct recovery_journal_block *block)  in initialize_recovery_block()  argument
663 	 * Ensure that a block is large enough to store RECOVERY_JOURNAL_ENTRIES_PER_BLOCK entries.  in initialize_recovery_block()
670 	 * Allocate a full block for the journal block even though not all of the space is used  in initialize_recovery_block()
671 	 * since the VIO needs to write a full disk block.  in initialize_recovery_block()
678 					 VIO_PRIORITY_HIGH, block, 1, data, &block->vio);  in initialize_recovery_block()
684 	list_add_tail(&block->list_node, &journal->free_tail_blocks);  in initialize_recovery_block()
685 	block->journal = journal;  in initialize_recovery_block()
691  *                                 was decoded from the super block.
738 		struct recovery_journal_block *block = &journal->blocks[i];  in vdo_decode_recovery_journal()  local
740 		result = initialize_recovery_block(vdo, journal, block);  in vdo_decode_recovery_journal()
811 		struct recovery_journal_block *block = &journal->blocks[i];  in vdo_free_recovery_journal()  local
813 		vdo_free(vdo_forget(block->vio.data));  in vdo_free_recovery_journal()
814 		free_vio_components(&block->vio);  in vdo_free_recovery_journal()
824  * @tail: The new tail block sequence number.
826  * @block_map_data_blocks: The new number of block map data blocks.
842  * vdo_get_journal_block_map_data_blocks_used() - Get the number of block map pages, allocated from
846  * Return: The number of block map pages allocated from slabs.
868  * @block_map: The block map for this VDO.
880  *                                 block.
895 		 * If the journal is saved, we should start one past the active block (since the  in vdo_record_recovery_journal()
896 		 * active block is not guaranteed to be empty).  in vdo_record_recovery_journal()
902 		 * block that might have entries that need to be applied.  in vdo_record_recovery_journal()
911  * get_block_header() - Get a pointer to the packed journal block header in the block buffer.
912  * @block: The recovery block.
914  * Return: The block's header.
917 get_block_header(const struct recovery_journal_block *block)  in get_block_header()  argument
919 	return (struct packed_journal_header *) block->vio.data;  in get_block_header()
923  * set_active_sector() - Set the current sector of the current block and initialize it.
924  * @block: The block to update.
927 static void set_active_sector(struct recovery_journal_block *block, void *sector)  in set_active_sector()  argument
929 	block->sector = sector;  in set_active_sector()
930 	block->sector->check_byte = get_block_header(block)->check_byte;  in set_active_sector()
931 	block->sector->recovery_count = block->journal->recovery_count;  in set_active_sector()
932 	block->sector->entry_count = 0;  in set_active_sector()
945 	struct recovery_journal_block *block;  in advance_tail()  local
947 	block = journal->active_block = pop_free_list(journal);  in advance_tail()
948 	if (block == NULL)  in advance_tail()
951 	list_move_tail(&block->list_node, &journal->active_tail_blocks);  in advance_tail()
964 	header = get_block_header(block);  in advance_tail()
965 	memset(block->vio.data, 0x0, VDO_BLOCK_SIZE);  in advance_tail()
966 	block->sequence_number = journal->tail;  in advance_tail()
967 	block->entry_count = 0;  in advance_tail()
968 	block->uncommitted_entry_count = 0;  in advance_tail()
969 	block->block_number = vdo_get_recovery_journal_block_number(journal,  in advance_tail()
973 	set_active_sector(block, vdo_get_journal_block_sector(header, 1));  in advance_tail()
999  * prepare_to_assign_entry() - Prepare the currently active block to receive an entry and check
1017 		/* Cannot use this block since the journal is full. */  in prepare_to_assign_entry()
1023 	 * Don't allow the new block to be reaped until all of its entries have been committed to  in prepare_to_assign_entry()
1024 	 * the block map and until the journal block has been fully committed as well. Because the  in prepare_to_assign_entry()
1025 	 * block map update is done only after any slab journal entries have been made, the  in prepare_to_assign_entry()
1026 	 * per-entry lock for the block map entry serves to protect those as well.  in prepare_to_assign_entry()
1035  * schedule_block_write() - Queue a block for writing.
1037  * @block: The block which is now ready to write.
1039  * The block is expected to be full. If the block is currently writing, this is a noop as the block
1040  * will be queued for writing when the write finishes. The block must not currently be queued for
1044 				 struct recovery_journal_block *block)  in schedule_block_write()  argument
1046 	if (!block->committing)  in schedule_block_write()
1047 		vdo_waitq_enqueue_waiter(&journal->pending_writes, &block->write_waiter);  in schedule_block_write()
1049 	 * At the end of adding entries, or discovering this partial block is now full and ready to  in schedule_block_write()
1055  * release_journal_block_reference() - Release a reference to a journal block.
1056  * @block: The journal block from which to release a reference.
1058 static void release_journal_block_reference(struct recovery_journal_block *block)  in release_journal_block_reference()  argument
1060 	vdo_release_recovery_journal_block_reference(block->journal,  in release_journal_block_reference()
1061 						     block->sequence_number,  in release_journal_block_reference()
1080  * assign_entry() - Assign an entry waiter to the active block.
1087 	struct recovery_journal_block *block = context;  in assign_entry()  local
1088 	struct recovery_journal *journal = block->journal;  in assign_entry()
1092 		.sequence_number = block->sequence_number,  in assign_entry()
1093 		.entry_count = block->entry_count,  in assign_entry()
1099 	if (!vdo_waitq_has_waiters(&block->entry_waiters))  in assign_entry()
1102 	vdo_waitq_enqueue_waiter(&block->entry_waiters, &data_vio->waiter);  in assign_entry()
1103 	block->entry_count++;  in assign_entry()
1104 	block->uncommitted_entry_count++;  in assign_entry()
1107 	if (is_block_full(block)) {  in assign_entry()
1109 		 * The block is full, so we can write it anytime henceforth. If it is already  in assign_entry()
1112 		schedule_block_write(journal, block);  in assign_entry()
1139  * recycle_journal_block() - Prepare an in-memory journal block to be reused now that it has been
1141  * @block: The block to be recycled.
1143 static void recycle_journal_block(struct recovery_journal_block *block)  in recycle_journal_block()  argument
1145 	struct recovery_journal *journal = block->journal;  in recycle_journal_block()
1148 	list_move_tail(&block->list_node, &journal->free_tail_blocks);  in recycle_journal_block()
1151 	for (i = block->entry_count; i < journal->entries_per_block; i++)  in recycle_journal_block()
1152 		release_journal_block_reference(block);  in recycle_journal_block()
1155 	 * Release our own lock against reaping now that the block is completely committed, or  in recycle_journal_block()
1158 	if (block->entry_count > 0)  in recycle_journal_block()
1159 		release_journal_block_reference(block);  in recycle_journal_block()
1161 	if (block == journal->active_block)  in recycle_journal_block()
1211 	struct recovery_journal_block *block;  in notify_commit_waiters()  local
1213 	list_for_each_entry(block, &journal->active_tail_blocks, list_node) {  in notify_commit_waiters()
1214 		if (block->committing)  in notify_commit_waiters()
1217 		vdo_waitq_notify_all_waiters(&block->commit_waiters,  in notify_commit_waiters()
1220 			vdo_waitq_notify_all_waiters(&block->entry_waiters,  in notify_commit_waiters()
1223 		} else if (is_block_dirty(block) || !is_block_full(block)) {  in notify_commit_waiters()
1236 	struct recovery_journal_block *block, *tmp;  in recycle_journal_blocks()  local
1238 	list_for_each_entry_safe(block, tmp, &journal->active_tail_blocks, list_node) {  in recycle_journal_blocks()
1239 		if (block->committing) {  in recycle_journal_blocks()
1245 		    (is_block_dirty(block) || !is_block_full(block))) {  in recycle_journal_blocks()
1253 		recycle_journal_block(block);  in recycle_journal_blocks()
1259  * @completion: The completion of the VIO writing this block.
1261  * This is the callback registered by write_block(). If more entries accumulated in the block being
1266 	struct recovery_journal_block *block = completion->parent;  in complete_write()  local
1267 	struct recovery_journal *journal = block->journal;  in complete_write()
1274 	journal->events.entries.committed += block->entries_in_commit;  in complete_write()
1275 	block->uncommitted_entry_count -= block->entries_in_commit;  in complete_write()
1276 	block->entries_in_commit = 0;  in complete_write()
1277 	block->committing = false;  in complete_write()
1279 	/* If this block is the latest block to be acknowledged, record that fact. */  in complete_write()
1280 	if (block->sequence_number > journal->last_write_acknowledged)  in complete_write()
1281 		journal->last_write_acknowledged = block->sequence_number;  in complete_write()
1284 	VDO_ASSERT_LOG_ONLY((block->sequence_number >= last_active_block->sequence_number),  in complete_write()
1290 	 * Is this block now full? Reaping, and adding entries, might have already sent it off for  in complete_write()
1293 	if (is_block_dirty(block) && is_block_full(block))  in complete_write()
1294 		schedule_block_write(journal, block);  in complete_write()
1304 	struct recovery_journal_block *block = completion->parent;  in handle_write_error()  local
1305 	struct recovery_journal *journal = block->journal;  in handle_write_error()
1309 			       "cannot write recovery journal block %llu",  in handle_write_error()
1310 			       (unsigned long long) block->sequence_number);  in handle_write_error()
1318 	struct recovery_journal_block *block = vio->completion.parent;  in complete_write_endio()  local
1319 	struct recovery_journal *journal = block->journal;  in complete_write_endio()
1325  * add_queued_recovery_entries() - Actually add entries from the queue to the given block.
1326  * @block: The journal block.
1328 static void add_queued_recovery_entries(struct recovery_journal_block *block)  in add_queued_recovery_entries()  argument
1330 	while (vdo_waitq_has_waiters(&block->entry_waiters)) {  in add_queued_recovery_entries()
1332 			vdo_waiter_as_data_vio(vdo_waitq_dequeue_waiter(&block->entry_waiters));  in add_queued_recovery_entries()
1337 		if (block->sector->entry_count == RECOVERY_JOURNAL_ENTRIES_PER_SECTOR)  in add_queued_recovery_entries()
1338 			set_active_sector(block,  in add_queued_recovery_entries()
1339 					  (char *) block->sector + VDO_SECTOR_SIZE);  in add_queued_recovery_entries()
1342 		packed_entry = &block->sector->entries[block->sector->entry_count++];  in add_queued_recovery_entries()
1356 		data_vio->recovery_sequence_number = block->sequence_number;  in add_queued_recovery_entries()
1359 		vdo_waitq_enqueue_waiter(&block->commit_waiters, &data_vio->waiter);  in add_queued_recovery_entries()
1364  * write_block() - Issue a block for writing.
1370 	struct recovery_journal_block *block =  in write_block()  local
1372 	struct recovery_journal *journal = block->journal;  in write_block()
1373 	struct packed_journal_header *header = get_block_header(block);  in write_block()
1375 	if (block->committing || !vdo_waitq_has_waiters(&block->entry_waiters) ||  in write_block()
1379 	block->entries_in_commit = vdo_waitq_num_waiters(&block->entry_waiters);  in write_block()
1380 	add_queued_recovery_entries(block);  in write_block()
1384 	journal->events.entries.written += block->entries_in_commit;  in write_block()
1388 	header->entry_count = __cpu_to_le16(block->entry_count);  in write_block()
1390 	block->committing = true;  in write_block()
1395 	 * block itself is stable before allowing overwrites of the lbn's previous data.  in write_block()
1397 	vdo_submit_metadata_vio(&block->vio, journal->origin + block->block_number,  in write_block()
1411 	 * We call this function after adding entries to the journal and after finishing a block  in write_blocks()
1416 	 * no outstanding writes and some unwritten entries, we must issue a block, even if it's  in write_blocks()
1417 	 * the active block and it isn't full.  in write_blocks()
1426 	 * Do we need to write the active block? Only if we have no outstanding writes, even after  in write_blocks()
1439  *	      sequence number of the journal block in which the entry was
1515 	 * block is referenced.  in reap_recovery_journal()
1540 	 * If the block map head will advance, we must flush any block map page modified by the  in reap_recovery_journal()
1549  * vdo_acquire_recovery_journal_block_reference() - Acquire a reference to a recovery journal block
1552  * @sequence_number: The journal sequence number of the referenced block.
1591  *                                    journal block.
1593  * @sequence_number: The journal sequence number of the referenced block.
1717  * dump_recovery_block() - Dump the contents of the recovery block to the log.
1718  * @block: The block to dump.
1720 static void dump_recovery_block(const struct recovery_journal_block *block)  in dump_recovery_block()  argument
1723 		     (unsigned long long) block->sequence_number, block->entry_count,  in dump_recovery_block()
1724 		     (block->committing ? "committing" : "waiting"),  in dump_recovery_block()
1725 		     vdo_waitq_num_waiters(&block->entry_waiters),  in dump_recovery_block()
1726 		     vdo_waitq_num_waiters(&block->commit_waiters));  in dump_recovery_block()
1736 	const struct recovery_journal_block *block;  in vdo_dump_recovery_journal_statistics()  local
1760 	list_for_each_entry(block, &journal->active_tail_blocks, list_node)  in vdo_dump_recovery_journal_statistics()
1761 		dump_recovery_block(block);  in vdo_dump_recovery_journal_statistics()