8 #include "dm-verity-fec.h"
11 #define DM_MSG_PREFIX	"verity-fec"
18 	return v->fec && v->fec->dev;  in verity_fec_is_enabled()
38 	mod = do_div(offset, v->fec->rsn);  in fec_interleave()
39 	return offset + mod * (v->fec->rounds << v->data_dev_block_bits);  in fec_interleave()
57 	position = (index + rsb) * v->fec->roots;  in fec_read_parity()
58 	block = div64_u64_rem(position, v->fec->io_size, &rem);  in fec_read_parity()
61 	res = dm_bufio_read_with_ioprio(v->fec->bufio, block, buf, ioprio);  in fec_read_parity()
63 		DMERR("%s: FEC %llu: parity read failed (block %llu): %ld",  in fec_read_parity()
97 	return &fio->bufs[i][j * v->fec->rsn];  in fec_buffer_rs_block()
131 	 * one corrected target byte and consumes fec->roots parity bytes.  in fec_decode_bufs()
135 		for (j = 0; j < v->fec->roots - par_buf_offset; j++)  in fec_decode_bufs()
138 		res = decode_rs8(fio->rs, block, par_buf, v->fec->rsn,  in fec_decode_bufs()
153 		offset += (v->fec->roots - par_buf_offset);  in fec_decode_bufs()
155 		if (offset < v->fec->io_size && (offset + v->fec->roots) > v->fec->io_size) {  in fec_decode_bufs()
156 			par_buf_offset = v->fec->io_size - offset;  in fec_decode_bufs()
163 		if (offset >= v->fec->io_size) {  in fec_decode_bufs()
178 		DMERR_LIMIT("%s: FEC %llu: failed to correct: %d",  in fec_decode_bufs()
181 		DMWARN_LIMIT("%s: FEC %llu: corrected %d errors",  in fec_decode_bufs()
230 	for (i = 0; i < v->fec->rsn; i++) {  in fec_read_bufs()
231 		ileaved = fec_interleave(v, rsb * v->fec->rsn + i);  in fec_read_bufs()
241 		bufio = v->fec->data_bufio;  in fec_read_bufs()
250 			if (unlikely(block >= v->fec->hash_blocks))  in fec_read_bufs()
259 			DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld",  in fec_read_bufs()
265 			if (neras && *neras <= v->fec->roots)  in fec_read_bufs()
272 		if (bufio == v->fec->data_bufio &&  in fec_read_bufs()
281 			 * maximum number (i.e. fec->roots) of erasures  in fec_read_bufs()
283 			if (neras && *neras <= v->fec->roots &&  in fec_read_bufs()
309  * Allocate RS control structure and FEC buffers from preallocated mempools,
317 		fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO);  in fec_alloc_bufs()
323 		fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT);  in fec_alloc_bufs()
325 			DMERR("failed to allocate FEC buffer");  in fec_alloc_bufs()
335 		fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT);  in fec_alloc_bufs()
343 		fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO);  in fec_alloc_bufs()
357 		memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS);  in fec_init_bufs()
401 		DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)",  in fec_decode_rsb()
421 		DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name);  in verity_fec_decode()
431 	 * For RS(M, N), the continuous FEC data is divided into blocks of N  in verity_fec_decode()
441 	res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits);  in verity_fec_decode()
447 	rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits);  in verity_fec_decode()
474 	struct dm_verity_fec *f = io->v->fec;  in verity_fec_finish_io()
521 	       v->fec->dev->name,  in verity_fec_status_table()
522 	       (unsigned long long)v->fec->blocks,  in verity_fec_status_table()
523 	       (unsigned long long)v->fec->start,  in verity_fec_status_table()
524 	       v->fec->roots);  in verity_fec_status_table()
531 	struct dm_verity_fec *f = v->fec;  in verity_fec_dtr()
551 	v->fec = NULL;  in verity_fec_dtr()
558 	return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask);  in fec_rs_alloc()
588 		ti->error = "FEC feature arguments require a value";  in verity_fec_parse_opt_args()
596 		r = dm_get_device(ti, arg_value, BLK_OPEN_READ, &v->fec->dev);  in verity_fec_parse_opt_args()
598 			ti->error = "FEC device lookup failed";  in verity_fec_parse_opt_args()
609 		v->fec->blocks = num_ll;  in verity_fec_parse_opt_args()
618 		v->fec->start = num_ll;  in verity_fec_parse_opt_args()
627 		v->fec->roots = num_c;  in verity_fec_parse_opt_args()
630 		ti->error = "Unrecognized verity FEC feature request";  in verity_fec_parse_opt_args()
638  * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr.
646 		v->ti->error = "Cannot allocate FEC structure";  in verity_fec_ctr_alloc()
649 	v->fec = f;  in verity_fec_ctr_alloc()
660 	struct dm_verity_fec *f = v->fec;  in verity_fec_ctr()
671 	 * FEC is computed over data blocks, possible metadata, and  in verity_fec_ctr()
672 	 * hash blocks. In other words, FEC covers total of fec_blocks  in verity_fec_ctr()
678 	 * where all data is stored on the same device and FEC covers  in verity_fec_ctr()
692 		ti->error = "Block sizes must match to use FEC";  in verity_fec_ctr()
737 		ti->error = "Cannot initialize FEC bufio client";  in verity_fec_ctr()
743 	fec_blocks = div64_u64(f->rounds * f->roots, v->fec->roots << SECTOR_SHIFT);  in verity_fec_ctr()
745 		ti->error = "FEC device is too small";  in verity_fec_ctr()
753 		ti->error = "Cannot initialize FEC data bufio client";  in verity_fec_ctr()
774 		ti->error = "Cannot create FEC buffer cache";  in verity_fec_ctr()
783 		ti->error = "Cannot allocate FEC buffer prealloc pool";  in verity_fec_ctr()
789 		ti->error = "Cannot allocate FEC buffer extra pool";  in verity_fec_ctr()
797 		ti->error = "Cannot allocate FEC output pool";  in verity_fec_ctr()