Lines Matching +full:block +full:- +full:offset
1 // SPDX-License-Identifier: GPL-2.0-or-later
8 #include "dm-verity-fec.h"
11 #define DM_MSG_PREFIX "verity-fec"
18 return v->fec && v->fec->dev; in verity_fec_is_enabled()
28 ((char *)io + io->v->ti->per_io_data_size - sizeof(struct dm_verity_fec_io)); in fec_io()
32 * Return an interleaved offset for a byte in RS block.
34 static inline u64 fec_interleave(struct dm_verity *v, u64 offset) in fec_interleave() argument
38 mod = do_div(offset, v->fec->rsn); in fec_interleave()
39 return offset + mod * (v->fec->rounds << v->data_dev_block_bits); in fec_interleave()
43 * Read error-correcting codes for the requested RS block. Returns a pointer
44 * to the data block. Caller is responsible for releasing buf.
47 unsigned int *offset, unsigned int par_buf_offset, in fec_read_parity() argument
50 u64 position, block, rem; in fec_read_parity() local
53 /* We have already part of parity bytes read, skip to the next block */ in fec_read_parity()
57 position = (index + rsb) * v->fec->roots; in fec_read_parity()
58 block = div64_u64_rem(position, v->fec->io_size, &rem); in fec_read_parity()
59 *offset = par_buf_offset ? 0 : (unsigned int)rem; in fec_read_parity()
61 res = dm_bufio_read_with_ioprio(v->fec->bufio, block, buf, ioprio); in fec_read_parity()
63 DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", in fec_read_parity()
64 v->data_dev->name, (unsigned long long)rsb, in fec_read_parity()
65 (unsigned long long)block, PTR_ERR(res)); in fec_read_parity()
82 for (__i = 0; __i < (io)->nbufs; __i++)
84 /* Loop over each RS block in each allocated buffer. */
90 * Return a pointer to the current RS block when called inside
97 return &fio->bufs[i][j * v->fec->rsn]; in fec_buffer_rs_block()
101 * Return an index to the current RS block when called inside
110 * Decode all RS blocks from buffers and copy corrected bytes into fio->output
119 unsigned int n, i, j, offset, par_buf_offset = 0; in fec_decode_bufs() local
120 uint16_t par_buf[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; in fec_decode_bufs()
121 u8 *par, *block; in fec_decode_bufs() local
122 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); in fec_decode_bufs()
124 par = fec_read_parity(v, rsb, block_offset, &offset, in fec_decode_bufs()
125 par_buf_offset, &buf, bio->bi_ioprio); in fec_decode_bufs()
130 * Decode the RS blocks we have in bufs. Each RS block results in in fec_decode_bufs()
131 * one corrected target byte and consumes fec->roots parity bytes. in fec_decode_bufs()
134 block = fec_buffer_rs_block(v, fio, n, i); in fec_decode_bufs()
135 for (j = 0; j < v->fec->roots - par_buf_offset; j++) in fec_decode_bufs()
136 par_buf[par_buf_offset + j] = par[offset + j]; in fec_decode_bufs()
137 /* Decode an RS block using Reed-Solomon */ in fec_decode_bufs()
138 res = decode_rs8(fio->rs, block, par_buf, v->fec->rsn, in fec_decode_bufs()
139 NULL, neras, fio->erasures, 0, NULL); in fec_decode_bufs()
146 fio->output[block_offset] = block[byte_index]; in fec_decode_bufs()
149 if (block_offset >= 1 << v->data_dev_block_bits) in fec_decode_bufs()
152 /* Read the next block when we run out of parity bytes */ in fec_decode_bufs()
153 offset += (v->fec->roots - par_buf_offset); in fec_decode_bufs()
155 if (offset < v->fec->io_size && (offset + v->fec->roots) > v->fec->io_size) { in fec_decode_bufs()
156 par_buf_offset = v->fec->io_size - offset; in fec_decode_bufs()
158 par_buf[j] = par[offset + j]; in fec_decode_bufs()
159 offset += par_buf_offset; in fec_decode_bufs()
163 if (offset >= v->fec->io_size) { in fec_decode_bufs()
166 par = fec_read_parity(v, rsb, block_offset, &offset, in fec_decode_bufs()
167 par_buf_offset, &buf, bio->bi_ioprio); in fec_decode_bufs()
179 v->data_dev->name, (unsigned long long)rsb, r); in fec_decode_bufs()
182 v->data_dev->name, (unsigned long long)rsb, r); in fec_decode_bufs()
188 * Locate data block erasures using verity hashes.
193 if (unlikely(verity_hash(v, io, data, 1 << v->data_dev_block_bits, in fec_is_erasure()
198 v->digest_size) != 0; in fec_is_erasure()
202 * Read data blocks that are part of the RS block and deinterleave as much as
203 * fits into buffers. Check for erasure locations if @neras is non-NULL.
210 int i, j, target_index = -1; in fec_read_bufs()
214 u64 block, ileaved; in fec_read_bufs() local
218 struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); in fec_read_bufs()
223 if (WARN_ON(v->digest_size > sizeof(want_digest))) in fec_read_bufs()
224 return -EINVAL; in fec_read_bufs()
227 * read each of the rsn data blocks that are part of the RS block, and in fec_read_bufs()
230 for (i = 0; i < v->fec->rsn; i++) { in fec_read_bufs()
231 ileaved = fec_interleave(v, rsb * v->fec->rsn + i); in fec_read_bufs()
234 * target is the data block we want to correct, target_index is in fec_read_bufs()
235 * the index of this block within the rsn RS blocks in fec_read_bufs()
240 block = ileaved >> v->data_dev_block_bits; in fec_read_bufs()
241 bufio = v->fec->data_bufio; in fec_read_bufs()
243 if (block >= v->data_blocks) { in fec_read_bufs()
244 block -= v->data_blocks; in fec_read_bufs()
250 if (unlikely(block >= v->fec->hash_blocks)) in fec_read_bufs()
253 block += v->hash_start; in fec_read_bufs()
254 bufio = v->bufio; in fec_read_bufs()
257 bbuf = dm_bufio_read_with_ioprio(bufio, block, &buf, bio->bi_ioprio); in fec_read_bufs()
260 v->data_dev->name, in fec_read_bufs()
262 (unsigned long long)block, PTR_ERR(bbuf)); in fec_read_bufs()
264 /* assume the block is corrupted */ in fec_read_bufs()
265 if (neras && *neras <= v->fec->roots) in fec_read_bufs()
266 fio->erasures[(*neras)++] = i; in fec_read_bufs()
271 /* locate erasures if the block is on the data device */ in fec_read_bufs()
272 if (bufio == v->fec->data_bufio && in fec_read_bufs()
273 verity_hash_for_block(v, io, block, want_digest, in fec_read_bufs()
281 * maximum number (i.e. fec->roots) of erasures in fec_read_bufs()
283 if (neras && *neras <= v->fec->roots && in fec_read_bufs()
285 fio->erasures[(*neras)++] = i; in fec_read_bufs()
295 if (k >= 1 << v->data_dev_block_bits) in fec_read_bufs()
316 if (!fio->rs) in fec_alloc_bufs()
317 fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO); in fec_alloc_bufs()
320 if (fio->bufs[n]) in fec_alloc_bufs()
323 fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT); in fec_alloc_bufs()
324 if (unlikely(!fio->bufs[n])) { in fec_alloc_bufs()
326 return -ENOMEM; in fec_alloc_bufs()
332 if (fio->bufs[n]) in fec_alloc_bufs()
335 fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT); in fec_alloc_bufs()
337 if (unlikely(!fio->bufs[n])) in fec_alloc_bufs()
340 fio->nbufs = n; in fec_alloc_bufs()
342 if (!fio->output) in fec_alloc_bufs()
343 fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO); in fec_alloc_bufs()
357 memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS); in fec_init_bufs()
359 memset(fio->erasures, 0, sizeof(fio->erasures)); in fec_init_bufs()
363 * Decode all RS blocks in a single data block and return the target block
364 * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses
368 struct dm_verity_fec_io *fio, u64 rsb, u64 offset, in fec_decode_rsb() argument
378 for (pos = 0; pos < 1 << v->data_dev_block_bits; ) { in fec_decode_rsb()
381 r = fec_read_bufs(v, io, rsb, offset, pos, in fec_decode_rsb()
390 pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS; in fec_decode_rsb()
393 /* Always re-validate the corrected block against the expected hash */ in fec_decode_rsb()
394 r = verity_hash(v, io, fio->output, 1 << v->data_dev_block_bits, in fec_decode_rsb()
400 v->digest_size)) { in fec_decode_rsb()
402 v->data_dev->name, (unsigned long long)rsb, neras); in fec_decode_rsb()
403 return -EILSEQ; in fec_decode_rsb()
409 /* Correct errors in a block. Copies corrected block to dest. */
411 enum verity_block_type type, sector_t block, u8 *dest) in verity_fec_decode() argument
415 u64 offset, res, rsb; in verity_fec_decode() local
418 return -EOPNOTSUPP; in verity_fec_decode()
420 if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) { in verity_fec_decode()
421 DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name); in verity_fec_decode()
422 return -EIO; in verity_fec_decode()
425 fio->level++; in verity_fec_decode()
428 block = block - v->hash_start + v->data_blocks; in verity_fec_decode()
432 * bytes. Since block size may not be divisible by N, the last block in verity_fec_decode()
435 * Each byte of the block is covered by a different RS(M, N) code, in verity_fec_decode()
440 offset = block << v->data_dev_block_bits; in verity_fec_decode()
441 res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits); in verity_fec_decode()
444 * The base RS block we can feed to the interleaver to find out all in verity_fec_decode()
447 rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits); in verity_fec_decode()
450 * Locating erasures is slow, so attempt to recover the block without in verity_fec_decode()
454 r = fec_decode_rsb(v, io, fio, rsb, offset, false); in verity_fec_decode()
456 r = fec_decode_rsb(v, io, fio, rsb, offset, true); in verity_fec_decode()
461 memcpy(dest, fio->output, 1 << v->data_dev_block_bits); in verity_fec_decode()
464 fio->level--; in verity_fec_decode()
469 * Clean up per-bio data.
474 struct dm_verity_fec *f = io->v->fec; in verity_fec_finish_io()
477 if (!verity_fec_is_enabled(io->v)) in verity_fec_finish_io()
480 mempool_free(fio->rs, &f->rs_pool); in verity_fec_finish_io()
483 mempool_free(fio->bufs[n], &f->prealloc_pool); in verity_fec_finish_io()
486 mempool_free(fio->bufs[n], &f->extra_pool); in verity_fec_finish_io()
488 mempool_free(fio->output, &f->output_pool); in verity_fec_finish_io()
492 * Initialize per-bio data.
498 if (!verity_fec_is_enabled(io->v)) in verity_fec_init_io()
501 fio->rs = NULL; in verity_fec_init_io()
502 memset(fio->bufs, 0, sizeof(fio->bufs)); in verity_fec_init_io()
503 fio->nbufs = 0; in verity_fec_init_io()
504 fio->output = NULL; in verity_fec_init_io()
505 fio->level = 0; in verity_fec_init_io()
521 v->fec->dev->name, in verity_fec_status_table()
522 (unsigned long long)v->fec->blocks, in verity_fec_status_table()
523 (unsigned long long)v->fec->start, in verity_fec_status_table()
524 v->fec->roots); in verity_fec_status_table()
531 struct dm_verity_fec *f = v->fec; in verity_fec_dtr()
536 mempool_exit(&f->rs_pool); in verity_fec_dtr()
537 mempool_exit(&f->prealloc_pool); in verity_fec_dtr()
538 mempool_exit(&f->extra_pool); in verity_fec_dtr()
539 mempool_exit(&f->output_pool); in verity_fec_dtr()
540 kmem_cache_destroy(f->cache); in verity_fec_dtr()
542 if (f->data_bufio) in verity_fec_dtr()
543 dm_bufio_client_destroy(f->data_bufio); in verity_fec_dtr()
544 if (f->bufio) in verity_fec_dtr()
545 dm_bufio_client_destroy(f->bufio); in verity_fec_dtr()
547 if (f->dev) in verity_fec_dtr()
548 dm_put_device(v->ti, f->dev); in verity_fec_dtr()
551 v->fec = NULL; in verity_fec_dtr()
558 return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask); in fec_rs_alloc()
581 struct dm_target *ti = v->ti; in verity_fec_parse_opt_args()
588 ti->error = "FEC feature arguments require a value"; in verity_fec_parse_opt_args()
589 return -EINVAL; in verity_fec_parse_opt_args()
593 (*argc)--; in verity_fec_parse_opt_args()
596 r = dm_get_device(ti, arg_value, BLK_OPEN_READ, &v->fec->dev); in verity_fec_parse_opt_args()
598 ti->error = "FEC device lookup failed"; in verity_fec_parse_opt_args()
604 ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) in verity_fec_parse_opt_args()
605 >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { in verity_fec_parse_opt_args()
606 ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; in verity_fec_parse_opt_args()
607 return -EINVAL; in verity_fec_parse_opt_args()
609 v->fec->blocks = num_ll; in verity_fec_parse_opt_args()
613 ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >> in verity_fec_parse_opt_args()
614 (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) { in verity_fec_parse_opt_args()
615 ti->error = "Invalid " DM_VERITY_OPT_FEC_START; in verity_fec_parse_opt_args()
616 return -EINVAL; in verity_fec_parse_opt_args()
618 v->fec->start = num_ll; in verity_fec_parse_opt_args()
622 num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) || in verity_fec_parse_opt_args()
623 num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) { in verity_fec_parse_opt_args()
624 ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS; in verity_fec_parse_opt_args()
625 return -EINVAL; in verity_fec_parse_opt_args()
627 v->fec->roots = num_c; in verity_fec_parse_opt_args()
630 ti->error = "Unrecognized verity FEC feature request"; in verity_fec_parse_opt_args()
631 return -EINVAL; in verity_fec_parse_opt_args()
638 * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr.
646 v->ti->error = "Cannot allocate FEC structure"; in verity_fec_ctr_alloc()
647 return -ENOMEM; in verity_fec_ctr_alloc()
649 v->fec = f; in verity_fec_ctr_alloc()
660 struct dm_verity_fec *f = v->fec; in verity_fec_ctr()
661 struct dm_target *ti = v->ti; in verity_fec_ctr()
685 hash_blocks = v->hash_blocks - v->hash_start; in verity_fec_ctr()
688 * Require matching block sizes for data and hash devices for in verity_fec_ctr()
691 if (v->data_dev_block_bits != v->hash_dev_block_bits) { in verity_fec_ctr()
692 ti->error = "Block sizes must match to use FEC"; in verity_fec_ctr()
693 return -EINVAL; in verity_fec_ctr()
696 if (!f->roots) { in verity_fec_ctr()
697 ti->error = "Missing " DM_VERITY_OPT_FEC_ROOTS; in verity_fec_ctr()
698 return -EINVAL; in verity_fec_ctr()
700 f->rsn = DM_VERITY_FEC_RSM - f->roots; in verity_fec_ctr()
702 if (!f->blocks) { in verity_fec_ctr()
703 ti->error = "Missing " DM_VERITY_OPT_FEC_BLOCKS; in verity_fec_ctr()
704 return -EINVAL; in verity_fec_ctr()
707 f->rounds = f->blocks; in verity_fec_ctr()
708 if (sector_div(f->rounds, f->rsn)) in verity_fec_ctr()
709 f->rounds++; in verity_fec_ctr()
712 * Due to optional metadata, f->blocks can be larger than in verity_fec_ctr()
715 if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) { in verity_fec_ctr()
716 ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS; in verity_fec_ctr()
717 return -EINVAL; in verity_fec_ctr()
724 f->hash_blocks = f->blocks - v->data_blocks; in verity_fec_ctr()
725 if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) { in verity_fec_ctr()
726 ti->error = "Hash device is too small for " in verity_fec_ctr()
728 return -E2BIG; in verity_fec_ctr()
731 f->io_size = 1 << v->data_dev_block_bits; in verity_fec_ctr()
733 f->bufio = dm_bufio_client_create(f->dev->bdev, in verity_fec_ctr()
734 f->io_size, in verity_fec_ctr()
736 if (IS_ERR(f->bufio)) { in verity_fec_ctr()
737 ti->error = "Cannot initialize FEC bufio client"; in verity_fec_ctr()
738 return PTR_ERR(f->bufio); in verity_fec_ctr()
741 dm_bufio_set_sector_offset(f->bufio, f->start << (v->data_dev_block_bits - SECTOR_SHIFT)); in verity_fec_ctr()
743 fec_blocks = div64_u64(f->rounds * f->roots, v->fec->roots << SECTOR_SHIFT); in verity_fec_ctr()
744 if (dm_bufio_get_device_size(f->bufio) < fec_blocks) { in verity_fec_ctr()
745 ti->error = "FEC device is too small"; in verity_fec_ctr()
746 return -E2BIG; in verity_fec_ctr()
749 f->data_bufio = dm_bufio_client_create(v->data_dev->bdev, in verity_fec_ctr()
750 1 << v->data_dev_block_bits, in verity_fec_ctr()
752 if (IS_ERR(f->data_bufio)) { in verity_fec_ctr()
753 ti->error = "Cannot initialize FEC data bufio client"; in verity_fec_ctr()
754 return PTR_ERR(f->data_bufio); in verity_fec_ctr()
757 if (dm_bufio_get_device_size(f->data_bufio) < v->data_blocks) { in verity_fec_ctr()
758 ti->error = "Data device is too small"; in verity_fec_ctr()
759 return -E2BIG; in verity_fec_ctr()
763 ret = mempool_init(&f->rs_pool, num_online_cpus(), fec_rs_alloc, in verity_fec_ctr()
766 ti->error = "Cannot allocate RS pool"; in verity_fec_ctr()
770 f->cache = kmem_cache_create("dm_verity_fec_buffers", in verity_fec_ctr()
771 f->rsn << DM_VERITY_FEC_BUF_RS_BITS, in verity_fec_ctr()
773 if (!f->cache) { in verity_fec_ctr()
774 ti->error = "Cannot create FEC buffer cache"; in verity_fec_ctr()
775 return -ENOMEM; in verity_fec_ctr()
779 ret = mempool_init_slab_pool(&f->prealloc_pool, num_online_cpus() * in verity_fec_ctr()
781 f->cache); in verity_fec_ctr()
783 ti->error = "Cannot allocate FEC buffer prealloc pool"; in verity_fec_ctr()
787 ret = mempool_init_slab_pool(&f->extra_pool, 0, f->cache); in verity_fec_ctr()
789 ti->error = "Cannot allocate FEC buffer extra pool"; in verity_fec_ctr()
794 ret = mempool_init_kmalloc_pool(&f->output_pool, num_online_cpus(), in verity_fec_ctr()
795 1 << v->data_dev_block_bits); in verity_fec_ctr()
797 ti->error = "Cannot allocate FEC output pool"; in verity_fec_ctr()
801 /* Reserve space for our per-bio data */ in verity_fec_ctr()
802 ti->per_io_data_size += sizeof(struct dm_verity_fec_io); in verity_fec_ctr()