1 // SPDX-License-Identifier: GPL-2.0-only
3  * AES using the RISC-V vector crypto extensions.  Includes the bare block
4  * cipher and the ECB, CBC, CBC-CTS, CTR, and XTS modes.
9  * Copyright (C) 2023 SiFive, Inc.
10  * Author: Jerry Shih <jerry.shih@sifive.com>
69 	 * - zvkned's key expansion instructions don't support AES-192.  in riscv64_aes_setkey()
70 	 *   So, non-zvkned fallback code would be needed anyway.  in riscv64_aes_setkey()
72 	 * - Users of AES in Linux usually don't change keys frequently.  in riscv64_aes_setkey()
73 	 *   So, key expansion isn't performance-critical.  in riscv64_aes_setkey()
75 	 * - For single-block AES exposed as a "cipher" algorithm, it's  in riscv64_aes_setkey()
83 	 *   single-block AES encryption.  It's simplest to just use  in riscv64_aes_setkey()
133 /* AES-ECB */
149 					       nbytes & ~(AES_BLOCK_SIZE - 1));  in riscv64_aes_ecb_crypt()
153 					       nbytes & ~(AES_BLOCK_SIZE - 1));  in riscv64_aes_ecb_crypt()
155 		err = skcipher_walk_done(&walk, nbytes & (AES_BLOCK_SIZE - 1));  in riscv64_aes_ecb_crypt()
171 /* AES-CBC */
187 					       nbytes & ~(AES_BLOCK_SIZE - 1),  in riscv64_aes_cbc_crypt()
192 					       nbytes & ~(AES_BLOCK_SIZE - 1),  in riscv64_aes_cbc_crypt()
195 		err = skcipher_walk_done(&walk, nbytes & (AES_BLOCK_SIZE - 1));  in riscv64_aes_cbc_crypt()
211 /* AES-CBC-CTS */
224 	if (req->cryptlen < AES_BLOCK_SIZE)  in riscv64_aes_cbc_cts_crypt()
225 		return -EINVAL;  in riscv64_aes_cbc_cts_crypt()
232 	 * to the CBC-CTS assembly function.  This reduces overhead, especially  in riscv64_aes_cbc_cts_crypt()
234 	 * two blocks, then invoke CTS just for the ciphertext stealing.  in riscv64_aes_cbc_cts_crypt()
236 	if (unlikely(walk.nbytes != req->cryptlen)) {  in riscv64_aes_cbc_cts_crypt()
237 		cbc_len = round_down(req->cryptlen - AES_BLOCK_SIZE - 1,  in riscv64_aes_cbc_cts_crypt()
244 		skcipher_request_set_crypt(&subreq, req->src, req->dst,  in riscv64_aes_cbc_cts_crypt()
245 					   cbc_len, req->iv);  in riscv64_aes_cbc_cts_crypt()
249 		dst = src = scatterwalk_ffwd(sg_src, req->src, cbc_len);  in riscv64_aes_cbc_cts_crypt()
250 		if (req->dst != req->src)  in riscv64_aes_cbc_cts_crypt()
251 			dst = scatterwalk_ffwd(sg_dst, req->dst, cbc_len);  in riscv64_aes_cbc_cts_crypt()
253 					   req->cryptlen - cbc_len, req->iv);  in riscv64_aes_cbc_cts_crypt()
260 				 walk.nbytes, req->iv, enc);  in riscv64_aes_cbc_cts_crypt()
275 /* AES-CTR */
286 	/* Get the low 32-bit word of the 128-bit big endian counter. */  in riscv64_aes_ctr_crypt()
287 	ctr32 = get_unaligned_be32(req->iv + 12);  in riscv64_aes_ctr_crypt()
292 			/* Not the end yet, so keep the length block-aligned. */  in riscv64_aes_ctr_crypt()
303 			/* The low 32-bit word of the counter won't overflow. */  in riscv64_aes_ctr_crypt()
306 						    req->iv);  in riscv64_aes_ctr_crypt()
309 			 * The low 32-bit word of the counter will overflow.  in riscv64_aes_ctr_crypt()
315 					  (nblocks - ctr32) * AES_BLOCK_SIZE);  in riscv64_aes_ctr_crypt()
318 						    p1_nbytes, req->iv);  in riscv64_aes_ctr_crypt()
319 			crypto_inc(req->iv, 12);  in riscv64_aes_ctr_crypt()
326 					nbytes - p1_nbytes, req->iv);  in riscv64_aes_ctr_crypt()
331 		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);  in riscv64_aes_ctr_crypt()
337 /* AES-XTS */
350 	       riscv64_aes_setkey(&ctx->ctx1, key, keylen / 2) ?:  in riscv64_aes_xts_setkey()
351 	       riscv64_aes_setkey(&ctx->ctx2, key + keylen / 2, keylen / 2);  in riscv64_aes_xts_setkey()
358 	int tail = req->cryptlen % AES_BLOCK_SIZE;  in riscv64_aes_xts_crypt()
365 	if (req->cryptlen < AES_BLOCK_SIZE)  in riscv64_aes_xts_crypt()
366 		return -EINVAL;  in riscv64_aes_xts_crypt()
370 	aes_encrypt_zvkned(&ctx->ctx2, req->iv, req->iv);  in riscv64_aes_xts_crypt()
389 		skcipher_request_set_crypt(&subreq, req->src, req->dst,  in riscv64_aes_xts_crypt()
390 					   req->cryptlen - tail - AES_BLOCK_SIZE,  in riscv64_aes_xts_crypt()
391 					   req->iv);  in riscv64_aes_xts_crypt()
407 				&ctx->ctx1, walk.src.virt.addr,  in riscv64_aes_xts_crypt()
408 				walk.dst.virt.addr, nbytes, req->iv);  in riscv64_aes_xts_crypt()
411 				&ctx->ctx1, walk.src.virt.addr,  in riscv64_aes_xts_crypt()
412 				walk.dst.virt.addr, nbytes, req->iv);  in riscv64_aes_xts_crypt()
414 		err = skcipher_walk_done(&walk, walk.nbytes - nbytes);  in riscv64_aes_xts_crypt()
422 	dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);  in riscv64_aes_xts_crypt()
423 	if (req->dst != req->src)  in riscv64_aes_xts_crypt()
424 		dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);  in riscv64_aes_xts_crypt()
427 				   req->iv);  in riscv64_aes_xts_crypt()
436 			&ctx->ctx1, walk.src.virt.addr,  in riscv64_aes_xts_crypt()
437 			walk.dst.virt.addr, walk.nbytes, req->iv);  in riscv64_aes_xts_crypt()
440 			&ctx->ctx1, walk.src.virt.addr,  in riscv64_aes_xts_crypt()
441 			walk.dst.virt.addr, walk.nbytes, req->iv);  in riscv64_aes_xts_crypt()
465 	.cra_driver_name = "aes-riscv64-zvkned",
489 			.cra_driver_name = "ecb-aes-riscv64-zvkned",
504 			.cra_driver_name = "cbc-aes-riscv64-zvkned",
520 			.cra_driver_name = "cts-cbc-aes-riscv64-zvkned",
540 		.cra_driver_name = "ctr-aes-riscv64-zvkned-zvkb",
559 		.cra_driver_name = "xts-aes-riscv64-zvkned-zvbb-zvkg",
573 	int err = -ENODEV;  in riscv64_aes_mod_init()
629 MODULE_DESCRIPTION("AES-ECB/CBC/CTS/CTR/XTS (RISC-V accelerated)");
630 MODULE_AUTHOR("Jerry Shih <jerry.shih@sifive.com>");