1 // SPDX-License-Identifier: GPL-2.0-only
5  * Support for SAHARA cryptographic accelerator.
9  * Author: Javier Martin <javier.martin@vista-silicon.com>
11  * Based on omap-aes.c and tegra-aes.c
23 #include <linux/dma-mapping.h>
36 #define SAHARA_NAME				"sahara"
140 	/* AES-specific context */
153  * struct sahara_sha_reqctx - private data per request
155  * @rembuf: used to prepare one block_size-aligned request
156  * @context: hw-specific context for request. Digest is extracted from this
157  * @mode: specifies what type of hw-descriptor needs to be built
159  * @context_size: length of hw-context for this request.
224 	writel(data, dev->regs_base + reg);  in sahara_write()
229 	return readl(dev->regs_base + reg);  in sahara_read()
238 	if (dev->flags & FLAGS_CBC) {  in sahara_aes_key_hdr()
243 	if (dev->flags & FLAGS_ENCRYPT) {  in sahara_aes_key_hdr()
278 	"Half-word transfer",
295 	"Input buffer non-empty",
316 	dev_err(dev->device, "%s: Error Register = 0x%08x\n", __func__, error);  in sahara_decode_error()
318 	dev_err(dev->device, "	- %s.\n", sahara_err_src[source]);  in sahara_decode_error()
322 			dev_err(dev->device, "		* DMA read.\n");  in sahara_decode_error()
324 			dev_err(dev->device, "		* DMA write.\n");  in sahara_decode_error()
326 		dev_err(dev->device, "		* %s.\n",  in sahara_decode_error()
328 		dev_err(dev->device, "		* %s.\n",  in sahara_decode_error()
331 		dev_err(dev->device, "		* %s.\n",  in sahara_decode_error()
333 		dev_err(dev->device, "		* %s.\n",  in sahara_decode_error()
336 	dev_err(dev->device, "\n");  in sahara_decode_error()
350 	dev_dbg(dev->device, "%s: Status Register = 0x%08x\n",  in sahara_decode_status()
353 	dev_dbg(dev->device, "	- State = %d:\n", state);  in sahara_decode_status()
355 		dev_dbg(dev->device, "		* Descriptor completed. IRQ pending.\n");  in sahara_decode_status()
357 	dev_dbg(dev->device, "		* %s.\n",  in sahara_decode_status()
361 		dev_dbg(dev->device, "	- DAR Full.\n");  in sahara_decode_status()
363 		dev_dbg(dev->device, "	- Error.\n");  in sahara_decode_status()
365 		dev_dbg(dev->device, "	- Secure.\n");  in sahara_decode_status()
367 		dev_dbg(dev->device, "	- Fail.\n");  in sahara_decode_status()
369 		dev_dbg(dev->device, "	- RNG Reseed Request.\n");  in sahara_decode_status()
371 		dev_dbg(dev->device, "	- RNG Active.\n");  in sahara_decode_status()
373 		dev_dbg(dev->device, "	- MDHA Active.\n");  in sahara_decode_status()
375 		dev_dbg(dev->device, "	- SKHA Active.\n");  in sahara_decode_status()
378 		dev_dbg(dev->device, "	- Batch Mode.\n");  in sahara_decode_status()
380 		dev_dbg(dev->device, "	- Dedicated Mode.\n");  in sahara_decode_status()
382 		dev_dbg(dev->device, "	- Debug Mode.\n");  in sahara_decode_status()
384 	dev_dbg(dev->device, "	- Internal state = 0x%02x\n",  in sahara_decode_status()
387 	dev_dbg(dev->device, "Current DAR: 0x%08x\n",  in sahara_decode_status()
389 	dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n",  in sahara_decode_status()
401 		dev_dbg(dev->device, "Descriptor (%d) (%pad):\n",  in sahara_dump_descriptors()
402 			i, &dev->hw_phys_desc[i]);  in sahara_dump_descriptors()
403 		dev_dbg(dev->device, "\thdr = 0x%08x\n", dev->hw_desc[i]->hdr);  in sahara_dump_descriptors()
404 		dev_dbg(dev->device, "\tlen1 = %u\n", dev->hw_desc[i]->len1);  in sahara_dump_descriptors()
405 		dev_dbg(dev->device, "\tp1 = 0x%08x\n", dev->hw_desc[i]->p1);  in sahara_dump_descriptors()
406 		dev_dbg(dev->device, "\tlen2 = %u\n", dev->hw_desc[i]->len2);  in sahara_dump_descriptors()
407 		dev_dbg(dev->device, "\tp2 = 0x%08x\n", dev->hw_desc[i]->p2);  in sahara_dump_descriptors()
408 		dev_dbg(dev->device, "\tnext = 0x%08x\n",  in sahara_dump_descriptors()
409 			dev->hw_desc[i]->next);  in sahara_dump_descriptors()
411 	dev_dbg(dev->device, "\n");  in sahara_dump_descriptors()
422 		dev_dbg(dev->device, "Link (%d) (%pad):\n",  in sahara_dump_links()
423 			i, &dev->hw_phys_link[i]);  in sahara_dump_links()
424 		dev_dbg(dev->device, "\tlen = %u\n", dev->hw_link[i]->len);  in sahara_dump_links()
425 		dev_dbg(dev->device, "\tp = 0x%08x\n", dev->hw_link[i]->p);  in sahara_dump_links()
426 		dev_dbg(dev->device, "\tnext = 0x%08x\n",  in sahara_dump_links()
427 			dev->hw_link[i]->next);  in sahara_dump_links()
429 	dev_dbg(dev->device, "\n");  in sahara_dump_links()
434 	struct sahara_ctx *ctx = dev->ctx;  in sahara_hw_descriptor_create()
441 	memcpy(dev->key_base, ctx->key, ctx->keylen);  in sahara_hw_descriptor_create()
443 	if (dev->flags & FLAGS_CBC) {  in sahara_hw_descriptor_create()
444 		dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE;  in sahara_hw_descriptor_create()
445 		dev->hw_desc[idx]->p1 = dev->iv_phys_base;  in sahara_hw_descriptor_create()
447 		dev->hw_desc[idx]->len1 = 0;  in sahara_hw_descriptor_create()
448 		dev->hw_desc[idx]->p1 = 0;  in sahara_hw_descriptor_create()
450 	dev->hw_desc[idx]->len2 = ctx->keylen;  in sahara_hw_descriptor_create()
451 	dev->hw_desc[idx]->p2 = dev->key_phys_base;  in sahara_hw_descriptor_create()
452 	dev->hw_desc[idx]->next = dev->hw_phys_desc[1];  in sahara_hw_descriptor_create()
453 	dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev);  in sahara_hw_descriptor_create()
458 	dev->nb_in_sg = sg_nents_for_len(dev->in_sg, dev->total);  in sahara_hw_descriptor_create()
459 	if (dev->nb_in_sg < 0) {  in sahara_hw_descriptor_create()
460 		dev_err(dev->device, "Invalid numbers of src SG.\n");  in sahara_hw_descriptor_create()
461 		return dev->nb_in_sg;  in sahara_hw_descriptor_create()
463 	dev->nb_out_sg = sg_nents_for_len(dev->out_sg, dev->total);  in sahara_hw_descriptor_create()
464 	if (dev->nb_out_sg < 0) {  in sahara_hw_descriptor_create()
465 		dev_err(dev->device, "Invalid numbers of dst SG.\n");  in sahara_hw_descriptor_create()
466 		return dev->nb_out_sg;  in sahara_hw_descriptor_create()
468 	if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) {  in sahara_hw_descriptor_create()
469 		dev_err(dev->device, "not enough hw links (%d)\n",  in sahara_hw_descriptor_create()
470 			dev->nb_in_sg + dev->nb_out_sg);  in sahara_hw_descriptor_create()
471 		return -EINVAL;  in sahara_hw_descriptor_create()
474 	ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg,  in sahara_hw_descriptor_create()
477 		dev_err(dev->device, "couldn't map in sg\n");  in sahara_hw_descriptor_create()
478 		return -EINVAL;  in sahara_hw_descriptor_create()
481 	ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg,  in sahara_hw_descriptor_create()
484 		dev_err(dev->device, "couldn't map out sg\n");  in sahara_hw_descriptor_create()
489 	dev->hw_desc[idx]->p1 = dev->hw_phys_link[0];  in sahara_hw_descriptor_create()
490 	sg = dev->in_sg;  in sahara_hw_descriptor_create()
491 	len = dev->total;  in sahara_hw_descriptor_create()
492 	for (i = 0; i < dev->nb_in_sg; i++) {  in sahara_hw_descriptor_create()
493 		dev->hw_link[i]->len = min(len, sg->length);  in sahara_hw_descriptor_create()
494 		dev->hw_link[i]->p = sg->dma_address;  in sahara_hw_descriptor_create()
495 		if (i == (dev->nb_in_sg - 1)) {  in sahara_hw_descriptor_create()
496 			dev->hw_link[i]->next = 0;  in sahara_hw_descriptor_create()
498 			len -= min(len, sg->length);  in sahara_hw_descriptor_create()
499 			dev->hw_link[i]->next = dev->hw_phys_link[i + 1];  in sahara_hw_descriptor_create()
505 	dev->hw_desc[idx]->p2 = dev->hw_phys_link[i];  in sahara_hw_descriptor_create()
506 	sg = dev->out_sg;  in sahara_hw_descriptor_create()
507 	len = dev->total;  in sahara_hw_descriptor_create()
508 	for (j = i; j < dev->nb_out_sg + i; j++) {  in sahara_hw_descriptor_create()
509 		dev->hw_link[j]->len = min(len, sg->length);  in sahara_hw_descriptor_create()
510 		dev->hw_link[j]->p = sg->dma_address;  in sahara_hw_descriptor_create()
511 		if (j == (dev->nb_out_sg + i - 1)) {  in sahara_hw_descriptor_create()
512 			dev->hw_link[j]->next = 0;  in sahara_hw_descriptor_create()
514 			len -= min(len, sg->length);  in sahara_hw_descriptor_create()
515 			dev->hw_link[j]->next = dev->hw_phys_link[j + 1];  in sahara_hw_descriptor_create()
521 	dev->hw_desc[idx]->hdr = sahara_aes_data_link_hdr(dev);  in sahara_hw_descriptor_create()
522 	dev->hw_desc[idx]->len1 = dev->total;  in sahara_hw_descriptor_create()
523 	dev->hw_desc[idx]->len2 = dev->total;  in sahara_hw_descriptor_create()
524 	dev->hw_desc[idx]->next = 0;  in sahara_hw_descriptor_create()
529 	sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);  in sahara_hw_descriptor_create()
534 	dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,  in sahara_hw_descriptor_create()
537 	return -EINVAL;  in sahara_hw_descriptor_create()
547 	if (rctx->mode & FLAGS_ENCRYPT) {  in sahara_aes_cbc_update_iv()
548 		sg_pcopy_to_buffer(req->dst, sg_nents(req->dst), req->iv,  in sahara_aes_cbc_update_iv()
549 				   ivsize, req->cryptlen - ivsize);  in sahara_aes_cbc_update_iv()
551 		memcpy(req->iv, rctx->iv_out, ivsize);  in sahara_aes_cbc_update_iv()
565 	dev_dbg(dev->device,  in sahara_aes_process()
567 		req->cryptlen, req->src, req->dst);  in sahara_aes_process()
570 	dev->total = req->cryptlen;  in sahara_aes_process()
571 	dev->in_sg = req->src;  in sahara_aes_process()
572 	dev->out_sg = req->dst;  in sahara_aes_process()
576 	rctx->mode &= FLAGS_MODE_MASK;  in sahara_aes_process()
577 	dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode;  in sahara_aes_process()
579 	if ((dev->flags & FLAGS_CBC) && req->iv) {  in sahara_aes_process()
582 		memcpy(dev->iv_base, req->iv, ivsize);  in sahara_aes_process()
584 		if (!(dev->flags & FLAGS_ENCRYPT)) {  in sahara_aes_process()
585 			sg_pcopy_to_buffer(req->src, sg_nents(req->src),  in sahara_aes_process()
586 					   rctx->iv_out, ivsize,  in sahara_aes_process()
587 					   req->cryptlen - ivsize);  in sahara_aes_process()
592 	dev->ctx = ctx;  in sahara_aes_process()
594 	reinit_completion(&dev->dma_completion);  in sahara_aes_process()
598 		return -EINVAL;  in sahara_aes_process()
600 	time_left = wait_for_completion_timeout(&dev->dma_completion,  in sahara_aes_process()
603 	dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,  in sahara_aes_process()
605 	dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,  in sahara_aes_process()
609 		dev_err(dev->device, "AES timeout\n");  in sahara_aes_process()
610 		return -ETIMEDOUT;  in sahara_aes_process()
613 	if ((dev->flags & FLAGS_CBC) && req->iv)  in sahara_aes_process()
624 	ctx->keylen = keylen;  in sahara_aes_setkey()
626 	/* SAHARA only supports 128bit keys */  in sahara_aes_setkey()
628 		memcpy(ctx->key, key, keylen);  in sahara_aes_setkey()
633 		return -EINVAL;  in sahara_aes_setkey()
638 	crypto_skcipher_clear_flags(ctx->fallback, CRYPTO_TFM_REQ_MASK);  in sahara_aes_setkey()
639 	crypto_skcipher_set_flags(ctx->fallback, tfm->base.crt_flags &  in sahara_aes_setkey()
641 	return crypto_skcipher_setkey(ctx->fallback, key, keylen);  in sahara_aes_setkey()
650 	skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback);  in sahara_aes_fallback()
651 	skcipher_request_set_callback(&rctx->fallback_req,  in sahara_aes_fallback()
652 				      req->base.flags,  in sahara_aes_fallback()
653 				      req->base.complete,  in sahara_aes_fallback()
654 				      req->base.data);  in sahara_aes_fallback()
655 	skcipher_request_set_crypt(&rctx->fallback_req, req->src,  in sahara_aes_fallback()
656 				   req->dst, req->cryptlen, req->iv);  in sahara_aes_fallback()
659 		return crypto_skcipher_encrypt(&rctx->fallback_req);  in sahara_aes_fallback()
661 	return crypto_skcipher_decrypt(&rctx->fallback_req);  in sahara_aes_fallback()
671 	if (!req->cryptlen)  in sahara_aes_crypt()
674 	if (unlikely(ctx->keylen != AES_KEYSIZE_128))  in sahara_aes_crypt()
677 	dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n",  in sahara_aes_crypt()
678 		req->cryptlen, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC));  in sahara_aes_crypt()
680 	if (!IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE))  in sahara_aes_crypt()
681 		return -EINVAL;  in sahara_aes_crypt()
683 	rctx->mode = mode;  in sahara_aes_crypt()
685 	return crypto_transfer_skcipher_request_to_engine(dev->engine, req);  in sahara_aes_crypt()
710 	const char *name = crypto_tfm_alg_name(&tfm->base);  in sahara_aes_init_tfm()
713 	ctx->fallback = crypto_alloc_skcipher(name, 0,  in sahara_aes_init_tfm()
715 	if (IS_ERR(ctx->fallback)) {  in sahara_aes_init_tfm()
717 		return PTR_ERR(ctx->fallback);  in sahara_aes_init_tfm()
721 					 crypto_skcipher_reqsize(ctx->fallback));  in sahara_aes_init_tfm()
730 	crypto_free_skcipher(ctx->fallback);  in sahara_aes_exit_tfm()
738 	hdr = rctx->mode;  in sahara_sha_init_hdr()
740 	if (rctx->first) {  in sahara_sha_init_hdr()
747 	if (rctx->last)  in sahara_sha_init_hdr()
765 	dev->in_sg = rctx->in_sg;  in sahara_sha_hw_links_create()
767 	dev->nb_in_sg = sg_nents_for_len(dev->in_sg, rctx->total);  in sahara_sha_hw_links_create()
768 	if (dev->nb_in_sg < 0) {  in sahara_sha_hw_links_create()
769 		dev_err(dev->device, "Invalid numbers of src SG.\n");  in sahara_sha_hw_links_create()
770 		return dev->nb_in_sg;  in sahara_sha_hw_links_create()
772 	if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) {  in sahara_sha_hw_links_create()
773 		dev_err(dev->device, "not enough hw links (%d)\n",  in sahara_sha_hw_links_create()
774 			dev->nb_in_sg + dev->nb_out_sg);  in sahara_sha_hw_links_create()
775 		return -EINVAL;  in sahara_sha_hw_links_create()
778 	sg = dev->in_sg;  in sahara_sha_hw_links_create()
779 	ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg, DMA_TO_DEVICE);  in sahara_sha_hw_links_create()
781 		return -EFAULT;  in sahara_sha_hw_links_create()
783 	len = rctx->total;  in sahara_sha_hw_links_create()
784 	for (i = start; i < dev->nb_in_sg + start; i++) {  in sahara_sha_hw_links_create()
785 		dev->hw_link[i]->len = min(len, sg->length);  in sahara_sha_hw_links_create()
786 		dev->hw_link[i]->p = sg->dma_address;  in sahara_sha_hw_links_create()
787 		if (i == (dev->nb_in_sg + start - 1)) {  in sahara_sha_hw_links_create()
788 			dev->hw_link[i]->next = 0;  in sahara_sha_hw_links_create()
790 			len -= min(len, sg->length);  in sahara_sha_hw_links_create()
791 			dev->hw_link[i]->next = dev->hw_phys_link[i + 1];  in sahara_sha_hw_links_create()
807 	if (rctx->first)  in sahara_sha_hw_data_descriptor_create()
809 		dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);  in sahara_sha_hw_data_descriptor_create()
812 		dev->hw_desc[index]->hdr = SAHARA_HDR_MDHA_HASH;  in sahara_sha_hw_data_descriptor_create()
814 	dev->hw_desc[index]->len1 = rctx->total;  in sahara_sha_hw_data_descriptor_create()
815 	if (dev->hw_desc[index]->len1 == 0) {  in sahara_sha_hw_data_descriptor_create()
817 		dev->hw_desc[index]->p1 = 0;  in sahara_sha_hw_data_descriptor_create()
818 		rctx->sg_in_idx = 0;  in sahara_sha_hw_data_descriptor_create()
821 		dev->hw_desc[index]->p1 = dev->hw_phys_link[index];  in sahara_sha_hw_data_descriptor_create()
824 		rctx->sg_in_idx = index;  in sahara_sha_hw_data_descriptor_create()
829 	dev->hw_desc[index]->p2 = dev->hw_phys_link[i];  in sahara_sha_hw_data_descriptor_create()
832 	result_len = rctx->context_size;  in sahara_sha_hw_data_descriptor_create()
833 	dev->hw_link[i]->p = dev->context_phys_base;  in sahara_sha_hw_data_descriptor_create()
835 	dev->hw_link[i]->len = result_len;  in sahara_sha_hw_data_descriptor_create()
836 	dev->hw_desc[index]->len2 = result_len;  in sahara_sha_hw_data_descriptor_create()
838 	dev->hw_link[i]->next = 0;  in sahara_sha_hw_data_descriptor_create()
857 	dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);  in sahara_sha_hw_context_descriptor_create()
859 	dev->hw_desc[index]->len1 = rctx->context_size;  in sahara_sha_hw_context_descriptor_create()
860 	dev->hw_desc[index]->p1 = dev->hw_phys_link[index];  in sahara_sha_hw_context_descriptor_create()
861 	dev->hw_desc[index]->len2 = 0;  in sahara_sha_hw_context_descriptor_create()
862 	dev->hw_desc[index]->p2 = 0;  in sahara_sha_hw_context_descriptor_create()
864 	dev->hw_link[index]->len = rctx->context_size;  in sahara_sha_hw_context_descriptor_create()
865 	dev->hw_link[index]->p = dev->context_phys_base;  in sahara_sha_hw_context_descriptor_create()
866 	dev->hw_link[index]->next = 0;  in sahara_sha_hw_context_descriptor_create()
882 	len = rctx->buf_cnt + req->nbytes;  in sahara_sha_prepare_request()
885 	if (!rctx->last && (len < block_size)) {  in sahara_sha_prepare_request()
887 		scatterwalk_map_and_copy(rctx->buf + rctx->buf_cnt, req->src,  in sahara_sha_prepare_request()
888 					 0, req->nbytes, 0);  in sahara_sha_prepare_request()
889 		rctx->buf_cnt += req->nbytes;  in sahara_sha_prepare_request()
895 	if (rctx->buf_cnt)  in sahara_sha_prepare_request()
896 		memcpy(rctx->rembuf, rctx->buf, rctx->buf_cnt);  in sahara_sha_prepare_request()
899 	hash_later = rctx->last ? 0 : len & (block_size - 1);  in sahara_sha_prepare_request()
901 		unsigned int offset = req->nbytes - hash_later;  in sahara_sha_prepare_request()
903 		scatterwalk_map_and_copy(rctx->buf, req->src, offset,  in sahara_sha_prepare_request()
907 	rctx->total = len - hash_later;  in sahara_sha_prepare_request()
909 	if (rctx->buf_cnt && req->nbytes) {  in sahara_sha_prepare_request()
910 		sg_init_table(rctx->in_sg_chain, 2);  in sahara_sha_prepare_request()
911 		sg_set_buf(rctx->in_sg_chain, rctx->rembuf, rctx->buf_cnt);  in sahara_sha_prepare_request()
912 		sg_chain(rctx->in_sg_chain, 2, req->src);  in sahara_sha_prepare_request()
913 		rctx->in_sg = rctx->in_sg_chain;  in sahara_sha_prepare_request()
915 	} else if (rctx->buf_cnt) {  in sahara_sha_prepare_request()
916 		rctx->in_sg = rctx->in_sg_chain;  in sahara_sha_prepare_request()
917 		sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt);  in sahara_sha_prepare_request()
920 		rctx->in_sg = req->src;  in sahara_sha_prepare_request()
924 	rctx->buf_cnt = hash_later;  in sahara_sha_prepare_request()
926 	return -EINPROGRESS;  in sahara_sha_prepare_request()
940 	if (rctx->first) {  in sahara_sha_process()
945 		dev->hw_desc[0]->next = 0;  in sahara_sha_process()
946 		rctx->first = 0;  in sahara_sha_process()
948 		memcpy(dev->context_base, rctx->context, rctx->context_size);  in sahara_sha_process()
951 		dev->hw_desc[0]->next = dev->hw_phys_desc[1];  in sahara_sha_process()
956 		dev->hw_desc[1]->next = 0;  in sahara_sha_process()
962 	reinit_completion(&dev->dma_completion);  in sahara_sha_process()
964 	sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);  in sahara_sha_process()
966 	time_left = wait_for_completion_timeout(&dev->dma_completion,  in sahara_sha_process()
969 	if (rctx->sg_in_idx)  in sahara_sha_process()
970 		dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,  in sahara_sha_process()
974 		dev_err(dev->device, "SHA timeout\n");  in sahara_sha_process()
975 		return -ETIMEDOUT;  in sahara_sha_process()
978 	memcpy(rctx->context, dev->context_base, rctx->context_size);  in sahara_sha_process()
980 	if (req->result && rctx->last)  in sahara_sha_process()
981 		memcpy(req->result, rctx->context, rctx->digest_size);  in sahara_sha_process()
991 	if (crypto_tfm_alg_type(async_req->tfm) == CRYPTO_ALG_TYPE_AHASH) {  in sahara_do_one_request()
1015 	if (!req->nbytes && !last)  in sahara_sha_enqueue()
1018 	rctx->last = last;  in sahara_sha_enqueue()
1020 	return crypto_transfer_hash_request_to_engine(dev->engine, req);  in sahara_sha_enqueue()
1032 		rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA1;  in sahara_sha_init()
1033 		rctx->digest_size = SHA1_DIGEST_SIZE;  in sahara_sha_init()
1036 		rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA256;  in sahara_sha_init()
1037 		rctx->digest_size = SHA256_DIGEST_SIZE;  in sahara_sha_init()
1040 		return -EINVAL;  in sahara_sha_init()
1043 	rctx->context_size = rctx->digest_size + 4;  in sahara_sha_init()
1044 	rctx->first = 1;  in sahara_sha_init()
1056 	req->nbytes = 0;  in sahara_sha_final()
1102 		.base.cra_driver_name	= "sahara-ecb-aes",
1124 		.base.cra_driver_name	= "sahara-cbc-aes",
1161 			.cra_driver_name	= "sahara-sha1",
1192 			.cra_driver_name	= "sahara-sha256",
1226 	complete(&dev->dma_completion);  in sahara_irq_handler()
1245 	if (dev->version > SAHARA_VERSION_3) {  in sahara_register_algs()
1268 	if (dev->version > SAHARA_VERSION_3)  in sahara_unregister_algs()
1274 	{ .compatible = "fsl,imx53-sahara" },
1275 	{ .compatible = "fsl,imx27-sahara" },
1288 	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);  in sahara_probe()
1290 		return -ENOMEM;  in sahara_probe()
1292 	dev->device = &pdev->dev;  in sahara_probe()
1296 	dev->regs_base = devm_platform_ioremap_resource(pdev, 0);  in sahara_probe()
1297 	if (IS_ERR(dev->regs_base))  in sahara_probe()
1298 		return PTR_ERR(dev->regs_base);  in sahara_probe()
1305 	err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler,  in sahara_probe()
1306 			       0, dev_name(&pdev->dev), dev);  in sahara_probe()
1308 		return dev_err_probe(&pdev->dev, err,  in sahara_probe()
1312 	dev->clk_ipg = devm_clk_get_enabled(&pdev->dev, "ipg");  in sahara_probe()
1313 	if (IS_ERR(dev->clk_ipg))  in sahara_probe()
1314 		return dev_err_probe(&pdev->dev, PTR_ERR(dev->clk_ipg),  in sahara_probe()
1317 	dev->clk_ahb = devm_clk_get_enabled(&pdev->dev, "ahb");  in sahara_probe()
1318 	if (IS_ERR(dev->clk_ahb))  in sahara_probe()
1319 		return dev_err_probe(&pdev->dev, PTR_ERR(dev->clk_ahb),  in sahara_probe()
1323 	dev->hw_desc[0] = dmam_alloc_coherent(&pdev->dev,  in sahara_probe()
1325 			&dev->hw_phys_desc[0], GFP_KERNEL);  in sahara_probe()
1326 	if (!dev->hw_desc[0])  in sahara_probe()
1327 		return -ENOMEM;  in sahara_probe()
1328 	dev->hw_desc[1] = dev->hw_desc[0] + 1;  in sahara_probe()
1329 	dev->hw_phys_desc[1] = dev->hw_phys_desc[0] +  in sahara_probe()
1333 	dev->key_base = dmam_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128,  in sahara_probe()
1334 				&dev->key_phys_base, GFP_KERNEL);  in sahara_probe()
1335 	if (!dev->key_base)  in sahara_probe()
1336 		return -ENOMEM;  in sahara_probe()
1337 	dev->iv_base = dev->key_base + AES_KEYSIZE_128;  in sahara_probe()
1338 	dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128;  in sahara_probe()
1341 	dev->context_base = dmam_alloc_coherent(&pdev->dev,  in sahara_probe()
1343 					&dev->context_phys_base, GFP_KERNEL);  in sahara_probe()
1344 	if (!dev->context_base)  in sahara_probe()
1345 		return -ENOMEM;  in sahara_probe()
1348 	dev->hw_link[0] = dmam_alloc_coherent(&pdev->dev,  in sahara_probe()
1350 			&dev->hw_phys_link[0], GFP_KERNEL);  in sahara_probe()
1351 	if (!dev->hw_link[0])  in sahara_probe()
1352 		return -ENOMEM;  in sahara_probe()
1354 		dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] +  in sahara_probe()
1356 		dev->hw_link[i] = dev->hw_link[i - 1] + 1;  in sahara_probe()
1361 	dev->engine = crypto_engine_alloc_init(&pdev->dev, true);  in sahara_probe()
1362 	if (!dev->engine)  in sahara_probe()
1363 		return -ENOMEM;  in sahara_probe()
1365 	err = crypto_engine_start(dev->engine);  in sahara_probe()
1367 		crypto_engine_exit(dev->engine);  in sahara_probe()
1368 		return dev_err_probe(&pdev->dev, err,  in sahara_probe()
1372 	init_completion(&dev->dma_completion);  in sahara_probe()
1375 	if (of_device_is_compatible(pdev->dev.of_node, "fsl,imx27-sahara")) {  in sahara_probe()
1377 			err = -ENODEV;  in sahara_probe()
1378 	} else if (of_device_is_compatible(pdev->dev.of_node,  in sahara_probe()
1379 			"fsl,imx53-sahara")) {  in sahara_probe()
1381 			err = -ENODEV;  in sahara_probe()
1384 	if (err == -ENODEV) {  in sahara_probe()
1385 		dev_err_probe(&pdev->dev, err,  in sahara_probe()
1386 			      "SAHARA version %d not supported\n", version);  in sahara_probe()
1390 	dev->version = version;  in sahara_probe()
1404 	dev_info(&pdev->dev, "SAHARA version %d initialized\n", version);  in sahara_probe()
1409 	crypto_engine_exit(dev->engine);  in sahara_probe()
1418 	crypto_engine_exit(dev->engine);  in sahara_remove()
1434 MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>");