1 // SPDX-License-Identifier: GPL-2.0-only
3  * Support for OMAP DES and Triple DES HW acceleration.
22 #include <linux/dma-mapping.h>
36 #include "omap-crypto.h"
42 #define DES_REG_KEY(dd, x)		((dd)->pdata->key_ofs - \
45 #define DES_REG_IV(dd, x)		((dd)->pdata->iv_ofs + ((x) * 0x04))
47 #define DES_REG_CTRL(dd)		((dd)->pdata->ctrl_ofs)
54 #define DES_REG_DATA_N(dd, x)		((dd)->pdata->data_ofs + ((x) * 0x04))
56 #define DES_REG_REV(dd)			((dd)->pdata->rev_ofs)
58 #define DES_REG_MASK(dd)		((dd)->pdata->mask_ofs)
62 #define DES_REG_IRQ_STATUS(dd)         ((dd)->pdata->irq_status_ofs)
63 #define DES_REG_IRQ_ENABLE(dd)         ((dd)->pdata->irq_enable_ofs)
170 	 _read_ret = __raw_readl(dd->io_base + offset);          \
178 	return __raw_readl(dd->io_base + offset);  in omap_des_read()
187 		__raw_writel(value, dd->io_base + offset);              \
193 	__raw_writel(value, dd->io_base + offset);  in omap_des_write()
211 	for (; count--; value++, offset += 4)  in omap_des_write_n()
224 	err = pm_runtime_resume_and_get(dd->dev);  in omap_des_hw_init()
226 		dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err);  in omap_des_hw_init()
230 	if (!(dd->flags & FLAGS_INIT)) {  in omap_des_hw_init()
231 		dd->flags |= FLAGS_INIT;  in omap_des_hw_init()
232 		dd->err = 0;  in omap_des_hw_init()
248 	key32 = dd->ctx->keylen / sizeof(u32);  in omap_des_write_ctrl()
253 			       __le32_to_cpu(dd->ctx->key[i]));  in omap_des_write_ctrl()
256 	if ((dd->flags & FLAGS_CBC) && dd->req->iv)  in omap_des_write_ctrl()
257 		omap_des_write_n(dd, DES_REG_IV(dd, 0), (void *)dd->req->iv, 2);  in omap_des_write_ctrl()
259 	if (dd->flags & FLAGS_CBC)  in omap_des_write_ctrl()
261 	if (dd->flags & FLAGS_ENCRYPT)  in omap_des_write_ctrl()
279 	val = dd->pdata->dma_start;  in omap_des_dma_trigger_omap4()
281 	if (dd->dma_lch_out != NULL)  in omap_des_dma_trigger_omap4()
282 		val |= dd->pdata->dma_enable_out;  in omap_des_dma_trigger_omap4()
283 	if (dd->dma_lch_in != NULL)  in omap_des_dma_trigger_omap4()
284 		val |= dd->pdata->dma_enable_in;  in omap_des_dma_trigger_omap4()
286 	mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |  in omap_des_dma_trigger_omap4()
287 	       dd->pdata->dma_start;  in omap_des_dma_trigger_omap4()
296 	mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |  in omap_des_dma_stop()
297 	       dd->pdata->dma_start;  in omap_des_dma_stop()
307 	if (!ctx->dd) {  in omap_des_find_dev()
313 		ctx->dd = dd;  in omap_des_find_dev()
316 		dd = ctx->dd;  in omap_des_find_dev()
327 	/* dma_lch_out - completed */  in omap_des_dma_out_callback()
328 	tasklet_schedule(&dd->done_task);  in omap_des_dma_out_callback()
335 	dd->dma_lch_out = NULL;  in omap_des_dma_init()
336 	dd->dma_lch_in = NULL;  in omap_des_dma_init()
338 	dd->dma_lch_in = dma_request_chan(dd->dev, "rx");  in omap_des_dma_init()
339 	if (IS_ERR(dd->dma_lch_in)) {  in omap_des_dma_init()
340 		dev_err(dd->dev, "Unable to request in DMA channel\n");  in omap_des_dma_init()
341 		return PTR_ERR(dd->dma_lch_in);  in omap_des_dma_init()
344 	dd->dma_lch_out = dma_request_chan(dd->dev, "tx");  in omap_des_dma_init()
345 	if (IS_ERR(dd->dma_lch_out)) {  in omap_des_dma_init()
346 		dev_err(dd->dev, "Unable to request out DMA channel\n");  in omap_des_dma_init()
347 		err = PTR_ERR(dd->dma_lch_out);  in omap_des_dma_init()
354 	dma_release_channel(dd->dma_lch_in);  in omap_des_dma_init()
361 	if (dd->pio_only)  in omap_des_dma_cleanup()
364 	dma_release_channel(dd->dma_lch_out);  in omap_des_dma_cleanup()
365 	dma_release_channel(dd->dma_lch_in);  in omap_des_dma_cleanup()
373 	struct omap_des_dev *dd = ctx->dd;  in omap_des_crypt_dma()
378 	if (dd->pio_only) {  in omap_des_crypt_dma()
379 		dd->in_sg_offset = 0;  in omap_des_crypt_dma()
380 		dd->out_sg_offset = 0;  in omap_des_crypt_dma()
388 	dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE);  in omap_des_crypt_dma()
392 	cfg.src_addr = dd->phys_base + DES_REG_DATA_N(dd, 0);  in omap_des_crypt_dma()
393 	cfg.dst_addr = dd->phys_base + DES_REG_DATA_N(dd, 0);  in omap_des_crypt_dma()
400 	ret = dmaengine_slave_config(dd->dma_lch_in, &cfg);  in omap_des_crypt_dma()
402 		dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",  in omap_des_crypt_dma()
407 	tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len,  in omap_des_crypt_dma()
411 		dev_err(dd->dev, "IN prep_slave_sg() failed\n");  in omap_des_crypt_dma()
412 		return -EINVAL;  in omap_des_crypt_dma()
416 	tx_in->callback_param = dd;  in omap_des_crypt_dma()
419 	ret = dmaengine_slave_config(dd->dma_lch_out, &cfg);  in omap_des_crypt_dma()
421 		dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",  in omap_des_crypt_dma()
426 	tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, out_sg_len,  in omap_des_crypt_dma()
430 		dev_err(dd->dev, "OUT prep_slave_sg() failed\n");  in omap_des_crypt_dma()
431 		return -EINVAL;  in omap_des_crypt_dma()
434 	tx_out->callback = omap_des_dma_out_callback;  in omap_des_crypt_dma()
435 	tx_out->callback_param = dd;  in omap_des_crypt_dma()
440 	dma_async_issue_pending(dd->dma_lch_in);  in omap_des_crypt_dma()
441 	dma_async_issue_pending(dd->dma_lch_out);  in omap_des_crypt_dma()
444 	dd->pdata->trigger(dd, dd->total);  in omap_des_crypt_dma()
452 					crypto_skcipher_reqtfm(dd->req));  in omap_des_crypt_dma_start()
455 	pr_debug("total: %zd\n", dd->total);  in omap_des_crypt_dma_start()
457 	if (!dd->pio_only) {  in omap_des_crypt_dma_start()
458 		err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len,  in omap_des_crypt_dma_start()
461 			dev_err(dd->dev, "dma_map_sg() error\n");  in omap_des_crypt_dma_start()
462 			return -EINVAL;  in omap_des_crypt_dma_start()
465 		err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len,  in omap_des_crypt_dma_start()
468 			dev_err(dd->dev, "dma_map_sg() error\n");  in omap_des_crypt_dma_start()
469 			return -EINVAL;  in omap_des_crypt_dma_start()
473 	err = omap_des_crypt_dma(tfm, dd->in_sg, dd->out_sg, dd->in_sg_len,  in omap_des_crypt_dma_start()
474 				 dd->out_sg_len);  in omap_des_crypt_dma_start()
475 	if (err && !dd->pio_only) {  in omap_des_crypt_dma_start()
476 		dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);  in omap_des_crypt_dma_start()
477 		dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,  in omap_des_crypt_dma_start()
486 	struct skcipher_request *req = dd->req;  in omap_des_finish_req()
490 	crypto_finalize_skcipher_request(dd->engine, req, err);  in omap_des_finish_req()
492 	pm_runtime_mark_last_busy(dd->dev);  in omap_des_finish_req()
493 	pm_runtime_put_autosuspend(dd->dev);  in omap_des_finish_req()
498 	pr_debug("total: %zd\n", dd->total);  in omap_des_crypt_dma_stop()
502 	dmaengine_terminate_all(dd->dma_lch_in);  in omap_des_crypt_dma_stop()
503 	dmaengine_terminate_all(dd->dma_lch_out);  in omap_des_crypt_dma_stop()
512 		return crypto_transfer_skcipher_request_to_engine(dd->engine, req);  in omap_des_handle_queue()
527 	dd->req = req;  in omap_des_prepare_req()
528 	dd->total = req->cryptlen;  in omap_des_prepare_req()
529 	dd->total_save = req->cryptlen;  in omap_des_prepare_req()
530 	dd->in_sg = req->src;  in omap_des_prepare_req()
531 	dd->out_sg = req->dst;  in omap_des_prepare_req()
532 	dd->orig_out = req->dst;  in omap_des_prepare_req()
535 	if (req->src == req->dst)  in omap_des_prepare_req()
538 	ret = omap_crypto_align_sg(&dd->in_sg, dd->total, DES_BLOCK_SIZE,  in omap_des_prepare_req()
539 				   &dd->in_sgl, flags,  in omap_des_prepare_req()
540 				   FLAGS_IN_DATA_ST_SHIFT, &dd->flags);  in omap_des_prepare_req()
544 	ret = omap_crypto_align_sg(&dd->out_sg, dd->total, DES_BLOCK_SIZE,  in omap_des_prepare_req()
545 				   &dd->out_sgl, 0,  in omap_des_prepare_req()
546 				   FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);  in omap_des_prepare_req()
550 	dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total);  in omap_des_prepare_req()
551 	if (dd->in_sg_len < 0)  in omap_des_prepare_req()
552 		return dd->in_sg_len;  in omap_des_prepare_req()
554 	dd->out_sg_len = sg_nents_for_len(dd->out_sg, dd->total);  in omap_des_prepare_req()
555 	if (dd->out_sg_len < 0)  in omap_des_prepare_req()
556 		return dd->out_sg_len;  in omap_des_prepare_req()
560 	rctx->mode &= FLAGS_MODE_MASK;  in omap_des_prepare_req()
561 	dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;  in omap_des_prepare_req()
563 	dd->ctx = ctx;  in omap_des_prepare_req()
564 	ctx->dd = dd;  in omap_des_prepare_req()
578 		return -ENODEV;  in omap_des_crypt_req()
591 	if (!dd->pio_only) {  in omap_des_done_task()
592 		dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,  in omap_des_done_task()
594 		dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);  in omap_des_done_task()
595 		dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,  in omap_des_done_task()
600 	omap_crypto_cleanup(&dd->in_sgl, NULL, 0, dd->total_save,  in omap_des_done_task()
601 			    FLAGS_IN_DATA_ST_SHIFT, dd->flags);  in omap_des_done_task()
603 	omap_crypto_cleanup(&dd->out_sgl, dd->orig_out, 0, dd->total_save,  in omap_des_done_task()
604 			    FLAGS_OUT_DATA_ST_SHIFT, dd->flags);  in omap_des_done_task()
606 	if ((dd->flags & FLAGS_CBC) && dd->req->iv)  in omap_des_done_task()
608 			((u32 *)dd->req->iv)[i] =  in omap_des_done_task()
623 	pr_debug("nbytes: %d, enc: %d, cbc: %d\n", req->cryptlen,  in omap_des_crypt()
627 	if (!req->cryptlen)  in omap_des_crypt()
630 	if (!IS_ALIGNED(req->cryptlen, DES_BLOCK_SIZE))  in omap_des_crypt()
631 		return -EINVAL;  in omap_des_crypt()
635 		return -ENODEV;  in omap_des_crypt()
637 	rctx->mode = mode;  in omap_des_crypt()
656 	memcpy(ctx->key, key, keylen);  in omap_des_setkey()
657 	ctx->keylen = keylen;  in omap_des_setkey()
674 	memcpy(ctx->key, key, keylen);  in omap_des3_setkey()
675 	ctx->keylen = keylen;  in omap_des3_setkey()
715 		.base.cra_driver_name	= "ecb-des-omap",
735 		.base.cra_driver_name	= "cbc-des-omap",
756 		.base.cra_driver_name	= "ecb-des3-omap",
776 		.base.cra_driver_name	= "cbc-des3-omap",
834 		BUG_ON(!dd->in_sg);  in omap_des_irq()
836 		BUG_ON(dd->in_sg_offset > dd->in_sg->length);  in omap_des_irq()
838 		src = sg_virt(dd->in_sg) + dd->in_sg_offset;  in omap_des_irq()
842 			dd->in_sg_offset += 4;  in omap_des_irq()
843 			if (dd->in_sg_offset == dd->in_sg->length) {  in omap_des_irq()
844 				dd->in_sg = sg_next(dd->in_sg);  in omap_des_irq()
845 				if (dd->in_sg) {  in omap_des_irq()
846 					dd->in_sg_offset = 0;  in omap_des_irq()
847 					src = sg_virt(dd->in_sg);  in omap_des_irq()
864 		BUG_ON(!dd->out_sg);  in omap_des_irq()
866 		BUG_ON(dd->out_sg_offset > dd->out_sg->length);  in omap_des_irq()
868 		dst = sg_virt(dd->out_sg) + dd->out_sg_offset;  in omap_des_irq()
872 			dd->out_sg_offset += 4;  in omap_des_irq()
873 			if (dd->out_sg_offset == dd->out_sg->length) {  in omap_des_irq()
874 				dd->out_sg = sg_next(dd->out_sg);  in omap_des_irq()
875 				if (dd->out_sg) {  in omap_des_irq()
876 					dd->out_sg_offset = 0;  in omap_des_irq()
877 					dst = sg_virt(dd->out_sg);  in omap_des_irq()
884 		BUG_ON(dd->total < DES_BLOCK_SIZE);  in omap_des_irq()
886 		dd->total -= DES_BLOCK_SIZE;  in omap_des_irq()
892 		if (!dd->total)  in omap_des_irq()
894 			tasklet_schedule(&dd->done_task);  in omap_des_irq()
905 		.compatible	= "ti,omap4-des",
916 	dd->pdata = of_device_get_match_data(&pdev->dev);  in omap_des_get_of()
917 	if (!dd->pdata) {  in omap_des_get_of()
918 		dev_err(&pdev->dev, "no compatible OF match\n");  in omap_des_get_of()
919 		return -EINVAL;  in omap_des_get_of()
928 	return -EINVAL;  in omap_des_get_of()
935 	/* non-DT devices get pdata from pdev */  in omap_des_get_pdev()
936 	dd->pdata = pdev->dev.platform_data;  in omap_des_get_pdev()
943 	struct device *dev = &pdev->dev;  in omap_des_probe()
947 	int err = -ENOMEM, i, j, irq = -1;  in omap_des_probe()
955 	dd->dev = dev;  in omap_des_probe()
958 	err = (dev->of_node) ? omap_des_get_of(dd, pdev) :  in omap_des_probe()
963 	dd->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);  in omap_des_probe()
964 	if (IS_ERR(dd->io_base)) {  in omap_des_probe()
965 		err = PTR_ERR(dd->io_base);  in omap_des_probe()
968 	dd->phys_base = res->start;  in omap_des_probe()
976 		dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err);  in omap_des_probe()
987 		 (reg & dd->pdata->major_mask) >> dd->pdata->major_shift,  in omap_des_probe()
988 		 (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift);  in omap_des_probe()
990 	tasklet_init(&dd->done_task, omap_des_done_task, (unsigned long)dd);  in omap_des_probe()
993 	if (err == -EPROBE_DEFER) {  in omap_des_probe()
996 		dd->pio_only = 1;  in omap_des_probe()
1007 			dev_err(dev, "Unable to grab omap-des IRQ\n");  in omap_des_probe()
1013 	INIT_LIST_HEAD(&dd->list);  in omap_des_probe()
1015 	list_add_tail(&dd->list, &dev_list);  in omap_des_probe()
1019 	dd->engine = crypto_engine_alloc_init(dev, 1);  in omap_des_probe()
1020 	if (!dd->engine) {  in omap_des_probe()
1021 		err = -ENOMEM;  in omap_des_probe()
1025 	err = crypto_engine_start(dd->engine);  in omap_des_probe()
1029 	for (i = 0; i < dd->pdata->algs_info_size; i++) {  in omap_des_probe()
1030 		for (j = 0; j < dd->pdata->algs_info[i].size; j++) {  in omap_des_probe()
1031 			algp = &dd->pdata->algs_info[i].algs_list[j];  in omap_des_probe()
1033 			pr_debug("reg alg: %s\n", algp->base.base.cra_name);  in omap_des_probe()
1039 			dd->pdata->algs_info[i].registered++;  in omap_des_probe()
1046 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)  in omap_des_probe()
1047 		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)  in omap_des_probe()
1049 					&dd->pdata->algs_info[i].algs_list[j]);  in omap_des_probe()
1052 	if (dd->engine)  in omap_des_probe()
1053 		crypto_engine_exit(dd->engine);  in omap_des_probe()
1057 	tasklet_kill(&dd->done_task);  in omap_des_probe()
1073 	list_del(&dd->list);  in omap_des_remove()
1076 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)  in omap_des_remove()
1077 		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)  in omap_des_remove()
1079 					&dd->pdata->algs_info[i].algs_list[j]);  in omap_des_remove()
1081 	tasklet_kill(&dd->done_task);  in omap_des_remove()
1083 	pm_runtime_disable(dd->dev);  in omap_des_remove()
1112 		.name	= "omap-des",