88 #define A7_PDMA_IN_CMD_FLUSH_STAT		BIT(1)
150 	unsigned char short_descr : 1;
151 	unsigned char pad1        : 1;
152 	unsigned char eop         : 1;
153 	unsigned char intr        : 1;
155 	unsigned char pad2        : 1;
176 	unsigned char pad1        : 1;
177 	unsigned char pad2        : 1;
178 	unsigned char eop         : 1;
189 #define ARTPEC6_CRYPTO_HASH_SHA1	1
193 #define ARTPEC6_CRYPTO_CIPHER_AES_ECB	1
213  *  |  0   |  1   |  2   |~~| 11 EOP|  12  |  ....
439 				   1,  in artpec6_crypto_copy_bounce_buffers()
451 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_busy()  local
452 	int fifo_count = ac->pending_count;  in artpec6_crypto_busy()
459 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_submit()  local
462 	spin_lock_bh(&ac->queue_lock);  in artpec6_crypto_submit()
465 		list_add_tail(&req->list, &ac->pending);  in artpec6_crypto_submit()
469 		list_add_tail(&req->list, &ac->queue);  in artpec6_crypto_submit()
474 	spin_unlock_bh(&ac->queue_lock);  in artpec6_crypto_submit()
481 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_start_dma()  local
482 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_start_dma()
483 	void __iomem *base = ac->base;  in artpec6_crypto_start_dma()
490 	ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) |  in artpec6_crypto_start_dma()
493 	statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) |  in artpec6_crypto_start_dma()
496 	outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) |  in artpec6_crypto_start_dma()
512 	ac->pending_count++;  in artpec6_crypto_start_dma()
529 	return should_fail(&artpec6_crypto_fail_dma_array_full, 1);  in fault_inject_dma_descr()
570  * @len: The length of the data, must be between 1 to 7 bytes
588 	} else if (len > 7 || len < 1) {  in artpec6_crypto_setup_out_descr_short()
594 	d->ctrl.short_descr = 1;  in artpec6_crypto_setup_out_descr_short()
663 	dma->stat[dma->in_cnt - 1] = 0;  in artpec6_crypto_dma_map_descs()
792 	base = bbuf + 1;  in artpec6_crypto_alloc_bounce()
853 			chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1);  in artpec6_crypto_setup_sg_descrs_in()
905 			sg_pcopy_to_buffer(walk->sg, 1, buf, chunk,  in artpec6_crypto_setup_sg_descrs_out()
964 	d = &dma->out[dma->out_cnt-1];  in artpec6_crypto_terminate_out_descrs()
965 	d->ctrl.eop = 1;  in artpec6_crypto_terminate_out_descrs()
987 	d = &dma->in[dma->in_cnt-1];  in artpec6_crypto_terminate_in_descrs()
988 	d->ctrl.intr = 1;  in artpec6_crypto_terminate_in_descrs()
1022 	target -= 1;  in create_hash_pad()
1023 	diff = dgstlen & (mod - 1);  in create_hash_pad()
1026 	memset(dst + 1, 0, pad_bytes);  in create_hash_pad()
1030 		memset(dst + 1 + pad_bytes, 0, 8);  in create_hash_pad()
1031 		memcpy(dst + 1 + pad_bytes + 8, &bits, 8);  in create_hash_pad()
1033 		memcpy(dst + 1 + pad_bytes, &bits, 8);  in create_hash_pad()
1036 	return pad_bytes + size_bytes + 1;  in create_hash_pad()
1045 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_common_init()  local
1051 	common->dma = kmem_cache_alloc(ac->dma_cache, flags);  in artpec6_crypto_common_init()
1074 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_common_destroy()  local
1078 	kmem_cache_free(ac->dma_cache, common->dma);  in artpec6_crypto_common_destroy()
1145 		req_ctx->decrypt = 1;  in artpec6_crypto_decrypt()
1317 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_prepare_hash()  local
1318 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_prepare_hash()
1386 	error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);  in artpec6_crypto_prepare_hash()
1672 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_prepare_crypto()  local
1673 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_prepare_crypto()
1777 	ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);  in artpec6_crypto_prepare_crypto()
1807 							     ac->pad_buffer,  in artpec6_crypto_prepare_crypto()
1813 							    ac->pad_buffer, pad,  in artpec6_crypto_prepare_crypto()
1840 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_prepare_aead()  local
1841 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_prepare_aead()
1903 	ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);  in artpec6_crypto_prepare_aead()
1943 							     ac->zero_buffer,  in artpec6_crypto_prepare_aead()
1960 							     ac->zero_buffer,  in artpec6_crypto_prepare_aead()
1994 							    ac->pad_buffer,  in artpec6_crypto_prepare_aead()
2025 					ac->pad_buffer,  in artpec6_crypto_prepare_aead()
2045 static void artpec6_crypto_process_queue(struct artpec6_crypto *ac,  in artpec6_crypto_process_queue()  argument
2050 	while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) {  in artpec6_crypto_process_queue()
2051 		req = list_first_entry(&ac->queue,  in artpec6_crypto_process_queue()
2054 		list_move_tail(&req->list, &ac->pending);  in artpec6_crypto_process_queue()
2067 	if (ac->pending_count)  in artpec6_crypto_process_queue()
2068 		mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100));  in artpec6_crypto_process_queue()
2070 		del_timer(&ac->timer);  in artpec6_crypto_process_queue()
2075 	struct artpec6_crypto *ac = from_timer(ac, t, timer);  in artpec6_crypto_timeout()  local
2079 	tasklet_schedule(&ac->task);  in artpec6_crypto_timeout()
2084 	struct artpec6_crypto *ac = (struct artpec6_crypto *)data;  in artpec6_crypto_task()  local
2093 	if (list_empty(&ac->pending)) {  in artpec6_crypto_task()
2098 	spin_lock(&ac->queue_lock);  in artpec6_crypto_task()
2100 	list_for_each_entry_safe(req, n, &ac->pending, list) {  in artpec6_crypto_task()
2105 		stataddr = dma->stat_dma_addr + 4 * (req->dma->in_cnt - 1);  in artpec6_crypto_task()
2111 		stat = req->dma->stat[req->dma->in_cnt-1];  in artpec6_crypto_task()
2122 		if (should_fail(&artpec6_crypto_fail_status_read, 1))  in artpec6_crypto_task()
2130 		ac->pending_count--;  in artpec6_crypto_task()
2133 	artpec6_crypto_process_queue(ac, &complete_in_progress);  in artpec6_crypto_task()
2135 	spin_unlock(&ac->queue_lock);  in artpec6_crypto_task()
2208 			print_hex_dump_debug("ref:", DUMP_PREFIX_ADDRESS, 32, 1,  in artpec6_crypto_complete_aead()
2210 			print_hex_dump_debug("out:", DUMP_PREFIX_ADDRESS, 32, 1,  in artpec6_crypto_complete_aead()
2264 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_init_hash()  local
2265 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_init_hash()
2379 	return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1);  in artpec6_crypto_hmac_sha256_init()
2386 	artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1);  in artpec6_crypto_hmac_sha256_digest()
2441 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_hash_export()  local
2442 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_hash_export()
2469 	struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);  in artpec6_crypto_hash_import()  local
2470 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_hash_import()
2491 static int init_crypto_hw(struct artpec6_crypto *ac)  in init_crypto_hw()  argument
2493 	enum artpec6_crypto_variant variant = ac->variant;  in init_crypto_hw()
2494 	void __iomem *base = ac->base;  in init_crypto_hw()
2551 static void artpec6_crypto_disable_hw(struct artpec6_crypto *ac)  in artpec6_crypto_disable_hw()  argument
2553 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_disable_hw()
2554 	void __iomem *base = ac->base;  in artpec6_crypto_disable_hw()
2572 	struct artpec6_crypto *ac = dev_id;  in artpec6_crypto_irq()  local
2573 	enum artpec6_crypto_variant variant = ac->variant;  in artpec6_crypto_irq()
2574 	void __iomem *base = ac->base;  in artpec6_crypto_irq()
2615 		tasklet_schedule(&ac->task);  in artpec6_crypto_irq()
2624 	/* SHA-1 */
2728 			.cra_blocksize = 1,
2772 			.cra_blocksize = 1,
2804 			.cra_blocksize = 1,
2847 	struct artpec6_crypto *ac;  in artpec6_crypto_probe()  local
2870 	ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto),  in artpec6_crypto_probe()
2872 	if (!ac)  in artpec6_crypto_probe()
2875 	platform_set_drvdata(pdev, ac);  in artpec6_crypto_probe()
2876 	ac->variant = variant;  in artpec6_crypto_probe()
2878 	spin_lock_init(&ac->queue_lock);  in artpec6_crypto_probe()
2879 	INIT_LIST_HEAD(&ac->queue);  in artpec6_crypto_probe()
2880 	INIT_LIST_HEAD(&ac->pending);  in artpec6_crypto_probe()
2881 	timer_setup(&ac->timer, artpec6_crypto_timeout, 0);  in artpec6_crypto_probe()
2883 	ac->base = base;  in artpec6_crypto_probe()
2885 	ac->dma_cache = kmem_cache_create("artpec6_crypto_dma",  in artpec6_crypto_probe()
2890 	if (!ac->dma_cache)  in artpec6_crypto_probe()
2897 	tasklet_init(&ac->task, artpec6_crypto_task,  in artpec6_crypto_probe()
2898 		     (unsigned long)ac);  in artpec6_crypto_probe()
2900 	ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX,  in artpec6_crypto_probe()
2902 	if (!ac->pad_buffer)  in artpec6_crypto_probe()
2904 	ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX);  in artpec6_crypto_probe()
2906 	ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX,  in artpec6_crypto_probe()
2908 	if (!ac->zero_buffer)  in artpec6_crypto_probe()
2910 	ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX);  in artpec6_crypto_probe()
2912 	err = init_crypto_hw(ac);  in artpec6_crypto_probe()
2917 			       "artpec6-crypto", ac);  in artpec6_crypto_probe()
2948 	artpec6_crypto_disable_hw(ac);  in artpec6_crypto_probe()
2950 	kmem_cache_destroy(ac->dma_cache);  in artpec6_crypto_probe()
2956 	struct artpec6_crypto *ac = platform_get_drvdata(pdev);  in artpec6_crypto_remove()  local
2963 	tasklet_disable(&ac->task);  in artpec6_crypto_remove()
2964 	devm_free_irq(&pdev->dev, irq, ac);  in artpec6_crypto_remove()
2965 	tasklet_kill(&ac->task);  in artpec6_crypto_remove()
2966 	del_timer_sync(&ac->timer);  in artpec6_crypto_remove()
2968 	artpec6_crypto_disable_hw(ac);  in artpec6_crypto_remove()
2970 	kmem_cache_destroy(ac->dma_cache);  in artpec6_crypto_remove()