1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * AMD AE4DMA driver
4 *
5 * Copyright (c) 2024, Advanced Micro Devices, Inc.
6 * All Rights Reserved.
7 *
8 * Author: Basavaraj Natikar <[email protected]>
9 */
10
11 #include "ae4dma.h"
12
13 static unsigned int max_hw_q = 1;
14 module_param(max_hw_q, uint, 0444);
15 MODULE_PARM_DESC(max_hw_q, "max hw queues supported by engine (any non-zero value, default: 1)");
16
ae4_pending_work(struct work_struct * work)17 static void ae4_pending_work(struct work_struct *work)
18 {
19 struct ae4_cmd_queue *ae4cmd_q = container_of(work, struct ae4_cmd_queue, p_work.work);
20 struct pt_cmd_queue *cmd_q = &ae4cmd_q->cmd_q;
21 struct pt_cmd *cmd;
22 u32 cridx;
23
24 for (;;) {
25 wait_event_interruptible(ae4cmd_q->q_w,
26 ((atomic64_read(&ae4cmd_q->done_cnt)) <
27 atomic64_read(&ae4cmd_q->intr_cnt)));
28
29 atomic64_inc(&ae4cmd_q->done_cnt);
30
31 mutex_lock(&ae4cmd_q->cmd_lock);
32 cridx = readl(cmd_q->reg_control + AE4_RD_IDX_OFF);
33 while ((ae4cmd_q->dridx != cridx) && !list_empty(&ae4cmd_q->cmd)) {
34 cmd = list_first_entry(&ae4cmd_q->cmd, struct pt_cmd, entry);
35 list_del(&cmd->entry);
36
37 ae4_check_status_error(ae4cmd_q, ae4cmd_q->dridx);
38 cmd->pt_cmd_callback(cmd->data, cmd->ret);
39
40 ae4cmd_q->q_cmd_count--;
41 ae4cmd_q->dridx = (ae4cmd_q->dridx + 1) % CMD_Q_LEN;
42
43 complete_all(&ae4cmd_q->cmp);
44 }
45 mutex_unlock(&ae4cmd_q->cmd_lock);
46 }
47 }
48
ae4_core_irq_handler(int irq,void * data)49 static irqreturn_t ae4_core_irq_handler(int irq, void *data)
50 {
51 struct ae4_cmd_queue *ae4cmd_q = data;
52 struct pt_cmd_queue *cmd_q;
53 struct pt_device *pt;
54 u32 status;
55
56 cmd_q = &ae4cmd_q->cmd_q;
57 pt = cmd_q->pt;
58
59 pt->total_interrupts++;
60 atomic64_inc(&ae4cmd_q->intr_cnt);
61
62 status = readl(cmd_q->reg_control + AE4_INTR_STS_OFF);
63 if (status & BIT(0)) {
64 status &= GENMASK(31, 1);
65 writel(status, cmd_q->reg_control + AE4_INTR_STS_OFF);
66 }
67
68 wake_up(&ae4cmd_q->q_w);
69
70 return IRQ_HANDLED;
71 }
72
ae4_destroy_work(struct ae4_device * ae4)73 void ae4_destroy_work(struct ae4_device *ae4)
74 {
75 struct ae4_cmd_queue *ae4cmd_q;
76 int i;
77
78 for (i = 0; i < ae4->cmd_q_count; i++) {
79 ae4cmd_q = &ae4->ae4cmd_q[i];
80
81 if (!ae4cmd_q->pws)
82 break;
83
84 cancel_delayed_work_sync(&ae4cmd_q->p_work);
85 destroy_workqueue(ae4cmd_q->pws);
86 }
87 }
88
ae4_core_init(struct ae4_device * ae4)89 int ae4_core_init(struct ae4_device *ae4)
90 {
91 struct pt_device *pt = &ae4->pt;
92 struct ae4_cmd_queue *ae4cmd_q;
93 struct device *dev = pt->dev;
94 struct pt_cmd_queue *cmd_q;
95 int i, ret = 0;
96
97 writel(max_hw_q, pt->io_regs);
98
99 for (i = 0; i < max_hw_q; i++) {
100 ae4cmd_q = &ae4->ae4cmd_q[i];
101 ae4cmd_q->id = ae4->cmd_q_count;
102 ae4->cmd_q_count++;
103
104 cmd_q = &ae4cmd_q->cmd_q;
105 cmd_q->pt = pt;
106
107 cmd_q->reg_control = pt->io_regs + ((i + 1) * AE4_Q_SZ);
108
109 ret = devm_request_irq(dev, ae4->ae4_irq[i], ae4_core_irq_handler, 0,
110 dev_name(pt->dev), ae4cmd_q);
111 if (ret)
112 return ret;
113
114 cmd_q->qsize = Q_SIZE(sizeof(struct ae4dma_desc));
115
116 cmd_q->qbase = dmam_alloc_coherent(dev, cmd_q->qsize, &cmd_q->qbase_dma,
117 GFP_KERNEL);
118 if (!cmd_q->qbase)
119 return -ENOMEM;
120 }
121
122 for (i = 0; i < ae4->cmd_q_count; i++) {
123 ae4cmd_q = &ae4->ae4cmd_q[i];
124
125 cmd_q = &ae4cmd_q->cmd_q;
126
127 cmd_q->reg_control = pt->io_regs + ((i + 1) * AE4_Q_SZ);
128
129 /* Update the device registers with queue information. */
130 writel(CMD_Q_LEN, cmd_q->reg_control + AE4_MAX_IDX_OFF);
131
132 cmd_q->qdma_tail = cmd_q->qbase_dma;
133 writel(lower_32_bits(cmd_q->qdma_tail), cmd_q->reg_control + AE4_Q_BASE_L_OFF);
134 writel(upper_32_bits(cmd_q->qdma_tail), cmd_q->reg_control + AE4_Q_BASE_H_OFF);
135
136 INIT_LIST_HEAD(&ae4cmd_q->cmd);
137 init_waitqueue_head(&ae4cmd_q->q_w);
138
139 ae4cmd_q->pws = alloc_ordered_workqueue("ae4dma_%d", WQ_MEM_RECLAIM, ae4cmd_q->id);
140 if (!ae4cmd_q->pws) {
141 ae4_destroy_work(ae4);
142 return -ENOMEM;
143 }
144 INIT_DELAYED_WORK(&ae4cmd_q->p_work, ae4_pending_work);
145 queue_delayed_work(ae4cmd_q->pws, &ae4cmd_q->p_work, usecs_to_jiffies(100));
146
147 init_completion(&ae4cmd_q->cmp);
148 }
149
150 ret = pt_dmaengine_register(pt);
151 if (ret)
152 ae4_destroy_work(ae4);
153 else
154 ptdma_debugfs_setup(pt);
155
156 return ret;
157 }
158