1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
4 */
5
6 #include <drm/amdxdna_accel.h>
7 #include <drm/drm_device.h>
8 #include <drm/drm_drv.h>
9 #include <drm/drm_gem_shmem_helper.h>
10 #include <drm/drm_managed.h>
11 #include <drm/drm_print.h>
12 #include <drm/gpu_scheduler.h>
13 #include <linux/errno.h>
14 #include <linux/firmware.h>
15 #include <linux/iommu.h>
16 #include <linux/iopoll.h>
17 #include <linux/pci.h>
18 #include <linux/xarray.h>
19
20 #include "aie2_msg_priv.h"
21 #include "aie2_pci.h"
22 #include "aie2_solver.h"
23 #include "amdxdna_ctx.h"
24 #include "amdxdna_gem.h"
25 #include "amdxdna_mailbox.h"
26 #include "amdxdna_pci_drv.h"
27
28 static int aie2_max_col = XRS_MAX_COL;
29 module_param(aie2_max_col, uint, 0600);
30 MODULE_PARM_DESC(aie2_max_col, "Maximum column could be used");
31
32 /*
33 * The management mailbox channel is allocated by firmware.
34 * The related register and ring buffer information is on SRAM BAR.
35 * This struct is the register layout.
36 */
37 #define MGMT_MBOX_MAGIC 0x55504e5f /* _NPU */
38 struct mgmt_mbox_chann_info {
39 __u32 x2i_tail;
40 __u32 x2i_head;
41 __u32 x2i_buf;
42 __u32 x2i_buf_sz;
43 __u32 i2x_tail;
44 __u32 i2x_head;
45 __u32 i2x_buf;
46 __u32 i2x_buf_sz;
47 __u32 magic;
48 __u32 msi_id;
49 __u32 prot_major;
50 __u32 prot_minor;
51 __u32 rsvd[4];
52 };
53
aie2_check_protocol(struct amdxdna_dev_hdl * ndev,u32 fw_major,u32 fw_minor)54 static int aie2_check_protocol(struct amdxdna_dev_hdl *ndev, u32 fw_major, u32 fw_minor)
55 {
56 struct amdxdna_dev *xdna = ndev->xdna;
57
58 /*
59 * The driver supported mailbox behavior is defined by
60 * ndev->priv->protocol_major and protocol_minor.
61 *
62 * When protocol_major and fw_major are different, it means driver
63 * and firmware are incompatible.
64 */
65 if (ndev->priv->protocol_major != fw_major) {
66 XDNA_ERR(xdna, "Incompatible firmware protocol major %d minor %d",
67 fw_major, fw_minor);
68 return -EINVAL;
69 }
70
71 /*
72 * When protocol_minor is greater then fw_minor, that means driver
73 * relies on operation the installed firmware does not support.
74 */
75 if (ndev->priv->protocol_minor > fw_minor) {
76 XDNA_ERR(xdna, "Firmware minor version smaller than supported");
77 return -EINVAL;
78 }
79 return 0;
80 }
81
aie2_dump_chann_info_debug(struct amdxdna_dev_hdl * ndev)82 static void aie2_dump_chann_info_debug(struct amdxdna_dev_hdl *ndev)
83 {
84 struct amdxdna_dev *xdna = ndev->xdna;
85
86 XDNA_DBG(xdna, "i2x tail 0x%x", ndev->mgmt_i2x.mb_tail_ptr_reg);
87 XDNA_DBG(xdna, "i2x head 0x%x", ndev->mgmt_i2x.mb_head_ptr_reg);
88 XDNA_DBG(xdna, "i2x ringbuf 0x%x", ndev->mgmt_i2x.rb_start_addr);
89 XDNA_DBG(xdna, "i2x rsize 0x%x", ndev->mgmt_i2x.rb_size);
90 XDNA_DBG(xdna, "x2i tail 0x%x", ndev->mgmt_x2i.mb_tail_ptr_reg);
91 XDNA_DBG(xdna, "x2i head 0x%x", ndev->mgmt_x2i.mb_head_ptr_reg);
92 XDNA_DBG(xdna, "x2i ringbuf 0x%x", ndev->mgmt_x2i.rb_start_addr);
93 XDNA_DBG(xdna, "x2i rsize 0x%x", ndev->mgmt_x2i.rb_size);
94 XDNA_DBG(xdna, "x2i chann index 0x%x", ndev->mgmt_chan_idx);
95 XDNA_DBG(xdna, "mailbox protocol major 0x%x", ndev->mgmt_prot_major);
96 XDNA_DBG(xdna, "mailbox protocol minor 0x%x", ndev->mgmt_prot_minor);
97 }
98
aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl * ndev)99 static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev)
100 {
101 struct mgmt_mbox_chann_info info_regs;
102 struct xdna_mailbox_chann_res *i2x;
103 struct xdna_mailbox_chann_res *x2i;
104 u32 addr, off;
105 u32 *reg;
106 int ret;
107 int i;
108
109 /*
110 * Once firmware is alive, it will write management channel
111 * information in SRAM BAR and write the address of that information
112 * at FW_ALIVE_OFF offset in SRMA BAR.
113 *
114 * Read a non-zero value from FW_ALIVE_OFF implies that firmware
115 * is alive.
116 */
117 ret = readx_poll_timeout(readl, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF),
118 addr, addr, AIE2_INTERVAL, AIE2_TIMEOUT);
119 if (ret || !addr)
120 return -ETIME;
121
122 off = AIE2_SRAM_OFF(ndev, addr);
123 reg = (u32 *)&info_regs;
124 for (i = 0; i < sizeof(info_regs) / sizeof(u32); i++)
125 reg[i] = readl(ndev->sram_base + off + i * sizeof(u32));
126
127 if (info_regs.magic != MGMT_MBOX_MAGIC) {
128 XDNA_ERR(ndev->xdna, "Invalid mbox magic 0x%x", info_regs.magic);
129 ret = -EINVAL;
130 goto done;
131 }
132
133 i2x = &ndev->mgmt_i2x;
134 x2i = &ndev->mgmt_x2i;
135
136 i2x->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_head);
137 i2x->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_tail);
138 i2x->rb_start_addr = AIE2_SRAM_OFF(ndev, info_regs.i2x_buf);
139 i2x->rb_size = info_regs.i2x_buf_sz;
140
141 x2i->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_head);
142 x2i->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.x2i_tail);
143 x2i->rb_start_addr = AIE2_SRAM_OFF(ndev, info_regs.x2i_buf);
144 x2i->rb_size = info_regs.x2i_buf_sz;
145
146 ndev->mgmt_chan_idx = info_regs.msi_id;
147 ndev->mgmt_prot_major = info_regs.prot_major;
148 ndev->mgmt_prot_minor = info_regs.prot_minor;
149
150 ret = aie2_check_protocol(ndev, ndev->mgmt_prot_major, ndev->mgmt_prot_minor);
151
152 done:
153 aie2_dump_chann_info_debug(ndev);
154
155 /* Must clear address at FW_ALIVE_OFF */
156 writel(0, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF));
157
158 return ret;
159 }
160
aie2_runtime_cfg(struct amdxdna_dev_hdl * ndev,enum rt_config_category category,u32 * val)161 int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev,
162 enum rt_config_category category, u32 *val)
163 {
164 const struct rt_config *cfg;
165 u32 value;
166 int ret;
167
168 for (cfg = ndev->priv->rt_config; cfg->type; cfg++) {
169 if (cfg->category != category)
170 continue;
171
172 value = val ? *val : cfg->value;
173 ret = aie2_set_runtime_cfg(ndev, cfg->type, value);
174 if (ret) {
175 XDNA_ERR(ndev->xdna, "Set type %d value %d failed",
176 cfg->type, value);
177 return ret;
178 }
179 }
180
181 return 0;
182 }
183
aie2_xdna_reset(struct amdxdna_dev_hdl * ndev)184 static int aie2_xdna_reset(struct amdxdna_dev_hdl *ndev)
185 {
186 int ret;
187
188 ret = aie2_suspend_fw(ndev);
189 if (ret) {
190 XDNA_ERR(ndev->xdna, "Suspend firmware failed");
191 return ret;
192 }
193
194 ret = aie2_resume_fw(ndev);
195 if (ret) {
196 XDNA_ERR(ndev->xdna, "Resume firmware failed");
197 return ret;
198 }
199
200 return 0;
201 }
202
aie2_mgmt_fw_init(struct amdxdna_dev_hdl * ndev)203 static int aie2_mgmt_fw_init(struct amdxdna_dev_hdl *ndev)
204 {
205 int ret;
206
207 ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_INIT, NULL);
208 if (ret) {
209 XDNA_ERR(ndev->xdna, "Runtime config failed");
210 return ret;
211 }
212
213 ret = aie2_assign_mgmt_pasid(ndev, 0);
214 if (ret) {
215 XDNA_ERR(ndev->xdna, "Can not assign PASID");
216 return ret;
217 }
218
219 ret = aie2_xdna_reset(ndev);
220 if (ret) {
221 XDNA_ERR(ndev->xdna, "Reset firmware failed");
222 return ret;
223 }
224
225 if (!ndev->async_events)
226 return 0;
227
228 ret = aie2_error_async_events_send(ndev);
229 if (ret) {
230 XDNA_ERR(ndev->xdna, "Send async events failed");
231 return ret;
232 }
233
234 return 0;
235 }
236
aie2_mgmt_fw_query(struct amdxdna_dev_hdl * ndev)237 static int aie2_mgmt_fw_query(struct amdxdna_dev_hdl *ndev)
238 {
239 int ret;
240
241 ret = aie2_query_firmware_version(ndev, &ndev->xdna->fw_ver);
242 if (ret) {
243 XDNA_ERR(ndev->xdna, "query firmware version failed");
244 return ret;
245 }
246
247 ret = aie2_query_aie_version(ndev, &ndev->version);
248 if (ret) {
249 XDNA_ERR(ndev->xdna, "Query AIE version failed");
250 return ret;
251 }
252
253 ret = aie2_query_aie_metadata(ndev, &ndev->metadata);
254 if (ret) {
255 XDNA_ERR(ndev->xdna, "Query AIE metadata failed");
256 return ret;
257 }
258
259 return 0;
260 }
261
aie2_mgmt_fw_fini(struct amdxdna_dev_hdl * ndev)262 static void aie2_mgmt_fw_fini(struct amdxdna_dev_hdl *ndev)
263 {
264 if (aie2_suspend_fw(ndev))
265 XDNA_ERR(ndev->xdna, "Suspend_fw failed");
266 XDNA_DBG(ndev->xdna, "Firmware suspended");
267 }
268
aie2_xrs_load(void * cb_arg,struct xrs_action_load * action)269 static int aie2_xrs_load(void *cb_arg, struct xrs_action_load *action)
270 {
271 struct amdxdna_hwctx *hwctx = cb_arg;
272 struct amdxdna_dev *xdna;
273 int ret;
274
275 xdna = hwctx->client->xdna;
276
277 hwctx->start_col = action->part.start_col;
278 hwctx->num_col = action->part.ncols;
279 ret = aie2_create_context(xdna->dev_handle, hwctx);
280 if (ret)
281 XDNA_ERR(xdna, "create context failed, ret %d", ret);
282
283 return ret;
284 }
285
aie2_xrs_unload(void * cb_arg)286 static int aie2_xrs_unload(void *cb_arg)
287 {
288 struct amdxdna_hwctx *hwctx = cb_arg;
289 struct amdxdna_dev *xdna;
290 int ret;
291
292 xdna = hwctx->client->xdna;
293
294 ret = aie2_destroy_context(xdna->dev_handle, hwctx);
295 if (ret)
296 XDNA_ERR(xdna, "destroy context failed, ret %d", ret);
297
298 return ret;
299 }
300
aie2_xrs_set_dft_dpm_level(struct drm_device * ddev,u32 dpm_level)301 static int aie2_xrs_set_dft_dpm_level(struct drm_device *ddev, u32 dpm_level)
302 {
303 struct amdxdna_dev *xdna = to_xdna_dev(ddev);
304 struct amdxdna_dev_hdl *ndev;
305
306 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
307
308 ndev = xdna->dev_handle;
309 ndev->dft_dpm_level = dpm_level;
310 if (ndev->pw_mode != POWER_MODE_DEFAULT || ndev->dpm_level == dpm_level)
311 return 0;
312
313 return ndev->priv->hw_ops.set_dpm(ndev, dpm_level);
314 }
315
316 static struct xrs_action_ops aie2_xrs_actions = {
317 .load = aie2_xrs_load,
318 .unload = aie2_xrs_unload,
319 .set_dft_dpm_level = aie2_xrs_set_dft_dpm_level,
320 };
321
aie2_hw_stop(struct amdxdna_dev * xdna)322 static void aie2_hw_stop(struct amdxdna_dev *xdna)
323 {
324 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
325 struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
326
327 if (ndev->dev_status <= AIE2_DEV_INIT) {
328 XDNA_ERR(xdna, "device is already stopped");
329 return;
330 }
331
332 aie2_mgmt_fw_fini(ndev);
333 xdna_mailbox_stop_channel(ndev->mgmt_chann);
334 xdna_mailbox_destroy_channel(ndev->mgmt_chann);
335 ndev->mgmt_chann = NULL;
336 drmm_kfree(&xdna->ddev, ndev->mbox);
337 ndev->mbox = NULL;
338 aie2_psp_stop(ndev->psp_hdl);
339 aie2_smu_fini(ndev);
340 pci_disable_device(pdev);
341
342 ndev->dev_status = AIE2_DEV_INIT;
343 }
344
aie2_hw_start(struct amdxdna_dev * xdna)345 static int aie2_hw_start(struct amdxdna_dev *xdna)
346 {
347 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
348 struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
349 struct xdna_mailbox_res mbox_res;
350 u32 xdna_mailbox_intr_reg;
351 int mgmt_mb_irq, ret;
352
353 if (ndev->dev_status >= AIE2_DEV_START) {
354 XDNA_INFO(xdna, "device is already started");
355 return 0;
356 }
357
358 ret = pci_enable_device(pdev);
359 if (ret) {
360 XDNA_ERR(xdna, "failed to enable device, ret %d", ret);
361 return ret;
362 }
363 pci_set_master(pdev);
364
365 ret = aie2_smu_init(ndev);
366 if (ret) {
367 XDNA_ERR(xdna, "failed to init smu, ret %d", ret);
368 goto disable_dev;
369 }
370
371 ret = aie2_psp_start(ndev->psp_hdl);
372 if (ret) {
373 XDNA_ERR(xdna, "failed to start psp, ret %d", ret);
374 goto fini_smu;
375 }
376
377 ret = aie2_get_mgmt_chann_info(ndev);
378 if (ret) {
379 XDNA_ERR(xdna, "firmware is not alive");
380 goto stop_psp;
381 }
382
383 mbox_res.ringbuf_base = ndev->sram_base;
384 mbox_res.ringbuf_size = pci_resource_len(pdev, xdna->dev_info->sram_bar);
385 mbox_res.mbox_base = ndev->mbox_base;
386 mbox_res.mbox_size = MBOX_SIZE(ndev);
387 mbox_res.name = "xdna_mailbox";
388 ndev->mbox = xdnam_mailbox_create(&xdna->ddev, &mbox_res);
389 if (!ndev->mbox) {
390 XDNA_ERR(xdna, "failed to create mailbox device");
391 ret = -ENODEV;
392 goto stop_psp;
393 }
394
395 mgmt_mb_irq = pci_irq_vector(pdev, ndev->mgmt_chan_idx);
396 if (mgmt_mb_irq < 0) {
397 ret = mgmt_mb_irq;
398 XDNA_ERR(xdna, "failed to alloc irq vector, ret %d", ret);
399 goto stop_psp;
400 }
401
402 xdna_mailbox_intr_reg = ndev->mgmt_i2x.mb_head_ptr_reg + 4;
403 ndev->mgmt_chann = xdna_mailbox_create_channel(ndev->mbox,
404 &ndev->mgmt_x2i,
405 &ndev->mgmt_i2x,
406 xdna_mailbox_intr_reg,
407 mgmt_mb_irq);
408 if (!ndev->mgmt_chann) {
409 XDNA_ERR(xdna, "failed to create management mailbox channel");
410 ret = -EINVAL;
411 goto stop_psp;
412 }
413
414 ret = aie2_pm_init(ndev);
415 if (ret) {
416 XDNA_ERR(xdna, "failed to init pm, ret %d", ret);
417 goto destroy_mgmt_chann;
418 }
419
420 ret = aie2_mgmt_fw_init(ndev);
421 if (ret) {
422 XDNA_ERR(xdna, "initial mgmt firmware failed, ret %d", ret);
423 goto destroy_mgmt_chann;
424 }
425
426 ndev->dev_status = AIE2_DEV_START;
427
428 return 0;
429
430 destroy_mgmt_chann:
431 xdna_mailbox_stop_channel(ndev->mgmt_chann);
432 xdna_mailbox_destroy_channel(ndev->mgmt_chann);
433 stop_psp:
434 aie2_psp_stop(ndev->psp_hdl);
435 fini_smu:
436 aie2_smu_fini(ndev);
437 disable_dev:
438 pci_disable_device(pdev);
439
440 return ret;
441 }
442
aie2_init(struct amdxdna_dev * xdna)443 static int aie2_init(struct amdxdna_dev *xdna)
444 {
445 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
446 void __iomem *tbl[PCI_NUM_RESOURCES] = {0};
447 struct init_config xrs_cfg = { 0 };
448 struct amdxdna_dev_hdl *ndev;
449 struct psp_config psp_conf;
450 const struct firmware *fw;
451 unsigned long bars = 0;
452 int i, nvec, ret;
453
454 ndev = drmm_kzalloc(&xdna->ddev, sizeof(*ndev), GFP_KERNEL);
455 if (!ndev)
456 return -ENOMEM;
457
458 ndev->priv = xdna->dev_info->dev_priv;
459 ndev->xdna = xdna;
460
461 ret = request_firmware(&fw, ndev->priv->fw_path, &pdev->dev);
462 if (ret) {
463 XDNA_ERR(xdna, "failed to request_firmware %s, ret %d",
464 ndev->priv->fw_path, ret);
465 return ret;
466 }
467
468 ret = pcim_enable_device(pdev);
469 if (ret) {
470 XDNA_ERR(xdna, "pcim enable device failed, ret %d", ret);
471 goto release_fw;
472 }
473
474 for (i = 0; i < PSP_MAX_REGS; i++)
475 set_bit(PSP_REG_BAR(ndev, i), &bars);
476
477 set_bit(xdna->dev_info->sram_bar, &bars);
478 set_bit(xdna->dev_info->smu_bar, &bars);
479 set_bit(xdna->dev_info->mbox_bar, &bars);
480
481 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
482 if (!test_bit(i, &bars))
483 continue;
484 tbl[i] = pcim_iomap(pdev, i, 0);
485 if (!tbl[i]) {
486 XDNA_ERR(xdna, "map bar %d failed", i);
487 ret = -ENOMEM;
488 goto release_fw;
489 }
490 }
491
492 ndev->sram_base = tbl[xdna->dev_info->sram_bar];
493 ndev->smu_base = tbl[xdna->dev_info->smu_bar];
494 ndev->mbox_base = tbl[xdna->dev_info->mbox_bar];
495
496 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
497 if (ret) {
498 XDNA_ERR(xdna, "Failed to set DMA mask: %d", ret);
499 goto release_fw;
500 }
501
502 nvec = pci_msix_vec_count(pdev);
503 if (nvec <= 0) {
504 XDNA_ERR(xdna, "does not get number of interrupt vector");
505 ret = -EINVAL;
506 goto release_fw;
507 }
508
509 ret = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
510 if (ret < 0) {
511 XDNA_ERR(xdna, "failed to alloc irq vectors, ret %d", ret);
512 goto release_fw;
513 }
514
515 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
516 if (ret) {
517 XDNA_ERR(xdna, "Enable PASID failed, ret %d", ret);
518 goto free_irq;
519 }
520
521 psp_conf.fw_size = fw->size;
522 psp_conf.fw_buf = fw->data;
523 for (i = 0; i < PSP_MAX_REGS; i++)
524 psp_conf.psp_regs[i] = tbl[PSP_REG_BAR(ndev, i)] + PSP_REG_OFF(ndev, i);
525 ndev->psp_hdl = aie2m_psp_create(&xdna->ddev, &psp_conf);
526 if (!ndev->psp_hdl) {
527 XDNA_ERR(xdna, "failed to create psp");
528 ret = -ENOMEM;
529 goto disable_sva;
530 }
531 xdna->dev_handle = ndev;
532
533 ret = aie2_hw_start(xdna);
534 if (ret) {
535 XDNA_ERR(xdna, "start npu failed, ret %d", ret);
536 goto disable_sva;
537 }
538
539 ret = aie2_mgmt_fw_query(ndev);
540 if (ret) {
541 XDNA_ERR(xdna, "Query firmware failed, ret %d", ret);
542 goto stop_hw;
543 }
544 ndev->total_col = min(aie2_max_col, ndev->metadata.cols);
545
546 xrs_cfg.clk_list.num_levels = ndev->max_dpm_level + 1;
547 for (i = 0; i < xrs_cfg.clk_list.num_levels; i++)
548 xrs_cfg.clk_list.cu_clk_list[i] = ndev->priv->dpm_clk_tbl[i].hclk;
549 xrs_cfg.sys_eff_factor = 1;
550 xrs_cfg.ddev = &xdna->ddev;
551 xrs_cfg.actions = &aie2_xrs_actions;
552 xrs_cfg.total_col = ndev->total_col;
553
554 xdna->xrs_hdl = xrsm_init(&xrs_cfg);
555 if (!xdna->xrs_hdl) {
556 XDNA_ERR(xdna, "Initialize resolver failed");
557 ret = -EINVAL;
558 goto stop_hw;
559 }
560
561 ret = aie2_error_async_events_alloc(ndev);
562 if (ret) {
563 XDNA_ERR(xdna, "Allocate async events failed, ret %d", ret);
564 goto stop_hw;
565 }
566
567 ret = aie2_error_async_events_send(ndev);
568 if (ret) {
569 XDNA_ERR(xdna, "Send async events failed, ret %d", ret);
570 goto async_event_free;
571 }
572
573 /* Issue a command to make sure firmware handled async events */
574 ret = aie2_query_firmware_version(ndev, &ndev->xdna->fw_ver);
575 if (ret) {
576 XDNA_ERR(xdna, "Re-query firmware version failed");
577 goto async_event_free;
578 }
579
580 release_firmware(fw);
581 return 0;
582
583 async_event_free:
584 aie2_error_async_events_free(ndev);
585 stop_hw:
586 aie2_hw_stop(xdna);
587 disable_sva:
588 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
589 free_irq:
590 pci_free_irq_vectors(pdev);
591 release_fw:
592 release_firmware(fw);
593
594 return ret;
595 }
596
aie2_fini(struct amdxdna_dev * xdna)597 static void aie2_fini(struct amdxdna_dev *xdna)
598 {
599 struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
600 struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
601
602 aie2_hw_stop(xdna);
603 aie2_error_async_events_free(ndev);
604 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA);
605 pci_free_irq_vectors(pdev);
606 }
607
aie2_get_aie_status(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)608 static int aie2_get_aie_status(struct amdxdna_client *client,
609 struct amdxdna_drm_get_info *args)
610 {
611 struct amdxdna_drm_query_aie_status status;
612 struct amdxdna_dev *xdna = client->xdna;
613 struct amdxdna_dev_hdl *ndev;
614 int ret;
615
616 ndev = xdna->dev_handle;
617 if (copy_from_user(&status, u64_to_user_ptr(args->buffer), sizeof(status))) {
618 XDNA_ERR(xdna, "Failed to copy AIE request into kernel");
619 return -EFAULT;
620 }
621
622 if (ndev->metadata.cols * ndev->metadata.size < status.buffer_size) {
623 XDNA_ERR(xdna, "Invalid buffer size. Given Size: %u. Need Size: %u.",
624 status.buffer_size, ndev->metadata.cols * ndev->metadata.size);
625 return -EINVAL;
626 }
627
628 ret = aie2_query_status(ndev, u64_to_user_ptr(status.buffer),
629 status.buffer_size, &status.cols_filled);
630 if (ret) {
631 XDNA_ERR(xdna, "Failed to get AIE status info. Ret: %d", ret);
632 return ret;
633 }
634
635 if (copy_to_user(u64_to_user_ptr(args->buffer), &status, sizeof(status))) {
636 XDNA_ERR(xdna, "Failed to copy AIE request info to user space");
637 return -EFAULT;
638 }
639
640 return 0;
641 }
642
aie2_get_aie_metadata(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)643 static int aie2_get_aie_metadata(struct amdxdna_client *client,
644 struct amdxdna_drm_get_info *args)
645 {
646 struct amdxdna_drm_query_aie_metadata *meta;
647 struct amdxdna_dev *xdna = client->xdna;
648 struct amdxdna_dev_hdl *ndev;
649 int ret = 0;
650
651 ndev = xdna->dev_handle;
652 meta = kzalloc(sizeof(*meta), GFP_KERNEL);
653 if (!meta)
654 return -ENOMEM;
655
656 meta->col_size = ndev->metadata.size;
657 meta->cols = ndev->metadata.cols;
658 meta->rows = ndev->metadata.rows;
659
660 meta->version.major = ndev->metadata.version.major;
661 meta->version.minor = ndev->metadata.version.minor;
662
663 meta->core.row_count = ndev->metadata.core.row_count;
664 meta->core.row_start = ndev->metadata.core.row_start;
665 meta->core.dma_channel_count = ndev->metadata.core.dma_channel_count;
666 meta->core.lock_count = ndev->metadata.core.lock_count;
667 meta->core.event_reg_count = ndev->metadata.core.event_reg_count;
668
669 meta->mem.row_count = ndev->metadata.mem.row_count;
670 meta->mem.row_start = ndev->metadata.mem.row_start;
671 meta->mem.dma_channel_count = ndev->metadata.mem.dma_channel_count;
672 meta->mem.lock_count = ndev->metadata.mem.lock_count;
673 meta->mem.event_reg_count = ndev->metadata.mem.event_reg_count;
674
675 meta->shim.row_count = ndev->metadata.shim.row_count;
676 meta->shim.row_start = ndev->metadata.shim.row_start;
677 meta->shim.dma_channel_count = ndev->metadata.shim.dma_channel_count;
678 meta->shim.lock_count = ndev->metadata.shim.lock_count;
679 meta->shim.event_reg_count = ndev->metadata.shim.event_reg_count;
680
681 if (copy_to_user(u64_to_user_ptr(args->buffer), meta, sizeof(*meta)))
682 ret = -EFAULT;
683
684 kfree(meta);
685 return ret;
686 }
687
aie2_get_aie_version(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)688 static int aie2_get_aie_version(struct amdxdna_client *client,
689 struct amdxdna_drm_get_info *args)
690 {
691 struct amdxdna_drm_query_aie_version version;
692 struct amdxdna_dev *xdna = client->xdna;
693 struct amdxdna_dev_hdl *ndev;
694
695 ndev = xdna->dev_handle;
696 version.major = ndev->version.major;
697 version.minor = ndev->version.minor;
698
699 if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version)))
700 return -EFAULT;
701
702 return 0;
703 }
704
aie2_get_firmware_version(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)705 static int aie2_get_firmware_version(struct amdxdna_client *client,
706 struct amdxdna_drm_get_info *args)
707 {
708 struct amdxdna_drm_query_firmware_version version;
709 struct amdxdna_dev *xdna = client->xdna;
710
711 version.major = xdna->fw_ver.major;
712 version.minor = xdna->fw_ver.minor;
713 version.patch = xdna->fw_ver.sub;
714 version.build = xdna->fw_ver.build;
715
716 if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version)))
717 return -EFAULT;
718
719 return 0;
720 }
721
aie2_get_power_mode(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)722 static int aie2_get_power_mode(struct amdxdna_client *client,
723 struct amdxdna_drm_get_info *args)
724 {
725 struct amdxdna_drm_get_power_mode mode = {};
726 struct amdxdna_dev *xdna = client->xdna;
727 struct amdxdna_dev_hdl *ndev;
728
729 ndev = xdna->dev_handle;
730 mode.power_mode = ndev->pw_mode;
731
732 if (copy_to_user(u64_to_user_ptr(args->buffer), &mode, sizeof(mode)))
733 return -EFAULT;
734
735 return 0;
736 }
737
aie2_get_clock_metadata(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)738 static int aie2_get_clock_metadata(struct amdxdna_client *client,
739 struct amdxdna_drm_get_info *args)
740 {
741 struct amdxdna_drm_query_clock_metadata *clock;
742 struct amdxdna_dev *xdna = client->xdna;
743 struct amdxdna_dev_hdl *ndev;
744 int ret = 0;
745
746 ndev = xdna->dev_handle;
747 clock = kzalloc(sizeof(*clock), GFP_KERNEL);
748 if (!clock)
749 return -ENOMEM;
750
751 snprintf(clock->mp_npu_clock.name, sizeof(clock->mp_npu_clock.name),
752 "MP-NPU Clock");
753 clock->mp_npu_clock.freq_mhz = ndev->npuclk_freq;
754 snprintf(clock->h_clock.name, sizeof(clock->h_clock.name), "H Clock");
755 clock->h_clock.freq_mhz = ndev->hclk_freq;
756
757 if (copy_to_user(u64_to_user_ptr(args->buffer), clock, sizeof(*clock)))
758 ret = -EFAULT;
759
760 kfree(clock);
761 return ret;
762 }
763
aie2_get_hwctx_status(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)764 static int aie2_get_hwctx_status(struct amdxdna_client *client,
765 struct amdxdna_drm_get_info *args)
766 {
767 struct amdxdna_drm_query_hwctx __user *buf;
768 struct amdxdna_dev *xdna = client->xdna;
769 struct amdxdna_drm_query_hwctx *tmp;
770 struct amdxdna_client *tmp_client;
771 struct amdxdna_hwctx *hwctx;
772 unsigned long hwctx_id;
773 bool overflow = false;
774 u32 req_bytes = 0;
775 u32 hw_i = 0;
776 int ret = 0;
777 int idx;
778
779 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
780
781 tmp = kzalloc(sizeof(*tmp), GFP_KERNEL);
782 if (!tmp)
783 return -ENOMEM;
784
785 buf = u64_to_user_ptr(args->buffer);
786 list_for_each_entry(tmp_client, &xdna->client_list, node) {
787 idx = srcu_read_lock(&tmp_client->hwctx_srcu);
788 amdxdna_for_each_hwctx(tmp_client, hwctx_id, hwctx) {
789 req_bytes += sizeof(*tmp);
790 if (args->buffer_size < req_bytes) {
791 /* Continue iterating to get the required size */
792 overflow = true;
793 continue;
794 }
795
796 memset(tmp, 0, sizeof(*tmp));
797 tmp->pid = tmp_client->pid;
798 tmp->context_id = hwctx->id;
799 tmp->start_col = hwctx->start_col;
800 tmp->num_col = hwctx->num_col;
801 tmp->command_submissions = hwctx->priv->seq;
802 tmp->command_completions = hwctx->priv->completed;
803
804 if (copy_to_user(&buf[hw_i], tmp, sizeof(*tmp))) {
805 ret = -EFAULT;
806 srcu_read_unlock(&tmp_client->hwctx_srcu, idx);
807 goto out;
808 }
809 hw_i++;
810 }
811 srcu_read_unlock(&tmp_client->hwctx_srcu, idx);
812 }
813
814 if (overflow) {
815 XDNA_ERR(xdna, "Invalid buffer size. Given: %u Need: %u.",
816 args->buffer_size, req_bytes);
817 ret = -EINVAL;
818 }
819
820 out:
821 kfree(tmp);
822 args->buffer_size = req_bytes;
823 return ret;
824 }
825
aie2_get_info(struct amdxdna_client * client,struct amdxdna_drm_get_info * args)826 static int aie2_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_info *args)
827 {
828 struct amdxdna_dev *xdna = client->xdna;
829 int ret, idx;
830
831 if (!drm_dev_enter(&xdna->ddev, &idx))
832 return -ENODEV;
833
834 switch (args->param) {
835 case DRM_AMDXDNA_QUERY_AIE_STATUS:
836 ret = aie2_get_aie_status(client, args);
837 break;
838 case DRM_AMDXDNA_QUERY_AIE_METADATA:
839 ret = aie2_get_aie_metadata(client, args);
840 break;
841 case DRM_AMDXDNA_QUERY_AIE_VERSION:
842 ret = aie2_get_aie_version(client, args);
843 break;
844 case DRM_AMDXDNA_QUERY_CLOCK_METADATA:
845 ret = aie2_get_clock_metadata(client, args);
846 break;
847 case DRM_AMDXDNA_QUERY_HW_CONTEXTS:
848 ret = aie2_get_hwctx_status(client, args);
849 break;
850 case DRM_AMDXDNA_QUERY_FIRMWARE_VERSION:
851 ret = aie2_get_firmware_version(client, args);
852 break;
853 case DRM_AMDXDNA_GET_POWER_MODE:
854 ret = aie2_get_power_mode(client, args);
855 break;
856 default:
857 XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
858 ret = -EOPNOTSUPP;
859 }
860 XDNA_DBG(xdna, "Got param %d", args->param);
861
862 drm_dev_exit(idx);
863 return ret;
864 }
865
aie2_set_power_mode(struct amdxdna_client * client,struct amdxdna_drm_set_state * args)866 static int aie2_set_power_mode(struct amdxdna_client *client,
867 struct amdxdna_drm_set_state *args)
868 {
869 struct amdxdna_drm_set_power_mode power_state;
870 enum amdxdna_power_mode_type power_mode;
871 struct amdxdna_dev *xdna = client->xdna;
872
873 if (copy_from_user(&power_state, u64_to_user_ptr(args->buffer),
874 sizeof(power_state))) {
875 XDNA_ERR(xdna, "Failed to copy power mode request into kernel");
876 return -EFAULT;
877 }
878
879 if (XDNA_MBZ_DBG(xdna, power_state.pad, sizeof(power_state.pad)))
880 return -EINVAL;
881
882 power_mode = power_state.power_mode;
883 if (power_mode > POWER_MODE_TURBO) {
884 XDNA_ERR(xdna, "Invalid power mode %d", power_mode);
885 return -EINVAL;
886 }
887
888 return aie2_pm_set_mode(xdna->dev_handle, power_mode);
889 }
890
aie2_set_state(struct amdxdna_client * client,struct amdxdna_drm_set_state * args)891 static int aie2_set_state(struct amdxdna_client *client,
892 struct amdxdna_drm_set_state *args)
893 {
894 struct amdxdna_dev *xdna = client->xdna;
895 int ret, idx;
896
897 if (!drm_dev_enter(&xdna->ddev, &idx))
898 return -ENODEV;
899
900 switch (args->param) {
901 case DRM_AMDXDNA_SET_POWER_MODE:
902 ret = aie2_set_power_mode(client, args);
903 break;
904 default:
905 XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
906 ret = -EOPNOTSUPP;
907 break;
908 }
909
910 drm_dev_exit(idx);
911 return ret;
912 }
913
914 const struct amdxdna_dev_ops aie2_ops = {
915 .init = aie2_init,
916 .fini = aie2_fini,
917 .resume = aie2_hw_start,
918 .suspend = aie2_hw_stop,
919 .get_aie_info = aie2_get_info,
920 .set_aie_state = aie2_set_state,
921 .hwctx_init = aie2_hwctx_init,
922 .hwctx_fini = aie2_hwctx_fini,
923 .hwctx_config = aie2_hwctx_config,
924 .cmd_submit = aie2_cmd_submit,
925 .hmm_invalidate = aie2_hmm_invalidate,
926 .hwctx_suspend = aie2_hwctx_suspend,
927 .hwctx_resume = aie2_hwctx_resume,
928 };
929