Lines Matching +full:num +full:- +full:tx +full:- +full:queues
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * DMA driver for AMD Queue-based DMA Subsystem
5 * Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
10 #include <linux/dma-mapping.h>
19 #define CHAN_STR(q) (((q)->dir == DMA_MEM_TO_DEV) ? "H2C" : "C2H")
20 #define QDMA_REG_OFF(d, r) ((d)->roffs[r].off)
43 idx = qdev->qintr_rings[qdev->qintr_ring_idx++].ridx; in qdma_get_intr_ring_idx()
44 qdev->qintr_ring_idx %= qdev->qintr_ring_num; in qdma_get_intr_ring_idx()
52 const struct qdma_reg_field *f = &qdev->rfields[field]; in qdma_get_field()
56 low_pos = f->lsb / BITS_PER_TYPE(*data); in qdma_get_field()
57 hi_pos = f->msb / BITS_PER_TYPE(*data); in qdma_get_field()
60 low_bit = f->lsb % BITS_PER_TYPE(*data); in qdma_get_field()
61 hi_bit = f->msb % BITS_PER_TYPE(*data); in qdma_get_field()
65 low_bit = f->lsb % BITS_PER_TYPE(*data); in qdma_get_field()
66 hi_bit = low_bit + (f->msb - f->lsb); in qdma_get_field()
72 hi_bit = f->msb % BITS_PER_TYPE(*data); in qdma_get_field()
75 low_bit = f->msb - f->lsb - hi_bit; in qdma_get_field()
77 low_bit -= 32; in qdma_get_field()
78 value |= (u64)data[hi_pos - 1] << low_bit; in qdma_get_field()
79 mask = GENMASK(31, 32 - low_bit); in qdma_get_field()
80 value |= (data[hi_pos - 2] & mask) >> low_bit; in qdma_get_field()
89 const struct qdma_reg_field *f = &qdev->rfields[field]; in qdma_set_field()
92 low_pos = f->lsb / BITS_PER_TYPE(*data); in qdma_set_field()
93 hi_pos = f->msb / BITS_PER_TYPE(*data); in qdma_set_field()
94 low_bit = f->lsb % BITS_PER_TYPE(*data); in qdma_set_field()
98 data[low_pos++] |= (u32)(value >> (32 - low_bit)); in qdma_set_field()
100 data[low_pos] |= (u32)(value >> (64 - low_bit)); in qdma_set_field()
106 const struct qdma_reg *r = &qdev->roffs[reg]; in qdma_reg_write()
109 if (r->count > 1) in qdma_reg_write()
110 ret = regmap_bulk_write(qdev->regmap, r->off, data, r->count); in qdma_reg_write()
112 ret = regmap_write(qdev->regmap, r->off, *data); in qdma_reg_write()
120 const struct qdma_reg *r = &qdev->roffs[reg]; in qdma_reg_read()
123 if (r->count > 1) in qdma_reg_read()
124 ret = regmap_bulk_read(qdev->regmap, r->off, data, r->count); in qdma_reg_read()
126 ret = regmap_read(qdev->regmap, r->off, data); in qdma_reg_read()
146 ret = regmap_read_poll_timeout(qdev->regmap, in qdma_context_cmd_execute()
185 qdma_set_field(qdev, data, QDMA_REGF_DESC_BASE, ctxt->desc_base); in qdma_prep_sw_desc_context()
186 qdma_set_field(qdev, data, QDMA_REGF_IRQ_VEC, ctxt->vec); in qdma_prep_sw_desc_context()
187 qdma_set_field(qdev, data, QDMA_REGF_FUNCTION_ID, qdev->fid); in qdma_prep_sw_desc_context()
207 qdma_set_field(qdev, data, QDMA_REGF_INTR_AGG_BASE, ctxt->agg_base); in qdma_prep_intr_context()
208 qdma_set_field(qdev, data, QDMA_REGF_INTR_VECTOR, ctxt->vec); in qdma_prep_intr_context()
209 qdma_set_field(qdev, data, QDMA_REGF_INTR_SIZE, ctxt->size); in qdma_prep_intr_context()
210 qdma_set_field(qdev, data, QDMA_REGF_INTR_VALID, ctxt->valid); in qdma_prep_intr_context()
211 qdma_set_field(qdev, data, QDMA_REGF_INTR_COLOR, ctxt->color); in qdma_prep_intr_context()
212 qdma_set_field(qdev, data, QDMA_REGF_INTR_FUNCTION_ID, qdev->fid); in qdma_prep_intr_context()
220 qdma_set_field(qdev, data, QDMA_REGF_QUEUE_BASE, ctxt->qbase); in qdma_prep_fmap_context()
221 qdma_set_field(qdev, data, QDMA_REGF_QUEUE_MAX, ctxt->qmax); in qdma_prep_fmap_context()
237 mutex_lock(&qdev->ctxt_lock); in qdma_prog_context()
255 mutex_unlock(&qdev->ctxt_lock); in qdma_prog_context()
278 return -EBUSY; in qdma_check_queue_status()
298 struct qdma_device *qdev = queue->qdev; in qdma_clear_queue_context()
300 int ret, num, i; in qdma_clear_queue_context() local
302 if (queue->dir == DMA_MEM_TO_DEV) { in qdma_clear_queue_context()
304 num = ARRAY_SIZE(h2c_types); in qdma_clear_queue_context()
307 num = ARRAY_SIZE(c2h_types); in qdma_clear_queue_context()
309 for (i = 0; i < num; i++) { in qdma_clear_queue_context()
311 queue->qid, NULL); in qdma_clear_queue_context()
328 qdev->fid, NULL); in qdma_setup_fmap_context()
335 fmap.qmax = qdev->chan_num * 2; in qdma_setup_fmap_context()
338 qdev->fid, ctxt); in qdma_setup_fmap_context()
368 * Enable or disable memory-mapped DMA engines
383 struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev); in qdma_get_hw_info()
392 if (pdata->max_mm_channels * 2 > value) { in qdma_get_hw_info()
393 qdma_err(qdev, "not enough hw queues %d", value); in qdma_get_hw_info()
394 return -EINVAL; in qdma_get_hw_info()
396 qdev->chan_num = pdata->max_mm_channels; in qdma_get_hw_info()
398 ret = qdma_reg_read(qdev, &qdev->fid, QDMA_REGO_FUNC_ID); in qdma_get_hw_info()
403 qdev->chan_num, qdev->fid); in qdma_get_hw_info()
410 struct qdma_device *qdev = queue->qdev; in qdma_update_pidx()
412 return regmap_write(qdev->regmap, queue->pidx_reg, in qdma_update_pidx()
419 struct qdma_device *qdev = queue->qdev; in qdma_update_cidx()
421 return regmap_write(qdev->regmap, queue->cidx_reg, in qdma_update_cidx()
426 * qdma_free_vdesc - Free descriptor
439 struct qdma_queue *q, **queues; in qdma_alloc_queues() local
444 queues = &qdev->h2c_queues; in qdma_alloc_queues()
447 queues = &qdev->c2h_queues; in qdma_alloc_queues()
451 *queues = devm_kcalloc(&qdev->pdev->dev, qdev->chan_num, sizeof(*q), in qdma_alloc_queues()
453 if (!*queues) in qdma_alloc_queues()
454 return -ENOMEM; in qdma_alloc_queues()
456 for (i = 0; i < qdev->chan_num; i++) { in qdma_alloc_queues()
461 q = &(*queues)[i]; in qdma_alloc_queues()
462 q->ring_size = QDMA_DEFAULT_RING_SIZE; in qdma_alloc_queues()
463 q->idx_mask = q->ring_size - 2; in qdma_alloc_queues()
464 q->qdev = qdev; in qdma_alloc_queues()
465 q->dir = dir; in qdma_alloc_queues()
466 q->qid = i; in qdma_alloc_queues()
467 q->pidx_reg = pidx_base + i * QDMA_DMAP_REG_STRIDE; in qdma_alloc_queues()
468 q->cidx_reg = QDMA_REG_OFF(qdev, QDMA_REGO_INTR_CIDX) + in qdma_alloc_queues()
470 q->vchan.desc_free = qdma_free_vdesc; in qdma_alloc_queues()
471 vchan_init(&q->vchan, &qdev->dma_dev); in qdma_alloc_queues()
482 ret = regmap_read(qdev->regmap, QDMA_IDENTIFIER_REGOFF, &value); in qdma_device_verify()
489 return -ENODEV; in qdma_device_verify()
491 qdev->rfields = qdma_regfs_default; in qdma_device_verify()
492 qdev->roffs = qdma_regos_default; in qdma_device_verify()
516 /* Enable memory-mapped DMA engine in both directions */ in qdma_device_setup()
525 qdma_err(qdev, "Failed to alloc H2C queues, ret %d", ret); in qdma_device_setup()
531 qdma_err(qdev, "Failed to alloc C2H queues, ret %d", ret); in qdma_device_setup()
539 * qdma_free_queue_resources() - Free queue resources
545 struct qdma_device *qdev = queue->qdev; in qdma_free_queue_resources()
549 vchan_free_chan_resources(&queue->vchan); in qdma_free_queue_resources()
550 pdata = dev_get_platdata(&qdev->pdev->dev); in qdma_free_queue_resources()
551 dma_free_coherent(pdata->dma_dev, queue->ring_size * QDMA_MM_DESC_SIZE, in qdma_free_queue_resources()
552 queue->desc_base, queue->dma_desc_base); in qdma_free_queue_resources()
556 * qdma_alloc_queue_resources() - Allocate queue resources
562 struct qdma_device *qdev = queue->qdev; in qdma_alloc_queue_resources()
572 pdata = dev_get_platdata(&qdev->pdev->dev); in qdma_alloc_queue_resources()
573 size = queue->ring_size * QDMA_MM_DESC_SIZE; in qdma_alloc_queue_resources()
574 queue->desc_base = dma_alloc_coherent(pdata->dma_dev, size, in qdma_alloc_queue_resources()
575 &queue->dma_desc_base, in qdma_alloc_queue_resources()
577 if (!queue->desc_base) { in qdma_alloc_queue_resources()
579 return -ENOMEM; in qdma_alloc_queue_resources()
584 desc.desc_base = queue->dma_desc_base; in qdma_alloc_queue_resources()
585 ret = qdma_setup_queue_context(qdev, &desc, queue->dir, queue->qid); in qdma_alloc_queue_resources()
588 chan->name); in qdma_alloc_queue_resources()
589 dma_free_coherent(pdata->dma_dev, size, queue->desc_base, in qdma_alloc_queue_resources()
590 queue->dma_desc_base); in qdma_alloc_queue_resources()
594 queue->pidx = 0; in qdma_alloc_queue_resources()
595 queue->cidx = 0; in qdma_alloc_queue_resources()
605 return info->dir == queue->dir; in qdma_filter_fn()
610 struct qdma_device *qdev = queue->qdev; in qdma_xfer_start()
613 if (!vchan_next_desc(&queue->vchan)) in qdma_xfer_start()
617 queue->issued_vdesc->pidx, CHAN_STR(queue), queue->qid); in qdma_xfer_start()
619 ret = qdma_update_pidx(queue, queue->issued_vdesc->pidx); in qdma_xfer_start()
622 queue->pidx, CHAN_STR(queue), queue->qid); in qdma_xfer_start()
633 spin_lock_irqsave(&queue->vchan.lock, flags); in qdma_issue_pending()
634 if (vchan_issue_pending(&queue->vchan)) { in qdma_issue_pending()
635 if (queue->submitted_vdesc) { in qdma_issue_pending()
636 queue->issued_vdesc = queue->submitted_vdesc; in qdma_issue_pending()
637 queue->submitted_vdesc = NULL; in qdma_issue_pending()
642 spin_unlock_irqrestore(&queue->vchan.lock, flags); in qdma_issue_pending()
649 if (((q->pidx + 1) & q->idx_mask) == q->cidx) in qdma_get_desc()
652 desc = q->desc_base + q->pidx; in qdma_get_desc()
653 q->pidx = (q->pidx + 1) & q->idx_mask; in qdma_get_desc()
667 if (!vdesc->sg_len) in qdma_hw_enqueue()
670 if (q->dir == DMA_MEM_TO_DEV) { in qdma_hw_enqueue()
671 dst = &vdesc->dev_addr; in qdma_hw_enqueue()
675 src = &vdesc->dev_addr; in qdma_hw_enqueue()
678 for_each_sg(vdesc->sgl, sg, vdesc->sg_len, i) { in qdma_hw_enqueue()
679 addr = sg_dma_address(sg) + vdesc->sg_off; in qdma_hw_enqueue()
680 rest = sg_dma_len(sg) - vdesc->sg_off; in qdma_hw_enqueue()
685 ret = -EBUSY; in qdma_hw_enqueue()
689 desc->src_addr = cpu_to_le64(*src); in qdma_hw_enqueue()
690 desc->dst_addr = cpu_to_le64(*dst); in qdma_hw_enqueue()
691 desc->len = cpu_to_le32(len); in qdma_hw_enqueue()
693 vdesc->dev_addr += len; in qdma_hw_enqueue()
694 vdesc->sg_off += len; in qdma_hw_enqueue()
695 vdesc->pending_descs++; in qdma_hw_enqueue()
697 rest -= len; in qdma_hw_enqueue()
699 vdesc->sg_off = 0; in qdma_hw_enqueue()
702 vdesc->sg_len -= i; in qdma_hw_enqueue()
703 vdesc->pidx = q->pidx; in qdma_hw_enqueue()
709 struct virt_dma_chan *vc = &q->vchan; in qdma_fill_pending_vdesc()
714 if (!list_empty(&vc->desc_issued)) { in qdma_fill_pending_vdesc()
715 vd = &q->issued_vdesc->vdesc; in qdma_fill_pending_vdesc()
716 list_for_each_entry_from(vd, &vc->desc_issued, node) { in qdma_fill_pending_vdesc()
720 q->issued_vdesc = vdesc; in qdma_fill_pending_vdesc()
724 q->issued_vdesc = vdesc; in qdma_fill_pending_vdesc()
727 if (list_empty(&vc->desc_submitted)) in qdma_fill_pending_vdesc()
730 if (q->submitted_vdesc) in qdma_fill_pending_vdesc()
731 vd = &q->submitted_vdesc->vdesc; in qdma_fill_pending_vdesc()
733 vd = list_first_entry(&vc->desc_submitted, typeof(*vd), node); in qdma_fill_pending_vdesc()
735 list_for_each_entry_from(vd, &vc->desc_submitted, node) { in qdma_fill_pending_vdesc()
741 q->submitted_vdesc = vdesc; in qdma_fill_pending_vdesc()
744 static dma_cookie_t qdma_tx_submit(struct dma_async_tx_descriptor *tx) in qdma_tx_submit() argument
746 struct virt_dma_chan *vc = to_virt_chan(tx->chan); in qdma_tx_submit()
747 struct qdma_queue *q = to_qdma_queue(&vc->chan); in qdma_tx_submit()
752 vd = container_of(tx, struct virt_dma_desc, tx); in qdma_tx_submit()
753 spin_lock_irqsave(&vc->lock, flags); in qdma_tx_submit()
754 cookie = dma_cookie_assign(tx); in qdma_tx_submit()
756 list_move_tail(&vd->node, &vc->desc_submitted); in qdma_tx_submit()
758 spin_unlock_irqrestore(&vc->lock, flags); in qdma_tx_submit()
769 struct dma_async_tx_descriptor *tx; in qdma_prep_device_sg() local
775 vdesc->sgl = sgl; in qdma_prep_device_sg()
776 vdesc->sg_len = sg_len; in qdma_prep_device_sg()
778 vdesc->dev_addr = q->cfg.dst_addr; in qdma_prep_device_sg()
780 vdesc->dev_addr = q->cfg.src_addr; in qdma_prep_device_sg()
782 tx = vchan_tx_prep(&q->vchan, &vdesc->vdesc, flags); in qdma_prep_device_sg()
783 tx->tx_submit = qdma_tx_submit; in qdma_prep_device_sg()
785 return tx; in qdma_prep_device_sg()
793 memcpy(&q->cfg, cfg, sizeof(*cfg)); in qdma_device_config()
802 qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_FUNC, qdev->fid); in qdma_arm_err_intr()
803 qdma_set_field(qdev, &value, QDMA_REGF_ERR_INT_VEC, qdev->err_irq_idx); in qdma_arm_err_intr()
842 qdev = intr->qdev; in qdma_queue_isr()
843 index = intr->cidx; in qdma_queue_isr()
850 intr_ent = le64_to_cpu(intr->base[index]); in qdma_queue_isr()
852 if (color != intr->color) in qdma_queue_isr()
857 q = qdev->c2h_queues; in qdma_queue_isr()
859 q = qdev->h2c_queues; in qdma_queue_isr()
864 spin_lock_irqsave(&q->vchan.lock, flags); in qdma_queue_isr()
865 comp_desc = (cidx - q->cidx) & q->idx_mask; in qdma_queue_isr()
867 vd = vchan_next_desc(&q->vchan); in qdma_queue_isr()
872 while (comp_desc > vdesc->pending_descs) { in qdma_queue_isr()
873 list_del(&vd->node); in qdma_queue_isr()
875 comp_desc -= vdesc->pending_descs; in qdma_queue_isr()
876 vd = vchan_next_desc(&q->vchan); in qdma_queue_isr()
879 vdesc->pending_descs -= comp_desc; in qdma_queue_isr()
880 if (!vdesc->pending_descs && QDMA_VDESC_QUEUED(vdesc)) { in qdma_queue_isr()
881 list_del(&vd->node); in qdma_queue_isr()
884 q->cidx = cidx; in qdma_queue_isr()
890 spin_unlock_irqrestore(&q->vchan.lock, flags); in qdma_queue_isr()
899 intr->color = !intr->color; in qdma_queue_isr()
907 qdma_dbg(qdev, "update intr ring%d %d", intr->ridx, index); in qdma_queue_isr()
913 intr->cidx = index; in qdma_queue_isr()
915 ret = qdma_update_cidx(q, intr->ridx, index); in qdma_queue_isr()
927 struct device *dev = &qdev->pdev->dev; in qdma_init_error_irq()
931 vec = qdev->queue_irq_start - 1; in qdma_init_error_irq()
934 IRQF_ONESHOT, "amd-qdma-error", qdev); in qdma_init_error_irq()
949 struct qdma_platdata *pdata = dev_get_platdata(&qdev->pdev->dev); in qdmam_alloc_qintr_rings()
950 struct device *dev = &qdev->pdev->dev; in qdmam_alloc_qintr_rings()
957 qdev->qintr_ring_num = qdev->queue_irq_num; in qdmam_alloc_qintr_rings()
958 qdev->qintr_rings = devm_kcalloc(dev, qdev->qintr_ring_num, in qdmam_alloc_qintr_rings()
959 sizeof(*qdev->qintr_rings), in qdmam_alloc_qintr_rings()
961 if (!qdev->qintr_rings) in qdmam_alloc_qintr_rings()
962 return -ENOMEM; in qdmam_alloc_qintr_rings()
964 vector = qdev->queue_irq_start; in qdmam_alloc_qintr_rings()
965 for (i = 0; i < qdev->qintr_ring_num; i++, vector++) { in qdmam_alloc_qintr_rings()
966 ring = &qdev->qintr_rings[i]; in qdmam_alloc_qintr_rings()
967 ring->qdev = qdev; in qdmam_alloc_qintr_rings()
968 ring->msix_id = qdev->err_irq_idx + i + 1; in qdmam_alloc_qintr_rings()
969 ring->ridx = i; in qdmam_alloc_qintr_rings()
970 ring->color = 1; in qdmam_alloc_qintr_rings()
971 ring->base = dmam_alloc_coherent(pdata->dma_dev, in qdmam_alloc_qintr_rings()
973 &ring->dev_base, GFP_KERNEL); in qdmam_alloc_qintr_rings()
974 if (!ring->base) { in qdmam_alloc_qintr_rings()
976 return -ENOMEM; in qdmam_alloc_qintr_rings()
978 intr_ctxt.agg_base = QDMA_INTR_RING_BASE(ring->dev_base); in qdmam_alloc_qintr_rings()
979 intr_ctxt.size = (QDMA_INTR_RING_SIZE - 1) / 4096; in qdmam_alloc_qintr_rings()
980 intr_ctxt.vec = ring->msix_id; in qdmam_alloc_qintr_rings()
984 QDMA_CTXT_CLEAR, ring->ridx, NULL); in qdmam_alloc_qintr_rings()
992 QDMA_CTXT_WRITE, ring->ridx, ctxt); in qdmam_alloc_qintr_rings()
1000 "amd-qdma-queue", ring); in qdmam_alloc_qintr_rings()
1034 dma_async_device_unregister(&qdev->dma_dev); in amd_qdma_remove()
1036 mutex_destroy(&qdev->ctxt_lock); in amd_qdma_remove()
1041 struct qdma_platdata *pdata = dev_get_platdata(&pdev->dev); in amd_qdma_probe()
1047 qdev = devm_kzalloc(&pdev->dev, sizeof(*qdev), GFP_KERNEL); in amd_qdma_probe()
1049 return -ENOMEM; in amd_qdma_probe()
1052 qdev->pdev = pdev; in amd_qdma_probe()
1053 mutex_init(&qdev->ctxt_lock); in amd_qdma_probe()
1058 ret = -ENODEV; in amd_qdma_probe()
1061 qdev->err_irq_idx = pdata->irq_index; in amd_qdma_probe()
1062 qdev->queue_irq_start = res->start + 1; in amd_qdma_probe()
1063 qdev->queue_irq_num = resource_size(res) - 1; in amd_qdma_probe()
1072 qdev->regmap = devm_regmap_init_mmio(&pdev->dev, regs, in amd_qdma_probe()
1074 if (IS_ERR(qdev->regmap)) { in amd_qdma_probe()
1075 ret = PTR_ERR(qdev->regmap); in amd_qdma_probe()
1088 INIT_LIST_HEAD(&qdev->dma_dev.channels); in amd_qdma_probe()
1100 dma_cap_set(DMA_SLAVE, qdev->dma_dev.cap_mask); in amd_qdma_probe()
1101 dma_cap_set(DMA_PRIVATE, qdev->dma_dev.cap_mask); in amd_qdma_probe()
1103 qdev->dma_dev.dev = &pdev->dev; in amd_qdma_probe()
1104 qdev->dma_dev.filter.map = pdata->device_map; in amd_qdma_probe()
1105 qdev->dma_dev.filter.mapcnt = qdev->chan_num * 2; in amd_qdma_probe()
1106 qdev->dma_dev.filter.fn = qdma_filter_fn; in amd_qdma_probe()
1107 qdev->dma_dev.device_alloc_chan_resources = qdma_alloc_queue_resources; in amd_qdma_probe()
1108 qdev->dma_dev.device_free_chan_resources = qdma_free_queue_resources; in amd_qdma_probe()
1109 qdev->dma_dev.device_prep_slave_sg = qdma_prep_device_sg; in amd_qdma_probe()
1110 qdev->dma_dev.device_config = qdma_device_config; in amd_qdma_probe()
1111 qdev->dma_dev.device_issue_pending = qdma_issue_pending; in amd_qdma_probe()
1112 qdev->dma_dev.device_tx_status = dma_cookie_status; in amd_qdma_probe()
1113 qdev->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); in amd_qdma_probe()
1115 ret = dma_async_device_register(&qdev->dma_dev); in amd_qdma_probe()
1126 mutex_destroy(&qdev->ctxt_lock); in amd_qdma_probe()
1133 .name = "amd-qdma",