Lines Matching +full:dma +full:- +full:maxburst
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Core driver for the Intel integrated DMA 64-bit
12 #include <linux/dma-mapping.h>
19 #include <linux/dma/idma64.h>
26 /* ---------------------------------------------------------------------- */
30 return &chan->dev->device; in chan2dev()
33 /* ---------------------------------------------------------------------- */
41 channel_clear_bit(idma64, MASK(XFER), idma64->all_chan_mask); in idma64_off()
42 channel_clear_bit(idma64, MASK(BLOCK), idma64->all_chan_mask); in idma64_off()
43 channel_clear_bit(idma64, MASK(SRC_TRAN), idma64->all_chan_mask); in idma64_off()
44 channel_clear_bit(idma64, MASK(DST_TRAN), idma64->all_chan_mask); in idma64_off()
45 channel_clear_bit(idma64, MASK(ERROR), idma64->all_chan_mask); in idma64_off()
49 } while (dma_readl(idma64, CFG) & IDMA64_CFG_DMA_EN && --count); in idma64_off()
57 /* ---------------------------------------------------------------------- */
71 channel_set_bit(idma64, MASK(XFER), idma64c->mask); in idma64_chan_init()
72 channel_set_bit(idma64, MASK(ERROR), idma64c->mask); in idma64_chan_init()
77 * The iDMA is turned off in ->probe() and looses context during system in idma64_chan_init()
86 channel_clear_bit(idma64, CH_EN, idma64c->mask); in idma64_chan_stop()
91 struct idma64_desc *desc = idma64c->desc; in idma64_chan_start()
92 struct idma64_hw_desc *hw = &desc->hw[0]; in idma64_chan_start()
100 channel_writeq(idma64c, LLP, hw->llp); in idma64_chan_start()
102 channel_set_bit(idma64, CH_EN, idma64c->mask); in idma64_chan_start()
107 struct idma64 *idma64 = to_idma64(idma64c->vchan.chan.device); in idma64_stop_transfer()
114 struct idma64 *idma64 = to_idma64(idma64c->vchan.chan.device); in idma64_start_transfer()
118 vdesc = vchan_next_desc(&idma64c->vchan); in idma64_start_transfer()
120 idma64c->desc = NULL; in idma64_start_transfer()
124 list_del(&vdesc->node); in idma64_start_transfer()
125 idma64c->desc = to_idma64_desc(vdesc); in idma64_start_transfer()
134 /* ---------------------------------------------------------------------- */
139 struct idma64_chan *idma64c = &idma64->chan[c]; in idma64_chan_irq()
143 stat = this_cpu_ptr(idma64c->vchan.chan.local); in idma64_chan_irq()
145 spin_lock(&idma64c->vchan.lock); in idma64_chan_irq()
146 desc = idma64c->desc; in idma64_chan_irq()
149 dma_writel(idma64, CLEAR(ERROR), idma64c->mask); in idma64_chan_irq()
150 desc->status = DMA_ERROR; in idma64_chan_irq()
152 dma_writel(idma64, CLEAR(XFER), idma64c->mask); in idma64_chan_irq()
153 desc->status = DMA_COMPLETE; in idma64_chan_irq()
154 vchan_cookie_complete(&desc->vdesc); in idma64_chan_irq()
155 stat->bytes_transferred += desc->length; in idma64_chan_irq()
159 /* idma64_start_transfer() updates idma64c->desc */ in idma64_chan_irq()
160 if (idma64c->desc == NULL || desc->status == DMA_ERROR) in idma64_chan_irq()
163 spin_unlock(&idma64c->vchan.lock); in idma64_chan_irq()
174 /* Since IRQ may be shared, check if DMA controller is powered on */ in idma64_irq()
178 dev_vdbg(idma64->dma.dev, "%s: status=%#x\n", __func__, status); in idma64_irq()
180 /* Check if we have any interrupt from the DMA controller */ in idma64_irq()
187 for (i = 0; i < idma64->dma.chancnt; i++) in idma64_irq()
193 /* ---------------------------------------------------------------------- */
203 desc->hw = kcalloc(ndesc, sizeof(*desc->hw), GFP_NOWAIT); in idma64_alloc_desc()
204 if (!desc->hw) { in idma64_alloc_desc()
217 if (desc->ndesc) { in idma64_desc_free()
218 unsigned int i = desc->ndesc; in idma64_desc_free()
221 hw = &desc->hw[--i]; in idma64_desc_free()
222 dma_pool_free(idma64c->pool, hw->lli, hw->llp); in idma64_desc_free()
226 kfree(desc->hw); in idma64_desc_free()
232 struct idma64_chan *idma64c = to_idma64_chan(vdesc->tx.chan); in idma64_vdesc_free()
241 struct idma64_lli *lli = hw->lli; in idma64_hw_desc_fill()
243 u32 ctlhi = IDMA64C_CTLH_BLOCK_TS(hw->len); in idma64_hw_desc_fill()
248 sar = hw->phys; in idma64_hw_desc_fill()
249 dar = config->dst_addr; in idma64_hw_desc_fill()
252 src_width = __ffs(sar | hw->len | 4); in idma64_hw_desc_fill()
253 dst_width = __ffs(config->dst_addr_width); in idma64_hw_desc_fill()
255 sar = config->src_addr; in idma64_hw_desc_fill()
256 dar = hw->phys; in idma64_hw_desc_fill()
259 src_width = __ffs(config->src_addr_width); in idma64_hw_desc_fill()
260 dst_width = __ffs(dar | hw->len | 4); in idma64_hw_desc_fill()
263 lli->sar = sar; in idma64_hw_desc_fill()
264 lli->dar = dar; in idma64_hw_desc_fill()
266 lli->ctlhi = ctlhi; in idma64_hw_desc_fill()
267 lli->ctllo = ctllo | in idma64_hw_desc_fill()
268 IDMA64C_CTLL_SRC_MSIZE(config->src_maxburst) | in idma64_hw_desc_fill()
269 IDMA64C_CTLL_DST_MSIZE(config->dst_maxburst) | in idma64_hw_desc_fill()
273 lli->llp = llp; in idma64_hw_desc_fill()
279 struct dma_slave_config *config = &idma64c->config; in idma64_desc_fill()
280 unsigned int i = desc->ndesc; in idma64_desc_fill()
281 struct idma64_hw_desc *hw = &desc->hw[i - 1]; in idma64_desc_fill()
282 struct idma64_lli *lli = hw->lli; in idma64_desc_fill()
287 hw = &desc->hw[--i]; in idma64_desc_fill()
288 idma64_hw_desc_fill(hw, config, desc->direction, llp); in idma64_desc_fill()
289 llp = hw->llp; in idma64_desc_fill()
290 desc->length += hw->len; in idma64_desc_fill()
294 lli->ctllo |= IDMA64C_CTLL_INT_EN; in idma64_desc_fill()
297 lli->ctllo &= ~(IDMA64C_CTLL_LLP_S_EN | IDMA64C_CTLL_LLP_D_EN); in idma64_desc_fill()
315 struct idma64_hw_desc *hw = &desc->hw[i]; in idma64_prep_slave_sg()
317 /* Allocate DMA capable memory for hardware descriptor */ in idma64_prep_slave_sg()
318 hw->lli = dma_pool_alloc(idma64c->pool, GFP_NOWAIT, &hw->llp); in idma64_prep_slave_sg()
319 if (!hw->lli) { in idma64_prep_slave_sg()
320 desc->ndesc = i; in idma64_prep_slave_sg()
325 hw->phys = sg_dma_address(sg); in idma64_prep_slave_sg()
326 hw->len = sg_dma_len(sg); in idma64_prep_slave_sg()
329 desc->ndesc = sg_len; in idma64_prep_slave_sg()
330 desc->direction = direction; in idma64_prep_slave_sg()
331 desc->status = DMA_IN_PROGRESS; in idma64_prep_slave_sg()
334 return vchan_tx_prep(&idma64c->vchan, &desc->vdesc, flags); in idma64_prep_slave_sg()
342 spin_lock_irqsave(&idma64c->vchan.lock, flags); in idma64_issue_pending()
343 if (vchan_issue_pending(&idma64c->vchan) && !idma64c->desc) in idma64_issue_pending()
345 spin_unlock_irqrestore(&idma64c->vchan.lock, flags); in idma64_issue_pending()
350 struct idma64_desc *desc = idma64c->desc; in idma64_active_desc_size()
352 size_t bytes = desc->length; in idma64_active_desc_size()
358 hw = &desc->hw[i]; in idma64_active_desc_size()
359 if (hw->llp == llp) in idma64_active_desc_size()
361 bytes -= hw->len; in idma64_active_desc_size()
362 } while (++i < desc->ndesc); in idma64_active_desc_size()
368 bytes += desc->hw[--i].len; in idma64_active_desc_size()
370 return bytes - IDMA64C_CTLH_BLOCK_TS(ctlhi); in idma64_active_desc_size()
386 spin_lock_irqsave(&idma64c->vchan.lock, flags); in idma64_tx_status()
387 vdesc = vchan_find_desc(&idma64c->vchan, cookie); in idma64_tx_status()
388 if (idma64c->desc && cookie == idma64c->desc->vdesc.tx.cookie) { in idma64_tx_status()
391 status = idma64c->desc->status; in idma64_tx_status()
393 bytes = to_idma64_desc(vdesc)->length; in idma64_tx_status()
396 spin_unlock_irqrestore(&idma64c->vchan.lock, flags); in idma64_tx_status()
401 static void convert_burst(u32 *maxburst) in convert_burst() argument
403 if (*maxburst) in convert_burst()
404 *maxburst = __fls(*maxburst); in convert_burst()
406 *maxburst = 0; in convert_burst()
414 memcpy(&idma64c->config, config, sizeof(idma64c->config)); in idma64_slave_config()
416 convert_burst(&idma64c->config.src_maxburst); in idma64_slave_config()
417 convert_burst(&idma64c->config.dst_maxburst); in idma64_slave_config()
437 } while (!(cfglo & IDMA64C_CFGL_FIFO_EMPTY) && --count); in idma64_chan_deactivate()
453 spin_lock_irqsave(&idma64c->vchan.lock, flags); in idma64_pause()
454 if (idma64c->desc && idma64c->desc->status == DMA_IN_PROGRESS) { in idma64_pause()
456 idma64c->desc->status = DMA_PAUSED; in idma64_pause()
458 spin_unlock_irqrestore(&idma64c->vchan.lock, flags); in idma64_pause()
468 spin_lock_irqsave(&idma64c->vchan.lock, flags); in idma64_resume()
469 if (idma64c->desc && idma64c->desc->status == DMA_PAUSED) { in idma64_resume()
470 idma64c->desc->status = DMA_IN_PROGRESS; in idma64_resume()
473 spin_unlock_irqrestore(&idma64c->vchan.lock, flags); in idma64_resume()
484 spin_lock_irqsave(&idma64c->vchan.lock, flags); in idma64_terminate_all()
487 if (idma64c->desc) { in idma64_terminate_all()
488 idma64_vdesc_free(&idma64c->desc->vdesc); in idma64_terminate_all()
489 idma64c->desc = NULL; in idma64_terminate_all()
491 vchan_get_all_descriptors(&idma64c->vchan, &head); in idma64_terminate_all()
492 spin_unlock_irqrestore(&idma64c->vchan.lock, flags); in idma64_terminate_all()
494 vchan_dma_desc_free_list(&idma64c->vchan, &head); in idma64_terminate_all()
502 vchan_synchronize(&idma64c->vchan); in idma64_synchronize()
510 idma64c->pool = dma_pool_create(dev_name(chan2dev(chan)), in idma64_alloc_chan_resources()
511 chan->device->dev, in idma64_alloc_chan_resources()
513 if (!idma64c->pool) { in idma64_alloc_chan_resources()
515 return -ENOMEM; in idma64_alloc_chan_resources()
526 dma_pool_destroy(idma64c->pool); in idma64_free_chan_resources()
527 idma64c->pool = NULL; in idma64_free_chan_resources()
530 /* ---------------------------------------------------------------------- */
544 idma64 = devm_kzalloc(chip->dev, sizeof(*idma64), GFP_KERNEL); in idma64_probe()
546 return -ENOMEM; in idma64_probe()
548 idma64->regs = chip->regs; in idma64_probe()
549 chip->idma64 = idma64; in idma64_probe()
551 idma64->chan = devm_kcalloc(chip->dev, nr_chan, sizeof(*idma64->chan), in idma64_probe()
553 if (!idma64->chan) in idma64_probe()
554 return -ENOMEM; in idma64_probe()
556 idma64->all_chan_mask = (1 << nr_chan) - 1; in idma64_probe()
561 ret = devm_request_irq(chip->dev, chip->irq, idma64_irq, IRQF_SHARED, in idma64_probe()
562 dev_name(chip->dev), idma64); in idma64_probe()
566 INIT_LIST_HEAD(&idma64->dma.channels); in idma64_probe()
568 struct idma64_chan *idma64c = &idma64->chan[i]; in idma64_probe()
570 idma64c->vchan.desc_free = idma64_vdesc_free; in idma64_probe()
571 vchan_init(&idma64c->vchan, &idma64->dma); in idma64_probe()
573 idma64c->regs = idma64->regs + i * IDMA64_CH_LENGTH; in idma64_probe()
574 idma64c->mask = BIT(i); in idma64_probe()
577 dma_cap_set(DMA_SLAVE, idma64->dma.cap_mask); in idma64_probe()
578 dma_cap_set(DMA_PRIVATE, idma64->dma.cap_mask); in idma64_probe()
580 idma64->dma.device_alloc_chan_resources = idma64_alloc_chan_resources; in idma64_probe()
581 idma64->dma.device_free_chan_resources = idma64_free_chan_resources; in idma64_probe()
583 idma64->dma.device_prep_slave_sg = idma64_prep_slave_sg; in idma64_probe()
585 idma64->dma.device_issue_pending = idma64_issue_pending; in idma64_probe()
586 idma64->dma.device_tx_status = idma64_tx_status; in idma64_probe()
588 idma64->dma.device_config = idma64_slave_config; in idma64_probe()
589 idma64->dma.device_pause = idma64_pause; in idma64_probe()
590 idma64->dma.device_resume = idma64_resume; in idma64_probe()
591 idma64->dma.device_terminate_all = idma64_terminate_all; in idma64_probe()
592 idma64->dma.device_synchronize = idma64_synchronize; in idma64_probe()
594 idma64->dma.src_addr_widths = IDMA64_BUSWIDTHS; in idma64_probe()
595 idma64->dma.dst_addr_widths = IDMA64_BUSWIDTHS; in idma64_probe()
596 idma64->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); in idma64_probe()
597 idma64->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; in idma64_probe()
599 idma64->dma.dev = chip->sysdev; in idma64_probe()
601 dma_set_max_seg_size(idma64->dma.dev, IDMA64C_CTLH_BLOCK_TS_MASK); in idma64_probe()
603 ret = dma_async_device_register(&idma64->dma); in idma64_probe()
607 dev_info(chip->dev, "Found Intel integrated DMA 64-bit\n"); in idma64_probe()
613 struct idma64 *idma64 = chip->idma64; in idma64_remove()
616 dma_async_device_unregister(&idma64->dma); in idma64_remove()
622 devm_free_irq(chip->dev, chip->irq, idma64); in idma64_remove()
624 for (i = 0; i < idma64->dma.chancnt; i++) { in idma64_remove()
625 struct idma64_chan *idma64c = &idma64->chan[i]; in idma64_remove()
627 tasklet_kill(&idma64c->vchan.task); in idma64_remove()
631 /* ---------------------------------------------------------------------- */
636 struct device *dev = &pdev->dev; in idma64_platform_probe()
637 struct device *sysdev = dev->parent; in idma64_platform_probe()
642 return -ENOMEM; in idma64_platform_probe()
644 chip->irq = platform_get_irq(pdev, 0); in idma64_platform_probe()
645 if (chip->irq < 0) in idma64_platform_probe()
646 return chip->irq; in idma64_platform_probe()
648 chip->regs = devm_platform_ioremap_resource(pdev, 0); in idma64_platform_probe()
649 if (IS_ERR(chip->regs)) in idma64_platform_probe()
650 return PTR_ERR(chip->regs); in idma64_platform_probe()
656 chip->dev = dev; in idma64_platform_probe()
657 chip->sysdev = sysdev; in idma64_platform_probe()
678 idma64_off(chip->idma64); in idma64_pm_suspend()
686 idma64_on(chip->idma64); in idma64_pm_resume()