Lines Matching +full:axi4 +full:- +full:lite
1 // SPDX-License-Identifier: GPL-2.0-or-later
5 * Copyright (C) 2010-2014 Xilinx, Inc. All rights reserved.
11 * core that provides high-bandwidth direct memory access between memory
12 * and AXI4-Stream type video target peripherals. The core provides efficient
18 * registers are accessed through an AXI4-Lite slave interface.
21 * provides high-bandwidth one dimensional direct memory access between memory
22 * and AXI4-Stream target peripherals. It supports one receive and one
25 * The AXI CDMA, is a soft IP, which provides high-bandwidth Direct Memory
26 * Access (DMA) between a memory-mapped source address and a memory-mapped
30 * Xilinx IP that provides high-bandwidth direct memory access between
31 * memory and AXI4-Stream target peripherals. It provides scatter gather
50 #include <linux/io-64-nonatomic-lo-hi.h>
224 * struct xilinx_vdma_desc_hw - Hardware Descriptor
245 * struct xilinx_axidma_desc_hw - Hardware Descriptor for AXI DMA
254 * @app: APP Fields @0x20 - 0x30
269 * struct xilinx_aximcdma_desc_hw - Hardware Descriptor for AXI MCDMA
278 * @app: APP Fields @0x20 - 0x30
293 * struct xilinx_cdma_desc_hw - Hardware Descriptor
315 * struct xilinx_vdma_tx_segment - Descriptor segment
327 * struct xilinx_axidma_tx_segment - Descriptor segment
339 * struct xilinx_aximcdma_tx_segment - Descriptor segment
351 * struct xilinx_cdma_tx_segment - Descriptor segment
363 * struct xilinx_dma_tx_descriptor - Per Transaction structure
381 * struct xilinx_dma_chan - Driver specific DMA channel structure
461 * enum xdma_ip_type - DMA IP type.
486 * struct xilinx_dma_device - DMA device structure
495 * @axi_clk: DMA Axi4-lite interace clock
531 readl_poll_timeout_atomic(chan->xdev->regs + chan->ctrl_offset + reg, \
537 return ioread32(chan->xdev->regs + reg); in dma_read()
542 iowrite32(value, chan->xdev->regs + reg); in dma_write()
548 dma_write(chan, chan->desc_offset + reg, value); in vdma_desc_write()
553 return dma_read(chan, chan->ctrl_offset + reg); in dma_ctrl_read()
559 dma_write(chan, chan->ctrl_offset + reg, value); in dma_ctrl_write()
575 * vdma_desc_write_64 - 64-bit descriptor write
589 writel(value_lsb, chan->xdev->regs + chan->desc_offset + reg); in vdma_desc_write_64()
592 writel(value_msb, chan->xdev->regs + chan->desc_offset + reg + 4); in vdma_desc_write_64()
597 lo_hi_writeq(value, chan->xdev->regs + chan->ctrl_offset + reg); in dma_writeq()
603 if (chan->ext_addr) in xilinx_write()
614 if (chan->ext_addr) { in xilinx_axidma_buf()
615 hw->buf_addr = lower_32_bits(buf_addr + sg_used + period_len); in xilinx_axidma_buf()
616 hw->buf_addr_msb = upper_32_bits(buf_addr + sg_used + in xilinx_axidma_buf()
619 hw->buf_addr = buf_addr + sg_used + period_len; in xilinx_axidma_buf()
627 if (chan->ext_addr) { in xilinx_aximcdma_buf()
628 hw->buf_addr = lower_32_bits(buf_addr + sg_used); in xilinx_aximcdma_buf()
629 hw->buf_addr_msb = upper_32_bits(buf_addr + sg_used); in xilinx_aximcdma_buf()
631 hw->buf_addr = buf_addr + sg_used; in xilinx_aximcdma_buf()
636 * xilinx_dma_get_metadata_ptr- Populate metadata pointer and payload length
649 seg = list_first_entry(&desc->segments, in xilinx_dma_get_metadata_ptr()
651 return seg->hw.app; in xilinx_dma_get_metadata_ptr()
658 /* -----------------------------------------------------------------------------
663 * xilinx_vdma_alloc_tx_segment - Allocate transaction segment
674 segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys); in xilinx_vdma_alloc_tx_segment()
678 segment->phys = phys; in xilinx_vdma_alloc_tx_segment()
684 * xilinx_cdma_alloc_tx_segment - Allocate transaction segment
695 segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys); in xilinx_cdma_alloc_tx_segment()
699 segment->phys = phys; in xilinx_cdma_alloc_tx_segment()
705 * xilinx_axidma_alloc_tx_segment - Allocate transaction segment
716 spin_lock_irqsave(&chan->lock, flags); in xilinx_axidma_alloc_tx_segment()
717 if (!list_empty(&chan->free_seg_list)) { in xilinx_axidma_alloc_tx_segment()
718 segment = list_first_entry(&chan->free_seg_list, in xilinx_axidma_alloc_tx_segment()
721 list_del(&segment->node); in xilinx_axidma_alloc_tx_segment()
723 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_axidma_alloc_tx_segment()
726 dev_dbg(chan->dev, "Could not find free tx segment\n"); in xilinx_axidma_alloc_tx_segment()
732 * xilinx_aximcdma_alloc_tx_segment - Allocate transaction segment
743 spin_lock_irqsave(&chan->lock, flags); in xilinx_aximcdma_alloc_tx_segment()
744 if (!list_empty(&chan->free_seg_list)) { in xilinx_aximcdma_alloc_tx_segment()
745 segment = list_first_entry(&chan->free_seg_list, in xilinx_aximcdma_alloc_tx_segment()
748 list_del(&segment->node); in xilinx_aximcdma_alloc_tx_segment()
750 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_aximcdma_alloc_tx_segment()
757 u32 next_desc = hw->next_desc; in xilinx_dma_clean_hw_desc()
758 u32 next_desc_msb = hw->next_desc_msb; in xilinx_dma_clean_hw_desc()
762 hw->next_desc = next_desc; in xilinx_dma_clean_hw_desc()
763 hw->next_desc_msb = next_desc_msb; in xilinx_dma_clean_hw_desc()
768 u32 next_desc = hw->next_desc; in xilinx_mcdma_clean_hw_desc()
769 u32 next_desc_msb = hw->next_desc_msb; in xilinx_mcdma_clean_hw_desc()
773 hw->next_desc = next_desc; in xilinx_mcdma_clean_hw_desc()
774 hw->next_desc_msb = next_desc_msb; in xilinx_mcdma_clean_hw_desc()
778 * xilinx_dma_free_tx_segment - Free transaction segment
785 xilinx_dma_clean_hw_desc(&segment->hw); in xilinx_dma_free_tx_segment()
787 list_add_tail(&segment->node, &chan->free_seg_list); in xilinx_dma_free_tx_segment()
791 * xilinx_mcdma_free_tx_segment - Free transaction segment
799 xilinx_mcdma_clean_hw_desc(&segment->hw); in xilinx_mcdma_free_tx_segment()
801 list_add_tail(&segment->node, &chan->free_seg_list); in xilinx_mcdma_free_tx_segment()
805 * xilinx_cdma_free_tx_segment - Free transaction segment
812 dma_pool_free(chan->desc_pool, segment, segment->phys); in xilinx_cdma_free_tx_segment()
816 * xilinx_vdma_free_tx_segment - Free transaction segment
823 dma_pool_free(chan->desc_pool, segment, segment->phys); in xilinx_vdma_free_tx_segment()
827 * xilinx_dma_alloc_tx_descriptor - Allocate transaction descriptor
841 INIT_LIST_HEAD(&desc->segments); in xilinx_dma_alloc_tx_descriptor()
847 * xilinx_dma_free_tx_descriptor - Free transaction descriptor
863 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { in xilinx_dma_free_tx_descriptor()
864 list_for_each_entry_safe(segment, next, &desc->segments, node) { in xilinx_dma_free_tx_descriptor()
865 list_del(&segment->node); in xilinx_dma_free_tx_descriptor()
868 } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { in xilinx_dma_free_tx_descriptor()
870 &desc->segments, node) { in xilinx_dma_free_tx_descriptor()
871 list_del(&cdma_segment->node); in xilinx_dma_free_tx_descriptor()
874 } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { in xilinx_dma_free_tx_descriptor()
876 &desc->segments, node) { in xilinx_dma_free_tx_descriptor()
877 list_del(&axidma_segment->node); in xilinx_dma_free_tx_descriptor()
882 &desc->segments, node) { in xilinx_dma_free_tx_descriptor()
883 list_del(&aximcdma_segment->node); in xilinx_dma_free_tx_descriptor()
894 * xilinx_dma_free_desc_list - Free descriptors list
904 list_del(&desc->node); in xilinx_dma_free_desc_list()
910 * xilinx_dma_free_descriptors - Free channel descriptors
917 spin_lock_irqsave(&chan->lock, flags); in xilinx_dma_free_descriptors()
919 xilinx_dma_free_desc_list(chan, &chan->pending_list); in xilinx_dma_free_descriptors()
920 xilinx_dma_free_desc_list(chan, &chan->done_list); in xilinx_dma_free_descriptors()
921 xilinx_dma_free_desc_list(chan, &chan->active_list); in xilinx_dma_free_descriptors()
923 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dma_free_descriptors()
927 * xilinx_dma_free_chan_resources - Free channel resources
935 dev_dbg(chan->dev, "Free all channel resources.\n"); in xilinx_dma_free_chan_resources()
939 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { in xilinx_dma_free_chan_resources()
940 spin_lock_irqsave(&chan->lock, flags); in xilinx_dma_free_chan_resources()
941 INIT_LIST_HEAD(&chan->free_seg_list); in xilinx_dma_free_chan_resources()
942 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dma_free_chan_resources()
945 dma_free_coherent(chan->dev, sizeof(*chan->seg_v) * in xilinx_dma_free_chan_resources()
946 XILINX_DMA_NUM_DESCS, chan->seg_v, in xilinx_dma_free_chan_resources()
947 chan->seg_p); in xilinx_dma_free_chan_resources()
950 dma_free_coherent(chan->dev, sizeof(*chan->cyclic_seg_v), in xilinx_dma_free_chan_resources()
951 chan->cyclic_seg_v, chan->cyclic_seg_p); in xilinx_dma_free_chan_resources()
954 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { in xilinx_dma_free_chan_resources()
955 spin_lock_irqsave(&chan->lock, flags); in xilinx_dma_free_chan_resources()
956 INIT_LIST_HEAD(&chan->free_seg_list); in xilinx_dma_free_chan_resources()
957 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dma_free_chan_resources()
960 dma_free_coherent(chan->dev, sizeof(*chan->seg_mv) * in xilinx_dma_free_chan_resources()
961 XILINX_DMA_NUM_DESCS, chan->seg_mv, in xilinx_dma_free_chan_resources()
962 chan->seg_p); in xilinx_dma_free_chan_resources()
965 if (chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA && in xilinx_dma_free_chan_resources()
966 chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIMCDMA) { in xilinx_dma_free_chan_resources()
967 dma_pool_destroy(chan->desc_pool); in xilinx_dma_free_chan_resources()
968 chan->desc_pool = NULL; in xilinx_dma_free_chan_resources()
974 * xilinx_dma_get_residue - Compute residue for a given descriptor
992 list_for_each(entry, &desc->segments) { in xilinx_dma_get_residue()
993 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { in xilinx_dma_get_residue()
997 cdma_hw = &cdma_seg->hw; in xilinx_dma_get_residue()
998 residue += (cdma_hw->control - cdma_hw->status) & in xilinx_dma_get_residue()
999 chan->xdev->max_buffer_len; in xilinx_dma_get_residue()
1000 } else if (chan->xdev->dma_config->dmatype == in xilinx_dma_get_residue()
1005 axidma_hw = &axidma_seg->hw; in xilinx_dma_get_residue()
1006 residue += (axidma_hw->control - axidma_hw->status) & in xilinx_dma_get_residue()
1007 chan->xdev->max_buffer_len; in xilinx_dma_get_residue()
1013 aximcdma_hw = &aximcdma_seg->hw; in xilinx_dma_get_residue()
1015 (aximcdma_hw->control - aximcdma_hw->status) & in xilinx_dma_get_residue()
1016 chan->xdev->max_buffer_len; in xilinx_dma_get_residue()
1024 * xilinx_dma_chan_handle_cyclic - Cyclic dma callback
1035 dmaengine_desc_get_callback(&desc->async_tx, &cb); in xilinx_dma_chan_handle_cyclic()
1037 spin_unlock_irqrestore(&chan->lock, *flags); in xilinx_dma_chan_handle_cyclic()
1039 spin_lock_irqsave(&chan->lock, *flags); in xilinx_dma_chan_handle_cyclic()
1044 * xilinx_dma_chan_desc_cleanup - Clean channel descriptors
1052 spin_lock_irqsave(&chan->lock, flags); in xilinx_dma_chan_desc_cleanup()
1054 list_for_each_entry_safe(desc, next, &chan->done_list, node) { in xilinx_dma_chan_desc_cleanup()
1057 if (desc->cyclic) { in xilinx_dma_chan_desc_cleanup()
1063 list_del(&desc->node); in xilinx_dma_chan_desc_cleanup()
1065 if (unlikely(desc->err)) { in xilinx_dma_chan_desc_cleanup()
1066 if (chan->direction == DMA_DEV_TO_MEM) in xilinx_dma_chan_desc_cleanup()
1074 result.residue = desc->residue; in xilinx_dma_chan_desc_cleanup()
1077 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dma_chan_desc_cleanup()
1078 dmaengine_desc_get_callback_invoke(&desc->async_tx, &result); in xilinx_dma_chan_desc_cleanup()
1079 spin_lock_irqsave(&chan->lock, flags); in xilinx_dma_chan_desc_cleanup()
1082 dma_run_dependencies(&desc->async_tx); in xilinx_dma_chan_desc_cleanup()
1089 if (chan->terminating) in xilinx_dma_chan_desc_cleanup()
1093 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dma_chan_desc_cleanup()
1097 * xilinx_dma_do_tasklet - Schedule completion tasklet
1108 * xilinx_dma_alloc_chan_resources - Allocate channel resources
1119 if (chan->desc_pool) in xilinx_dma_alloc_chan_resources()
1126 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { in xilinx_dma_alloc_chan_resources()
1128 chan->seg_v = dma_alloc_coherent(chan->dev, in xilinx_dma_alloc_chan_resources()
1129 sizeof(*chan->seg_v) * XILINX_DMA_NUM_DESCS, in xilinx_dma_alloc_chan_resources()
1130 &chan->seg_p, GFP_KERNEL); in xilinx_dma_alloc_chan_resources()
1131 if (!chan->seg_v) { in xilinx_dma_alloc_chan_resources()
1132 dev_err(chan->dev, in xilinx_dma_alloc_chan_resources()
1134 chan->id); in xilinx_dma_alloc_chan_resources()
1135 return -ENOMEM; in xilinx_dma_alloc_chan_resources()
1143 chan->cyclic_seg_v = dma_alloc_coherent(chan->dev, in xilinx_dma_alloc_chan_resources()
1144 sizeof(*chan->cyclic_seg_v), in xilinx_dma_alloc_chan_resources()
1145 &chan->cyclic_seg_p, in xilinx_dma_alloc_chan_resources()
1147 if (!chan->cyclic_seg_v) { in xilinx_dma_alloc_chan_resources()
1148 dev_err(chan->dev, in xilinx_dma_alloc_chan_resources()
1150 dma_free_coherent(chan->dev, sizeof(*chan->seg_v) * in xilinx_dma_alloc_chan_resources()
1151 XILINX_DMA_NUM_DESCS, chan->seg_v, in xilinx_dma_alloc_chan_resources()
1152 chan->seg_p); in xilinx_dma_alloc_chan_resources()
1153 return -ENOMEM; in xilinx_dma_alloc_chan_resources()
1155 chan->cyclic_seg_v->phys = chan->cyclic_seg_p; in xilinx_dma_alloc_chan_resources()
1158 chan->seg_v[i].hw.next_desc = in xilinx_dma_alloc_chan_resources()
1159 lower_32_bits(chan->seg_p + sizeof(*chan->seg_v) * in xilinx_dma_alloc_chan_resources()
1161 chan->seg_v[i].hw.next_desc_msb = in xilinx_dma_alloc_chan_resources()
1162 upper_32_bits(chan->seg_p + sizeof(*chan->seg_v) * in xilinx_dma_alloc_chan_resources()
1164 chan->seg_v[i].phys = chan->seg_p + in xilinx_dma_alloc_chan_resources()
1165 sizeof(*chan->seg_v) * i; in xilinx_dma_alloc_chan_resources()
1166 list_add_tail(&chan->seg_v[i].node, in xilinx_dma_alloc_chan_resources()
1167 &chan->free_seg_list); in xilinx_dma_alloc_chan_resources()
1169 } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { in xilinx_dma_alloc_chan_resources()
1171 chan->seg_mv = dma_alloc_coherent(chan->dev, in xilinx_dma_alloc_chan_resources()
1172 sizeof(*chan->seg_mv) * in xilinx_dma_alloc_chan_resources()
1174 &chan->seg_p, GFP_KERNEL); in xilinx_dma_alloc_chan_resources()
1175 if (!chan->seg_mv) { in xilinx_dma_alloc_chan_resources()
1176 dev_err(chan->dev, in xilinx_dma_alloc_chan_resources()
1178 chan->id); in xilinx_dma_alloc_chan_resources()
1179 return -ENOMEM; in xilinx_dma_alloc_chan_resources()
1182 chan->seg_mv[i].hw.next_desc = in xilinx_dma_alloc_chan_resources()
1183 lower_32_bits(chan->seg_p + sizeof(*chan->seg_mv) * in xilinx_dma_alloc_chan_resources()
1185 chan->seg_mv[i].hw.next_desc_msb = in xilinx_dma_alloc_chan_resources()
1186 upper_32_bits(chan->seg_p + sizeof(*chan->seg_mv) * in xilinx_dma_alloc_chan_resources()
1188 chan->seg_mv[i].phys = chan->seg_p + in xilinx_dma_alloc_chan_resources()
1189 sizeof(*chan->seg_mv) * i; in xilinx_dma_alloc_chan_resources()
1190 list_add_tail(&chan->seg_mv[i].node, in xilinx_dma_alloc_chan_resources()
1191 &chan->free_seg_list); in xilinx_dma_alloc_chan_resources()
1193 } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { in xilinx_dma_alloc_chan_resources()
1194 chan->desc_pool = dma_pool_create("xilinx_cdma_desc_pool", in xilinx_dma_alloc_chan_resources()
1195 chan->dev, in xilinx_dma_alloc_chan_resources()
1200 chan->desc_pool = dma_pool_create("xilinx_vdma_desc_pool", in xilinx_dma_alloc_chan_resources()
1201 chan->dev, in xilinx_dma_alloc_chan_resources()
1207 if (!chan->desc_pool && in xilinx_dma_alloc_chan_resources()
1208 ((chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA) && in xilinx_dma_alloc_chan_resources()
1209 chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIMCDMA)) { in xilinx_dma_alloc_chan_resources()
1210 dev_err(chan->dev, in xilinx_dma_alloc_chan_resources()
1212 chan->id); in xilinx_dma_alloc_chan_resources()
1213 return -ENOMEM; in xilinx_dma_alloc_chan_resources()
1218 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { in xilinx_dma_alloc_chan_resources()
1226 if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg) in xilinx_dma_alloc_chan_resources()
1234 * xilinx_dma_calc_copysize - Calculate the amount of data to copy
1246 copy = min_t(size_t, size - done, in xilinx_dma_calc_copysize()
1247 chan->xdev->max_buffer_len); in xilinx_dma_calc_copysize()
1250 chan->xdev->common.copy_align) { in xilinx_dma_calc_copysize()
1256 (1 << chan->xdev->common.copy_align)); in xilinx_dma_calc_copysize()
1262 * xilinx_dma_tx_status - Get DMA transaction status
1283 spin_lock_irqsave(&chan->lock, flags); in xilinx_dma_tx_status()
1284 if (!list_empty(&chan->active_list)) { in xilinx_dma_tx_status()
1285 desc = list_last_entry(&chan->active_list, in xilinx_dma_tx_status()
1291 if (chan->has_sg && chan->xdev->dma_config->dmatype != XDMA_TYPE_VDMA) in xilinx_dma_tx_status()
1294 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dma_tx_status()
1302 * xilinx_dma_stop_transfer - Halt DMA channel
1320 * xilinx_cdma_stop_transfer - Wait for the current transfer to complete
1335 * xilinx_dma_start - Start DMA channel
1351 dev_err(chan->dev, "Cannot start channel %p: %x\n", in xilinx_dma_start()
1354 chan->err = true; in xilinx_dma_start()
1359 * xilinx_vdma_start_transfer - Starts VDMA transfer
1364 struct xilinx_vdma_config *config = &chan->config; in xilinx_vdma_start_transfer()
1371 if (chan->err) in xilinx_vdma_start_transfer()
1374 if (!chan->idle) in xilinx_vdma_start_transfer()
1377 if (list_empty(&chan->pending_list)) in xilinx_vdma_start_transfer()
1380 desc = list_first_entry(&chan->pending_list, in xilinx_vdma_start_transfer()
1384 if (chan->has_vflip) { in xilinx_vdma_start_transfer()
1387 reg |= config->vflip_en; in xilinx_vdma_start_transfer()
1394 if (config->frm_cnt_en) in xilinx_vdma_start_transfer()
1400 if (config->park) in xilinx_vdma_start_transfer()
1407 if (config->park) { in xilinx_vdma_start_transfer()
1408 j = chan->desc_submitcount; in xilinx_vdma_start_transfer()
1410 if (chan->direction == DMA_MEM_TO_DEV) { in xilinx_vdma_start_transfer()
1423 if (chan->err) in xilinx_vdma_start_transfer()
1427 if (chan->desc_submitcount < chan->num_frms) in xilinx_vdma_start_transfer()
1428 i = chan->desc_submitcount; in xilinx_vdma_start_transfer()
1430 list_for_each_entry(segment, &desc->segments, node) { in xilinx_vdma_start_transfer()
1431 if (chan->ext_addr) in xilinx_vdma_start_transfer()
1434 segment->hw.buf_addr, in xilinx_vdma_start_transfer()
1435 segment->hw.buf_addr_msb); in xilinx_vdma_start_transfer()
1439 segment->hw.buf_addr); in xilinx_vdma_start_transfer()
1448 vdma_desc_write(chan, XILINX_DMA_REG_HSIZE, last->hw.hsize); in xilinx_vdma_start_transfer()
1450 last->hw.stride); in xilinx_vdma_start_transfer()
1451 vdma_desc_write(chan, XILINX_DMA_REG_VSIZE, last->hw.vsize); in xilinx_vdma_start_transfer()
1453 chan->desc_submitcount++; in xilinx_vdma_start_transfer()
1454 chan->desc_pendingcount--; in xilinx_vdma_start_transfer()
1455 list_move_tail(&desc->node, &chan->active_list); in xilinx_vdma_start_transfer()
1456 if (chan->desc_submitcount == chan->num_frms) in xilinx_vdma_start_transfer()
1457 chan->desc_submitcount = 0; in xilinx_vdma_start_transfer()
1459 chan->idle = false; in xilinx_vdma_start_transfer()
1463 * xilinx_cdma_start_transfer - Starts cdma transfer
1472 if (chan->err) in xilinx_cdma_start_transfer()
1475 if (!chan->idle) in xilinx_cdma_start_transfer()
1478 if (list_empty(&chan->pending_list)) in xilinx_cdma_start_transfer()
1481 head_desc = list_first_entry(&chan->pending_list, in xilinx_cdma_start_transfer()
1483 tail_desc = list_last_entry(&chan->pending_list, in xilinx_cdma_start_transfer()
1485 tail_segment = list_last_entry(&tail_desc->segments, in xilinx_cdma_start_transfer()
1488 if (chan->desc_pendingcount <= XILINX_DMA_COALESCE_MAX) { in xilinx_cdma_start_transfer()
1490 ctrl_reg |= chan->desc_pendingcount << in xilinx_cdma_start_transfer()
1495 if (chan->has_sg) { in xilinx_cdma_start_transfer()
1503 head_desc->async_tx.phys); in xilinx_cdma_start_transfer()
1507 tail_segment->phys); in xilinx_cdma_start_transfer()
1513 segment = list_first_entry(&head_desc->segments, in xilinx_cdma_start_transfer()
1517 hw = &segment->hw; in xilinx_cdma_start_transfer()
1520 xilinx_prep_dma_addr_t(hw->src_addr)); in xilinx_cdma_start_transfer()
1522 xilinx_prep_dma_addr_t(hw->dest_addr)); in xilinx_cdma_start_transfer()
1526 hw->control & chan->xdev->max_buffer_len); in xilinx_cdma_start_transfer()
1529 list_splice_tail_init(&chan->pending_list, &chan->active_list); in xilinx_cdma_start_transfer()
1530 chan->desc_pendingcount = 0; in xilinx_cdma_start_transfer()
1531 chan->idle = false; in xilinx_cdma_start_transfer()
1535 * xilinx_dma_start_transfer - Starts DMA transfer
1544 if (chan->err) in xilinx_dma_start_transfer()
1547 if (list_empty(&chan->pending_list)) in xilinx_dma_start_transfer()
1550 if (!chan->idle) in xilinx_dma_start_transfer()
1553 head_desc = list_first_entry(&chan->pending_list, in xilinx_dma_start_transfer()
1555 tail_desc = list_last_entry(&chan->pending_list, in xilinx_dma_start_transfer()
1557 tail_segment = list_last_entry(&tail_desc->segments, in xilinx_dma_start_transfer()
1562 if (chan->desc_pendingcount <= XILINX_DMA_COALESCE_MAX) { in xilinx_dma_start_transfer()
1564 reg |= chan->desc_pendingcount << in xilinx_dma_start_transfer()
1569 if (chan->has_sg) in xilinx_dma_start_transfer()
1571 head_desc->async_tx.phys); in xilinx_dma_start_transfer()
1573 reg |= chan->irq_delay << XILINX_DMA_CR_DELAY_SHIFT; in xilinx_dma_start_transfer()
1578 if (chan->err) in xilinx_dma_start_transfer()
1582 if (chan->has_sg) { in xilinx_dma_start_transfer()
1583 if (chan->cyclic) in xilinx_dma_start_transfer()
1585 chan->cyclic_seg_v->phys); in xilinx_dma_start_transfer()
1588 tail_segment->phys); in xilinx_dma_start_transfer()
1593 segment = list_first_entry(&head_desc->segments, in xilinx_dma_start_transfer()
1596 hw = &segment->hw; in xilinx_dma_start_transfer()
1599 xilinx_prep_dma_addr_t(hw->buf_addr)); in xilinx_dma_start_transfer()
1603 hw->control & chan->xdev->max_buffer_len); in xilinx_dma_start_transfer()
1606 list_splice_tail_init(&chan->pending_list, &chan->active_list); in xilinx_dma_start_transfer()
1607 chan->desc_pendingcount = 0; in xilinx_dma_start_transfer()
1608 chan->idle = false; in xilinx_dma_start_transfer()
1612 * xilinx_mcdma_start_transfer - Starts MCDMA transfer
1626 if (chan->err) in xilinx_mcdma_start_transfer()
1629 if (!chan->idle) in xilinx_mcdma_start_transfer()
1632 if (list_empty(&chan->pending_list)) in xilinx_mcdma_start_transfer()
1635 head_desc = list_first_entry(&chan->pending_list, in xilinx_mcdma_start_transfer()
1637 tail_desc = list_last_entry(&chan->pending_list, in xilinx_mcdma_start_transfer()
1639 tail_segment = list_last_entry(&tail_desc->segments, in xilinx_mcdma_start_transfer()
1642 reg = dma_ctrl_read(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest)); in xilinx_mcdma_start_transfer()
1644 if (chan->desc_pendingcount <= XILINX_MCDMA_COALESCE_MAX) { in xilinx_mcdma_start_transfer()
1646 reg |= chan->desc_pendingcount << in xilinx_mcdma_start_transfer()
1651 dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg); in xilinx_mcdma_start_transfer()
1654 xilinx_write(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET(chan->tdest), in xilinx_mcdma_start_transfer()
1655 head_desc->async_tx.phys); in xilinx_mcdma_start_transfer()
1659 reg |= BIT(chan->tdest); in xilinx_mcdma_start_transfer()
1663 reg = dma_ctrl_read(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest)); in xilinx_mcdma_start_transfer()
1665 dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg); in xilinx_mcdma_start_transfer()
1669 if (chan->err) in xilinx_mcdma_start_transfer()
1673 xilinx_write(chan, XILINX_MCDMA_CHAN_TDESC_OFFSET(chan->tdest), in xilinx_mcdma_start_transfer()
1674 tail_segment->phys); in xilinx_mcdma_start_transfer()
1676 list_splice_tail_init(&chan->pending_list, &chan->active_list); in xilinx_mcdma_start_transfer()
1677 chan->desc_pendingcount = 0; in xilinx_mcdma_start_transfer()
1678 chan->idle = false; in xilinx_mcdma_start_transfer()
1682 * xilinx_dma_issue_pending - Issue pending transactions
1690 spin_lock_irqsave(&chan->lock, flags); in xilinx_dma_issue_pending()
1691 chan->start_transfer(chan); in xilinx_dma_issue_pending()
1692 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dma_issue_pending()
1696 * xilinx_dma_device_config - Configure the DMA channel
1709 * xilinx_dma_complete_descriptor - Mark the active descriptor as complete
1719 if (list_empty(&chan->active_list)) in xilinx_dma_complete_descriptor()
1722 list_for_each_entry_safe(desc, next, &chan->active_list, node) { in xilinx_dma_complete_descriptor()
1723 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { in xilinx_dma_complete_descriptor()
1726 seg = list_last_entry(&desc->segments, in xilinx_dma_complete_descriptor()
1728 if (!(seg->hw.status & XILINX_DMA_BD_COMP_MASK) && chan->has_sg) in xilinx_dma_complete_descriptor()
1731 if (chan->has_sg && chan->xdev->dma_config->dmatype != in xilinx_dma_complete_descriptor()
1733 desc->residue = xilinx_dma_get_residue(chan, desc); in xilinx_dma_complete_descriptor()
1735 desc->residue = 0; in xilinx_dma_complete_descriptor()
1736 desc->err = chan->err; in xilinx_dma_complete_descriptor()
1738 list_del(&desc->node); in xilinx_dma_complete_descriptor()
1739 if (!desc->cyclic) in xilinx_dma_complete_descriptor()
1740 dma_cookie_complete(&desc->async_tx); in xilinx_dma_complete_descriptor()
1741 list_add_tail(&desc->node, &chan->done_list); in xilinx_dma_complete_descriptor()
1746 * xilinx_dma_reset - Reset DMA channel
1764 dev_err(chan->dev, "reset timeout, cr %x, sr %x\n", in xilinx_dma_reset()
1767 return -ETIMEDOUT; in xilinx_dma_reset()
1770 chan->err = false; in xilinx_dma_reset()
1771 chan->idle = true; in xilinx_dma_reset()
1772 chan->desc_pendingcount = 0; in xilinx_dma_reset()
1773 chan->desc_submitcount = 0; in xilinx_dma_reset()
1779 * xilinx_dma_chan_reset - Reset DMA channel and enable interrupts
1801 * xilinx_mcdma_irq_handler - MCDMA Interrupt handler
1812 if (chan->direction == DMA_DEV_TO_MEM) in xilinx_mcdma_irq_handler()
1824 if (chan->direction == DMA_DEV_TO_MEM) in xilinx_mcdma_irq_handler()
1825 chan_offset = chan->xdev->dma_config->max_channels / 2; in xilinx_mcdma_irq_handler()
1827 chan_offset = chan_offset + (chan_id - 1); in xilinx_mcdma_irq_handler()
1828 chan = chan->xdev->chan[chan_offset]; in xilinx_mcdma_irq_handler()
1830 status = dma_ctrl_read(chan, XILINX_MCDMA_CHAN_SR_OFFSET(chan->tdest)); in xilinx_mcdma_irq_handler()
1834 dma_ctrl_write(chan, XILINX_MCDMA_CHAN_SR_OFFSET(chan->tdest), in xilinx_mcdma_irq_handler()
1838 dev_err(chan->dev, "Channel %p has errors %x cdr %x tdr %x\n", in xilinx_mcdma_irq_handler()
1842 (chan->tdest)), in xilinx_mcdma_irq_handler()
1844 (chan->tdest))); in xilinx_mcdma_irq_handler()
1845 chan->err = true; in xilinx_mcdma_irq_handler()
1853 dev_dbg(chan->dev, "Inter-packet latency too long\n"); in xilinx_mcdma_irq_handler()
1857 spin_lock(&chan->lock); in xilinx_mcdma_irq_handler()
1859 chan->idle = true; in xilinx_mcdma_irq_handler()
1860 chan->start_transfer(chan); in xilinx_mcdma_irq_handler()
1861 spin_unlock(&chan->lock); in xilinx_mcdma_irq_handler()
1864 tasklet_hi_schedule(&chan->tasklet); in xilinx_mcdma_irq_handler()
1869 * xilinx_dma_irq_handler - DMA Interrupt handler
1901 if (!chan->flush_on_fsync || in xilinx_dma_irq_handler()
1903 dev_err(chan->dev, in xilinx_dma_irq_handler()
1908 chan->err = true; in xilinx_dma_irq_handler()
1914 spin_lock(&chan->lock); in xilinx_dma_irq_handler()
1916 chan->idle = true; in xilinx_dma_irq_handler()
1917 chan->start_transfer(chan); in xilinx_dma_irq_handler()
1918 spin_unlock(&chan->lock); in xilinx_dma_irq_handler()
1921 tasklet_schedule(&chan->tasklet); in xilinx_dma_irq_handler()
1926 * append_desc_queue - Queuing descriptor
1939 if (list_empty(&chan->pending_list)) in append_desc_queue()
1946 tail_desc = list_last_entry(&chan->pending_list, in append_desc_queue()
1948 if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { in append_desc_queue()
1949 tail_segment = list_last_entry(&tail_desc->segments, in append_desc_queue()
1952 tail_segment->hw.next_desc = (u32)desc->async_tx.phys; in append_desc_queue()
1953 } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { in append_desc_queue()
1954 cdma_tail_segment = list_last_entry(&tail_desc->segments, in append_desc_queue()
1957 cdma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys; in append_desc_queue()
1958 } else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { in append_desc_queue()
1959 axidma_tail_segment = list_last_entry(&tail_desc->segments, in append_desc_queue()
1962 axidma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys; in append_desc_queue()
1965 list_last_entry(&tail_desc->segments, in append_desc_queue()
1968 aximcdma_tail_segment->hw.next_desc = (u32)desc->async_tx.phys; in append_desc_queue()
1976 list_add_tail(&desc->node, &chan->pending_list); in append_desc_queue()
1977 chan->desc_pendingcount++; in append_desc_queue()
1979 if (chan->has_sg && (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) in append_desc_queue()
1980 && unlikely(chan->desc_pendingcount > chan->num_frms)) { in append_desc_queue()
1981 dev_dbg(chan->dev, "desc pendingcount is too high\n"); in append_desc_queue()
1982 chan->desc_pendingcount = chan->num_frms; in append_desc_queue()
1987 * xilinx_dma_tx_submit - Submit DMA transaction
1995 struct xilinx_dma_chan *chan = to_xilinx_chan(tx->chan); in xilinx_dma_tx_submit()
2000 if (chan->cyclic) { in xilinx_dma_tx_submit()
2002 return -EBUSY; in xilinx_dma_tx_submit()
2005 if (chan->err) { in xilinx_dma_tx_submit()
2015 spin_lock_irqsave(&chan->lock, flags); in xilinx_dma_tx_submit()
2022 if (desc->cyclic) in xilinx_dma_tx_submit()
2023 chan->cyclic = true; in xilinx_dma_tx_submit()
2025 chan->terminating = false; in xilinx_dma_tx_submit()
2027 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dma_tx_submit()
2033 * xilinx_vdma_dma_prep_interleaved - prepare a descriptor for a
2051 if (!is_slave_direction(xt->dir)) in xilinx_vdma_dma_prep_interleaved()
2054 if (!xt->numf || !xt->sgl[0].size) in xilinx_vdma_dma_prep_interleaved()
2057 if (xt->numf & ~XILINX_DMA_VSIZE_MASK || in xilinx_vdma_dma_prep_interleaved()
2058 xt->sgl[0].size & ~XILINX_DMA_HSIZE_MASK) in xilinx_vdma_dma_prep_interleaved()
2061 if (xt->frame_size != 1) in xilinx_vdma_dma_prep_interleaved()
2069 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); in xilinx_vdma_dma_prep_interleaved()
2070 desc->async_tx.tx_submit = xilinx_dma_tx_submit; in xilinx_vdma_dma_prep_interleaved()
2071 async_tx_ack(&desc->async_tx); in xilinx_vdma_dma_prep_interleaved()
2079 hw = &segment->hw; in xilinx_vdma_dma_prep_interleaved()
2080 hw->vsize = xt->numf; in xilinx_vdma_dma_prep_interleaved()
2081 hw->hsize = xt->sgl[0].size; in xilinx_vdma_dma_prep_interleaved()
2082 hw->stride = (xt->sgl[0].icg + xt->sgl[0].size) << in xilinx_vdma_dma_prep_interleaved()
2084 hw->stride |= chan->config.frm_dly << in xilinx_vdma_dma_prep_interleaved()
2087 if (xt->dir != DMA_MEM_TO_DEV) { in xilinx_vdma_dma_prep_interleaved()
2088 if (chan->ext_addr) { in xilinx_vdma_dma_prep_interleaved()
2089 hw->buf_addr = lower_32_bits(xt->dst_start); in xilinx_vdma_dma_prep_interleaved()
2090 hw->buf_addr_msb = upper_32_bits(xt->dst_start); in xilinx_vdma_dma_prep_interleaved()
2092 hw->buf_addr = xt->dst_start; in xilinx_vdma_dma_prep_interleaved()
2095 if (chan->ext_addr) { in xilinx_vdma_dma_prep_interleaved()
2096 hw->buf_addr = lower_32_bits(xt->src_start); in xilinx_vdma_dma_prep_interleaved()
2097 hw->buf_addr_msb = upper_32_bits(xt->src_start); in xilinx_vdma_dma_prep_interleaved()
2099 hw->buf_addr = xt->src_start; in xilinx_vdma_dma_prep_interleaved()
2104 list_add_tail(&segment->node, &desc->segments); in xilinx_vdma_dma_prep_interleaved()
2107 segment = list_first_entry(&desc->segments, in xilinx_vdma_dma_prep_interleaved()
2109 desc->async_tx.phys = segment->phys; in xilinx_vdma_dma_prep_interleaved()
2111 return &desc->async_tx; in xilinx_vdma_dma_prep_interleaved()
2119 * xilinx_cdma_prep_memcpy - prepare descriptors for a memcpy transaction
2137 if (!len || len > chan->xdev->max_buffer_len) in xilinx_cdma_prep_memcpy()
2144 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); in xilinx_cdma_prep_memcpy()
2145 desc->async_tx.tx_submit = xilinx_dma_tx_submit; in xilinx_cdma_prep_memcpy()
2152 hw = &segment->hw; in xilinx_cdma_prep_memcpy()
2153 hw->control = len; in xilinx_cdma_prep_memcpy()
2154 hw->src_addr = dma_src; in xilinx_cdma_prep_memcpy()
2155 hw->dest_addr = dma_dst; in xilinx_cdma_prep_memcpy()
2156 if (chan->ext_addr) { in xilinx_cdma_prep_memcpy()
2157 hw->src_addr_msb = upper_32_bits(dma_src); in xilinx_cdma_prep_memcpy()
2158 hw->dest_addr_msb = upper_32_bits(dma_dst); in xilinx_cdma_prep_memcpy()
2162 list_add_tail(&segment->node, &desc->segments); in xilinx_cdma_prep_memcpy()
2164 desc->async_tx.phys = segment->phys; in xilinx_cdma_prep_memcpy()
2165 hw->next_desc = segment->phys; in xilinx_cdma_prep_memcpy()
2167 return &desc->async_tx; in xilinx_cdma_prep_memcpy()
2175 * xilinx_dma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
2207 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); in xilinx_dma_prep_slave_sg()
2208 desc->async_tx.tx_submit = xilinx_dma_tx_submit; in xilinx_dma_prep_slave_sg()
2229 hw = &segment->hw; in xilinx_dma_prep_slave_sg()
2235 hw->control = copy; in xilinx_dma_prep_slave_sg()
2237 if (chan->direction == DMA_MEM_TO_DEV) { in xilinx_dma_prep_slave_sg()
2239 memcpy(hw->app, app_w, sizeof(u32) * in xilinx_dma_prep_slave_sg()
2249 list_add_tail(&segment->node, &desc->segments); in xilinx_dma_prep_slave_sg()
2253 segment = list_first_entry(&desc->segments, in xilinx_dma_prep_slave_sg()
2255 desc->async_tx.phys = segment->phys; in xilinx_dma_prep_slave_sg()
2258 if (chan->direction == DMA_MEM_TO_DEV) { in xilinx_dma_prep_slave_sg()
2259 segment->hw.control |= XILINX_DMA_BD_SOP; in xilinx_dma_prep_slave_sg()
2260 segment = list_last_entry(&desc->segments, in xilinx_dma_prep_slave_sg()
2263 segment->hw.control |= XILINX_DMA_BD_EOP; in xilinx_dma_prep_slave_sg()
2266 if (chan->xdev->has_axistream_connected) in xilinx_dma_prep_slave_sg()
2267 desc->async_tx.metadata_ops = &xilinx_dma_metadata_ops; in xilinx_dma_prep_slave_sg()
2269 return &desc->async_tx; in xilinx_dma_prep_slave_sg()
2277 * xilinx_dma_prep_dma_cyclic - prepare descriptors for a DMA_SLAVE transaction
2316 chan->direction = direction; in xilinx_dma_prep_dma_cyclic()
2317 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); in xilinx_dma_prep_dma_cyclic()
2318 desc->async_tx.tx_submit = xilinx_dma_tx_submit; in xilinx_dma_prep_dma_cyclic()
2337 hw = &segment->hw; in xilinx_dma_prep_dma_cyclic()
2340 hw->control = copy; in xilinx_dma_prep_dma_cyclic()
2343 prev->hw.next_desc = segment->phys; in xilinx_dma_prep_dma_cyclic()
2352 list_add_tail(&segment->node, &desc->segments); in xilinx_dma_prep_dma_cyclic()
2356 head_segment = list_first_entry(&desc->segments, in xilinx_dma_prep_dma_cyclic()
2358 desc->async_tx.phys = head_segment->phys; in xilinx_dma_prep_dma_cyclic()
2360 desc->cyclic = true; in xilinx_dma_prep_dma_cyclic()
2365 segment = list_last_entry(&desc->segments, in xilinx_dma_prep_dma_cyclic()
2368 segment->hw.next_desc = (u32) head_segment->phys; in xilinx_dma_prep_dma_cyclic()
2372 head_segment->hw.control |= XILINX_DMA_BD_SOP; in xilinx_dma_prep_dma_cyclic()
2373 segment->hw.control |= XILINX_DMA_BD_EOP; in xilinx_dma_prep_dma_cyclic()
2376 return &desc->async_tx; in xilinx_dma_prep_dma_cyclic()
2384 * xilinx_mcdma_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
2417 dma_async_tx_descriptor_init(&desc->async_tx, &chan->common); in xilinx_mcdma_prep_slave_sg()
2418 desc->async_tx.tx_submit = xilinx_dma_tx_submit; in xilinx_mcdma_prep_slave_sg()
2437 copy = min_t(size_t, sg_dma_len(sg) - sg_used, in xilinx_mcdma_prep_slave_sg()
2438 chan->xdev->max_buffer_len); in xilinx_mcdma_prep_slave_sg()
2439 hw = &segment->hw; in xilinx_mcdma_prep_slave_sg()
2444 hw->control = copy; in xilinx_mcdma_prep_slave_sg()
2446 if (chan->direction == DMA_MEM_TO_DEV && app_w) { in xilinx_mcdma_prep_slave_sg()
2447 memcpy(hw->app, app_w, sizeof(u32) * in xilinx_mcdma_prep_slave_sg()
2456 list_add_tail(&segment->node, &desc->segments); in xilinx_mcdma_prep_slave_sg()
2460 segment = list_first_entry(&desc->segments, in xilinx_mcdma_prep_slave_sg()
2462 desc->async_tx.phys = segment->phys; in xilinx_mcdma_prep_slave_sg()
2465 if (chan->direction == DMA_MEM_TO_DEV) { in xilinx_mcdma_prep_slave_sg()
2466 segment->hw.control |= XILINX_MCDMA_BD_SOP; in xilinx_mcdma_prep_slave_sg()
2467 segment = list_last_entry(&desc->segments, in xilinx_mcdma_prep_slave_sg()
2470 segment->hw.control |= XILINX_MCDMA_BD_EOP; in xilinx_mcdma_prep_slave_sg()
2473 return &desc->async_tx; in xilinx_mcdma_prep_slave_sg()
2482 * xilinx_dma_terminate_all - Halt the channel and free descriptors
2493 if (!chan->cyclic) { in xilinx_dma_terminate_all()
2494 err = chan->stop_transfer(chan); in xilinx_dma_terminate_all()
2496 dev_err(chan->dev, "Cannot stop channel %p: %x\n", in xilinx_dma_terminate_all()
2499 chan->err = true; in xilinx_dma_terminate_all()
2505 chan->terminating = true; in xilinx_dma_terminate_all()
2507 chan->idle = true; in xilinx_dma_terminate_all()
2509 if (chan->cyclic) { in xilinx_dma_terminate_all()
2513 chan->cyclic = false; in xilinx_dma_terminate_all()
2516 if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg) in xilinx_dma_terminate_all()
2527 tasklet_kill(&chan->tasklet); in xilinx_dma_synchronize()
2531 * xilinx_vdma_channel_set_config - Configure VDMA channel
2532 * Run-time configuration for Axi VDMA, supports:
2534 * . configure interrupt coalescing and inter-packet delay threshold
2549 if (cfg->reset) in xilinx_vdma_channel_set_config()
2554 chan->config.frm_dly = cfg->frm_dly; in xilinx_vdma_channel_set_config()
2555 chan->config.park = cfg->park; in xilinx_vdma_channel_set_config()
2558 chan->config.gen_lock = cfg->gen_lock; in xilinx_vdma_channel_set_config()
2559 chan->config.master = cfg->master; in xilinx_vdma_channel_set_config()
2562 if (cfg->gen_lock && chan->genlock) { in xilinx_vdma_channel_set_config()
2565 dmacr |= cfg->master << XILINX_DMA_DMACR_MASTER_SHIFT; in xilinx_vdma_channel_set_config()
2568 chan->config.frm_cnt_en = cfg->frm_cnt_en; in xilinx_vdma_channel_set_config()
2569 chan->config.vflip_en = cfg->vflip_en; in xilinx_vdma_channel_set_config()
2571 if (cfg->park) in xilinx_vdma_channel_set_config()
2572 chan->config.park_frm = cfg->park_frm; in xilinx_vdma_channel_set_config()
2574 chan->config.park_frm = -1; in xilinx_vdma_channel_set_config()
2576 chan->config.coalesc = cfg->coalesc; in xilinx_vdma_channel_set_config()
2577 chan->config.delay = cfg->delay; in xilinx_vdma_channel_set_config()
2579 if (cfg->coalesc <= XILINX_DMA_DMACR_FRAME_COUNT_MAX) { in xilinx_vdma_channel_set_config()
2581 dmacr |= cfg->coalesc << XILINX_DMA_DMACR_FRAME_COUNT_SHIFT; in xilinx_vdma_channel_set_config()
2582 chan->config.coalesc = cfg->coalesc; in xilinx_vdma_channel_set_config()
2585 if (cfg->delay <= XILINX_DMA_DMACR_DELAY_MAX) { in xilinx_vdma_channel_set_config()
2587 dmacr |= cfg->delay << XILINX_DMA_DMACR_DELAY_SHIFT; in xilinx_vdma_channel_set_config()
2588 chan->config.delay = cfg->delay; in xilinx_vdma_channel_set_config()
2593 dmacr |= cfg->ext_fsync << XILINX_DMA_DMACR_FSYNCSRC_SHIFT; in xilinx_vdma_channel_set_config()
2601 /* -----------------------------------------------------------------------------
2606 * xilinx_dma_chan_remove - Per Channel remove function
2615 if (chan->irq > 0) in xilinx_dma_chan_remove()
2616 free_irq(chan->irq, chan); in xilinx_dma_chan_remove()
2618 tasklet_kill(&chan->tasklet); in xilinx_dma_chan_remove()
2620 list_del(&chan->common.device_node); in xilinx_dma_chan_remove()
2631 *axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk"); in axidma_clk_init()
2633 return dev_err_probe(&pdev->dev, PTR_ERR(*axi_clk), "failed to get axi_aclk\n"); in axidma_clk_init()
2635 *tx_clk = devm_clk_get(&pdev->dev, "m_axi_mm2s_aclk"); in axidma_clk_init()
2639 *rx_clk = devm_clk_get(&pdev->dev, "m_axi_s2mm_aclk"); in axidma_clk_init()
2643 *sg_clk = devm_clk_get(&pdev->dev, "m_axi_sg_aclk"); in axidma_clk_init()
2649 dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n", err); in axidma_clk_init()
2655 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); in axidma_clk_init()
2661 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); in axidma_clk_init()
2667 dev_err(&pdev->dev, "failed to enable sg_clk (%d)\n", err); in axidma_clk_init()
2693 *axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk"); in axicdma_clk_init()
2695 return dev_err_probe(&pdev->dev, PTR_ERR(*axi_clk), "failed to get axi_aclk\n"); in axicdma_clk_init()
2697 *dev_clk = devm_clk_get(&pdev->dev, "m_axi_aclk"); in axicdma_clk_init()
2699 return dev_err_probe(&pdev->dev, PTR_ERR(*dev_clk), "failed to get dev_clk\n"); in axicdma_clk_init()
2703 dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n", err); in axicdma_clk_init()
2709 dev_err(&pdev->dev, "failed to enable dev_clk (%d)\n", err); in axicdma_clk_init()
2727 *axi_clk = devm_clk_get(&pdev->dev, "s_axi_lite_aclk"); in axivdma_clk_init()
2729 return dev_err_probe(&pdev->dev, PTR_ERR(*axi_clk), "failed to get axi_aclk\n"); in axivdma_clk_init()
2731 *tx_clk = devm_clk_get(&pdev->dev, "m_axi_mm2s_aclk"); in axivdma_clk_init()
2735 *txs_clk = devm_clk_get(&pdev->dev, "m_axis_mm2s_aclk"); in axivdma_clk_init()
2739 *rx_clk = devm_clk_get(&pdev->dev, "m_axi_s2mm_aclk"); in axivdma_clk_init()
2743 *rxs_clk = devm_clk_get(&pdev->dev, "s_axis_s2mm_aclk"); in axivdma_clk_init()
2749 dev_err(&pdev->dev, "failed to enable axi_clk (%d)\n", in axivdma_clk_init()
2756 dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err); in axivdma_clk_init()
2762 dev_err(&pdev->dev, "failed to enable txs_clk (%d)\n", err); in axivdma_clk_init()
2768 dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err); in axivdma_clk_init()
2774 dev_err(&pdev->dev, "failed to enable rxs_clk (%d)\n", err); in axivdma_clk_init()
2794 clk_disable_unprepare(xdev->rxs_clk); in xdma_disable_allclks()
2795 clk_disable_unprepare(xdev->rx_clk); in xdma_disable_allclks()
2796 clk_disable_unprepare(xdev->txs_clk); in xdma_disable_allclks()
2797 clk_disable_unprepare(xdev->tx_clk); in xdma_disable_allclks()
2798 clk_disable_unprepare(xdev->axi_clk); in xdma_disable_allclks()
2802 * xilinx_dma_chan_probe - Per Channel Probing
2820 chan = devm_kzalloc(xdev->dev, sizeof(*chan), GFP_KERNEL); in xilinx_dma_chan_probe()
2822 return -ENOMEM; in xilinx_dma_chan_probe()
2824 chan->dev = xdev->dev; in xilinx_dma_chan_probe()
2825 chan->xdev = xdev; in xilinx_dma_chan_probe()
2826 chan->desc_pendingcount = 0x0; in xilinx_dma_chan_probe()
2827 chan->ext_addr = xdev->ext_addr; in xilinx_dma_chan_probe()
2833 chan->idle = true; in xilinx_dma_chan_probe()
2835 spin_lock_init(&chan->lock); in xilinx_dma_chan_probe()
2836 INIT_LIST_HEAD(&chan->pending_list); in xilinx_dma_chan_probe()
2837 INIT_LIST_HEAD(&chan->done_list); in xilinx_dma_chan_probe()
2838 INIT_LIST_HEAD(&chan->active_list); in xilinx_dma_chan_probe()
2839 INIT_LIST_HEAD(&chan->free_seg_list); in xilinx_dma_chan_probe()
2842 has_dre = of_property_read_bool(node, "xlnx,include-dre"); in xilinx_dma_chan_probe()
2844 of_property_read_u8(node, "xlnx,irq-delay", &chan->irq_delay); in xilinx_dma_chan_probe()
2846 chan->genlock = of_property_read_bool(node, "xlnx,genlock-mode"); in xilinx_dma_chan_probe()
2850 dev_err(xdev->dev, "missing xlnx,datawidth property\n"); in xilinx_dma_chan_probe()
2860 xdev->common.copy_align = (enum dmaengine_alignment)fls(width - 1); in xilinx_dma_chan_probe()
2862 if (of_device_is_compatible(node, "xlnx,axi-vdma-mm2s-channel") || in xilinx_dma_chan_probe()
2863 of_device_is_compatible(node, "xlnx,axi-dma-mm2s-channel") || in xilinx_dma_chan_probe()
2864 of_device_is_compatible(node, "xlnx,axi-cdma-channel")) { in xilinx_dma_chan_probe()
2865 chan->direction = DMA_MEM_TO_DEV; in xilinx_dma_chan_probe()
2866 chan->id = xdev->mm2s_chan_id++; in xilinx_dma_chan_probe()
2867 chan->tdest = chan->id; in xilinx_dma_chan_probe()
2869 chan->ctrl_offset = XILINX_DMA_MM2S_CTRL_OFFSET; in xilinx_dma_chan_probe()
2870 if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { in xilinx_dma_chan_probe()
2871 chan->desc_offset = XILINX_VDMA_MM2S_DESC_OFFSET; in xilinx_dma_chan_probe()
2872 chan->config.park = 1; in xilinx_dma_chan_probe()
2874 if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH || in xilinx_dma_chan_probe()
2875 xdev->flush_on_fsync == XILINX_DMA_FLUSH_MM2S) in xilinx_dma_chan_probe()
2876 chan->flush_on_fsync = true; in xilinx_dma_chan_probe()
2879 "xlnx,axi-vdma-s2mm-channel") || in xilinx_dma_chan_probe()
2881 "xlnx,axi-dma-s2mm-channel")) { in xilinx_dma_chan_probe()
2882 chan->direction = DMA_DEV_TO_MEM; in xilinx_dma_chan_probe()
2883 chan->id = xdev->s2mm_chan_id++; in xilinx_dma_chan_probe()
2884 chan->tdest = chan->id - xdev->dma_config->max_channels / 2; in xilinx_dma_chan_probe()
2885 chan->has_vflip = of_property_read_bool(node, in xilinx_dma_chan_probe()
2886 "xlnx,enable-vert-flip"); in xilinx_dma_chan_probe()
2887 if (chan->has_vflip) { in xilinx_dma_chan_probe()
2888 chan->config.vflip_en = dma_read(chan, in xilinx_dma_chan_probe()
2893 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) in xilinx_dma_chan_probe()
2894 chan->ctrl_offset = XILINX_MCDMA_S2MM_CTRL_OFFSET; in xilinx_dma_chan_probe()
2896 chan->ctrl_offset = XILINX_DMA_S2MM_CTRL_OFFSET; in xilinx_dma_chan_probe()
2898 if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { in xilinx_dma_chan_probe()
2899 chan->desc_offset = XILINX_VDMA_S2MM_DESC_OFFSET; in xilinx_dma_chan_probe()
2900 chan->config.park = 1; in xilinx_dma_chan_probe()
2902 if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH || in xilinx_dma_chan_probe()
2903 xdev->flush_on_fsync == XILINX_DMA_FLUSH_S2MM) in xilinx_dma_chan_probe()
2904 chan->flush_on_fsync = true; in xilinx_dma_chan_probe()
2907 dev_err(xdev->dev, "Invalid channel compatible node\n"); in xilinx_dma_chan_probe()
2908 return -EINVAL; in xilinx_dma_chan_probe()
2912 chan->irq = of_irq_get(node, chan->tdest); in xilinx_dma_chan_probe()
2913 if (chan->irq < 0) in xilinx_dma_chan_probe()
2914 return dev_err_probe(xdev->dev, chan->irq, "failed to get irq\n"); in xilinx_dma_chan_probe()
2915 err = request_irq(chan->irq, xdev->dma_config->irq_handler, in xilinx_dma_chan_probe()
2916 IRQF_SHARED, "xilinx-dma-controller", chan); in xilinx_dma_chan_probe()
2918 dev_err(xdev->dev, "unable to request IRQ %d\n", chan->irq); in xilinx_dma_chan_probe()
2922 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { in xilinx_dma_chan_probe()
2923 chan->start_transfer = xilinx_dma_start_transfer; in xilinx_dma_chan_probe()
2924 chan->stop_transfer = xilinx_dma_stop_transfer; in xilinx_dma_chan_probe()
2925 } else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { in xilinx_dma_chan_probe()
2926 chan->start_transfer = xilinx_mcdma_start_transfer; in xilinx_dma_chan_probe()
2927 chan->stop_transfer = xilinx_dma_stop_transfer; in xilinx_dma_chan_probe()
2928 } else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { in xilinx_dma_chan_probe()
2929 chan->start_transfer = xilinx_cdma_start_transfer; in xilinx_dma_chan_probe()
2930 chan->stop_transfer = xilinx_cdma_stop_transfer; in xilinx_dma_chan_probe()
2932 chan->start_transfer = xilinx_vdma_start_transfer; in xilinx_dma_chan_probe()
2933 chan->stop_transfer = xilinx_dma_stop_transfer; in xilinx_dma_chan_probe()
2937 if (xdev->dma_config->dmatype != XDMA_TYPE_VDMA) { in xilinx_dma_chan_probe()
2938 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA || in xilinx_dma_chan_probe()
2941 chan->has_sg = true; in xilinx_dma_chan_probe()
2942 dev_dbg(chan->dev, "ch %d: SG %s\n", chan->id, in xilinx_dma_chan_probe()
2943 chan->has_sg ? "enabled" : "disabled"); in xilinx_dma_chan_probe()
2947 tasklet_setup(&chan->tasklet, xilinx_dma_do_tasklet); in xilinx_dma_chan_probe()
2953 chan->common.device = &xdev->common; in xilinx_dma_chan_probe()
2955 list_add_tail(&chan->common.device_node, &xdev->common.channels); in xilinx_dma_chan_probe()
2956 xdev->chan[chan->id] = chan; in xilinx_dma_chan_probe()
2961 dev_err(xdev->dev, "Reset channel failed\n"); in xilinx_dma_chan_probe()
2969 * xilinx_dma_child_probe - Per child node probe
2970 * It get number of dma-channels per child node from
2971 * device-tree and initializes all the channels.
2984 ret = of_property_read_u32(node, "dma-channels", &nr_channels); in xilinx_dma_child_probe()
2985 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA && ret < 0) in xilinx_dma_child_probe()
2986 dev_warn(xdev->dev, "missing dma-channels property\n"); in xilinx_dma_child_probe()
2998 * of_dma_xilinx_xlate - Translation function
3007 struct xilinx_dma_device *xdev = ofdma->of_dma_data; in of_dma_xilinx_xlate()
3008 int chan_id = dma_spec->args[0]; in of_dma_xilinx_xlate()
3010 if (chan_id >= xdev->dma_config->max_channels || !xdev->chan[chan_id]) in of_dma_xilinx_xlate()
3013 return dma_get_slave_channel(&xdev->chan[chan_id]->common); in of_dma_xilinx_xlate()
3044 { .compatible = "xlnx,axi-dma-1.00.a", .data = &axidma_config },
3045 { .compatible = "xlnx,axi-cdma-1.00.a", .data = &axicdma_config },
3046 { .compatible = "xlnx,axi-vdma-1.00.a", .data = &axivdma_config },
3047 { .compatible = "xlnx,axi-mcdma-1.00.a", .data = &aximcdma_config },
3053 * xilinx_dma_probe - Driver probe function
3063 struct device_node *node = pdev->dev.of_node; in xilinx_dma_probe()
3065 struct device_node *child, *np = pdev->dev.of_node; in xilinx_dma_probe()
3070 xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL); in xilinx_dma_probe()
3072 return -ENOMEM; in xilinx_dma_probe()
3074 xdev->dev = &pdev->dev; in xilinx_dma_probe()
3079 if (match && match->data) { in xilinx_dma_probe()
3080 xdev->dma_config = match->data; in xilinx_dma_probe()
3081 clk_init = xdev->dma_config->clk_init; in xilinx_dma_probe()
3085 err = clk_init(pdev, &xdev->axi_clk, &xdev->tx_clk, &xdev->txs_clk, in xilinx_dma_probe()
3086 &xdev->rx_clk, &xdev->rxs_clk); in xilinx_dma_probe()
3091 xdev->regs = devm_platform_ioremap_resource(pdev, 0); in xilinx_dma_probe()
3092 if (IS_ERR(xdev->regs)) { in xilinx_dma_probe()
3093 err = PTR_ERR(xdev->regs); in xilinx_dma_probe()
3097 xdev->max_buffer_len = GENMASK(XILINX_DMA_MAX_TRANS_LEN_MAX - 1, 0); in xilinx_dma_probe()
3098 xdev->s2mm_chan_id = xdev->dma_config->max_channels / 2; in xilinx_dma_probe()
3100 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA || in xilinx_dma_probe()
3101 xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { in xilinx_dma_probe()
3102 if (!of_property_read_u32(node, "xlnx,sg-length-width", in xilinx_dma_probe()
3106 dev_warn(xdev->dev, in xilinx_dma_probe()
3107 "invalid xlnx,sg-length-width property value. Using default width\n"); in xilinx_dma_probe()
3110 dev_warn(xdev->dev, "Please ensure that IP supports buffer length > 23 bits\n"); in xilinx_dma_probe()
3111 xdev->max_buffer_len = in xilinx_dma_probe()
3112 GENMASK(len_width - 1, 0); in xilinx_dma_probe()
3117 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { in xilinx_dma_probe()
3118 xdev->has_axistream_connected = in xilinx_dma_probe()
3119 of_property_read_bool(node, "xlnx,axistream-connected"); in xilinx_dma_probe()
3122 if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { in xilinx_dma_probe()
3123 err = of_property_read_u32(node, "xlnx,num-fstores", in xilinx_dma_probe()
3126 dev_err(xdev->dev, in xilinx_dma_probe()
3127 "missing xlnx,num-fstores property\n"); in xilinx_dma_probe()
3131 err = of_property_read_u32(node, "xlnx,flush-fsync", in xilinx_dma_probe()
3132 &xdev->flush_on_fsync); in xilinx_dma_probe()
3134 dev_warn(xdev->dev, in xilinx_dma_probe()
3135 "missing xlnx,flush-fsync property\n"); in xilinx_dma_probe()
3140 dev_warn(xdev->dev, "missing xlnx,addrwidth property\n"); in xilinx_dma_probe()
3143 xdev->ext_addr = true; in xilinx_dma_probe()
3145 xdev->ext_addr = false; in xilinx_dma_probe()
3148 if (xdev->has_axistream_connected) in xilinx_dma_probe()
3149 xdev->common.desc_metadata_modes = DESC_METADATA_ENGINE; in xilinx_dma_probe()
3152 err = dma_set_mask_and_coherent(xdev->dev, DMA_BIT_MASK(addr_width)); in xilinx_dma_probe()
3154 dev_err(xdev->dev, "DMA mask error %d\n", err); in xilinx_dma_probe()
3159 xdev->common.dev = &pdev->dev; in xilinx_dma_probe()
3161 INIT_LIST_HEAD(&xdev->common.channels); in xilinx_dma_probe()
3162 if (!(xdev->dma_config->dmatype == XDMA_TYPE_CDMA)) { in xilinx_dma_probe()
3163 dma_cap_set(DMA_SLAVE, xdev->common.cap_mask); in xilinx_dma_probe()
3164 dma_cap_set(DMA_PRIVATE, xdev->common.cap_mask); in xilinx_dma_probe()
3167 xdev->common.device_alloc_chan_resources = in xilinx_dma_probe()
3169 xdev->common.device_free_chan_resources = in xilinx_dma_probe()
3171 xdev->common.device_terminate_all = xilinx_dma_terminate_all; in xilinx_dma_probe()
3172 xdev->common.device_synchronize = xilinx_dma_synchronize; in xilinx_dma_probe()
3173 xdev->common.device_tx_status = xilinx_dma_tx_status; in xilinx_dma_probe()
3174 xdev->common.device_issue_pending = xilinx_dma_issue_pending; in xilinx_dma_probe()
3175 xdev->common.device_config = xilinx_dma_device_config; in xilinx_dma_probe()
3176 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) { in xilinx_dma_probe()
3177 dma_cap_set(DMA_CYCLIC, xdev->common.cap_mask); in xilinx_dma_probe()
3178 xdev->common.device_prep_slave_sg = xilinx_dma_prep_slave_sg; in xilinx_dma_probe()
3179 xdev->common.device_prep_dma_cyclic = in xilinx_dma_probe()
3182 xdev->common.residue_granularity = in xilinx_dma_probe()
3184 } else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) { in xilinx_dma_probe()
3185 dma_cap_set(DMA_MEMCPY, xdev->common.cap_mask); in xilinx_dma_probe()
3186 xdev->common.device_prep_dma_memcpy = xilinx_cdma_prep_memcpy; in xilinx_dma_probe()
3188 xdev->common.residue_granularity = in xilinx_dma_probe()
3190 } else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) { in xilinx_dma_probe()
3191 xdev->common.device_prep_slave_sg = xilinx_mcdma_prep_slave_sg; in xilinx_dma_probe()
3193 xdev->common.device_prep_interleaved_dma = in xilinx_dma_probe()
3208 if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) { in xilinx_dma_probe()
3209 for (i = 0; i < xdev->dma_config->max_channels; i++) in xilinx_dma_probe()
3210 if (xdev->chan[i]) in xilinx_dma_probe()
3211 xdev->chan[i]->num_frms = num_frames; in xilinx_dma_probe()
3215 err = dma_async_device_register(&xdev->common); in xilinx_dma_probe()
3217 dev_err(xdev->dev, "failed to register the dma device\n"); in xilinx_dma_probe()
3224 dev_err(&pdev->dev, "Unable to register DMA to DT\n"); in xilinx_dma_probe()
3225 dma_async_device_unregister(&xdev->common); in xilinx_dma_probe()
3229 if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) in xilinx_dma_probe()
3230 dev_info(&pdev->dev, "Xilinx AXI DMA Engine Driver Probed!!\n"); in xilinx_dma_probe()
3231 else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) in xilinx_dma_probe()
3232 dev_info(&pdev->dev, "Xilinx AXI CDMA Engine Driver Probed!!\n"); in xilinx_dma_probe()
3233 else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) in xilinx_dma_probe()
3234 dev_info(&pdev->dev, "Xilinx AXI MCDMA Engine Driver Probed!!\n"); in xilinx_dma_probe()
3236 dev_info(&pdev->dev, "Xilinx AXI VDMA Engine Driver Probed!!\n"); in xilinx_dma_probe()
3241 for (i = 0; i < xdev->dma_config->max_channels; i++) in xilinx_dma_probe()
3242 if (xdev->chan[i]) in xilinx_dma_probe()
3243 xilinx_dma_chan_remove(xdev->chan[i]); in xilinx_dma_probe()
3251 * xilinx_dma_remove - Driver remove function
3259 of_dma_controller_free(pdev->dev.of_node); in xilinx_dma_remove()
3261 dma_async_device_unregister(&xdev->common); in xilinx_dma_remove()
3263 for (i = 0; i < xdev->dma_config->max_channels; i++) in xilinx_dma_remove()
3264 if (xdev->chan[i]) in xilinx_dma_remove()
3265 xilinx_dma_chan_remove(xdev->chan[i]); in xilinx_dma_remove()
3272 .name = "xilinx-vdma",