Lines Matching +full:map +full:- +full:to +full:- +full:dma +full:- +full:channel
1 // SPDX-License-Identifier: GPL-2.0
3 * Test driver to test endpoint functionality
18 #include <linux/pci-epc.h>
19 #include <linux/pci-epf.h>
96 epf_test->transfer_status = in pci_epf_test_dma_callback()
97 dmaengine_tx_status(epf_test->transfer_chan, in pci_epf_test_dma_callback()
98 epf_test->transfer_cookie, &state); in pci_epf_test_dma_callback()
99 if (epf_test->transfer_status == DMA_COMPLETE || in pci_epf_test_dma_callback()
100 epf_test->transfer_status == DMA_ERROR) in pci_epf_test_dma_callback()
101 complete(&epf_test->transfer_complete); in pci_epf_test_dma_callback()
105 * pci_epf_test_data_transfer() - Function that uses dmaengine API to transfer
109 * address given by pci_epc_mem_alloc_addr or DMA mapping APIs.
111 * address given by pci_epc_mem_alloc_addr or DMA mapping APIs.
114 * @dir: DMA transfer direction
116 * Function that uses dmaengine API to transfer data between PCIe EP and remote
118 * by pci_epc_mem_alloc_addr or the one obtained using DMA mapping APIs.
128 epf_test->dma_chan_tx : epf_test->dma_chan_rx; in pci_epf_test_data_transfer()
131 struct pci_epf *epf = epf_test->epf; in pci_epf_test_data_transfer()
134 struct device *dev = &epf->dev; in pci_epf_test_data_transfer()
138 dev_err(dev, "Invalid DMA memcpy channel\n"); in pci_epf_test_data_transfer()
139 return -EINVAL; in pci_epf_test_data_transfer()
142 if (epf_test->dma_private) { in pci_epf_test_data_transfer()
150 dev_err(dev, "DMA slave config fail\n"); in pci_epf_test_data_transfer()
151 return -EIO; in pci_epf_test_data_transfer()
161 dev_err(dev, "Failed to prepare DMA memcpy\n"); in pci_epf_test_data_transfer()
162 return -EIO; in pci_epf_test_data_transfer()
165 reinit_completion(&epf_test->transfer_complete); in pci_epf_test_data_transfer()
166 epf_test->transfer_chan = chan; in pci_epf_test_data_transfer()
167 tx->callback = pci_epf_test_dma_callback; in pci_epf_test_data_transfer()
168 tx->callback_param = epf_test; in pci_epf_test_data_transfer()
169 epf_test->transfer_cookie = dmaengine_submit(tx); in pci_epf_test_data_transfer()
171 ret = dma_submit_error(epf_test->transfer_cookie); in pci_epf_test_data_transfer()
173 dev_err(dev, "Failed to do DMA tx_submit %d\n", ret); in pci_epf_test_data_transfer()
178 ret = wait_for_completion_interruptible(&epf_test->transfer_complete); in pci_epf_test_data_transfer()
180 dev_err(dev, "DMA wait_for_completion interrupted\n"); in pci_epf_test_data_transfer()
184 if (epf_test->transfer_status == DMA_ERROR) { in pci_epf_test_data_transfer()
185 dev_err(dev, "DMA transfer failed\n"); in pci_epf_test_data_transfer()
186 ret = -EIO; in pci_epf_test_data_transfer()
208 return chan->device->dev == filter->dev in epf_dma_filter_fn()
209 && (filter->dma_mask & caps.directions); in epf_dma_filter_fn()
213 * pci_epf_test_init_dma_chan() - Function to initialize EPF test DMA channel
216 * Function to initialize EPF test DMA channel.
220 struct pci_epf *epf = epf_test->epf; in pci_epf_test_init_dma_chan()
221 struct device *dev = &epf->dev; in pci_epf_test_init_dma_chan()
227 filter.dev = epf->epc->dev.parent; in pci_epf_test_init_dma_chan()
234 dev_info(dev, "Failed to get private DMA rx channel. Falling back to generic one\n"); in pci_epf_test_init_dma_chan()
238 epf_test->dma_chan_rx = dma_chan; in pci_epf_test_init_dma_chan()
244 dev_info(dev, "Failed to get private DMA tx channel. Falling back to generic one\n"); in pci_epf_test_init_dma_chan()
248 epf_test->dma_chan_tx = dma_chan; in pci_epf_test_init_dma_chan()
249 epf_test->dma_private = true; in pci_epf_test_init_dma_chan()
251 init_completion(&epf_test->transfer_complete); in pci_epf_test_init_dma_chan()
256 dma_release_channel(epf_test->dma_chan_rx); in pci_epf_test_init_dma_chan()
257 epf_test->dma_chan_rx = NULL; in pci_epf_test_init_dma_chan()
266 if (ret != -EPROBE_DEFER) in pci_epf_test_init_dma_chan()
267 dev_err(dev, "Failed to get DMA channel\n"); in pci_epf_test_init_dma_chan()
270 init_completion(&epf_test->transfer_complete); in pci_epf_test_init_dma_chan()
272 epf_test->dma_chan_tx = epf_test->dma_chan_rx = dma_chan; in pci_epf_test_init_dma_chan()
278 * pci_epf_test_clean_dma_chan() - Function to cleanup EPF test DMA channel
281 * Helper to cleanup EPF test DMA channel.
285 if (!epf_test->dma_supported) in pci_epf_test_clean_dma_chan()
288 dma_release_channel(epf_test->dma_chan_tx); in pci_epf_test_clean_dma_chan()
289 if (epf_test->dma_chan_tx == epf_test->dma_chan_rx) { in pci_epf_test_clean_dma_chan()
290 epf_test->dma_chan_tx = NULL; in pci_epf_test_clean_dma_chan()
291 epf_test->dma_chan_rx = NULL; in pci_epf_test_clean_dma_chan()
295 dma_release_channel(epf_test->dma_chan_rx); in pci_epf_test_clean_dma_chan()
296 epf_test->dma_chan_rx = NULL; in pci_epf_test_clean_dma_chan()
302 struct timespec64 *end, bool dma) in pci_epf_test_print_rate() argument
312 dev_info(&epf_test->epf->dev, in pci_epf_test_print_rate()
313 "%s => Size: %llu B, DMA: %s, Time: %llu.%09u s, Rate: %llu KB/s\n", in pci_epf_test_print_rate()
314 op, size, dma ? "YES" : "NO", in pci_epf_test_print_rate()
323 struct pci_epf *epf = epf_test->epf; in pci_epf_test_copy()
324 struct pci_epc *epc = epf->epc; in pci_epf_test_copy()
325 struct device *dev = &epf->dev; in pci_epf_test_copy()
327 u64 src_addr = le64_to_cpu(reg->src_addr); in pci_epf_test_copy()
328 u64 dst_addr = le64_to_cpu(reg->dst_addr); in pci_epf_test_copy()
331 u32 flags = le32_to_cpu(reg->flags); in pci_epf_test_copy()
335 orig_size = copy_size = le32_to_cpu(reg->size); in pci_epf_test_copy()
338 if (!dma_has_cap(DMA_MEMCPY, epf_test->dma_chan_tx->device->cap_mask)) { in pci_epf_test_copy()
339 dev_err(dev, "DMA controller doesn't support MEMCPY\n"); in pci_epf_test_copy()
340 ret = -EINVAL; in pci_epf_test_copy()
346 ret = -ENOMEM; in pci_epf_test_copy()
353 ret = pci_epc_mem_map(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_copy()
356 dev_err(dev, "Failed to map source address\n"); in pci_epf_test_copy()
361 ret = pci_epc_mem_map(epf->epc, epf->func_no, epf->vfunc_no, in pci_epf_test_copy()
364 dev_err(dev, "Failed to map destination address\n"); in pci_epf_test_copy()
366 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_copy()
389 copy_size -= map_size; in pci_epf_test_copy()
393 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &dst_map); in pci_epf_test_copy()
394 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &src_map); in pci_epf_test_copy()
403 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &dst_map); in pci_epf_test_copy()
404 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &src_map); in pci_epf_test_copy()
415 reg->status = cpu_to_le32(status); in pci_epf_test_copy()
424 struct pci_epc_map map; in pci_epf_test_read() local
427 struct pci_epf *epf = epf_test->epf; in pci_epf_test_read()
428 struct pci_epc *epc = epf->epc; in pci_epf_test_read()
429 struct device *dev = &epf->dev; in pci_epf_test_read()
430 struct device *dma_dev = epf->epc->dev.parent; in pci_epf_test_read()
431 u64 src_addr = le64_to_cpu(reg->src_addr); in pci_epf_test_read()
434 u32 flags = le32_to_cpu(reg->flags); in pci_epf_test_read()
435 u32 checksum = le32_to_cpu(reg->checksum); in pci_epf_test_read()
438 orig_size = src_size = le32_to_cpu(reg->size); in pci_epf_test_read()
442 ret = -ENOMEM; in pci_epf_test_read()
448 ret = pci_epc_mem_map(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_read()
449 src_addr, src_size, &map); in pci_epf_test_read()
451 dev_err(dev, "Failed to map address\n"); in pci_epf_test_read()
456 map_size = map.pci_size; in pci_epf_test_read()
462 "Failed to map destination buffer addr\n"); in pci_epf_test_read()
463 ret = -ENOMEM; in pci_epf_test_read()
469 dst_phys_addr, map.phys_addr, in pci_epf_test_read()
482 memcpy_fromio(buf, map.virt_addr, map_size); in pci_epf_test_read()
486 src_size -= map_size; in pci_epf_test_read()
490 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); in pci_epf_test_read()
499 ret = -EIO; in pci_epf_test_read()
503 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); in pci_epf_test_read()
513 reg->status = cpu_to_le32(status); in pci_epf_test_read()
521 struct pci_epc_map map; in pci_epf_test_write() local
524 struct pci_epf *epf = epf_test->epf; in pci_epf_test_write()
525 struct pci_epc *epc = epf->epc; in pci_epf_test_write()
526 struct device *dev = &epf->dev; in pci_epf_test_write()
527 struct device *dma_dev = epf->epc->dev.parent; in pci_epf_test_write()
528 u64 dst_addr = le64_to_cpu(reg->dst_addr); in pci_epf_test_write()
531 u32 flags = le32_to_cpu(reg->flags); in pci_epf_test_write()
534 orig_size = dst_size = le32_to_cpu(reg->size); in pci_epf_test_write()
538 ret = -ENOMEM; in pci_epf_test_write()
542 reg->checksum = cpu_to_le32(crc32_le(~0, dst_buf, dst_size)); in pci_epf_test_write()
546 ret = pci_epc_mem_map(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_write()
547 dst_addr, dst_size, &map); in pci_epf_test_write()
549 dev_err(dev, "Failed to map address\n"); in pci_epf_test_write()
554 map_size = map.pci_size; in pci_epf_test_write()
560 "Failed to map source buffer addr\n"); in pci_epf_test_write()
561 ret = -ENOMEM; in pci_epf_test_write()
568 map.phys_addr, src_phys_addr, in pci_epf_test_write()
582 memcpy_toio(map.virt_addr, buf, map_size); in pci_epf_test_write()
586 dst_size -= map_size; in pci_epf_test_write()
590 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); in pci_epf_test_write()
598 * wait 1ms inorder for the write to complete. Without this delay L3 in pci_epf_test_write()
605 pci_epc_mem_unmap(epc, epf->func_no, epf->vfunc_no, &map); in pci_epf_test_write()
615 reg->status = cpu_to_le32(status); in pci_epf_test_write()
621 struct pci_epf *epf = epf_test->epf; in pci_epf_test_raise_irq()
622 struct device *dev = &epf->dev; in pci_epf_test_raise_irq()
623 struct pci_epc *epc = epf->epc; in pci_epf_test_raise_irq()
624 u32 status = le32_to_cpu(reg->status); in pci_epf_test_raise_irq()
625 u32 irq_number = le32_to_cpu(reg->irq_number); in pci_epf_test_raise_irq()
626 u32 irq_type = le32_to_cpu(reg->irq_type); in pci_epf_test_raise_irq()
630 * Set the status before raising the IRQ to ensure that the host sees in pci_epf_test_raise_irq()
634 WRITE_ONCE(reg->status, cpu_to_le32(status)); in pci_epf_test_raise_irq()
638 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_raise_irq()
642 count = pci_epc_get_msi(epc, epf->func_no, epf->vfunc_no); in pci_epf_test_raise_irq()
648 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_raise_irq()
652 count = pci_epc_get_msix(epc, epf->func_no, epf->vfunc_no); in pci_epf_test_raise_irq()
658 pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_raise_irq()
662 dev_err(dev, "Failed to raise IRQ, unknown type\n"); in pci_epf_test_raise_irq()
672 struct pci_epf *epf = epf_test->epf; in pci_epf_test_cmd_handler()
673 struct device *dev = &epf->dev; in pci_epf_test_cmd_handler()
674 enum pci_barno test_reg_bar = epf_test->test_reg_bar; in pci_epf_test_cmd_handler()
675 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; in pci_epf_test_cmd_handler()
676 u32 irq_type = le32_to_cpu(reg->irq_type); in pci_epf_test_cmd_handler()
678 command = le32_to_cpu(READ_ONCE(reg->command)); in pci_epf_test_cmd_handler()
682 WRITE_ONCE(reg->command, 0); in pci_epf_test_cmd_handler()
683 WRITE_ONCE(reg->status, 0); in pci_epf_test_cmd_handler()
685 if ((le32_to_cpu(READ_ONCE(reg->flags)) & FLAG_USE_DMA) && in pci_epf_test_cmd_handler()
686 !epf_test->dma_supported) { in pci_epf_test_cmd_handler()
687 dev_err(dev, "Cannot transfer data using DMA\n"); in pci_epf_test_cmd_handler()
692 dev_err(dev, "Failed to detect IRQ type\n"); in pci_epf_test_cmd_handler()
720 queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler, in pci_epf_test_cmd_handler()
727 struct pci_epc *epc = epf->epc; in pci_epf_test_set_bar()
728 struct device *dev = &epf->dev; in pci_epf_test_set_bar()
730 enum pci_barno test_reg_bar = epf_test->test_reg_bar; in pci_epf_test_set_bar()
733 if (!epf_test->reg[bar]) in pci_epf_test_set_bar()
736 ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_set_bar()
737 &epf->bar[bar]); in pci_epf_test_set_bar()
739 pci_epf_free_space(epf, epf_test->reg[bar], bar, in pci_epf_test_set_bar()
741 dev_err(dev, "Failed to set BAR%d\n", bar); in pci_epf_test_set_bar()
753 struct pci_epc *epc = epf->epc; in pci_epf_test_clear_bar()
757 if (!epf_test->reg[bar]) in pci_epf_test_clear_bar()
760 pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_clear_bar()
761 &epf->bar[bar]); in pci_epf_test_clear_bar()
768 enum pci_barno test_reg_bar = epf_test->test_reg_bar; in pci_epf_test_set_capabilities()
769 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar]; in pci_epf_test_set_capabilities()
770 struct pci_epc *epc = epf->epc; in pci_epf_test_set_capabilities()
773 if (epc->ops->align_addr) in pci_epf_test_set_capabilities()
776 reg->caps = cpu_to_le32(caps); in pci_epf_test_set_capabilities()
782 struct pci_epf_header *header = epf->header; in pci_epf_test_epc_init()
783 const struct pci_epc_features *epc_features = epf_test->epc_features; in pci_epf_test_epc_init()
784 struct pci_epc *epc = epf->epc; in pci_epf_test_epc_init()
785 struct device *dev = &epf->dev; in pci_epf_test_epc_init()
789 epf_test->dma_supported = true; in pci_epf_test_epc_init()
793 epf_test->dma_supported = false; in pci_epf_test_epc_init()
795 if (epf->vfunc_no <= 1) { in pci_epf_test_epc_init()
796 ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header); in pci_epf_test_epc_init()
809 if (epc_features->msi_capable) { in pci_epf_test_epc_init()
810 ret = pci_epc_set_msi(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_epc_init()
811 epf->msi_interrupts); in pci_epf_test_epc_init()
818 if (epc_features->msix_capable) { in pci_epf_test_epc_init()
819 ret = pci_epc_set_msix(epc, epf->func_no, epf->vfunc_no, in pci_epf_test_epc_init()
820 epf->msix_interrupts, in pci_epf_test_epc_init()
821 epf_test->test_reg_bar, in pci_epf_test_epc_init()
822 epf_test->msix_table_offset); in pci_epf_test_epc_init()
824 dev_err(dev, "MSI-X configuration failed\n"); in pci_epf_test_epc_init()
829 linkup_notifier = epc_features->linkup_notifier; in pci_epf_test_epc_init()
831 queue_work(kpcitest_workqueue, &epf_test->cmd_handler.work); in pci_epf_test_epc_init()
840 cancel_delayed_work_sync(&epf_test->cmd_handler); in pci_epf_test_epc_deinit()
849 queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler, in pci_epf_test_link_up()
859 cancel_delayed_work_sync(&epf_test->cmd_handler); in pci_epf_test_link_down()
874 struct device *dev = &epf->dev; in pci_epf_test_alloc_space()
879 enum pci_barno test_reg_bar = epf_test->test_reg_bar; in pci_epf_test_alloc_space()
881 const struct pci_epc_features *epc_features = epf_test->epc_features; in pci_epf_test_alloc_space()
886 if (epc_features->msix_capable) { in pci_epf_test_alloc_space()
887 msix_table_size = PCI_MSIX_ENTRY_SIZE * epf->msix_interrupts; in pci_epf_test_alloc_space()
888 epf_test->msix_table_offset = test_reg_bar_size; in pci_epf_test_alloc_space()
889 /* Align to QWORD or 8 Bytes */ in pci_epf_test_alloc_space()
890 pba_size = ALIGN(DIV_ROUND_UP(epf->msix_interrupts, 8), 8); in pci_epf_test_alloc_space()
897 dev_err(dev, "Failed to allocated register space\n"); in pci_epf_test_alloc_space()
898 return -ENOMEM; in pci_epf_test_alloc_space()
900 epf_test->reg[test_reg_bar] = base; in pci_epf_test_alloc_space()
913 dev_err(dev, "Failed to allocate space for BAR%d\n", in pci_epf_test_alloc_space()
915 epf_test->reg[bar] = base; in pci_epf_test_alloc_space()
927 if (!epf_test->reg[bar]) in pci_epf_test_free_space()
930 pci_epf_free_space(epf, epf_test->reg[bar], bar, in pci_epf_test_free_space()
941 struct pci_epc *epc = epf->epc; in pci_epf_test_bind()
944 return -EINVAL; in pci_epf_test_bind()
946 epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no); in pci_epf_test_bind()
948 dev_err(&epf->dev, "epc_features not implemented\n"); in pci_epf_test_bind()
949 return -EOPNOTSUPP; in pci_epf_test_bind()
954 return -EINVAL; in pci_epf_test_bind()
956 epf_test->test_reg_bar = test_reg_bar; in pci_epf_test_bind()
957 epf_test->epc_features = epc_features; in pci_epf_test_bind()
969 struct pci_epc *epc = epf->epc; in pci_epf_test_unbind()
971 cancel_delayed_work_sync(&epf_test->cmd_handler); in pci_epf_test_unbind()
972 if (epc->init_complete) { in pci_epf_test_unbind()
990 struct device *dev = &epf->dev; in pci_epf_test_probe()
994 return -ENOMEM; in pci_epf_test_probe()
996 epf->header = &test_header; in pci_epf_test_probe()
997 epf_test->epf = epf; in pci_epf_test_probe()
999 INIT_DELAYED_WORK(&epf_test->cmd_handler, pci_epf_test_cmd_handler); in pci_epf_test_probe()
1001 epf->event_ops = &pci_epf_test_event_ops; in pci_epf_test_probe()
1027 pr_err("Failed to allocate the kpcitest work queue\n"); in pci_epf_test_init()
1028 return -ENOMEM; in pci_epf_test_init()
1034 pr_err("Failed to register pci epf test driver --> %d\n", ret); in pci_epf_test_init()