Lines Matching +full:pcie +full:- +full:ob
1 // SPDX-License-Identifier: GPL-2.0
3 * Endpoint Function Driver to implement Non-Transparent Bridge functionality
9 * Based on pci-epf-ntb.c
15 * +------------+ +---------------------------------------+
17 * +------------+ | +--------------+
20 * +------------+ | +--------------+
23 * +------------+ | +--------------+
28 * | | +---------------+ | NTB Driver |
29 * | | | PCI EP NTB |<------>| |
31 * +------------+ +---------------+ +--------------+
33 * | PCI Bus | <-----> | PCI EP Bus | | Virtual PCI |
35 * +------------+ +---------------+--------+--------------+
36 * PCIe Root Port PCI EP
44 #include <linux/pci-epc.h>
45 #include <linux/pci-epf.h>
79 * +--------------------------------------------------+ Base
87 * +-----------------------+--------------------------+ Base+spad_offset
92 * +-----------------------+--------------------------+ Base+spad_offset
97 * +-----------------------+--------------------------+
98 * Virtual PCI PCIe Endpoint
158 * epf_ntb_link_up() - Raise link_up interrupt to Virtual Host (VHOST)
170 ntb->reg->link_status |= LINK_STATUS_UP; in epf_ntb_link_up()
172 ntb->reg->link_status &= ~LINK_STATUS_UP; in epf_ntb_link_up()
174 ntb_link_event(&ntb->ntb); in epf_ntb_link_up()
179 * epf_ntb_configure_mw() - Configure the Outbound Address Space for VHOST
185 * +--------+ +-----------+
190 * | | +-----------+
192 * | NTB | -----------> | |
194 * | | +-----------+
197 * +--------+ +-----------+
209 phys_addr = ntb->vpci_mw_phy[mw]; in epf_ntb_configure_mw()
210 addr = ntb->reg->addr; in epf_ntb_configure_mw()
211 size = ntb->reg->size; in epf_ntb_configure_mw()
213 func_no = ntb->epf->func_no; in epf_ntb_configure_mw()
214 vfunc_no = ntb->epf->vfunc_no; in epf_ntb_configure_mw()
216 ret = pci_epc_map_addr(ntb->epf->epc, func_no, vfunc_no, phys_addr, addr, size); in epf_ntb_configure_mw()
218 dev_err(&ntb->epf->epc->dev, in epf_ntb_configure_mw()
224 * epf_ntb_teardown_mw() - Teardown the configured OB ATU
228 * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using
233 pci_epc_unmap_addr(ntb->epf->epc, in epf_ntb_teardown_mw()
234 ntb->epf->func_no, in epf_ntb_teardown_mw()
235 ntb->epf->vfunc_no, in epf_ntb_teardown_mw()
236 ntb->vpci_mw_phy[mw]); in epf_ntb_teardown_mw()
240 * epf_ntb_cmd_handler() - Handle commands provided by the NTB HOST
259 for (i = 1; i < ntb->db_count; i++) { in epf_ntb_cmd_handler()
260 if (ntb->epf_db[i]) { in epf_ntb_cmd_handler()
261 ntb->db |= 1 << (i - 1); in epf_ntb_cmd_handler()
262 ntb_db_event(&ntb->ntb, i); in epf_ntb_cmd_handler()
263 ntb->epf_db[i] = 0; in epf_ntb_cmd_handler()
267 ctrl = ntb->reg; in epf_ntb_cmd_handler()
268 command = ctrl->command; in epf_ntb_cmd_handler()
271 argument = ctrl->argument; in epf_ntb_cmd_handler()
273 ctrl->command = 0; in epf_ntb_cmd_handler()
274 ctrl->argument = 0; in epf_ntb_cmd_handler()
276 ctrl = ntb->reg; in epf_ntb_cmd_handler()
277 dev = &ntb->epf->dev; in epf_ntb_cmd_handler()
281 ctrl->command_status = COMMAND_STATUS_OK; in epf_ntb_cmd_handler()
284 ctrl->command_status = COMMAND_STATUS_OK; in epf_ntb_cmd_handler()
289 ctrl->command_status = COMMAND_STATUS_ERROR; in epf_ntb_cmd_handler()
291 ctrl->command_status = COMMAND_STATUS_OK; in epf_ntb_cmd_handler()
295 ctrl->command_status = COMMAND_STATUS_OK; in epf_ntb_cmd_handler()
298 ntb->linkup = true; in epf_ntb_cmd_handler()
301 ctrl->command_status = COMMAND_STATUS_ERROR; in epf_ntb_cmd_handler()
303 ctrl->command_status = COMMAND_STATUS_OK; in epf_ntb_cmd_handler()
306 ntb->linkup = false; in epf_ntb_cmd_handler()
309 ctrl->command_status = COMMAND_STATUS_ERROR; in epf_ntb_cmd_handler()
311 ctrl->command_status = COMMAND_STATUS_OK; in epf_ntb_cmd_handler()
319 queue_delayed_work(kpcintb_workqueue, &ntb->cmd_handler, in epf_ntb_cmd_handler()
324 * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR
345 barno = ntb->epf_ntb_bar[BAR_CONFIG]; in epf_ntb_config_sspad_bar_clear()
346 epf_bar = &ntb->epf->bar[barno]; in epf_ntb_config_sspad_bar_clear()
348 pci_epc_clear_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar); in epf_ntb_config_sspad_bar_clear()
352 * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR
371 dev = &ntb->epf->dev; in epf_ntb_config_sspad_bar_set()
372 func_no = ntb->epf->func_no; in epf_ntb_config_sspad_bar_set()
373 vfunc_no = ntb->epf->vfunc_no; in epf_ntb_config_sspad_bar_set()
374 barno = ntb->epf_ntb_bar[BAR_CONFIG]; in epf_ntb_config_sspad_bar_set()
375 epf_bar = &ntb->epf->bar[barno]; in epf_ntb_config_sspad_bar_set()
377 ret = pci_epc_set_bar(ntb->epf->epc, func_no, vfunc_no, epf_bar); in epf_ntb_config_sspad_bar_set()
386 * epf_ntb_config_spad_bar_free() - Free the physical memory associated with
394 barno = ntb->epf_ntb_bar[BAR_CONFIG]; in epf_ntb_config_spad_bar_free()
395 pci_epf_free_space(ntb->epf, ntb->reg, barno, 0); in epf_ntb_config_spad_bar_free()
399 * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad
405 * is obtained from "spad-count" configfs entry.
416 struct pci_epf *epf = ntb->epf; in epf_ntb_config_spad_bar_alloc()
417 struct device *dev = &epf->dev; in epf_ntb_config_spad_bar_alloc()
421 const struct pci_epc_features *epc_features = pci_epc_get_features(epf->epc, in epf_ntb_config_spad_bar_alloc()
422 epf->func_no, in epf_ntb_config_spad_bar_alloc()
423 epf->vfunc_no); in epf_ntb_config_spad_bar_alloc()
424 barno = ntb->epf_ntb_bar[BAR_CONFIG]; in epf_ntb_config_spad_bar_alloc()
425 size = epc_features->bar[barno].fixed_size; in epf_ntb_config_spad_bar_alloc()
426 align = epc_features->align; in epf_ntb_config_spad_bar_alloc()
429 return -EINVAL; in epf_ntb_config_spad_bar_alloc()
431 spad_count = ntb->spad_count; in epf_ntb_config_spad_bar_alloc()
447 return -EINVAL; in epf_ntb_config_spad_bar_alloc()
452 return -ENOMEM; in epf_ntb_config_spad_bar_alloc()
455 ntb->reg = base; in epf_ntb_config_spad_bar_alloc()
457 ctrl = ntb->reg; in epf_ntb_config_spad_bar_alloc()
458 ctrl->spad_offset = ctrl_size; in epf_ntb_config_spad_bar_alloc()
460 ctrl->spad_count = spad_count; in epf_ntb_config_spad_bar_alloc()
461 ctrl->num_mws = ntb->num_mws; in epf_ntb_config_spad_bar_alloc()
462 ntb->spad_size = spad_size; in epf_ntb_config_spad_bar_alloc()
464 ctrl->db_entry_size = sizeof(u32); in epf_ntb_config_spad_bar_alloc()
466 for (i = 0; i < ntb->db_count; i++) { in epf_ntb_config_spad_bar_alloc()
467 ntb->reg->db_data[i] = 1 + i; in epf_ntb_config_spad_bar_alloc()
468 ntb->reg->db_offset[i] = 0; in epf_ntb_config_spad_bar_alloc()
475 * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capability
478 * Configure MSI/MSI-X capability for each interface with number of
490 dev = &ntb->epf->dev; in epf_ntb_configure_interrupt()
492 epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no); in epf_ntb_configure_interrupt()
494 if (!(epc_features->msix_capable || epc_features->msi_capable)) { in epf_ntb_configure_interrupt()
495 dev_err(dev, "MSI or MSI-X is required for doorbell\n"); in epf_ntb_configure_interrupt()
496 return -EINVAL; in epf_ntb_configure_interrupt()
499 db_count = ntb->db_count; in epf_ntb_configure_interrupt()
502 return -EINVAL; in epf_ntb_configure_interrupt()
505 ntb->db_count = db_count; in epf_ntb_configure_interrupt()
507 if (epc_features->msi_capable) { in epf_ntb_configure_interrupt()
508 ret = pci_epc_set_msi(ntb->epf->epc, in epf_ntb_configure_interrupt()
509 ntb->epf->func_no, in epf_ntb_configure_interrupt()
510 ntb->epf->vfunc_no, in epf_ntb_configure_interrupt()
522 * epf_ntb_db_bar_init() - Configure Doorbell window BARs
530 struct device *dev = &ntb->epf->dev; in epf_ntb_db_bar_init()
535 size_t size = sizeof(u32) * ntb->db_count; in epf_ntb_db_bar_init()
537 epc_features = pci_epc_get_features(ntb->epf->epc, in epf_ntb_db_bar_init()
538 ntb->epf->func_no, in epf_ntb_db_bar_init()
539 ntb->epf->vfunc_no); in epf_ntb_db_bar_init()
540 barno = ntb->epf_ntb_bar[BAR_DB]; in epf_ntb_db_bar_init()
542 mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, epc_features, 0); in epf_ntb_db_bar_init()
544 dev_err(dev, "Failed to allocate OB address\n"); in epf_ntb_db_bar_init()
545 return -ENOMEM; in epf_ntb_db_bar_init()
548 ntb->epf_db = mw_addr; in epf_ntb_db_bar_init()
550 epf_bar = &ntb->epf->bar[barno]; in epf_ntb_db_bar_init()
552 ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar); in epf_ntb_db_bar_init()
560 pci_epf_free_space(ntb->epf, mw_addr, barno, 0); in epf_ntb_db_bar_init()
561 return -1; in epf_ntb_db_bar_init()
567 * epf_ntb_db_bar_clear() - Clear doorbell BAR and free memory
575 barno = ntb->epf_ntb_bar[BAR_DB]; in epf_ntb_db_bar_clear()
576 pci_epf_free_space(ntb->epf, ntb->epf_db, barno, 0); in epf_ntb_db_bar_clear()
577 pci_epc_clear_bar(ntb->epf->epc, in epf_ntb_db_bar_clear()
578 ntb->epf->func_no, in epf_ntb_db_bar_clear()
579 ntb->epf->vfunc_no, in epf_ntb_db_bar_clear()
580 &ntb->epf->bar[barno]); in epf_ntb_db_bar_clear()
584 * epf_ntb_mw_bar_init() - Configure Memory window BARs
595 struct device *dev = &ntb->epf->dev; in epf_ntb_mw_bar_init()
597 for (i = 0; i < ntb->num_mws; i++) { in epf_ntb_mw_bar_init()
598 size = ntb->mws_size[i]; in epf_ntb_mw_bar_init()
599 barno = ntb->epf_ntb_bar[BAR_MW0 + i]; in epf_ntb_mw_bar_init()
601 ntb->epf->bar[barno].barno = barno; in epf_ntb_mw_bar_init()
602 ntb->epf->bar[barno].size = size; in epf_ntb_mw_bar_init()
603 ntb->epf->bar[barno].addr = NULL; in epf_ntb_mw_bar_init()
604 ntb->epf->bar[barno].phys_addr = 0; in epf_ntb_mw_bar_init()
605 ntb->epf->bar[barno].flags |= upper_32_bits(size) ? in epf_ntb_mw_bar_init()
609 ret = pci_epc_set_bar(ntb->epf->epc, in epf_ntb_mw_bar_init()
610 ntb->epf->func_no, in epf_ntb_mw_bar_init()
611 ntb->epf->vfunc_no, in epf_ntb_mw_bar_init()
612 &ntb->epf->bar[barno]); in epf_ntb_mw_bar_init()
619 ntb->vpci_mw_addr[i] = pci_epc_mem_alloc_addr(ntb->epf->epc, in epf_ntb_mw_bar_init()
620 &ntb->vpci_mw_phy[i], in epf_ntb_mw_bar_init()
622 if (!ntb->vpci_mw_addr[i]) { in epf_ntb_mw_bar_init()
623 ret = -ENOMEM; in epf_ntb_mw_bar_init()
632 pci_epc_clear_bar(ntb->epf->epc, in epf_ntb_mw_bar_init()
633 ntb->epf->func_no, in epf_ntb_mw_bar_init()
634 ntb->epf->vfunc_no, in epf_ntb_mw_bar_init()
635 &ntb->epf->bar[barno]); in epf_ntb_mw_bar_init()
642 * epf_ntb_mw_bar_clear() - Clear Memory window BARs
652 barno = ntb->epf_ntb_bar[BAR_MW0 + i]; in epf_ntb_mw_bar_clear()
653 pci_epc_clear_bar(ntb->epf->epc, in epf_ntb_mw_bar_clear()
654 ntb->epf->func_no, in epf_ntb_mw_bar_clear()
655 ntb->epf->vfunc_no, in epf_ntb_mw_bar_clear()
656 &ntb->epf->bar[barno]); in epf_ntb_mw_bar_clear()
658 pci_epc_mem_free_addr(ntb->epf->epc, in epf_ntb_mw_bar_clear()
659 ntb->vpci_mw_phy[i], in epf_ntb_mw_bar_clear()
660 ntb->vpci_mw_addr[i], in epf_ntb_mw_bar_clear()
661 ntb->mws_size[i]); in epf_ntb_mw_bar_clear()
666 * epf_ntb_epc_destroy() - Cleanup NTB EPC interface
673 pci_epc_remove_epf(ntb->epf->epc, ntb->epf, 0); in epf_ntb_epc_destroy()
674 pci_epc_put(ntb->epf->epc); in epf_ntb_epc_destroy()
678 * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB
694 num_mws = ntb->num_mws; in epf_ntb_init_epc_bar()
695 dev = &ntb->epf->dev; in epf_ntb_init_epc_bar()
696 epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no); in epf_ntb_init_epc_bar()
705 ntb->epf_ntb_bar[bar] = barno; in epf_ntb_init_epc_bar()
712 ntb->num_mws = i; in epf_ntb_init_epc_bar()
715 ntb->epf_ntb_bar[bar] = barno; in epf_ntb_init_epc_bar()
722 * epf_ntb_epc_init() - Initialize NTB interface
739 epf = ntb->epf; in epf_ntb_epc_init()
740 dev = &epf->dev; in epf_ntb_epc_init()
741 epc = epf->epc; in epf_ntb_epc_init()
742 func_no = ntb->epf->func_no; in epf_ntb_epc_init()
743 vfunc_no = ntb->epf->vfunc_no; in epf_ntb_epc_init()
770 ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header); in epf_ntb_epc_init()
777 INIT_DELAYED_WORK(&ntb->cmd_handler, epf_ntb_cmd_handler); in epf_ntb_epc_init()
778 queue_work(kpcintb_workqueue, &ntb->cmd_handler.work); in epf_ntb_epc_init()
783 epf_ntb_mw_bar_clear(ntb, ntb->num_mws); in epf_ntb_epc_init()
795 * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces
802 epf_ntb_mw_bar_clear(ntb, ntb->num_mws); in epf_ntb_epc_cleanup()
814 return sprintf(page, "%d\n", ntb->_name); \
830 ntb->_name = val; \
841 struct device *dev = &ntb->epf->dev; \
845 return -EINVAL; \
847 if (win_no <= 0 || win_no > ntb->num_mws) { \
848 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
849 return -EINVAL; \
852 return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]); \
861 struct device *dev = &ntb->epf->dev; \
871 return -EINVAL; \
873 if (win_no <= 0 || win_no > ntb->num_mws) { \
874 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
875 return -EINVAL; \
878 ntb->mws_size[win_no - 1] = val; \
896 return -EINVAL; in epf_ntb_num_mws_store()
898 ntb->num_mws = val; in epf_ntb_num_mws_store()
954 * epf_ntb_add_cfs() - Add configfs directory specific to NTB
957 * config_items of a specific type that belong to a specific sub-system.
968 struct config_group *ntb_group = &ntb->group; in epf_ntb_add_cfs()
969 struct device *dev = &epf->dev; in epf_ntb_add_cfs()
1021 vpci_bus = pci_scan_bus(ndev->vbus_number, &vpci_ops, sysdata); in vpci_scan_bus()
1024 return -EINVAL; in vpci_scan_bus()
1032 /*==================== Virtual PCIe NTB driver ==========================*/
1038 return ndev->num_mws; in vntb_epf_mw_count()
1043 return ntb_ndev(ntb)->spad_count; in vntb_epf_spad_count()
1048 return ntb_ndev(ntb)->num_mws; in vntb_epf_peer_mw_count()
1053 return BIT_ULL(ntb_ndev(ntb)->db_count) - 1; in vntb_epf_db_valid_mask()
1070 dev = &ntb->ntb.dev; in vntb_epf_mw_set_trans()
1071 barno = ntb->epf_ntb_bar[BAR_MW0 + idx]; in vntb_epf_mw_set_trans()
1072 epf_bar = &ntb->epf->bar[barno]; in vntb_epf_mw_set_trans()
1073 epf_bar->phys_addr = addr; in vntb_epf_mw_set_trans()
1074 epf_bar->barno = barno; in vntb_epf_mw_set_trans()
1075 epf_bar->size = size; in vntb_epf_mw_set_trans()
1077 ret = pci_epc_set_bar(ntb->epf->epc, 0, 0, epf_bar); in vntb_epf_mw_set_trans()
1097 *base = ntb->vpci_mw_phy[idx]; in vntb_epf_peer_mw_get_addr()
1100 *size = ntb->mws_size[idx]; in vntb_epf_peer_mw_get_addr()
1115 int off = ntb->reg->spad_offset, ct = ntb->reg->spad_count * sizeof(u32); in vntb_epf_spad_read()
1117 void __iomem *base = (void __iomem *)ntb->reg; in vntb_epf_spad_read()
1126 struct epf_ntb_ctrl *ctrl = ntb->reg; in vntb_epf_spad_write()
1127 int off = ctrl->spad_offset, ct = ctrl->spad_count * sizeof(u32); in vntb_epf_spad_write()
1128 void __iomem *base = (void __iomem *)ntb->reg; in vntb_epf_spad_write()
1137 struct epf_ntb_ctrl *ctrl = ntb->reg; in vntb_epf_peer_spad_read()
1138 int off = ctrl->spad_offset; in vntb_epf_peer_spad_read()
1139 void __iomem *base = (void __iomem *)ntb->reg; in vntb_epf_peer_spad_read()
1149 struct epf_ntb_ctrl *ctrl = ntb->reg; in vntb_epf_peer_spad_write()
1150 int off = ctrl->spad_offset; in vntb_epf_peer_spad_write()
1151 void __iomem *base = (void __iomem *)ntb->reg; in vntb_epf_peer_spad_write()
1164 func_no = ntb->epf->func_no; in vntb_epf_peer_db_set()
1165 vfunc_no = ntb->epf->vfunc_no; in vntb_epf_peer_db_set()
1167 ret = pci_epc_raise_irq(ntb->epf->epc, func_no, vfunc_no, in vntb_epf_peer_db_set()
1170 dev_err(&ntb->ntb.dev, "Failed to raise IRQ\n"); in vntb_epf_peer_db_set()
1179 return ntb->db; in vntb_epf_db_read()
1196 *size_max = ntb->mws_size[idx]; in vntb_epf_mw_get_align()
1207 return ntb->reg->link_status; in vntb_epf_link_is_up()
1219 ntb->db &= ~db_bits; in vntb_epf_db_clear()
1254 struct epf_ntb *ndev = (struct epf_ntb *)pdev->sysdata; in pci_vntb_probe()
1255 struct device *dev = &pdev->dev; in pci_vntb_probe()
1257 ndev->ntb.pdev = pdev; in pci_vntb_probe()
1258 ndev->ntb.topo = NTB_TOPO_NONE; in pci_vntb_probe()
1259 ndev->ntb.ops = &vntb_epf_ops; in pci_vntb_probe()
1267 ret = ntb_register_device(&ndev->ntb); in pci_vntb_probe()
1285 .name = "pci-vntb",
1290 /* ============ PCIe EPF Driver Bind ====================*/
1293 * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality
1306 struct device *dev = &epf->dev; in epf_ntb_bind()
1309 if (!epf->epc) { in epf_ntb_bind()
1334 pci_space[0] = (ntb->vntb_pid << 16) | ntb->vntb_vid; in epf_ntb_bind()
1335 pci_vntb_table[0].vendor = ntb->vntb_vid; in epf_ntb_bind()
1336 pci_vntb_table[0].device = ntb->vntb_pid; in epf_ntb_bind()
1364 * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind()
1388 * epf_ntb_probe() - Probe NTB function driver
1403 dev = &epf->dev; in epf_ntb_probe()
1407 return -ENOMEM; in epf_ntb_probe()
1409 epf->header = &epf_ntb_header; in epf_ntb_probe()
1410 ntb->epf = epf; in epf_ntb_probe()
1411 ntb->vbus_number = 0xff; in epf_ntb_probe()
1414 dev_info(dev, "pci-ep epf driver loaded\n"); in epf_ntb_probe()
1442 pr_err("Failed to register pci epf ntb driver --> %d\n", ret); in epf_ntb_init()