Lines Matching +full:pcie +full:- +full:ob
1 // SPDX-License-Identifier: GPL-2.0
3 * Endpoint Function Driver to implement Non-Transparent Bridge functionality
10 * The PCI NTB function driver configures the SoC with multiple PCIe Endpoint
16 * +-------------+ +-------------+
20 * +------^------+ +------^------+
23 * +---------|-------------------------------------------------|---------+
24 * | +------v------+ +------v------+ |
28 * | | <-----------------------------------> | |
33 * | +-------------+ +-------------+ |
34 * +---------------------------------------------------------------------+
42 #include <linux/pci-epc.h>
43 #include <linux/pci-epf.h>
132 * epf_ntb_link_up() - Raise link_up interrupt to both the hosts
152 ntb_epc = ntb->epc[type]; in epf_ntb_link_up()
153 epc = ntb_epc->epc; in epf_ntb_link_up()
154 func_no = ntb_epc->func_no; in epf_ntb_link_up()
155 vfunc_no = ntb_epc->vfunc_no; in epf_ntb_link_up()
156 is_msix = ntb_epc->is_msix; in epf_ntb_link_up()
157 ctrl = ntb_epc->reg; in epf_ntb_link_up()
159 ctrl->link_status |= LINK_STATUS_UP; in epf_ntb_link_up()
161 ctrl->link_status &= ~LINK_STATUS_UP; in epf_ntb_link_up()
165 dev_err(&epc->dev, in epf_ntb_link_up()
176 * epf_ntb_configure_mw() - Configure the Outbound Address Space for one host
182 * +-----------------+ +---->+----------------+-----------+-----------------+
183 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 |
184 * +-----------------+ | +----------------+ +-----------------+
185 * | BAR1 | | | Doorbell 2 +---------+ | |
186 * +-----------------+----+ +----------------+ | | |
187 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+
188 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 |
189 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+
190 * +-----------------+ | |----------------+ | | | |
191 * | BAR4 | | | | | | +-----------------+
192 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3||
193 * | BAR5 | | | | | | +-----------------+
194 * +-----------------+ +---->-----------------+ | | | |
195 * EP CONTROLLER 1 | | | | +-----------------+
196 * | | | +---->+ MSI|X ADDRESS 4 |
197 * +----------------+ | +-----------------+
199 * (OB SPACE) | | |
200 * +-------> MW1 |
203 * (B) +-----------------+
209 * +-----------------+
213 * This function performs stage (B) in the above diagram (see MW1) i.e., map OB
227 * be provided by HOST2 in ctrl->addr and ctrl->size of epf_ntb_epc that is
231 * mapped to a single BAR (BAR2) above for 32-bit BARs. The exact BAR that's
248 ntb_epc = ntb->epc[type]; in epf_ntb_configure_mw()
249 epc = ntb_epc->epc; in epf_ntb_configure_mw()
251 peer_ntb_epc = ntb->epc[!type]; in epf_ntb_configure_mw()
252 peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET]; in epf_ntb_configure_mw()
253 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; in epf_ntb_configure_mw()
255 phys_addr = peer_epf_bar->phys_addr; in epf_ntb_configure_mw()
256 ctrl = ntb_epc->reg; in epf_ntb_configure_mw()
257 addr = ctrl->addr; in epf_ntb_configure_mw()
258 size = ctrl->size; in epf_ntb_configure_mw()
260 phys_addr += ctrl->mw1_offset; in epf_ntb_configure_mw()
262 if (size > ntb->mws_size[mw]) { in epf_ntb_configure_mw()
263 dev_err(&epc->dev, in epf_ntb_configure_mw()
266 ntb->mws_size[mw]); in epf_ntb_configure_mw()
267 ret = -EINVAL; in epf_ntb_configure_mw()
271 func_no = ntb_epc->func_no; in epf_ntb_configure_mw()
272 vfunc_no = ntb_epc->vfunc_no; in epf_ntb_configure_mw()
276 dev_err(&epc->dev, in epf_ntb_configure_mw()
286 * epf_ntb_teardown_mw() - Teardown the configured OB ATU
291 * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using
305 ntb_epc = ntb->epc[type]; in epf_ntb_teardown_mw()
306 epc = ntb_epc->epc; in epf_ntb_teardown_mw()
308 peer_ntb_epc = ntb->epc[!type]; in epf_ntb_teardown_mw()
309 peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET]; in epf_ntb_teardown_mw()
310 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; in epf_ntb_teardown_mw()
312 phys_addr = peer_epf_bar->phys_addr; in epf_ntb_teardown_mw()
313 ctrl = ntb_epc->reg; in epf_ntb_teardown_mw()
315 phys_addr += ctrl->mw1_offset; in epf_ntb_teardown_mw()
316 func_no = ntb_epc->func_no; in epf_ntb_teardown_mw()
317 vfunc_no = ntb_epc->vfunc_no; in epf_ntb_teardown_mw()
323 * epf_ntb_configure_msi() - Map OB address space to MSI address
328 *+-----------------+ +----->+----------------+-----------+-----------------+
329 *| BAR0 | | | Doorbell 1 +---+-------> MSI ADDRESS |
330 *+-----------------+ | +----------------+ | +-----------------+
331 *| BAR1 | | | Doorbell 2 +---+ | |
332 *+-----------------+----+ +----------------+ | | |
333 *| BAR2 | | Doorbell 3 +---+ | |
334 *+-----------------+----+ +----------------+ | | |
335 *| BAR3 | | | Doorbell 4 +---+ | |
336 *+-----------------+ | |----------------+ | |
338 *+-----------------+ | | MW1 | | |
340 *+-----------------+ +----->-----------------+ | |
343 * +----------------+ +-----------------+
345 * (OB SPACE) | |
349 * (B) +-----------------+
355 * +-----------------+
361 * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to
366 * 2) MSI-X address in the PCIe Address space
367 * 3) Number of MSI-X interrupts that has to be configured
375 * and maps the OB address (obtained in epf_ntb_alloc_peer_mem()) to the MSI
396 ntb_epc = ntb->epc[type]; in epf_ntb_configure_msi()
397 epc = ntb_epc->epc; in epf_ntb_configure_msi()
399 peer_ntb_epc = ntb->epc[!type]; in epf_ntb_configure_msi()
400 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; in epf_ntb_configure_msi()
401 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; in epf_ntb_configure_msi()
402 peer_ctrl = peer_ntb_epc->reg; in epf_ntb_configure_msi()
403 db_entry_size = peer_ctrl->db_entry_size; in epf_ntb_configure_msi()
405 phys_addr = peer_epf_bar->phys_addr; in epf_ntb_configure_msi()
406 func_no = ntb_epc->func_no; in epf_ntb_configure_msi()
407 vfunc_no = ntb_epc->vfunc_no; in epf_ntb_configure_msi()
412 dev_err(&epc->dev, "%s intf: Failed to map MSI IRQ\n", in epf_ntb_configure_msi()
418 peer_ctrl->db_data[i] = db_data | i; in epf_ntb_configure_msi()
419 peer_ctrl->db_offset[i] = db_offset; in epf_ntb_configure_msi()
426 * epf_ntb_configure_msix() - Map OB address space to MSI-X address
431 *+-----------------+ +----->+----------------+-----------+-----------------+
432 *| BAR0 | | | Doorbell 1 +-----------> MSI-X ADDRESS 1 |
433 *+-----------------+ | +----------------+ +-----------------+
434 *| BAR1 | | | Doorbell 2 +---------+ | |
435 *+-----------------+----+ +----------------+ | | |
436 *| BAR2 | | Doorbell 3 +-------+ | +-----------------+
437 *+-----------------+----+ +----------------+ | +-> MSI-X ADDRESS 2 |
438 *| BAR3 | | | Doorbell 4 +-----+ | +-----------------+
439 *+-----------------+ | |----------------+ | | | |
440 *| BAR4 | | | | | | +-----------------+
441 *+-----------------+ | | MW1 + | +-->+ MSI-X ADDRESS 3||
442 *| BAR5 | | | | | +-----------------+
443 *+-----------------+ +----->-----------------+ | | |
444 * EP CONTROLLER 1 | | | +-----------------+
445 * | | +---->+ MSI-X ADDRESS 4 |
446 * +----------------+ +-----------------+
448 * (OB SPACE) | |
452 * (B) +-----------------+
458 * +-----------------+
463 * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to
464 * doorbell to MSI-X address in PCI address space.
468 * 2) MSI-X address in the PCIe Address space
469 * 3) Number of MSI-X interrupts that has to be configured
476 * The MSI-X address is in the MSI-X table of EP CONTROLLER 2 and
477 * the count of doorbell is in ctrl->argument of epf_ntb_epc that is connected
478 * to HOST2. MSI-X table is stored memory mapped to ntb_epc->msix_bar and the
479 * offset is in ntb_epc->msix_table_offset. From this epf_ntb_configure_msix()
480 * gets the MSI-X address and data.
482 * epf_ntb_configure_msix() also stores the MSI-X data to raise each interrupt
505 ntb_epc = ntb->epc[type]; in epf_ntb_configure_msix()
506 epc = ntb_epc->epc; in epf_ntb_configure_msix()
508 epf_bar = &ntb_epc->epf_bar[ntb_epc->msix_bar]; in epf_ntb_configure_msix()
509 msix_tbl = epf_bar->addr + ntb_epc->msix_table_offset; in epf_ntb_configure_msix()
511 peer_ntb_epc = ntb->epc[!type]; in epf_ntb_configure_msix()
512 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; in epf_ntb_configure_msix()
513 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; in epf_ntb_configure_msix()
514 phys_addr = peer_epf_bar->phys_addr; in epf_ntb_configure_msix()
515 peer_ctrl = peer_ntb_epc->reg; in epf_ntb_configure_msix()
516 epc_features = ntb_epc->epc_features; in epf_ntb_configure_msix()
517 align = epc_features->align; in epf_ntb_configure_msix()
519 func_no = ntb_epc->func_no; in epf_ntb_configure_msix()
520 vfunc_no = ntb_epc->vfunc_no; in epf_ntb_configure_msix()
521 db_entry_size = peer_ctrl->db_entry_size; in epf_ntb_configure_msix()
529 dev_err(&epc->dev, in epf_ntb_configure_msix()
530 "%s intf: Failed to configure MSI-X IRQ\n", in epf_ntb_configure_msix()
535 peer_ctrl->db_data[i] = msg_data; in epf_ntb_configure_msix()
536 peer_ctrl->db_offset[i] = msix_tbl[i].msg_addr & (align - 1); in epf_ntb_configure_msix()
538 ntb_epc->is_msix = true; in epf_ntb_configure_msix()
544 * epf_ntb_configure_db() - Configure the Outbound Address Space for one host
549 * @msix: Indicates whether MSI-X or MSI should be used
563 return -EINVAL; in epf_ntb_configure_db()
565 ntb_epc = ntb->epc[type]; in epf_ntb_configure_db()
566 epc = ntb_epc->epc; in epf_ntb_configure_db()
574 dev_err(&epc->dev, "%s intf: Failed to configure DB\n", in epf_ntb_configure_db()
581 * epf_ntb_teardown_db() - Unmap address in OB address space to MSI/MSI-X
586 * Invoke pci_epc_unmap_addr() to unmap OB address to MSI/MSI-X address.
598 ntb_epc = ntb->epc[type]; in epf_ntb_teardown_db()
599 epc = ntb_epc->epc; in epf_ntb_teardown_db()
601 peer_ntb_epc = ntb->epc[!type]; in epf_ntb_teardown_db()
602 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; in epf_ntb_teardown_db()
603 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; in epf_ntb_teardown_db()
604 phys_addr = peer_epf_bar->phys_addr; in epf_ntb_teardown_db()
605 func_no = ntb_epc->func_no; in epf_ntb_teardown_db()
606 vfunc_no = ntb_epc->vfunc_no; in epf_ntb_teardown_db()
612 * epf_ntb_cmd_handler() - Handle commands provided by the NTB Host
633 ctrl = ntb_epc->reg; in epf_ntb_cmd_handler()
634 command = ctrl->command; in epf_ntb_cmd_handler()
637 argument = ctrl->argument; in epf_ntb_cmd_handler()
639 ctrl->command = 0; in epf_ntb_cmd_handler()
640 ctrl->argument = 0; in epf_ntb_cmd_handler()
642 ctrl = ntb_epc->reg; in epf_ntb_cmd_handler()
643 type = ntb_epc->type; in epf_ntb_cmd_handler()
644 ntb = ntb_epc->epf_ntb; in epf_ntb_cmd_handler()
645 dev = &ntb->epf->dev; in epf_ntb_cmd_handler()
653 ctrl->command_status = COMMAND_STATUS_ERROR; in epf_ntb_cmd_handler()
655 ctrl->command_status = COMMAND_STATUS_OK; in epf_ntb_cmd_handler()
659 ctrl->command_status = COMMAND_STATUS_OK; in epf_ntb_cmd_handler()
664 ctrl->command_status = COMMAND_STATUS_ERROR; in epf_ntb_cmd_handler()
666 ctrl->command_status = COMMAND_STATUS_OK; in epf_ntb_cmd_handler()
670 ctrl->command_status = COMMAND_STATUS_OK; in epf_ntb_cmd_handler()
673 ntb_epc->linkup = true; in epf_ntb_cmd_handler()
674 if (ntb->epc[PRIMARY_INTERFACE]->linkup && in epf_ntb_cmd_handler()
675 ntb->epc[SECONDARY_INTERFACE]->linkup) { in epf_ntb_cmd_handler()
678 ctrl->command_status = COMMAND_STATUS_ERROR; in epf_ntb_cmd_handler()
680 ctrl->command_status = COMMAND_STATUS_OK; in epf_ntb_cmd_handler()
683 ctrl->command_status = COMMAND_STATUS_OK; in epf_ntb_cmd_handler()
686 ntb_epc->linkup = false; in epf_ntb_cmd_handler()
689 ctrl->command_status = COMMAND_STATUS_ERROR; in epf_ntb_cmd_handler()
691 ctrl->command_status = COMMAND_STATUS_OK; in epf_ntb_cmd_handler()
700 queue_delayed_work(kpcintb_workqueue, &ntb_epc->cmd_handler, in epf_ntb_cmd_handler()
705 * epf_ntb_peer_spad_bar_clear() - Clear Peer Scratchpad BAR
709 *+-----------------+------->+------------------+ +-----------------+
711 *+-----------------+----+ +------------------+<-------+-----------------+
713 *+-----------------+ +-->+------------------+<-------+-----------------+
715 *+-----------------+ +-----------------+
717 *+-----------------+ +-----------------+
719 *+-----------------+ +-----------------+
721 *+-----------------+ +-----------------+
726 * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs).
732 * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG].
741 epc = ntb_epc->epc; in epf_ntb_peer_spad_bar_clear()
742 func_no = ntb_epc->func_no; in epf_ntb_peer_spad_bar_clear()
743 vfunc_no = ntb_epc->vfunc_no; in epf_ntb_peer_spad_bar_clear()
744 barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; in epf_ntb_peer_spad_bar_clear()
745 epf_bar = &ntb_epc->epf_bar[barno]; in epf_ntb_peer_spad_bar_clear()
750 * epf_ntb_peer_spad_bar_set() - Set peer scratchpad BAR
754 *+-----------------+------->+------------------+ +-----------------+
756 *+-----------------+----+ +------------------+<-------+-----------------+
758 *+-----------------+ +-->+------------------+<-------+-----------------+
760 *+-----------------+ +-----------------+
762 *+-----------------+ +-----------------+
764 *+-----------------+ +-----------------+
766 *+-----------------+ +-----------------+
771 * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs).
777 * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG].
791 dev = &ntb->epf->dev; in epf_ntb_peer_spad_bar_set()
793 peer_ntb_epc = ntb->epc[!type]; in epf_ntb_peer_spad_bar_set()
794 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]; in epf_ntb_peer_spad_bar_set()
795 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; in epf_ntb_peer_spad_bar_set()
797 ntb_epc = ntb->epc[type]; in epf_ntb_peer_spad_bar_set()
798 barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; in epf_ntb_peer_spad_bar_set()
799 epf_bar = &ntb_epc->epf_bar[barno]; in epf_ntb_peer_spad_bar_set()
800 func_no = ntb_epc->func_no; in epf_ntb_peer_spad_bar_set()
801 vfunc_no = ntb_epc->vfunc_no; in epf_ntb_peer_spad_bar_set()
802 epc = ntb_epc->epc; in epf_ntb_peer_spad_bar_set()
804 peer_spad_offset = peer_ntb_epc->reg->spad_offset; in epf_ntb_peer_spad_bar_set()
805 epf_bar->phys_addr = peer_epf_bar->phys_addr + peer_spad_offset; in epf_ntb_peer_spad_bar_set()
806 epf_bar->size = peer_ntb_epc->spad_size; in epf_ntb_peer_spad_bar_set()
807 epf_bar->barno = barno; in epf_ntb_peer_spad_bar_set()
808 epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32; in epf_ntb_peer_spad_bar_set()
821 * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR
825 * +-----------------+------->+------------------+ +-----------------+
827 * +-----------------+----+ +------------------+<-------+-----------------+
829 * +-----------------+ +-->+------------------+<-------+-----------------+
831 * +-----------------+ +-----------------+
833 * +-----------------+ +-----------------+
835 * +-----------------+ +-----------------+
837 * +-----------------+ +-----------------+
857 epc = ntb_epc->epc; in epf_ntb_config_sspad_bar_clear()
858 func_no = ntb_epc->func_no; in epf_ntb_config_sspad_bar_clear()
859 vfunc_no = ntb_epc->vfunc_no; in epf_ntb_config_sspad_bar_clear()
860 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; in epf_ntb_config_sspad_bar_clear()
861 epf_bar = &ntb_epc->epf_bar[barno]; in epf_ntb_config_sspad_bar_clear()
866 * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR
870 * +-----------------+------->+------------------+ +-----------------+
872 * +-----------------+----+ +------------------+<-------+-----------------+
874 * +-----------------+ +-->+------------------+<-------+-----------------+
876 * +-----------------+ +-----------------+
878 * +-----------------+ +-----------------+
880 * +-----------------+ +-----------------+
882 * +-----------------+ +-----------------+
905 ntb = ntb_epc->epf_ntb; in epf_ntb_config_sspad_bar_set()
906 dev = &ntb->epf->dev; in epf_ntb_config_sspad_bar_set()
908 epc = ntb_epc->epc; in epf_ntb_config_sspad_bar_set()
909 func_no = ntb_epc->func_no; in epf_ntb_config_sspad_bar_set()
910 vfunc_no = ntb_epc->vfunc_no; in epf_ntb_config_sspad_bar_set()
911 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; in epf_ntb_config_sspad_bar_set()
912 epf_bar = &ntb_epc->epf_bar[barno]; in epf_ntb_config_sspad_bar_set()
917 pci_epc_interface_string(ntb_epc->type)); in epf_ntb_config_sspad_bar_set()
925 * epf_ntb_config_spad_bar_free() - Free the physical memory associated with
929 * +-----------------+------->+------------------+ +-----------------+
931 * +-----------------+----+ +------------------+<-------+-----------------+
933 * +-----------------+ +-->+------------------+<-------+-----------------+
935 * +-----------------+ +-----------------+
937 * +-----------------+ +-----------------+
939 * +-----------------+ +-----------------+
941 * +-----------------+ +-----------------+
955 epf = ntb->epf; in epf_ntb_config_spad_bar_free()
957 ntb_epc = ntb->epc[type]; in epf_ntb_config_spad_bar_free()
958 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; in epf_ntb_config_spad_bar_free()
959 if (ntb_epc->reg) in epf_ntb_config_spad_bar_free()
960 pci_epf_free_space(epf, ntb_epc->reg, barno, type); in epf_ntb_config_spad_bar_free()
965 * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad
970 * +-----------------+------->+------------------+ +-----------------+
972 * +-----------------+----+ +------------------+<-------+-----------------+
974 * +-----------------+ +-->+------------------+<-------+-----------------+
976 * +-----------------+ +-----------------+
978 * +-----------------+ +-----------------+
980 * +-----------------+ +-----------------+
982 * +-----------------+ +-----------------+
987 * is obtained from "spad-count" configfs entry.
1009 epf = ntb->epf; in epf_ntb_config_spad_bar_alloc()
1010 dev = &epf->dev; in epf_ntb_config_spad_bar_alloc()
1011 ntb_epc = ntb->epc[type]; in epf_ntb_config_spad_bar_alloc()
1013 epc_features = ntb_epc->epc_features; in epf_ntb_config_spad_bar_alloc()
1014 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; in epf_ntb_config_spad_bar_alloc()
1015 size = epc_features->bar[barno].fixed_size; in epf_ntb_config_spad_bar_alloc()
1016 align = epc_features->align; in epf_ntb_config_spad_bar_alloc()
1018 peer_ntb_epc = ntb->epc[!type]; in epf_ntb_config_spad_bar_alloc()
1019 peer_epc_features = peer_ntb_epc->epc_features; in epf_ntb_config_spad_bar_alloc()
1020 peer_barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; in epf_ntb_config_spad_bar_alloc()
1021 peer_size = peer_epc_features->bar[peer_barno].fixed_size; in epf_ntb_config_spad_bar_alloc()
1025 return -EINVAL; in epf_ntb_config_spad_bar_alloc()
1027 spad_count = ntb->spad_count; in epf_ntb_config_spad_bar_alloc()
1032 msix_capable = epc_features->msix_capable; in epf_ntb_config_spad_bar_alloc()
1034 msix_table_size = PCI_MSIX_ENTRY_SIZE * ntb->db_count; in epf_ntb_config_spad_bar_alloc()
1036 ntb_epc->msix_table_offset = ctrl_size; in epf_ntb_config_spad_bar_alloc()
1037 ntb_epc->msix_bar = barno; in epf_ntb_config_spad_bar_alloc()
1039 pba_size = ALIGN(DIV_ROUND_UP(ntb->db_count, 8), 8); in epf_ntb_config_spad_bar_alloc()
1068 return -EINVAL; in epf_ntb_config_spad_bar_alloc()
1074 return -ENOMEM; in epf_ntb_config_spad_bar_alloc()
1077 ntb_epc->reg = base; in epf_ntb_config_spad_bar_alloc()
1079 ctrl = ntb_epc->reg; in epf_ntb_config_spad_bar_alloc()
1080 ctrl->spad_offset = ctrl_size; in epf_ntb_config_spad_bar_alloc()
1081 ctrl->spad_count = spad_count; in epf_ntb_config_spad_bar_alloc()
1082 ctrl->num_mws = ntb->num_mws; in epf_ntb_config_spad_bar_alloc()
1083 ctrl->db_entry_size = align ? align : 4; in epf_ntb_config_spad_bar_alloc()
1084 ntb_epc->spad_size = spad_size; in epf_ntb_config_spad_bar_alloc()
1090 * epf_ntb_config_spad_bar_alloc_interface() - Allocate memory for config +
1103 dev = &ntb->epf->dev; in epf_ntb_config_spad_bar_alloc_interface()
1118 * epf_ntb_free_peer_mem() - Free memory allocated in peers outbound address
1123 * +-----------------+ +---->+----------------+-----------+-----------------+
1124 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 |
1125 * +-----------------+ | +----------------+ +-----------------+
1126 * | BAR1 | | | Doorbell 2 +---------+ | |
1127 * +-----------------+----+ +----------------+ | | |
1128 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+
1129 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 |
1130 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+
1131 * +-----------------+ | |----------------+ | | | |
1132 * | BAR4 | | | | | | +-----------------+
1133 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3||
1134 * | BAR5 | | | | | | +-----------------+
1135 * +-----------------+ +---->-----------------+ | | | |
1136 * EP CONTROLLER 1 | | | | +-----------------+
1137 * | | | +---->+ MSI|X ADDRESS 4 |
1138 * +----------------+ | +-----------------+
1140 * (OB SPACE) | | |
1141 * +-------> MW1 |
1144 * (B) +-----------------+
1150 * +-----------------+
1154 * Free memory allocated in EP CONTROLLER 2 (OB SPACE) in the above diagram.
1168 epc = ntb_epc->epc; in epf_ntb_free_peer_mem()
1171 barno = ntb_epc->epf_ntb_bar[bar]; in epf_ntb_free_peer_mem()
1172 mw_addr = ntb_epc->mw_addr[barno]; in epf_ntb_free_peer_mem()
1173 epf_bar = &ntb_epc->epf_bar[barno]; in epf_ntb_free_peer_mem()
1174 phys_addr = epf_bar->phys_addr; in epf_ntb_free_peer_mem()
1175 size = epf_bar->size; in epf_ntb_free_peer_mem()
1178 ntb_epc->mw_addr[barno] = NULL; in epf_ntb_free_peer_mem()
1184 * epf_ntb_db_mw_bar_clear() - Clear doorbell and memory BAR
1188 * +-----------------+ +---->+----------------+-----------+-----------------+
1189 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 |
1190 * +-----------------+ | +----------------+ +-----------------+
1191 * | BAR1 | | | Doorbell 2 +---------+ | |
1192 * +-----------------+----+ +----------------+ | | |
1193 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+
1194 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 |
1195 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+
1196 * +-----------------+ | |----------------+ | | | |
1197 * | BAR4 | | | | | | +-----------------+
1198 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3||
1199 * | BAR5 | | | | | | +-----------------+
1200 * +-----------------+ +---->-----------------+ | | | |
1201 * EP CONTROLLER 1 | | | | +-----------------+
1202 * | | | +---->+ MSI|X ADDRESS 4 |
1203 * +----------------+ | +-----------------+
1205 * (OB SPACE) | | |
1206 * +-------> MW1 |
1209 * (B) +-----------------+
1215 * +-----------------+
1231 epc = ntb_epc->epc; in epf_ntb_db_mw_bar_clear()
1233 func_no = ntb_epc->func_no; in epf_ntb_db_mw_bar_clear()
1234 vfunc_no = ntb_epc->vfunc_no; in epf_ntb_db_mw_bar_clear()
1237 barno = ntb_epc->epf_ntb_bar[bar]; in epf_ntb_db_mw_bar_clear()
1238 epf_bar = &ntb_epc->epf_bar[barno]; in epf_ntb_db_mw_bar_clear()
1244 * epf_ntb_db_mw_bar_cleanup() - Clear doorbell/memory BAR and free memory
1257 ntb_epc = ntb->epc[type]; in epf_ntb_db_mw_bar_cleanup()
1258 peer_ntb_epc = ntb->epc[!type]; in epf_ntb_db_mw_bar_cleanup()
1265 * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capability
1269 * Configure MSI/MSI-X capability for each interface with number of
1284 ntb_epc = ntb->epc[type]; in epf_ntb_configure_interrupt()
1285 dev = &ntb->epf->dev; in epf_ntb_configure_interrupt()
1287 epc_features = ntb_epc->epc_features; in epf_ntb_configure_interrupt()
1288 msix_capable = epc_features->msix_capable; in epf_ntb_configure_interrupt()
1289 msi_capable = epc_features->msi_capable; in epf_ntb_configure_interrupt()
1292 dev_err(dev, "MSI or MSI-X is required for doorbell\n"); in epf_ntb_configure_interrupt()
1293 return -EINVAL; in epf_ntb_configure_interrupt()
1296 func_no = ntb_epc->func_no; in epf_ntb_configure_interrupt()
1297 vfunc_no = ntb_epc->vfunc_no; in epf_ntb_configure_interrupt()
1299 db_count = ntb->db_count; in epf_ntb_configure_interrupt()
1302 return -EINVAL; in epf_ntb_configure_interrupt()
1305 ntb->db_count = db_count; in epf_ntb_configure_interrupt()
1306 epc = ntb_epc->epc; in epf_ntb_configure_interrupt()
1319 ntb_epc->msix_bar, in epf_ntb_configure_interrupt()
1320 ntb_epc->msix_table_offset); in epf_ntb_configure_interrupt()
1331 * epf_ntb_alloc_peer_mem() - Allocate memory in peer's outbound address space
1339 * @size: Size of the address region that has to be allocated in peers OB SPACE
1342 * +-----------------+ +---->+----------------+-----------+-----------------+
1343 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 |
1344 * +-----------------+ | +----------------+ +-----------------+
1345 * | BAR1 | | | Doorbell 2 +---------+ | |
1346 * +-----------------+----+ +----------------+ | | |
1347 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+
1348 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 |
1349 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+
1350 * +-----------------+ | |----------------+ | | | |
1351 * | BAR4 | | | | | | +-----------------+
1352 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3||
1353 * | BAR5 | | | | | | +-----------------+
1354 * +-----------------+ +---->-----------------+ | | | |
1355 * EP CONTROLLER 1 | | | | +-----------------+
1356 * | | | +---->+ MSI|X ADDRESS 4 |
1357 * +----------------+ | +-----------------+
1359 * (OB SPACE) | | |
1360 * +-------> MW1 |
1363 * (B) +-----------------+
1369 * +-----------------+
1373 * Allocate memory in OB space of EP CONTROLLER 2 in the above diagram. Allocate
1390 epc_features = ntb_epc->epc_features; in epf_ntb_alloc_peer_mem()
1391 align = epc_features->align; in epf_ntb_alloc_peer_mem()
1401 peer_epc = peer_ntb_epc->epc; in epf_ntb_alloc_peer_mem()
1404 dev_err(dev, "%s intf: Failed to allocate OB address\n", in epf_ntb_alloc_peer_mem()
1405 pci_epc_interface_string(peer_ntb_epc->type)); in epf_ntb_alloc_peer_mem()
1406 return -ENOMEM; in epf_ntb_alloc_peer_mem()
1409 barno = ntb_epc->epf_ntb_bar[bar]; in epf_ntb_alloc_peer_mem()
1410 epf_bar = &ntb_epc->epf_bar[barno]; in epf_ntb_alloc_peer_mem()
1411 ntb_epc->mw_addr[barno] = mw_addr; in epf_ntb_alloc_peer_mem()
1413 epf_bar->phys_addr = phys_addr; in epf_ntb_alloc_peer_mem()
1414 epf_bar->size = size; in epf_ntb_alloc_peer_mem()
1415 epf_bar->barno = barno; in epf_ntb_alloc_peer_mem()
1416 epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32; in epf_ntb_alloc_peer_mem()
1422 * epf_ntb_db_mw_bar_init() - Configure Doorbell and Memory window BARs
1427 * memory in OB address space of HOST2 and configures BAR of HOST1
1446 ntb_epc = ntb->epc[type]; in epf_ntb_db_mw_bar_init()
1447 peer_ntb_epc = ntb->epc[!type]; in epf_ntb_db_mw_bar_init()
1449 dev = &ntb->epf->dev; in epf_ntb_db_mw_bar_init()
1450 epc_features = ntb_epc->epc_features; in epf_ntb_db_mw_bar_init()
1451 align = epc_features->align; in epf_ntb_db_mw_bar_init()
1452 func_no = ntb_epc->func_no; in epf_ntb_db_mw_bar_init()
1453 vfunc_no = ntb_epc->vfunc_no; in epf_ntb_db_mw_bar_init()
1454 epc = ntb_epc->epc; in epf_ntb_db_mw_bar_init()
1455 num_mws = ntb->num_mws; in epf_ntb_db_mw_bar_init()
1456 db_count = ntb->db_count; in epf_ntb_db_mw_bar_init()
1462 size = ALIGN(size, ntb->mws_size[i]); in epf_ntb_db_mw_bar_init()
1463 ctrl = ntb_epc->reg; in epf_ntb_db_mw_bar_init()
1464 ctrl->mw1_offset = size; in epf_ntb_db_mw_bar_init()
1465 size += ntb->mws_size[i]; in epf_ntb_db_mw_bar_init()
1467 size = ntb->mws_size[i]; in epf_ntb_db_mw_bar_init()
1478 barno = ntb_epc->epf_ntb_bar[bar]; in epf_ntb_db_mw_bar_init()
1479 epf_bar = &ntb_epc->epf_bar[barno]; in epf_ntb_db_mw_bar_init()
1498 * epf_ntb_epc_destroy_interface() - Cleanup NTB EPC interface
1515 epf = ntb->epf; in epf_ntb_epc_destroy_interface()
1516 ntb_epc = ntb->epc[type]; in epf_ntb_epc_destroy_interface()
1519 epc = ntb_epc->epc; in epf_ntb_epc_destroy_interface()
1525 * epf_ntb_epc_destroy() - Cleanup NTB EPC interface
1539 * epf_ntb_epc_create_interface() - Create and initialize NTB EPC interface
1557 dev = &ntb->epf->dev; in epf_ntb_epc_create_interface()
1561 return -ENOMEM; in epf_ntb_epc_create_interface()
1563 epf = ntb->epf; in epf_ntb_epc_create_interface()
1564 vfunc_no = epf->vfunc_no; in epf_ntb_epc_create_interface()
1566 func_no = epf->func_no; in epf_ntb_epc_create_interface()
1567 epf_bar = epf->bar; in epf_ntb_epc_create_interface()
1569 func_no = epf->sec_epc_func_no; in epf_ntb_epc_create_interface()
1570 epf_bar = epf->sec_epc_bar; in epf_ntb_epc_create_interface()
1573 ntb_epc->linkup = false; in epf_ntb_epc_create_interface()
1574 ntb_epc->epc = epc; in epf_ntb_epc_create_interface()
1575 ntb_epc->func_no = func_no; in epf_ntb_epc_create_interface()
1576 ntb_epc->vfunc_no = vfunc_no; in epf_ntb_epc_create_interface()
1577 ntb_epc->type = type; in epf_ntb_epc_create_interface()
1578 ntb_epc->epf_bar = epf_bar; in epf_ntb_epc_create_interface()
1579 ntb_epc->epf_ntb = ntb; in epf_ntb_epc_create_interface()
1583 return -EINVAL; in epf_ntb_epc_create_interface()
1584 ntb_epc->epc_features = epc_features; in epf_ntb_epc_create_interface()
1586 ntb->epc[type] = ntb_epc; in epf_ntb_epc_create_interface()
1592 * epf_ntb_epc_create() - Create and initialize NTB EPC interface
1606 epf = ntb->epf; in epf_ntb_epc_create()
1607 dev = &epf->dev; in epf_ntb_epc_create()
1609 ret = epf_ntb_epc_create_interface(ntb, epf->epc, PRIMARY_INTERFACE); in epf_ntb_epc_create()
1615 ret = epf_ntb_epc_create_interface(ntb, epf->sec_epc, in epf_ntb_epc_create()
1631 * epf_ntb_init_epc_bar_interface() - Identify BARs to be used for each of
1651 ntb_epc = ntb->epc[type]; in epf_ntb_init_epc_bar_interface()
1652 num_mws = ntb->num_mws; in epf_ntb_init_epc_bar_interface()
1653 dev = &ntb->epf->dev; in epf_ntb_init_epc_bar_interface()
1654 epc_features = ntb_epc->epc_features; in epf_ntb_init_epc_bar_interface()
1664 ntb_epc->epf_ntb_bar[bar] = barno; in epf_ntb_init_epc_bar_interface()
1671 ntb->num_mws = i; in epf_ntb_init_epc_bar_interface()
1674 ntb_epc->epf_ntb_bar[bar] = barno; in epf_ntb_init_epc_bar_interface()
1681 * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB
1695 dev = &ntb->epf->dev; in epf_ntb_init_epc_bar()
1709 * epf_ntb_epc_init_interface() - Initialize NTB interface
1727 ntb_epc = ntb->epc[type]; in epf_ntb_epc_init_interface()
1728 epf = ntb->epf; in epf_ntb_epc_init_interface()
1729 dev = &epf->dev; in epf_ntb_epc_init_interface()
1730 epc = ntb_epc->epc; in epf_ntb_epc_init_interface()
1731 func_no = ntb_epc->func_no; in epf_ntb_epc_init_interface()
1732 vfunc_no = ntb_epc->vfunc_no; in epf_ntb_epc_init_interface()
1734 ret = epf_ntb_config_sspad_bar_set(ntb->epc[type]); in epf_ntb_epc_init_interface()
1763 ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header); in epf_ntb_epc_init_interface()
1771 INIT_DELAYED_WORK(&ntb->epc[type]->cmd_handler, epf_ntb_cmd_handler); in epf_ntb_epc_init_interface()
1772 queue_work(kpcintb_workqueue, &ntb->epc[type]->cmd_handler.work); in epf_ntb_epc_init_interface()
1780 epf_ntb_peer_spad_bar_clear(ntb->epc[type]); in epf_ntb_epc_init_interface()
1783 epf_ntb_config_sspad_bar_clear(ntb->epc[type]); in epf_ntb_epc_init_interface()
1789 * epf_ntb_epc_cleanup_interface() - Cleanup NTB interface
1803 ntb_epc = ntb->epc[type]; in epf_ntb_epc_cleanup_interface()
1804 cancel_delayed_work(&ntb_epc->cmd_handler); in epf_ntb_epc_cleanup_interface()
1811 * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces
1825 * epf_ntb_epc_init() - Initialize all NTB interfaces
1837 dev = &ntb->epf->dev; in epf_ntb_epc_init()
1851 epf_ntb_epc_cleanup_interface(ntb, type - 1); in epf_ntb_epc_init()
1857 * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality
1868 struct device *dev = &epf->dev; in epf_ntb_bind()
1871 if (!epf->epc) { in epf_ntb_bind()
1876 if (!epf->sec_epc) { in epf_ntb_bind()
1919 * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind()
1940 return sysfs_emit(page, "%d\n", ntb->_name); \
1952 return -EINVAL; \
1954 ntb->_name = val; \
1969 return sysfs_emit(page, "%lld\n", ntb->mws_size[win_no - 1]); \
1978 struct device *dev = &ntb->epf->dev; \
1983 return -EINVAL; \
1986 return -EINVAL; \
1988 if (ntb->num_mws < win_no) { \
1989 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
1990 return -EINVAL; \
1993 ntb->mws_size[win_no - 1] = val; \
2006 return -EINVAL; in epf_ntb_num_mws_store()
2009 return -EINVAL; in epf_ntb_num_mws_store()
2011 ntb->num_mws = val; in epf_ntb_num_mws_store()
2055 * epf_ntb_add_cfs() - Add configfs directory specific to NTB
2058 * config_items of a specific type that belong to a specific sub-system.
2067 struct config_group *ntb_group = &ntb->group; in epf_ntb_add_cfs()
2068 struct device *dev = &epf->dev; in epf_ntb_add_cfs()
2076 * epf_ntb_probe() - Probe NTB function driver
2089 dev = &epf->dev; in epf_ntb_probe()
2093 return -ENOMEM; in epf_ntb_probe()
2095 epf->header = &epf_ntb_header; in epf_ntb_probe()
2096 ntb->epf = epf; in epf_ntb_probe()
2132 pr_err("Failed to register pci epf ntb driver --> %d\n", ret); in epf_ntb_init()