1 // SPDX-License-Identifier: GPL-2.0
13 #include <linux/pci-epc.h>
14 #include <linux/pci-epf.h>
15 #include <linux/pci-ep-cfs.h>
36  * pci_epc_put() - release the PCI endpoint controller
46 	module_put(epc->ops->owner);  in pci_epc_put()
47 	put_device(&epc->dev);  in pci_epc_put()
52  * pci_epc_get() - get the PCI endpoint controller
60 	int ret = -EINVAL;  in pci_epc_get()
69 	if (try_module_get(epc->ops->owner))  in pci_epc_get()
79  * pci_epc_get_first_free_bar() - helper to get first unreserved BAR
93  * pci_epc_get_next_free_bar() - helper to get unreserved BAR starting from @bar
108 	/* If 'bar - 1' is a 64-bit BAR, move to the next BAR */  in pci_epc_get_next_free_bar()
109 	if (bar > 0 && epc_features->bar[bar - 1].only_64bit)  in pci_epc_get_next_free_bar()
114 		if (epc_features->bar[i].type != BAR_RESERVED)  in pci_epc_get_next_free_bar()
125 	if (IS_ERR_OR_NULL(epc) || func_no >= epc->max_functions)  in pci_epc_function_is_valid()
128 	if (vfunc_no > 0 && (!epc->max_vfs || vfunc_no > epc->max_vfs[func_no]))  in pci_epc_function_is_valid()
135  * pci_epc_get_features() - get the features supported by EPC
154 	if (!epc->ops->get_features)  in pci_epc_get_features()
157 	mutex_lock(&epc->lock);  in pci_epc_get_features()
158 	epc_features = epc->ops->get_features(epc, func_no, vfunc_no);  in pci_epc_get_features()
159 	mutex_unlock(&epc->lock);  in pci_epc_get_features()
166  * pci_epc_stop() - stop the PCI link
173 	if (IS_ERR(epc) || !epc->ops->stop)  in pci_epc_stop()
176 	mutex_lock(&epc->lock);  in pci_epc_stop()
177 	epc->ops->stop(epc);  in pci_epc_stop()
178 	mutex_unlock(&epc->lock);  in pci_epc_stop()
183  * pci_epc_start() - start the PCI link
193 		return -EINVAL;  in pci_epc_start()
195 	if (!epc->ops->start)  in pci_epc_start()
198 	mutex_lock(&epc->lock);  in pci_epc_start()
199 	ret = epc->ops->start(epc);  in pci_epc_start()
200 	mutex_unlock(&epc->lock);  in pci_epc_start()
207  * pci_epc_raise_irq() - interrupt the host system
211  * @type: specify the type of interrupt; INTX, MSI or MSI-X
212  * @interrupt_num: the MSI or MSI-X interrupt number with range (1-N)
214  * Invoke to raise an INTX, MSI or MSI-X interrupt
222 		return -EINVAL;  in pci_epc_raise_irq()
224 	if (!epc->ops->raise_irq)  in pci_epc_raise_irq()
227 	mutex_lock(&epc->lock);  in pci_epc_raise_irq()
228 	ret = epc->ops->raise_irq(epc, func_no, vfunc_no, type, interrupt_num);  in pci_epc_raise_irq()
229 	mutex_unlock(&epc->lock);  in pci_epc_raise_irq()
236  * pci_epc_map_msi_irq() - Map physical address to MSI address and return
237  *                         MSI data
238  * @epc: the EPC device which has the MSI capability
242  * @interrupt_num: the MSI interrupt number with range (1-N)
244  * @msi_data: the data that should be written in order to raise MSI interrupt
246  * @msi_addr_offset: Offset of MSI address from the aligned outbound address
247  *                   to which the MSI address is mapped
249  * Invoke to map physical address to MSI address and return MSI data. The
263 		return -EINVAL;  in pci_epc_map_msi_irq()
265 	if (!epc->ops->map_msi_irq)  in pci_epc_map_msi_irq()
266 		return -EINVAL;  in pci_epc_map_msi_irq()
268 	mutex_lock(&epc->lock);  in pci_epc_map_msi_irq()
269 	ret = epc->ops->map_msi_irq(epc, func_no, vfunc_no, phys_addr,  in pci_epc_map_msi_irq()
272 	mutex_unlock(&epc->lock);  in pci_epc_map_msi_irq()
279  * pci_epc_get_msi() - get the number of MSI interrupt numbers allocated
280  * @epc: the EPC device to which MSI interrupts was requested
284  * Invoke to get the number of MSI interrupts allocated by the RC
293 	if (!epc->ops->get_msi)  in pci_epc_get_msi()
296 	mutex_lock(&epc->lock);  in pci_epc_get_msi()
297 	interrupt = epc->ops->get_msi(epc, func_no, vfunc_no);  in pci_epc_get_msi()
298 	mutex_unlock(&epc->lock);  in pci_epc_get_msi()
310  * pci_epc_set_msi() - set the number of MSI interrupt numbers required
311  * @epc: the EPC device on which MSI has to be configured
314  * @interrupts: number of MSI interrupts required by the EPF
316  * Invoke to set the required number of MSI interrupts.
324 		return -EINVAL;  in pci_epc_set_msi()
327 		return -EINVAL;  in pci_epc_set_msi()
329 	if (!epc->ops->set_msi)  in pci_epc_set_msi()
334 	mutex_lock(&epc->lock);  in pci_epc_set_msi()
335 	ret = epc->ops->set_msi(epc, func_no, vfunc_no, encode_int);  in pci_epc_set_msi()
336 	mutex_unlock(&epc->lock);  in pci_epc_set_msi()
343  * pci_epc_get_msix() - get the number of MSI-X interrupt numbers allocated
344  * @epc: the EPC device to which MSI-X interrupts was requested
348  * Invoke to get the number of MSI-X interrupts allocated by the RC
357 	if (!epc->ops->get_msix)  in pci_epc_get_msix()
360 	mutex_lock(&epc->lock);  in pci_epc_get_msix()
361 	interrupt = epc->ops->get_msix(epc, func_no, vfunc_no);  in pci_epc_get_msix()
362 	mutex_unlock(&epc->lock);  in pci_epc_get_msix()
372  * pci_epc_set_msix() - set the number of MSI-X interrupt numbers required
373  * @epc: the EPC device on which MSI-X has to be configured
376  * @interrupts: number of MSI-X interrupts required by the EPF
377  * @bir: BAR where the MSI-X table resides
378  * @offset: Offset pointing to the start of MSI-X table
380  * Invoke to set the required number of MSI-X interrupts.
388 		return -EINVAL;  in pci_epc_set_msix()
391 		return -EINVAL;  in pci_epc_set_msix()
393 	if (!epc->ops->set_msix)  in pci_epc_set_msix()
396 	mutex_lock(&epc->lock);  in pci_epc_set_msix()
397 	ret = epc->ops->set_msix(epc, func_no, vfunc_no, interrupts - 1, bir,  in pci_epc_set_msix()
399 	mutex_unlock(&epc->lock);  in pci_epc_set_msix()
406  * pci_epc_unmap_addr() - unmap CPU address from PCI address
420 	if (!epc->ops->unmap_addr)  in pci_epc_unmap_addr()
423 	mutex_lock(&epc->lock);  in pci_epc_unmap_addr()
424 	epc->ops->unmap_addr(epc, func_no, vfunc_no, phys_addr);  in pci_epc_unmap_addr()
425 	mutex_unlock(&epc->lock);  in pci_epc_unmap_addr()
430  * pci_epc_map_addr() - map CPU address to PCI address
438  * Invoke to map CPU address with PCI address.
446 		return -EINVAL;  in pci_epc_map_addr()
448 	if (!epc->ops->map_addr)  in pci_epc_map_addr()
451 	mutex_lock(&epc->lock);  in pci_epc_map_addr()
452 	ret = epc->ops->map_addr(epc, func_no, vfunc_no, phys_addr, pci_addr,  in pci_epc_map_addr()
454 	mutex_unlock(&epc->lock);  in pci_epc_map_addr()
461  * pci_epc_mem_map() - allocate and map a PCI address to a CPU address
466  * @pci_size: the number of bytes to map starting from @pci_addr
467  * @map: where to return the mapping information
469  * Allocate a controller memory address region and map it to a RC PCI address
476  * indicated by @map->pci_size. This size may be less than the requested
478  * @map->virt_addr (@map->phys_addr indicates the physical address).
480  * respectively indicated by @map->map_size and @map->virt_base (and
481  * @map->phys_base for the physical address of @map->virt_base).
486 		    u64 pci_addr, size_t pci_size, struct pci_epc_map *map)  in pci_epc_mem_map()  argument
493 		return -EINVAL;  in pci_epc_mem_map()
495 	if (!pci_size || !map)  in pci_epc_mem_map()
496 		return -EINVAL;  in pci_epc_mem_map()
499 	 * Align the PCI address to map. If the controller defines the  in pci_epc_mem_map()
500 	 * .align_addr() operation, use it to determine the PCI address to map  in pci_epc_mem_map()
504 	memset(map, 0, sizeof(*map));  in pci_epc_mem_map()
505 	map->pci_addr = pci_addr;  in pci_epc_mem_map()
506 	if (epc->ops->align_addr)  in pci_epc_mem_map()
507 		map->map_pci_addr =  in pci_epc_mem_map()
508 			epc->ops->align_addr(epc, pci_addr,  in pci_epc_mem_map()
511 		map->map_pci_addr = pci_addr;  in pci_epc_mem_map()
512 	map->map_size = map_size;  in pci_epc_mem_map()
513 	if (map->map_pci_addr + map->map_size < pci_addr + pci_size)  in pci_epc_mem_map()
514 		map->pci_size = map->map_pci_addr + map->map_size - pci_addr;  in pci_epc_mem_map()
516 		map->pci_size = pci_size;  in pci_epc_mem_map()
518 	map->virt_base = pci_epc_mem_alloc_addr(epc, &map->phys_base,  in pci_epc_mem_map()
519 						map->map_size);  in pci_epc_mem_map()
520 	if (!map->virt_base)  in pci_epc_mem_map()
521 		return -ENOMEM;  in pci_epc_mem_map()
523 	map->phys_addr = map->phys_base + map_offset;  in pci_epc_mem_map()
524 	map->virt_addr = map->virt_base + map_offset;  in pci_epc_mem_map()
526 	ret = pci_epc_map_addr(epc, func_no, vfunc_no, map->phys_base,  in pci_epc_mem_map()
527 			       map->map_pci_addr, map->map_size);  in pci_epc_mem_map()
529 		pci_epc_mem_free_addr(epc, map->phys_base, map->virt_base,  in pci_epc_mem_map()
530 				      map->map_size);  in pci_epc_mem_map()
539  * pci_epc_mem_unmap() - unmap and free a CPU address region
543  * @map: the mapping information
549 		       struct pci_epc_map *map)  in pci_epc_mem_unmap()  argument
554 	if (!map || !map->virt_base)  in pci_epc_mem_unmap()
557 	pci_epc_unmap_addr(epc, func_no, vfunc_no, map->phys_base);  in pci_epc_mem_unmap()
558 	pci_epc_mem_free_addr(epc, map->phys_base, map->virt_base,  in pci_epc_mem_unmap()
559 			      map->map_size);  in pci_epc_mem_unmap()
564  * pci_epc_clear_bar() - reset the BAR
578 	if (epf_bar->barno == BAR_5 &&  in pci_epc_clear_bar()
579 	    epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64)  in pci_epc_clear_bar()
582 	if (!epc->ops->clear_bar)  in pci_epc_clear_bar()
585 	mutex_lock(&epc->lock);  in pci_epc_clear_bar()
586 	epc->ops->clear_bar(epc, func_no, vfunc_no, epf_bar);  in pci_epc_clear_bar()
587 	mutex_unlock(&epc->lock);  in pci_epc_clear_bar()
592  * pci_epc_set_bar() - configure BAR in order for host to assign PCI addr space
604 	enum pci_barno bar = epf_bar->barno;  in pci_epc_set_bar()
605 	int flags = epf_bar->flags;  in pci_epc_set_bar()
610 		return -EINVAL;  in pci_epc_set_bar()
612 	if (epc_features->bar[bar].type == BAR_FIXED &&  in pci_epc_set_bar()
613 	    (epc_features->bar[bar].fixed_size != epf_bar->size))  in pci_epc_set_bar()
614 		return -EINVAL;  in pci_epc_set_bar()
616 	if (!is_power_of_2(epf_bar->size))  in pci_epc_set_bar()
617 		return -EINVAL;  in pci_epc_set_bar()
619 	if ((epf_bar->barno == BAR_5 && flags & PCI_BASE_ADDRESS_MEM_TYPE_64) ||  in pci_epc_set_bar()
622 	    (upper_32_bits(epf_bar->size) &&  in pci_epc_set_bar()
624 		return -EINVAL;  in pci_epc_set_bar()
626 	if (!epc->ops->set_bar)  in pci_epc_set_bar()
629 	mutex_lock(&epc->lock);  in pci_epc_set_bar()
630 	ret = epc->ops->set_bar(epc, func_no, vfunc_no, epf_bar);  in pci_epc_set_bar()
631 	mutex_unlock(&epc->lock);  in pci_epc_set_bar()
638  * pci_epc_write_header() - write standard configuration header
655 		return -EINVAL;  in pci_epc_write_header()
659 		return -EINVAL;  in pci_epc_write_header()
661 	if (!epc->ops->write_header)  in pci_epc_write_header()
664 	mutex_lock(&epc->lock);  in pci_epc_write_header()
665 	ret = epc->ops->write_header(epc, func_no, vfunc_no, header);  in pci_epc_write_header()
666 	mutex_unlock(&epc->lock);  in pci_epc_write_header()
673  * pci_epc_add_epf() - bind PCI endpoint function to an endpoint controller
690 	if (IS_ERR_OR_NULL(epc) || epf->is_vf)  in pci_epc_add_epf()
691 		return -EINVAL;  in pci_epc_add_epf()
693 	if (type == PRIMARY_INTERFACE && epf->epc)  in pci_epc_add_epf()
694 		return -EBUSY;  in pci_epc_add_epf()
696 	if (type == SECONDARY_INTERFACE && epf->sec_epc)  in pci_epc_add_epf()
697 		return -EBUSY;  in pci_epc_add_epf()
699 	mutex_lock(&epc->list_lock);  in pci_epc_add_epf()
700 	func_no = find_first_zero_bit(&epc->function_num_map,  in pci_epc_add_epf()
703 		ret = -EINVAL;  in pci_epc_add_epf()
707 	if (func_no > epc->max_functions - 1) {  in pci_epc_add_epf()
708 		dev_err(&epc->dev, "Exceeding max supported Function Number\n");  in pci_epc_add_epf()
709 		ret = -EINVAL;  in pci_epc_add_epf()
713 	set_bit(func_no, &epc->function_num_map);  in pci_epc_add_epf()
715 		epf->func_no = func_no;  in pci_epc_add_epf()
716 		epf->epc = epc;  in pci_epc_add_epf()
717 		list = &epf->list;  in pci_epc_add_epf()
719 		epf->sec_epc_func_no = func_no;  in pci_epc_add_epf()
720 		epf->sec_epc = epc;  in pci_epc_add_epf()
721 		list = &epf->sec_epc_list;  in pci_epc_add_epf()
724 	list_add_tail(list, &epc->pci_epf);  in pci_epc_add_epf()
726 	mutex_unlock(&epc->list_lock);  in pci_epc_add_epf()
733  * pci_epc_remove_epf() - remove PCI endpoint function from endpoint controller
750 	mutex_lock(&epc->list_lock);  in pci_epc_remove_epf()
752 		func_no = epf->func_no;  in pci_epc_remove_epf()
753 		list = &epf->list;  in pci_epc_remove_epf()
754 		epf->epc = NULL;  in pci_epc_remove_epf()
756 		func_no = epf->sec_epc_func_no;  in pci_epc_remove_epf()
757 		list = &epf->sec_epc_list;  in pci_epc_remove_epf()
758 		epf->sec_epc = NULL;  in pci_epc_remove_epf()
760 	clear_bit(func_no, &epc->function_num_map);  in pci_epc_remove_epf()
762 	mutex_unlock(&epc->list_lock);  in pci_epc_remove_epf()
767  * pci_epc_linkup() - Notify the EPF device that EPC device has established a
781 	mutex_lock(&epc->list_lock);  in pci_epc_linkup()
782 	list_for_each_entry(epf, &epc->pci_epf, list) {  in pci_epc_linkup()
783 		mutex_lock(&epf->lock);  in pci_epc_linkup()
784 		if (epf->event_ops && epf->event_ops->link_up)  in pci_epc_linkup()
785 			epf->event_ops->link_up(epf);  in pci_epc_linkup()
786 		mutex_unlock(&epf->lock);  in pci_epc_linkup()
788 	mutex_unlock(&epc->list_lock);  in pci_epc_linkup()
793  * pci_epc_linkdown() - Notify the EPF device that EPC device has dropped the
807 	mutex_lock(&epc->list_lock);  in pci_epc_linkdown()
808 	list_for_each_entry(epf, &epc->pci_epf, list) {  in pci_epc_linkdown()
809 		mutex_lock(&epf->lock);  in pci_epc_linkdown()
810 		if (epf->event_ops && epf->event_ops->link_down)  in pci_epc_linkdown()
811 			epf->event_ops->link_down(epf);  in pci_epc_linkdown()
812 		mutex_unlock(&epf->lock);  in pci_epc_linkdown()
814 	mutex_unlock(&epc->list_lock);  in pci_epc_linkdown()
819  * pci_epc_init_notify() - Notify the EPF device that EPC device initialization
833 	mutex_lock(&epc->list_lock);  in pci_epc_init_notify()
834 	list_for_each_entry(epf, &epc->pci_epf, list) {  in pci_epc_init_notify()
835 		mutex_lock(&epf->lock);  in pci_epc_init_notify()
836 		if (epf->event_ops && epf->event_ops->epc_init)  in pci_epc_init_notify()
837 			epf->event_ops->epc_init(epf);  in pci_epc_init_notify()
838 		mutex_unlock(&epf->lock);  in pci_epc_init_notify()
840 	epc->init_complete = true;  in pci_epc_init_notify()
841 	mutex_unlock(&epc->list_lock);  in pci_epc_init_notify()
846  * pci_epc_notify_pending_init() - Notify the pending EPC device initialization
857 	if (epc->init_complete) {  in pci_epc_notify_pending_init()
858 		mutex_lock(&epf->lock);  in pci_epc_notify_pending_init()
859 		if (epf->event_ops && epf->event_ops->epc_init)  in pci_epc_notify_pending_init()
860 			epf->event_ops->epc_init(epf);  in pci_epc_notify_pending_init()
861 		mutex_unlock(&epf->lock);  in pci_epc_notify_pending_init()
867  * pci_epc_deinit_notify() - Notify the EPF device about EPC deinitialization
879 	mutex_lock(&epc->list_lock);  in pci_epc_deinit_notify()
880 	list_for_each_entry(epf, &epc->pci_epf, list) {  in pci_epc_deinit_notify()
881 		mutex_lock(&epf->lock);  in pci_epc_deinit_notify()
882 		if (epf->event_ops && epf->event_ops->epc_deinit)  in pci_epc_deinit_notify()
883 			epf->event_ops->epc_deinit(epf);  in pci_epc_deinit_notify()
884 		mutex_unlock(&epf->lock);  in pci_epc_deinit_notify()
886 	epc->init_complete = false;  in pci_epc_deinit_notify()
887 	mutex_unlock(&epc->list_lock);  in pci_epc_deinit_notify()
892  * pci_epc_bus_master_enable_notify() - Notify the EPF device that the EPC
907 	mutex_lock(&epc->list_lock);  in pci_epc_bus_master_enable_notify()
908 	list_for_each_entry(epf, &epc->pci_epf, list) {  in pci_epc_bus_master_enable_notify()
909 		mutex_lock(&epf->lock);  in pci_epc_bus_master_enable_notify()
910 		if (epf->event_ops && epf->event_ops->bus_master_enable)  in pci_epc_bus_master_enable_notify()
911 			epf->event_ops->bus_master_enable(epf);  in pci_epc_bus_master_enable_notify()
912 		mutex_unlock(&epf->lock);  in pci_epc_bus_master_enable_notify()
914 	mutex_unlock(&epc->list_lock);  in pci_epc_bus_master_enable_notify()
919  * pci_epc_destroy() - destroy the EPC device
926 	pci_ep_cfs_remove_epc_group(epc->group);  in pci_epc_destroy()
928 	pci_bus_release_domain_nr(epc->dev.parent, epc->domain_nr);  in pci_epc_destroy()
930 	device_unregister(&epc->dev);  in pci_epc_destroy()
935  * devm_pci_epc_destroy() - destroy the EPC device
958  * __pci_epc_create() - create a new endpoint controller (EPC) device
973 		ret = -EINVAL;  in __pci_epc_create()
979 		ret = -ENOMEM;  in __pci_epc_create()
983 	mutex_init(&epc->lock);  in __pci_epc_create()
984 	mutex_init(&epc->list_lock);  in __pci_epc_create()
985 	INIT_LIST_HEAD(&epc->pci_epf);  in __pci_epc_create()
987 	device_initialize(&epc->dev);  in __pci_epc_create()
988 	epc->dev.class = &pci_epc_class;  in __pci_epc_create()
989 	epc->dev.parent = dev;  in __pci_epc_create()
990 	epc->dev.release = pci_epc_release;  in __pci_epc_create()
991 	epc->ops = ops;  in __pci_epc_create()
994 	epc->domain_nr = pci_bus_find_domain_nr(NULL, dev);  in __pci_epc_create()
1003 	ret = dev_set_name(&epc->dev, "%s", dev_name(dev));  in __pci_epc_create()
1007 	ret = device_add(&epc->dev);  in __pci_epc_create()
1011 	epc->group = pci_ep_cfs_add_epc_group(dev_name(dev));  in __pci_epc_create()
1016 	put_device(&epc->dev);  in __pci_epc_create()
1024  * __devm_pci_epc_create() - create a new endpoint controller (EPC) device
1042 		return ERR_PTR(-ENOMEM);  in __devm_pci_epc_create()