86 static void iommu_flush_dte_sync(struct amd_iommu *iommu, u16 devid);
88 static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid);
146 	struct dev_table_entry *ptr = &dev_table[dev_data->devid];  in update_dte256()
154 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
159 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
167 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
176 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
189 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
194 		iommu_flush_dte_sync(iommu, dev_data->devid);  in update_dte256()
214 	ptr = &dev_table[dev_data->devid];  in get_dte256()
256 			return p->devid;  in get_acpihid_device_id()
295 		u32 devid = get_acpihid_device_id(dev, NULL);  in get_device_segment()  local
297 		seg = PCI_SBDF_TO_SEGID(devid);  in get_device_segment()
304 void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid)  in amd_iommu_set_rlookup_table()  argument
308 	pci_seg->rlookup_table[devid] = iommu;  in amd_iommu_set_rlookup_table()
311 static struct amd_iommu *__rlookup_amd_iommu(u16 seg, u16 devid)  in __rlookup_amd_iommu()  argument
317 			return pci_seg->rlookup_table[devid];  in __rlookup_amd_iommu()
325 	int devid = get_device_sbdf_id(dev);  in rlookup_amd_iommu()  local
327 	if (devid < 0)  in rlookup_amd_iommu()
329 	return __rlookup_amd_iommu(seg, PCI_SBDF_TO_DEVID(devid));  in rlookup_amd_iommu()
332 static struct iommu_dev_data *alloc_dev_data(struct amd_iommu *iommu, u16 devid)  in alloc_dev_data()  argument
343 	dev_data->devid = devid;  in alloc_dev_data()
350 struct iommu_dev_data *search_dev_data(struct amd_iommu *iommu, u16 devid)  in search_dev_data()  argument
361 		if (dev_data->devid == devid)  in search_dev_data()
373 	u16 devid = pci_dev_id(pdev);  in clone_alias()  local
376 	if (devid == alias)  in clone_alias()
386 		pr_err("%s : Failed to get dev_data for 0x%x\n", __func__, devid);  in clone_alias()
446 static struct iommu_dev_data *find_dev_data(struct amd_iommu *iommu, u16 devid)  in find_dev_data()  argument
450 	dev_data = search_dev_data(iommu, devid);  in find_dev_data()
453 		dev_data = alloc_dev_data(iommu, devid);  in find_dev_data()
470 	int devid;  in acpihid_device_group()  local
472 	devid = get_acpihid_device_id(dev, &entry);  in acpihid_device_group()
473 	if (devid < 0)  in acpihid_device_group()
474 		return ERR_PTR(devid);  in acpihid_device_group()
477 		if ((devid == p->devid) && p->group)  in acpihid_device_group()
636 	int devid, sbdf;  in check_device()  local
644 	devid = PCI_SBDF_TO_DEVID(sbdf);  in check_device()
652 	if (devid > pci_seg->last_bdf)  in check_device()
661 	int devid, sbdf;  in iommu_init_device()  local
670 	devid = PCI_SBDF_TO_DEVID(sbdf);  in iommu_init_device()
671 	dev_data = find_dev_data(iommu, devid);  in iommu_init_device()
702 	int devid, sbdf;  in iommu_ignore_device()  local
708 	devid = PCI_SBDF_TO_DEVID(sbdf);  in iommu_ignore_device()
709 	pci_seg->rlookup_table[devid] = NULL;  in iommu_ignore_device()
710 	memset(&dev_table[devid], 0, sizeof(struct dev_table_entry));  in iommu_ignore_device()
722 static void dump_dte_entry(struct amd_iommu *iommu, u16 devid)  in dump_dte_entry()  argument
726 	struct iommu_dev_data *dev_data = find_dev_data(iommu, devid);  in dump_dte_entry()
746 	int devid, vmg_tag, flags;  in amd_iommu_report_rmp_hw_error()  local
750 	devid   = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;  in amd_iommu_report_rmp_hw_error()
755 	pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),  in amd_iommu_report_rmp_hw_error()
756 					   devid & 0xff);  in amd_iommu_report_rmp_hw_error()
767 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in amd_iommu_report_rmp_hw_error()
778 	int devid, flags_rmp, vmg_tag, flags;  in amd_iommu_report_rmp_fault()  local
782 	devid     = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;  in amd_iommu_report_rmp_fault()
788 	pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),  in amd_iommu_report_rmp_fault()
789 					   devid & 0xff);  in amd_iommu_report_rmp_fault()
800 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in amd_iommu_report_rmp_fault()
815 					u16 devid, u16 domain_id,  in amd_iommu_report_page_fault()  argument
821 	pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid),  in amd_iommu_report_page_fault()
822 					   devid & 0xff);  in amd_iommu_report_page_fault()
837 						   iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid),  in amd_iommu_report_page_fault()
838 						   PCI_FUNC(devid), domain_id);  in amd_iommu_report_page_fault()
856 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in amd_iommu_report_page_fault()
868 	int type, devid, flags, tag;  in iommu_print_event()  local
876 	devid   = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;  in iommu_print_event()
893 		amd_iommu_report_page_fault(iommu, devid, pasid, address, flags);  in iommu_print_event()
900 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in iommu_print_event()
902 		dump_dte_entry(iommu, devid);  in iommu_print_event()
907 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in iommu_print_event()
912 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in iommu_print_event()
925 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in iommu_print_event()
930 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in iommu_print_event()
943 			iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid),  in iommu_print_event()
1018 			pr_debug("%s: devid=%#x, ga_tag=%#x\n",  in iommu_poll_ga_log()
1183 static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)  in build_inv_dte()  argument
1186 	cmd->data[0] = devid;  in build_inv_dte()
1252 static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,  in build_inv_iotlb_pages()  argument
1260 	cmd->data[0]  = devid;  in build_inv_iotlb_pages()
1262 	cmd->data[1]  = devid;  in build_inv_iotlb_pages()
1274 static void build_complete_ppr(struct iommu_cmd *cmd, u16 devid, u32 pasid,  in build_complete_ppr()  argument
1279 	cmd->data[0]  = devid;  in build_complete_ppr()
1296 static void build_inv_irt(struct iommu_cmd *cmd, u16 devid)  in build_inv_irt()  argument
1299 	cmd->data[0] = devid;  in build_inv_irt()
1409 static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)  in iommu_flush_dte()  argument
1413 	build_inv_dte(&cmd, devid);  in iommu_flush_dte()
1418 static void iommu_flush_dte_sync(struct amd_iommu *iommu, u16 devid)  in iommu_flush_dte_sync()  argument
1422 	ret = iommu_flush_dte(iommu, devid);  in iommu_flush_dte_sync()
1429 	u32 devid;  in amd_iommu_flush_dte_all()  local
1432 	for (devid = 0; devid <= last_bdf; ++devid)  in amd_iommu_flush_dte_all()
1433 		iommu_flush_dte(iommu, devid);  in amd_iommu_flush_dte_all()
1478 static void iommu_flush_irt(struct amd_iommu *iommu, u16 devid)  in iommu_flush_irt()  argument
1482 	build_inv_irt(&cmd, devid);  in iommu_flush_irt()
1489 	u32 devid;  in amd_iommu_flush_irt_all()  local
1495 	for (devid = 0; devid <= last_bdf; devid++)  in amd_iommu_flush_irt_all()
1496 		iommu_flush_irt(iommu, devid);  in amd_iommu_flush_irt_all()
1522 	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address,  in device_flush_iotlb()
1553 		ret = iommu_flush_dte(iommu, dev_data->devid);  in device_flush_dte()
1558 	alias = pci_seg->alias_table[dev_data->devid];  in device_flush_dte()
1559 	if (alias != dev_data->devid) {  in device_flush_dte()
1778 	build_complete_ppr(&cmd, dev_data->devid, pasid, status,  in amd_iommu_complete_ppr()
2005 	pr_debug("%s: devid=%#x, glx=%#x, gcr3_tbl=%#llx\n",  in set_dte_gcr3_table()
2006 		 __func__, dev_data->devid, gcr3_info->glx,  in set_dte_gcr3_table()
2036 	struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid];  in set_dte_entry()
2077 	initial_dte = amd_iommu_get_ivhd_dte_flags(iommu->pci_seg->id, dev_data->devid);  in set_dte_entry()
2103 	struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid];  in clear_dte_entry()
2816 		dte = &get_dev_table(iommu)[dev_data->devid];  in amd_iommu_set_dirty_tracking()
2866 	int devid, sbdf;  in amd_iommu_get_resv_regions()  local
2872 	devid = PCI_SBDF_TO_DEVID(sbdf);  in amd_iommu_get_resv_regions()
2880 		if (devid < entry->devid_start || devid > entry->devid_end)  in amd_iommu_get_resv_regions()
3058 static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid)  in iommu_flush_irt_and_complete()  argument
3068 	build_inv_irt(&cmd, devid);  in iommu_flush_irt_and_complete()
3084 static void set_dte_irq_entry(struct amd_iommu *iommu, u16 devid,  in set_dte_irq_entry()  argument
3088 	struct dev_table_entry *dte = &get_dev_table(iommu)[devid];  in set_dte_irq_entry()
3089 	struct iommu_dev_data *dev_data = search_dev_data(iommu, devid);  in set_dte_irq_entry()
3106 static struct irq_remap_table *get_irq_table(struct amd_iommu *iommu, u16 devid)  in get_irq_table()  argument
3111 	if (WARN_ONCE(!pci_seg->rlookup_table[devid],  in get_irq_table()
3112 		      "%s: no iommu for devid %x:%x\n",  in get_irq_table()
3113 		      __func__, pci_seg->id, devid))  in get_irq_table()
3116 	table = pci_seg->irq_lookup_table[devid];  in get_irq_table()
3117 	if (WARN_ONCE(!table, "%s: no table for devid %x:%x\n",  in get_irq_table()
3118 		      __func__, pci_seg->id, devid))  in get_irq_table()
3148 static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid,  in set_remap_table_entry()  argument
3153 	pci_seg->irq_lookup_table[devid] = table;  in set_remap_table_entry()
3154 	set_dte_irq_entry(iommu, devid, table);  in set_remap_table_entry()
3155 	iommu_flush_dte(iommu, devid);  in set_remap_table_entry()
3177 					       u16 devid, struct pci_dev *pdev)  in alloc_irq_table()  argument
3188 	table = pci_seg->irq_lookup_table[devid];  in alloc_irq_table()
3192 	alias = pci_seg->alias_table[devid];  in alloc_irq_table()
3195 		set_remap_table_entry(iommu, devid, table);  in alloc_irq_table()
3207 	table = pci_seg->irq_lookup_table[devid];  in alloc_irq_table()
3213 		set_remap_table_entry(iommu, devid, table);  in alloc_irq_table()
3224 		set_remap_table_entry(iommu, devid, table);  in alloc_irq_table()
3226 	if (devid != alias)  in alloc_irq_table()
3242 static int alloc_irq_index(struct amd_iommu *iommu, u16 devid, int count,  in alloc_irq_index()  argument
3249 	table = alloc_irq_table(iommu, devid, pdev);  in alloc_irq_index()
3288 static int __modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,  in __modify_irte_ga()  argument
3296 	table = get_irq_table(iommu, devid);  in __modify_irte_ga()
3319 static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,  in modify_irte_ga()  argument
3324 	ret = __modify_irte_ga(iommu, devid, index, irte);  in modify_irte_ga()
3328 	iommu_flush_irt_and_complete(iommu, devid);  in modify_irte_ga()
3334 		       u16 devid, int index, union irte *irte)  in modify_irte()  argument
3339 	table = get_irq_table(iommu, devid);  in modify_irte()
3347 	iommu_flush_irt_and_complete(iommu, devid);  in modify_irte()
3352 static void free_irte(struct amd_iommu *iommu, u16 devid, int index)  in free_irte()  argument
3357 	table = get_irq_table(iommu, devid);  in free_irte()
3365 	iommu_flush_irt_and_complete(iommu, devid);  in free_irte()
3370 			 u8 vector, u32 dest_apicid, int devid)  in irte_prepare()  argument
3384 			    u8 vector, u32 dest_apicid, int devid)  in irte_ga_prepare()  argument
3398 static void irte_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)  in irte_activate()  argument
3403 	modify_irte(iommu, devid, index, irte);  in irte_activate()
3406 static void irte_ga_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)  in irte_ga_activate()  argument
3411 	modify_irte_ga(iommu, devid, index, irte);  in irte_ga_activate()
3414 static void irte_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)  in irte_deactivate()  argument
3419 	modify_irte(iommu, devid, index, irte);  in irte_deactivate()
3422 static void irte_ga_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)  in irte_ga_deactivate()  argument
3427 	modify_irte_ga(iommu, devid, index, irte);  in irte_ga_deactivate()
3430 static void irte_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,  in irte_set_affinity()  argument
3437 	modify_irte(iommu, devid, index, irte);  in irte_set_affinity()
3440 static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,  in irte_ga_set_affinity()  argument
3451 		modify_irte_ga(iommu, devid, index, irte);  in irte_ga_set_affinity()
3505 		return get_ioapic_devid(info->devid);  in get_devid()
3507 		return get_hpet_devid(info->devid);  in get_devid()
3536 				       int devid, int index, int sub_handle)  in irq_remapping_prepare_irte()  argument
3544 	data->irq_2_irte.devid = devid;  in irq_remapping_prepare_irte()
3548 				 irq_cfg->dest_apicid, devid);  in irq_remapping_prepare_irte()
3592 	int i, ret, devid, seg, sbdf;  in irq_remapping_alloc()  local
3605 	devid = PCI_SBDF_TO_DEVID(sbdf);  in irq_remapping_alloc()
3606 	iommu = __rlookup_amd_iommu(seg, devid);  in irq_remapping_alloc()
3617 		table = alloc_irq_table(iommu, devid, NULL);  in irq_remapping_alloc()
3637 		index = alloc_irq_index(iommu, devid, nr_irqs, align,  in irq_remapping_alloc()
3640 		index = alloc_irq_index(iommu, devid, nr_irqs, false, NULL);  in irq_remapping_alloc()
3673 		irq_data->hwirq = (devid << 16) + i;  in irq_remapping_alloc()
3676 		irq_remapping_prepare_irte(data, cfg, info, devid, index, i);  in irq_remapping_alloc()
3688 		free_irte(iommu, devid, index + i);  in irq_remapping_alloc()
3707 			free_irte(data->iommu, irte_info->devid, irte_info->index);  in irq_remapping_free()
3731 	iommu->irte_ops->activate(iommu, data->entry, irte_info->devid,  in irq_remapping_activate()
3745 		iommu->irte_ops->deactivate(iommu, data->entry, irte_info->devid,  in irq_remapping_deactivate()
3753 	int devid = -1;  in irq_remapping_select()  local
3759 		devid = get_ioapic_devid(fwspec->param[0]);  in irq_remapping_select()
3761 		devid = get_hpet_devid(fwspec->param[0]);  in irq_remapping_select()
3763 	if (devid < 0)  in irq_remapping_select()
3765 	iommu = __rlookup_amd_iommu((devid >> 16), (devid & 0xffff));  in irq_remapping_select()
3799 	return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,  in amd_iommu_activate_guest_mode()
3829 	return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,  in amd_iommu_deactivate_guest_mode()
3846 	dev_data = search_dev_data(ir_data->iommu, irte_info->devid);  in amd_ir_set_vcpu_affinity()
3901 	iommu->irte_ops->set_affinity(iommu, ir_data->entry, irte_info->devid,  in amd_ir_update_irte()
3997 	return __modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,  in amd_iommu_update_ga()