40 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
42 #define IOAPIC_RANGE_START	(0xfee00000)
43 #define IOAPIC_RANGE_END	(0xfeefffff)
44 #define IOVA_START_ADDR		(0x1000)
64 static int force_on = 0;
77 		return 0;  in root_entry_lctp()
89 		return 0;  in root_entry_uctp()
106 	return 0;  in device_rid_cmp_key()
156 	return 0;  in device_rbtree_insert()
207 int intel_iommu_enabled = 0;
246 			dmar_disabled = 0;  in intel_iommu_setup()
263 			intel_iommu_superpage = 0;  in intel_iommu_setup()
269 			intel_iommu_sm = 0;  in intel_iommu_setup()
302 	fl_sagaw = BIT(2) | (cap_fl5lp_support(iommu->cap) ? BIT(3) : 0);  in __iommu_calculate_sagaw()
322 	for (agaw = width_to_agaw(max_gaw); agaw >= 0; agaw--) {  in __iommu_calculate_agaw()
357 	unsigned long bitmap = 0;  in domain_super_pgsize_bitmap()
387 		if (devfn >= 0x80) {  in iommu_context_addr()
388 			devfn -= 0x80;  in iommu_context_addr()
450 	rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);  in quirk_ioat_snb_local_iommu()
456 	vtbar &= 0xffff0000;  in quirk_ioat_snb_local_iommu()
458 	/* we know that the this iommu should be at offset 0xa000 from vtbar */  in quirk_ioat_snb_local_iommu()
460 	if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) {  in quirk_ioat_snb_local_iommu()
492 	u16 segment = 0;  in device_lookup_iommu()
571 	for (i = 0; i < ROOT_ENTRY_NR; i++) {  in free_context_table()
572 		context = iommu_context_addr(iommu, i, 0, 0);  in free_context_table()
579 		context = iommu_context_addr(iommu, i, 0x80, 0);  in free_context_table()
599 		pr_info("pte level: %d, pte value: 0x%016llx\n", level, pte->val);  in pgtable_walk()
622 	u8 devfn = source_id & 0xff;  in dmar_fault_dump_ptes()
626 	pr_info("Dump %s table entries for IOVA 0x%llx\n", iommu->name, addr);  in dmar_fault_dump_ptes()
636 		pr_info("scalable mode root entry: hi 0x%016llx, low 0x%016llx\n",  in dmar_fault_dump_ptes()
639 		pr_info("root entry: 0x%016llx", rt_entry->lo);  in dmar_fault_dump_ptes()
642 	ctx_entry = iommu_context_addr(iommu, bus, devfn, 0);  in dmar_fault_dump_ptes()
648 	pr_info("context entry: hi 0x%016llx, low 0x%016llx\n",  in dmar_fault_dump_ptes()
676 	pr_info("pasid dir entry: 0x%016llx\n", pde->val);  in dmar_fault_dump_ptes()
686 	for (i = 0; i < ARRAY_SIZE(pte->val); i++)  in dmar_fault_dump_ptes()
687 		pr_info("pasid table entry[%d]: 0x%016llx\n", i, pte->val[i]);  in dmar_fault_dump_ptes()
698 		level = agaw_to_level((pte->val[0] >> 2) & 0x7);  in dmar_fault_dump_ptes()
699 		pgtable = phys_to_virt(pte->val[0] & VTD_PAGE_MASK);  in dmar_fault_dump_ptes()
744 			tmp = 0ULL;  in pfn_to_dma_pte()
880 			   domain->pgd, 0, start_pfn, last_pfn);  in dma_pte_free_pagetable()
883 	if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {  in dma_pte_free_pagetable()
971 			    domain->pgd, 0, start_pfn, last_pfn, freelist);  in domain_unmap()
974 	if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {  in domain_unmap()
996 	return 0;  in iommu_alloc_root_entry()
1027 	iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);  in iommu_set_root_entry()
1029 		qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0);  in iommu_set_root_entry()
1030 	iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);  in iommu_set_root_entry()
1056 	u64 val = 0;  in __iommu_flush_context()
1071 		pr_warn("%s: Unexpected context-cache invalidation type 0x%llx\n",  in __iommu_flush_context()
1091 	u64 val = 0, val_iva = 0;  in __iommu_flush_iotlb()
1108 		pr_warn("%s: Unexpected iotlb invalidation type 0x%llx\n",  in __iommu_flush_iotlb()
1129 	if (DMA_TLB_IAIG(val) == 0)  in __iommu_flush_iotlb()
1159  * IDs ranging from 0x4940 to 0x4943. It is exempted from risky_device()
1163 #define BUGGY_QAT_DEVID_MASK 0x4940
1169 	if ((pdev->device & 0xfffc) != BUGGY_QAT_DEVID_MASK)  in dev_needs_extra_dtlb_flush()
1199 		info->ats_enabled = 0;  in iommu_disable_pci_caps()
1280 	 * with domain-id 0, hence we need to pre-allocate it. We also  in iommu_init_domains()
1281 	 * use domain-id 0 as a marker for non-allocated domain-id, so  in iommu_init_domains()
1284 	set_bit(0, iommu->domain_ids);  in iommu_init_domains()
1287 	 * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid  in iommu_init_domains()
1296 	return 0;  in iommu_init_domains()
1359 		return 0;  in domain_attach_iommu()
1371 		return 0;  in domain_attach_iommu()
1393 	return 0;  in domain_attach_iommu()
1412 	if (--info->refcnt == 0) {  in domain_detach_iommu()
1426 		domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist);  in domain_exit()
1465 		iommu->flush.flush_iotlb(iommu, did_old, 0, 0,  in copied_context_tear_down()
1476  * domain #0, which we have to flush:
1482 		iommu->flush.flush_context(iommu, 0,  in context_present_cache_flush()
1486 		iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);  in context_present_cache_flush()
1513 	ret = 0;  in domain_context_mapping_one()
1534 	ret = 0;  in domain_context_mapping_one()
1550 					  PCI_BUS_NUM(alias), alias & 0xff);  in domain_context_mapping_cb()
1616 					      end_pfn << VTD_PAGE_SHIFT, 0);  in switch_to_super_page()
1632 	unsigned int largepage_lvl = 0;  in __domain_mapping()
1633 	unsigned long lvl_pages = 0;  in __domain_mapping()
1640 	if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)  in __domain_mapping()
1660 	while (nr_pages > 0) {  in __domain_mapping()
1693 		tmp = 0ULL;  in __domain_mapping()
1696 			pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",  in __domain_mapping()
1731 	return 0;  in __domain_mapping()
1741 	context = iommu_context_addr(iommu, bus, devfn, 0);  in domain_context_clear_one()
1797 	int level, flags = 0;  in domain_setup_first_level()
1832 		return 0;  in dmar_domain_attach_device()
1852 	return 0;  in dmar_domain_attach_device()
1907 	return 0;  in device_def_domain_type()
1950 	int tbl_idx, pos = 0, idx, devfn, ret = 0, did;  in copy_context_table()
1959 	for (devfn = 0; devfn < 256; devfn++) {  in copy_context_table()
1963 		if (idx == 0) {  in copy_context_table()
1975 			ret = 0;  in copy_context_table()
1976 			if (devfn < 0x80)  in copy_context_table()
1982 				if (ext && devfn == 0) {  in copy_context_table()
1984 					devfn = 0x7f;  in copy_context_table()
2001 			ret = 0;  in copy_context_table()
2011 		if (did >= 0 && did < cap_ndoms(iommu->cap))  in copy_context_table()
2071 	for (bus = 0; bus < 256; bus++) {  in copy_translation_tables()
2084 	for (bus = 0; bus < 256; bus++) {  in copy_translation_tables()
2106 	ret = 0;  in copy_translation_tables()
2238 	return 0;  in init_dmars()
2317 	return 0;  in init_iommu_hw()
2326 		iommu->flush.flush_context(iommu, 0, 0, 0,  in iommu_flush_all()
2328 		iommu->flush.flush_iotlb(iommu, 0, 0, 0,  in iommu_flush_all()
2357 	return 0;  in iommu_suspend()
2413 	return 0;  in rmrr_sanity_check()
2450 	return 0;  in dmar_parse_one_rmrr()
2469 		if (memcmp(atsr, tmp, atsr->header.length) == 0)  in dmar_find_atsr()
2482 		return 0;  in dmar_parse_one_atsr()
2487 		return 0;  in dmar_parse_one_atsr()
2500 	atsru->include_all = atsr->flags & 0x1;  in dmar_parse_one_atsr()
2513 	return 0;  in dmar_parse_one_atsr()
2535 	return 0;  in dmar_release_one_atsr()
2548 		return 0;  in dmar_check_one_atsr()
2556 	return 0;  in dmar_check_one_atsr()
2571 		if (memcmp(satc, tmp, satc->header.length) == 0)  in dmar_find_satc()
2584 		return 0;  in dmar_parse_one_satc()
2589 		return 0;  in dmar_parse_one_satc()
2597 	satcu->atc_required = satc->flags & 0x1;  in dmar_parse_one_satc()
2607 	return 0;  in dmar_parse_one_satc()
2622 	if (ret == 0)  in intel_iommu_add()
2635 		return 0;  in intel_iommu_add()
2655 	return 0;  in intel_iommu_add()
2666 	int ret = 0;  in dmar_iommu_hotplug()
2670 		return 0;  in dmar_iommu_hotplug()
2761 			return 0;  in dmar_ats_supported()
2780 	ret = 0;  in dmar_ats_supported()
2798 		return 0;  in dmar_iommu_notify_scope_dev()
2808 			if (ret < 0)  in dmar_iommu_notify_scope_dev()
2826 			if (ret > 0)  in dmar_iommu_notify_scope_dev()
2828 			else if (ret < 0)  in dmar_iommu_notify_scope_dev()
2843 			if (ret > 0)  in dmar_iommu_notify_scope_dev()
2845 			else if (ret < 0)  in dmar_iommu_notify_scope_dev()
2854 	return 0;  in dmar_iommu_notify_scope_dev()
2984 		return 0;  in platform_optin_force_iommu()
2996 	dmar_disabled = 0;  in platform_optin_force_iommu()
2997 	no_iommu = 0;  in platform_optin_force_iommu()
3008 	int i, ret = 0;  in probe_acpi_namespace_devices()
3036 	return 0;  in probe_acpi_namespace_devices()
3042 		return 0;  in tboot_force_iommu()
3047 	dmar_disabled = 0;  in tboot_force_iommu()
3048 	no_iommu = 0;  in tboot_force_iommu()
3073 	if (dmar_dev_scope_init() < 0) {  in intel_iommu_init()
3182 	return 0;  in intel_iommu_init()
3194 	domain_context_clear_one(info, PCI_BUS_NUM(alias), alias & 0xff);  in domain_context_clear_one_cb()
3195 	return 0;  in domain_context_clear_one_cb()
3253 	return 0;  in blocking_domain_attach_dev()
3271 		return 0;  in iommu_superpage_capability()
3323 	domain->domain.geometry.aperture_start = 0;  in paging_domain_alloc()
3447 	return 0;  in paging_domain_compatible()
3470 	int prot = 0;  in intel_iommu_map()
3528 	int level = 0;  in intel_iommu_unmap()
3534 		return 0;  in intel_iommu_unmap()
3581 	int level = 0;  in intel_iommu_iova_to_phys()
3582 	u64 phys = 0;  in intel_iommu_iova_to_phys()
3710 			 * treated as reserved, which should be set to 0.  in intel_iommu_probe_device()
3720 				if (features >= 0)  in intel_iommu_probe_device()
3782 		info->pasid_enabled = 0;  in intel_iommu_release_device()
3842 			reg = iommu_alloc_resv_region(0, 1UL << 24, prot,  in intel_iommu_get_resv_regions()
3853 				      0, IOMMU_RESV_MSI, GFP_KERNEL);  in intel_iommu_get_resv_regions()
3892 		return 0;  in intel_iommu_enable_sva()
3898 	return 0;  in intel_iommu_enable_sva()
3931 	return 0;  in context_flip_pri()
3973 	return 0;  in intel_iommu_enable_iopf()
4008 	info->pri_enabled = 0;  in intel_iommu_disable_iopf()
4010 	return 0;  in intel_iommu_disable_iopf()
4036 		return 0;  in intel_iommu_dev_disable_feat()
4072 	return 0;  in intel_iommu_iotlb_sync_map()
4119 	return 0;  in blocking_domain_set_dev_pasid()
4202 	return 0;  in intel_iommu_set_dev_pasid()
4234 	int ret = 0;  in device_set_dirty_tracking()
4262 	return 0;  in parent_domain_set_dirty_tracking()
4299 	return 0;  in intel_iommu_set_dirty_tracking()
4328 		int lvl = 0;  in intel_iommu_read_and_clear_dirty()
4343 	return 0;  in intel_iommu_read_and_clear_dirty()
4366 		return 0;  in context_setup_pass_through()
4386 	return 0;  in context_setup_pass_through()
4393 	return context_setup_pass_through(dev, PCI_BUS_NUM(alias), alias & 0xff);  in context_setup_pass_through_cb()
4416 		return 0;  in identity_domain_attach_dev()
4445 	return 0;  in identity_domain_set_dev_pasid()
4499 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_igfx);
4500 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_igfx);
4501 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_igfx);
4502 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_igfx);
4503 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_igfx);
4504 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_igfx);
4505 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_igfx);
4508 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1606, quirk_iommu_igfx);
4509 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160B, quirk_iommu_igfx);
4510 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160E, quirk_iommu_igfx);
4511 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1602, quirk_iommu_igfx);
4512 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160A, quirk_iommu_igfx);
4513 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160D, quirk_iommu_igfx);
4514 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1616, quirk_iommu_igfx);
4515 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161B, quirk_iommu_igfx);
4516 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161E, quirk_iommu_igfx);
4517 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1612, quirk_iommu_igfx);
4518 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161A, quirk_iommu_igfx);
4519 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161D, quirk_iommu_igfx);
4520 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1626, quirk_iommu_igfx);
4521 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162B, quirk_iommu_igfx);
4522 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162E, quirk_iommu_igfx);
4523 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1622, quirk_iommu_igfx);
4524 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162A, quirk_iommu_igfx);
4525 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162D, quirk_iommu_igfx);
4526 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1636, quirk_iommu_igfx);
4527 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163B, quirk_iommu_igfx);
4528 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163E, quirk_iommu_igfx);
4529 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1632, quirk_iommu_igfx);
4530 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163A, quirk_iommu_igfx);
4531 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163D, quirk_iommu_igfx);
4546 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4547 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4548 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4549 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4550 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4551 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4552 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4554 #define GGC 0x52
4555 #define GGC_MEMORY_SIZE_MASK	(0xf << 8)
4556 #define GGC_MEMORY_SIZE_NONE	(0x0 << 8)
4557 #define GGC_MEMORY_SIZE_1M	(0x1 << 8)
4558 #define GGC_MEMORY_SIZE_2M	(0x3 << 8)
4559 #define GGC_MEMORY_VT_ENABLED	(0x8 << 8)
4560 #define GGC_MEMORY_SIZE_2M_VT	(0x9 << 8)
4561 #define GGC_MEMORY_SIZE_3M_VT	(0xa << 8)
4562 #define GGC_MEMORY_SIZE_4M_VT	(0xb << 8)
4583 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4584 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4585 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4586 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4595 	ver = (dev->device >> 8) & 0xff;  in quirk_igfx_skip_te_disable()
4596 	if (ver != 0x45 && ver != 0x46 && ver != 0x4c &&  in quirk_igfx_skip_te_disable()
4597 	    ver != 0x4e && ver != 0x8a && ver != 0x98 &&  in quirk_igfx_skip_te_disable()
4598 	    ver != 0x9a && ver != 0xa7 && ver != 0x7d)  in quirk_igfx_skip_te_disable()
4622 	pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);  in check_tylersburg_isoch()
4636 	pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);  in check_tylersburg_isoch()
4645 	if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {  in check_tylersburg_isoch()
4657 	vtisochctrl &= 0x1c;  in check_tylersburg_isoch()
4660 	if (vtisochctrl == 0x10)  in check_tylersburg_isoch()
4723 #define ecmd_get_status_code(res)	(((res) & 0xff) >> 1)
4733  *  - 0: Command successful without any error;