Lines Matching +full:dte +full:- +full:mode
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
8 #define pr_fmt(fmt) "AMD-Vi: " fmt
14 #include <linux/pci-ats.h>
19 #include <linux/dma-map-ops.h>
20 #include <linux/dma-direct.h>
22 #include <linux/iommu-helper.h>
24 #include <linux/amd-iommu.h>
31 #include <linux/io-pgtable.h>
44 #include "../dma-iommu.h"
46 #include "../iommu-pages.h"
48 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
63 int amd_iommu_max_glx_val = -1;
101 * - Need cmpxchg16b instruction mainly for 128-bit store to DTE in amd_iommu_atomic128_set()
103 * protected by a spin_lock for this DTE). in amd_iommu_atomic128_set()
104 * - Neither need LOCK_PREFIX nor try loop because of the spin_lock. in amd_iommu_atomic128_set()
113 old.data128[1] = ptr->data128[1]; in write_dte_upper128()
117 * spin_lock(&dev_data->dte_lock) context. in write_dte_upper128()
119 new->data[2] &= ~DTE_DATA2_INTR_MASK; in write_dte_upper128()
120 new->data[2] |= old.data[2] & DTE_DATA2_INTR_MASK; in write_dte_upper128()
122 amd_iommu_atomic128_set(&ptr->data128[1], new->data128[1]); in write_dte_upper128()
127 amd_iommu_atomic128_set(&ptr->data128[0], new->data128[0]); in write_dte_lower128()
132 * IOMMU reads the entire Device Table entry in a single 256-bit transaction
133 * but the driver is programming DTE using 2 128-bit cmpxchg. So, the driver
135 * - DTE[V|GV] bit is being written last when setting.
136 * - DTE[V|GV] bit is being written first when clearing.
138 * This function is used only by code, which updates DMA translation part of the DTE.
146 struct dev_table_entry *ptr = &dev_table[dev_data->devid]; in update_dte256()
148 spin_lock_irqsave(&dev_data->dte_lock, flags); in update_dte256()
150 if (!(ptr->data[0] & DTE_FLAG_V)) { in update_dte256()
151 /* Existing DTE is not valid. */ in update_dte256()
154 iommu_flush_dte_sync(iommu, dev_data->devid); in update_dte256()
155 } else if (!(new->data[0] & DTE_FLAG_V)) { in update_dte256()
156 /* Existing DTE is valid. New DTE is not valid. */ in update_dte256()
159 iommu_flush_dte_sync(iommu, dev_data->devid); in update_dte256()
160 } else if (!FIELD_GET(DTE_FLAG_GV, ptr->data[0])) { in update_dte256()
163 * Existing DTE has no guest page table. in update_dte256()
167 iommu_flush_dte_sync(iommu, dev_data->devid); in update_dte256()
168 } else if (!FIELD_GET(DTE_FLAG_GV, new->data[0])) { in update_dte256()
171 * Existing DTE has guest page table, in update_dte256()
172 * new DTE has no guest page table, in update_dte256()
176 iommu_flush_dte_sync(iommu, dev_data->devid); in update_dte256()
177 } else if (FIELD_GET(DTE_GPT_LEVEL_MASK, ptr->data[2]) != in update_dte256()
178 FIELD_GET(DTE_GPT_LEVEL_MASK, new->data[2])) { in update_dte256()
182 * to upadte both upper and lower 128-bit value, which in update_dte256()
187 /* First disable DTE */ in update_dte256()
189 iommu_flush_dte_sync(iommu, dev_data->devid); in update_dte256()
191 /* Then update DTE */ in update_dte256()
194 iommu_flush_dte_sync(iommu, dev_data->devid); in update_dte256()
199 * update the lower 128-bit. So no need to disable DTE. in update_dte256()
204 spin_unlock_irqrestore(&dev_data->dte_lock, flags); in update_dte256()
208 struct dev_table_entry *dte) in get_dte256() argument
214 ptr = &dev_table[dev_data->devid]; in get_dte256()
216 spin_lock_irqsave(&dev_data->dte_lock, flags); in get_dte256()
217 dte->data128[0] = ptr->data128[0]; in get_dte256()
218 dte->data128[1] = ptr->data128[1]; in get_dte256()
219 spin_unlock_irqrestore(&dev_data->dte_lock, flags); in get_dte256()
224 return (pdom && (pdom->pd_mode == PD_MODE_V2)); in pdom_is_v2_pgtbl_mode()
229 return (pdom->domain.type == IOMMU_DOMAIN_IDENTITY); in pdom_is_in_pt_mode()
235 * or passthrough mode can be used for PASID.
249 return -ENODEV; in get_acpihid_device_id()
252 if (acpi_dev_hid_uid_match(adev, p->hid, in get_acpihid_device_id()
253 p->uid[0] ? p->uid : NULL)) { in get_acpihid_device_id()
256 return p->devid; in get_acpihid_device_id()
259 return -EINVAL; in get_acpihid_device_id()
277 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; in get_dev_table()
280 dev_table = pci_seg->dev_table; in get_dev_table()
293 seg = pci_domain_nr(pdev->bus); in get_device_segment()
306 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; in amd_iommu_set_rlookup_table()
308 pci_seg->rlookup_table[devid] = iommu; in amd_iommu_set_rlookup_table()
316 if (pci_seg->id == seg) in __rlookup_amd_iommu()
317 return pci_seg->rlookup_table[devid]; in __rlookup_amd_iommu()
335 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; in alloc_dev_data()
341 mutex_init(&dev_data->mutex); in alloc_dev_data()
342 spin_lock_init(&dev_data->dte_lock); in alloc_dev_data()
343 dev_data->devid = devid; in alloc_dev_data()
344 ratelimit_default_init(&dev_data->rs); in alloc_dev_data()
346 llist_add(&dev_data->dev_data_list, &pci_seg->dev_data_list); in alloc_dev_data()
354 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; in search_dev_data()
356 if (llist_empty(&pci_seg->dev_data_list)) in search_dev_data()
359 node = pci_seg->dev_data_list.first; in search_dev_data()
361 if (dev_data->devid == devid) in search_dev_data()
379 iommu = rlookup_amd_iommu(&pdev->dev); in clone_alias()
384 dev_data = dev_iommu_priv_get(&pdev->dev); in clone_alias()
387 ret = -EINVAL; in clone_alias()
396 ret = -EINVAL; in clone_alias()
419 clone_alias(pdev, iommu->pci_seg->alias_table[pci_dev_id(pdev)], NULL); in clone_aliases()
427 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; in setup_aliases()
438 ivrs_alias = pci_seg->alias_table[pci_dev_id(pdev)]; in setup_aliases()
440 PCI_BUS_NUM(ivrs_alias) == pdev->bus->number) in setup_aliases()
458 dev_data->defer_attach = true; in find_dev_data()
477 if ((devid == p->devid) && p->group) in acpihid_device_group()
478 entry->group = p->group; in acpihid_device_group()
481 if (!entry->group) in acpihid_device_group()
482 entry->group = generic_device_group(dev); in acpihid_device_group()
484 iommu_group_ref_get(entry->group); in acpihid_device_group()
486 return entry->group; in acpihid_device_group()
491 return (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP); in pdev_pasid_supported()
521 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); in pdev_enable_cap_ats()
522 int ret = -EINVAL; in pdev_enable_cap_ats()
524 if (dev_data->ats_enabled) in pdev_enable_cap_ats()
528 (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP)) { in pdev_enable_cap_ats()
531 dev_data->ats_enabled = 1; in pdev_enable_cap_ats()
532 dev_data->ats_qdep = pci_ats_queue_depth(pdev); in pdev_enable_cap_ats()
541 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); in pdev_disable_cap_ats()
543 if (dev_data->ats_enabled) { in pdev_disable_cap_ats()
545 dev_data->ats_enabled = 0; in pdev_disable_cap_ats()
551 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); in pdev_enable_cap_pri()
552 int ret = -EINVAL; in pdev_enable_cap_pri()
554 if (dev_data->pri_enabled) in pdev_enable_cap_pri()
557 if (!dev_data->ats_enabled) in pdev_enable_cap_pri()
560 if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) { in pdev_enable_cap_pri()
566 dev_data->pri_enabled = 1; in pdev_enable_cap_pri()
567 dev_data->pri_tlp = pci_prg_resp_pasid_required(pdev); in pdev_enable_cap_pri()
578 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); in pdev_disable_cap_pri()
580 if (dev_data->pri_enabled) { in pdev_disable_cap_pri()
582 dev_data->pri_enabled = 0; in pdev_disable_cap_pri()
588 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); in pdev_enable_cap_pasid()
589 int ret = -EINVAL; in pdev_enable_cap_pasid()
591 if (dev_data->pasid_enabled) in pdev_enable_cap_pasid()
594 if (dev_data->flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP) { in pdev_enable_cap_pasid()
595 /* Only allow access to user-accessible pages */ in pdev_enable_cap_pasid()
598 dev_data->pasid_enabled = 1; in pdev_enable_cap_pasid()
606 struct iommu_dev_data *dev_data = dev_iommu_priv_get(&pdev->dev); in pdev_disable_cap_pasid()
608 if (dev_data->pasid_enabled) { in pdev_disable_cap_pasid()
610 dev_data->pasid_enabled = 0; in pdev_disable_cap_pasid()
651 pci_seg = iommu->pci_seg; in check_device()
652 if (devid > pci_seg->last_bdf) in check_device()
673 return -ENOMEM; in iommu_init_device()
675 dev_data->dev = dev; in iommu_init_device()
685 * By default we use passthrough mode for IOMMUv2 capable device. in iommu_init_device()
688 * it'll be forced to go into translation mode. in iommu_init_device()
692 dev_data->flags = pdev_get_caps(to_pci_dev(dev)); in iommu_init_device()
700 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; in iommu_ignore_device()
709 pci_seg->rlookup_table[devid] = NULL; in iommu_ignore_device()
725 struct dev_table_entry dte; in dump_dte_entry() local
728 get_dte256(iommu, dev_data, &dte); in dump_dte_entry()
731 pr_err("DTE[%d]: %016llx\n", i, dte.data[i]); in dump_dte_entry()
740 pr_err("CMD[%d]: %08x\n", i, cmd->data[i]); in dump_command()
755 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid), in amd_iommu_report_rmp_hw_error()
758 dev_data = dev_iommu_priv_get(&pdev->dev); in amd_iommu_report_rmp_hw_error()
761 if (__ratelimit(&dev_data->rs)) { in amd_iommu_report_rmp_hw_error()
767 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), in amd_iommu_report_rmp_hw_error()
788 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid), in amd_iommu_report_rmp_fault()
791 dev_data = dev_iommu_priv_get(&pdev->dev); in amd_iommu_report_rmp_fault()
794 if (__ratelimit(&dev_data->rs)) { in amd_iommu_report_rmp_fault()
800 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), in amd_iommu_report_rmp_fault()
821 pdev = pci_get_domain_bus_and_slot(iommu->pci_seg->id, PCI_BUS_NUM(devid), in amd_iommu_report_page_fault()
824 dev_data = dev_iommu_priv_get(&pdev->dev); in amd_iommu_report_page_fault()
834 if (dev_data->domain == NULL) { in amd_iommu_report_page_fault()
837 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), in amd_iommu_report_page_fault()
842 if (!report_iommu_fault(&dev_data->domain->domain, in amd_iommu_report_page_fault()
843 &pdev->dev, address, in amd_iommu_report_page_fault()
850 if (__ratelimit(&dev_data->rs)) { in amd_iommu_report_page_fault()
856 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), in amd_iommu_report_page_fault()
867 struct device *dev = iommu->iommu.dev; in iommu_print_event()
900 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), in iommu_print_event()
907 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), in iommu_print_event()
912 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), in iommu_print_event()
925 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), in iommu_print_event()
930 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), in iommu_print_event()
943 iommu->pci_seg->id, PCI_BUS_NUM(devid), PCI_SLOT(devid), PCI_FUNC(devid), in iommu_print_event()
965 head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); in iommu_poll_events()
966 tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET); in iommu_poll_events()
969 iommu_print_event(iommu, iommu->evt_buf + head); in iommu_poll_events()
971 /* Update head pointer of hardware ring-buffer */ in iommu_poll_events()
973 writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); in iommu_poll_events()
993 if (iommu->ga_log == NULL) in iommu_poll_ga_log()
996 head = readl(iommu->mmio_base + MMIO_GA_HEAD_OFFSET); in iommu_poll_ga_log()
997 tail = readl(iommu->mmio_base + MMIO_GA_TAIL_OFFSET); in iommu_poll_ga_log()
1003 raw = (u64 *)(iommu->ga_log + head); in iommu_poll_ga_log()
1005 /* Avoid memcpy function-call overhead */ in iommu_poll_ga_log()
1008 /* Update head pointer of hardware ring-buffer */ in iommu_poll_ga_log()
1010 writel(head, iommu->mmio_base + MMIO_GA_HEAD_OFFSET); in iommu_poll_ga_log()
1038 dev_set_msi_domain(dev, iommu->ir_domain); in amd_iommu_set_pci_msi_domain()
1052 u32 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); in amd_iommu_handle_irq()
1057 writel(mask, iommu->mmio_base + MMIO_STATUS_OFFSET); in amd_iommu_handle_irq()
1061 iommu->index, evt_type); in amd_iommu_handle_irq()
1070 * When re-enabling interrupt (by writing 1 in amd_iommu_handle_irq()
1079 * again and re-clear the bits in amd_iommu_handle_irq()
1081 status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); in amd_iommu_handle_irq()
1138 while (*iommu->cmd_sem != data && i < LOOP_TIMEOUT) { in wait_on_sem()
1144 pr_alert("Completion-Wait loop timed out\n"); in wait_on_sem()
1145 return -EIO; in wait_on_sem()
1158 tail = iommu->cmd_buf_tail; in copy_cmd_to_buffer()
1159 target = iommu->cmd_buf + tail; in copy_cmd_to_buffer()
1163 iommu->cmd_buf_tail = tail; in copy_cmd_to_buffer()
1166 writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); in copy_cmd_to_buffer()
1173 u64 paddr = iommu_virt_to_phys((void *)iommu->cmd_sem); in build_completion_wait()
1176 cmd->data[0] = lower_32_bits(paddr) | CMD_COMPL_WAIT_STORE_MASK; in build_completion_wait()
1177 cmd->data[1] = upper_32_bits(paddr); in build_completion_wait()
1178 cmd->data[2] = lower_32_bits(data); in build_completion_wait()
1179 cmd->data[3] = upper_32_bits(data); in build_completion_wait()
1186 cmd->data[0] = devid; in build_inv_dte()
1203 end = address + size - 1; in build_inv_address()
1209 msb_diff = fls64(end ^ address) - 1; in build_inv_address()
1219 * The msb-bit must be clear on the address. Just set all the in build_inv_address()
1222 address |= (1ull << msb_diff) - 1; in build_inv_address()
1228 /* Set the size bit - we flush more than one 4kb page */ in build_inv_address()
1240 cmd->data[1] |= domid; in build_inv_iommu_pages()
1241 cmd->data[2] = lower_32_bits(inv_address); in build_inv_iommu_pages()
1242 cmd->data[3] = upper_32_bits(inv_address); in build_inv_iommu_pages()
1243 /* PDE bit - we want to flush everything, not only the PTEs */ in build_inv_iommu_pages()
1244 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK; in build_inv_iommu_pages()
1246 cmd->data[0] |= pasid; in build_inv_iommu_pages()
1247 cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK; in build_inv_iommu_pages()
1260 cmd->data[0] = devid; in build_inv_iotlb_pages()
1261 cmd->data[0] |= (qdep & 0xff) << 24; in build_inv_iotlb_pages()
1262 cmd->data[1] = devid; in build_inv_iotlb_pages()
1263 cmd->data[2] = lower_32_bits(inv_address); in build_inv_iotlb_pages()
1264 cmd->data[3] = upper_32_bits(inv_address); in build_inv_iotlb_pages()
1266 cmd->data[0] |= ((pasid >> 8) & 0xff) << 16; in build_inv_iotlb_pages()
1267 cmd->data[1] |= (pasid & 0xff) << 16; in build_inv_iotlb_pages()
1268 cmd->data[2] |= CMD_INV_IOMMU_PAGES_GN_MASK; in build_inv_iotlb_pages()
1279 cmd->data[0] = devid; in build_complete_ppr()
1281 cmd->data[1] = pasid; in build_complete_ppr()
1282 cmd->data[2] = CMD_INV_IOMMU_PAGES_GN_MASK; in build_complete_ppr()
1284 cmd->data[3] = tag & 0x1ff; in build_complete_ppr()
1285 cmd->data[3] |= (status & PPR_STATUS_MASK) << PPR_STATUS_SHIFT; in build_complete_ppr()
1299 cmd->data[0] = devid; in build_inv_irt()
1314 next_tail = (iommu->cmd_buf_tail + sizeof(*cmd)) % CMD_BUFFER_SIZE; in __iommu_queue_command_sync()
1316 left = (iommu->cmd_buf_head - next_tail) % CMD_BUFFER_SIZE; in __iommu_queue_command_sync()
1323 return -EIO; in __iommu_queue_command_sync()
1330 iommu->cmd_buf_head = readl(iommu->mmio_base + in __iommu_queue_command_sync()
1339 iommu->need_sync = sync; in __iommu_queue_command_sync()
1351 raw_spin_lock_irqsave(&iommu->lock, flags); in iommu_queue_command_sync()
1353 raw_spin_unlock_irqrestore(&iommu->lock, flags); in iommu_queue_command_sync()
1374 if (!iommu->need_sync) in iommu_completion_wait()
1377 data = atomic64_inc_return(&iommu->cmd_sem_val); in iommu_completion_wait()
1380 raw_spin_lock_irqsave(&iommu->lock, flags); in iommu_completion_wait()
1389 raw_spin_unlock_irqrestore(&iommu->lock, flags); in iommu_completion_wait()
1399 lockdep_assert_held(&domain->lock); in domain_flush_complete()
1405 xa_for_each(&domain->iommu_array, i, pdom_iommu_info) in domain_flush_complete()
1406 iommu_completion_wait(pdom_iommu_info->iommu); in domain_flush_complete()
1430 u16 last_bdf = iommu->pci_seg->last_bdf; in amd_iommu_flush_dte_all()
1445 u16 last_bdf = iommu->pci_seg->last_bdf; in amd_iommu_flush_tlb_all()
1490 u16 last_bdf = iommu->pci_seg->last_bdf; in amd_iommu_flush_irt_all()
1492 if (iommu->irtcachedis_enabled) in amd_iommu_flush_irt_all()
1513 * Command send function for flushing on-device TLB
1520 int qdep = dev_data->ats_qdep; in device_flush_iotlb()
1522 build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, in device_flush_iotlb()
1546 if (dev_is_pci(dev_data->dev)) in device_flush_dte()
1547 pdev = to_pci_dev(dev_data->dev); in device_flush_dte()
1553 ret = iommu_flush_dte(iommu, dev_data->devid); in device_flush_dte()
1557 pci_seg = iommu->pci_seg; in device_flush_dte()
1558 alias = pci_seg->alias_table[dev_data->devid]; in device_flush_dte()
1559 if (alias != dev_data->devid) { in device_flush_dte()
1565 if (dev_data->ats_enabled) { in device_flush_dte()
1581 lockdep_assert_held(&pdom->lock); in domain_flush_pages_v2()
1582 list_for_each_entry(dev_data, &pdom->dev_list, list) { in domain_flush_pages_v2()
1583 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); in domain_flush_pages_v2()
1584 u16 domid = dev_data->gcr3_info.domid; in domain_flush_pages_v2()
1603 lockdep_assert_held(&pdom->lock); in domain_flush_pages_v1()
1606 pdom->id, IOMMU_NO_PASID, false); in domain_flush_pages_v1()
1608 xa_for_each(&pdom->iommu_array, i, pdom_iommu_info) { in domain_flush_pages_v1()
1613 ret |= iommu_queue_command(pdom_iommu_info->iommu, &cmd); in domain_flush_pages_v1()
1631 lockdep_assert_held(&domain->lock); in __domain_flush_pages()
1640 list_for_each_entry(dev_data, &domain->dev_list, list) { in __domain_flush_pages()
1642 if (!dev_data->ats_enabled) in __domain_flush_pages()
1654 lockdep_assert_held(&domain->lock); in amd_iommu_domain_flush_pages()
1682 * size is always non-zero, but address might be zero, causing in amd_iommu_domain_flush_pages()
1685 * of the address on x86-32, cast to long when doing the check. in amd_iommu_domain_flush_pages()
1696 size -= flush_size; in amd_iommu_domain_flush_pages()
1703 /* Flush the whole IO/TLB for a given protection domain - including PDE */
1714 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); in amd_iommu_dev_flush_pasid_pages()
1717 dev_data->gcr3_info.domid, pasid, true); in amd_iommu_dev_flush_pasid_pages()
1720 if (dev_data->ats_enabled) in amd_iommu_dev_flush_pasid_pages()
1740 spin_lock_irqsave(&domain->lock, flags); in domain_flush_np_cache()
1742 spin_unlock_irqrestore(&domain->lock, flags); in domain_flush_np_cache()
1754 lockdep_assert_held(&domain->lock); in amd_iommu_update_and_flush_device_table()
1756 list_for_each_entry(dev_data, &domain->dev_list, list) { in amd_iommu_update_and_flush_device_table()
1757 struct amd_iommu *iommu = rlookup_amd_iommu(dev_data->dev); in amd_iommu_update_and_flush_device_table()
1760 clone_aliases(iommu, dev_data->dev); in amd_iommu_update_and_flush_device_table()
1763 list_for_each_entry(dev_data, &domain->dev_list, list) in amd_iommu_update_and_flush_device_table()
1778 build_complete_ppr(&cmd, dev_data->devid, pasid, status, in amd_iommu_complete_ppr()
1779 tag, dev_data->pri_tlp); in amd_iommu_complete_ppr()
1796 return ida_alloc_range(&pdom_ids, 1, MAX_DOMAIN_ID - 1, GFP_ATOMIC); in pdom_id_alloc()
1836 if (gcr3_info->glx == 2) in free_gcr3_table()
1837 free_gcr3_tbl_level2(gcr3_info->gcr3_tbl); in free_gcr3_table()
1838 else if (gcr3_info->glx == 1) in free_gcr3_table()
1839 free_gcr3_tbl_level1(gcr3_info->gcr3_tbl); in free_gcr3_table()
1841 WARN_ON_ONCE(gcr3_info->glx != 0); in free_gcr3_table()
1843 gcr3_info->glx = 0; in free_gcr3_table()
1846 pdom_id_free(gcr3_info->domid); in free_gcr3_table()
1848 iommu_free_page(gcr3_info->gcr3_tbl); in free_gcr3_table()
1849 gcr3_info->gcr3_tbl = NULL; in free_gcr3_table()
1853 * Number of GCR3 table levels required. Level must be 4-Kbyte
1860 if (pasids == -1) in get_gcr3_levels()
1865 return levels ? (DIV_ROUND_UP(levels, 9) - 1) : levels; in get_gcr3_levels()
1872 int nid = iommu ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE; in setup_gcr3_table()
1876 return -EINVAL; in setup_gcr3_table()
1878 if (gcr3_info->gcr3_tbl) in setup_gcr3_table()
1879 return -EBUSY; in setup_gcr3_table()
1884 return -ENOSPC; in setup_gcr3_table()
1885 gcr3_info->domid = domid; in setup_gcr3_table()
1887 gcr3_info->gcr3_tbl = iommu_alloc_page_node(nid, GFP_ATOMIC); in setup_gcr3_table()
1888 if (gcr3_info->gcr3_tbl == NULL) { in setup_gcr3_table()
1890 return -ENOMEM; in setup_gcr3_table()
1893 gcr3_info->glx = levels; in setup_gcr3_table()
1903 u64 *root = gcr3_info->gcr3_tbl; in __get_gcr3_pte()
1904 int level = gcr3_info->glx; in __get_gcr3_pte()
1927 level -= 1; in __get_gcr3_pte()
1936 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; in update_gcr3()
1941 return -ENOMEM; in update_gcr3()
1955 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; in amd_iommu_set_gcr3()
1958 iommu_group_mutex_assert(dev_data->dev); in amd_iommu_set_gcr3()
1964 gcr3_info->pasid_cnt++; in amd_iommu_set_gcr3()
1970 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; in amd_iommu_clear_gcr3()
1973 iommu_group_mutex_assert(dev_data->dev); in amd_iommu_clear_gcr3()
1979 gcr3_info->pasid_cnt--; in amd_iommu_clear_gcr3()
1986 /* All existing DTE must have V bit set */ in make_clear_dte()
1987 new->data128[0] = DTE_FLAG_V; in make_clear_dte()
1988 new->data128[1] = 0; in make_clear_dte()
1999 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; in set_dte_gcr3_table()
2002 if (!gcr3_info->gcr3_tbl) in set_dte_gcr3_table()
2006 __func__, dev_data->devid, gcr3_info->glx, in set_dte_gcr3_table()
2007 (unsigned long long)gcr3_info->gcr3_tbl); in set_dte_gcr3_table()
2009 gcr3 = iommu_virt_to_phys(gcr3_info->gcr3_tbl); in set_dte_gcr3_table()
2011 target->data[0] |= DTE_FLAG_GV | in set_dte_gcr3_table()
2012 FIELD_PREP(DTE_GLX, gcr3_info->glx) | in set_dte_gcr3_table()
2014 if (pdom_is_v2_pgtbl_mode(dev_data->domain)) in set_dte_gcr3_table()
2015 target->data[0] |= DTE_FLAG_GIOV; in set_dte_gcr3_table()
2017 target->data[1] |= FIELD_PREP(DTE_GCR3_30_15, gcr3 >> 15) | in set_dte_gcr3_table()
2022 target->data[2] |= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_5_LEVEL); in set_dte_gcr3_table()
2024 target->data[2] |= FIELD_PREP(DTE_GPT_LEVEL_MASK, GUEST_PGTABLE_4_LEVEL); in set_dte_gcr3_table()
2034 struct protection_domain *domain = dev_data->domain; in set_dte_entry()
2035 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; in set_dte_entry()
2036 struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid]; in set_dte_entry() local
2038 if (gcr3_info && gcr3_info->gcr3_tbl) in set_dte_entry()
2039 domid = dev_data->gcr3_info.domid; in set_dte_entry()
2041 domid = domain->id; in set_dte_entry()
2043 make_clear_dte(dev_data, dte, &new); in set_dte_entry()
2045 if (domain->iop.mode != PAGE_MODE_NONE) in set_dte_entry()
2046 new.data[0] |= iommu_virt_to_phys(domain->iop.root); in set_dte_entry()
2048 new.data[0] |= (domain->iop.mode & DEV_ENTRY_MODE_MASK) in set_dte_entry()
2054 * When SNP is enabled, we can only support TV=1 with non-zero domain ID. in set_dte_entry()
2055 * This is prevented by the SNP-enable and IOMMU_DOMAIN_IDENTITY check in in set_dte_entry()
2061 if (dev_data->ppr) in set_dte_entry()
2064 if (domain->dirty_tracking) in set_dte_entry()
2067 if (dev_data->ats_enabled) in set_dte_entry()
2070 old_domid = READ_ONCE(dte->data[1]) & DEV_DOMID_MASK; in set_dte_entry()
2074 * Restore cached persistent DTE bits, which can be set by information in set_dte_entry()
2077 initial_dte = amd_iommu_get_ivhd_dte_flags(iommu->pci_seg->id, dev_data->devid); in set_dte_entry()
2079 new.data128[0] |= initial_dte->data128[0]; in set_dte_entry()
2080 new.data128[1] |= initial_dte->data128[1]; in set_dte_entry()
2089 * the previous kernel--if so, it needs to flush the translation cache in set_dte_entry()
2098 * Clear DMA-remap related flags to block all DMA (blockeded domain)
2103 struct dev_table_entry *dte = &get_dev_table(iommu)[dev_data->devid]; in clear_dte_entry() local
2105 make_clear_dte(dev_data, dte, &new); in clear_dte_entry()
2109 /* Update and flush DTE for the given device */
2112 struct amd_iommu *iommu = get_amd_iommu_from_dev(dev_data->dev); in dev_update_dte()
2119 clone_aliases(iommu, dev_data->dev); in dev_update_dte()
2126 * in v2 page table mode then update GCR3[0].
2132 int max_pasids = dev_data->max_pasids; in init_gcr3_table()
2136 * If domain is in pt mode then setup GCR3 table only if device in init_gcr3_table()
2146 ret = setup_gcr3_table(&dev_data->gcr3_info, iommu, in init_gcr3_table()
2151 /* Setup GCR3[0] only if domain is setup with v2 page table mode */ in init_gcr3_table()
2155 ret = update_gcr3(dev_data, 0, iommu_virt_to_phys(pdom->iop.pgd), true); in init_gcr3_table()
2157 free_gcr3_table(&dev_data->gcr3_info); in init_gcr3_table()
2165 struct gcr3_tbl_info *gcr3_info = &dev_data->gcr3_info; in destroy_gcr3_table()
2170 if (gcr3_info->gcr3_tbl == NULL) in destroy_gcr3_table()
2183 spin_lock_irqsave(&pdom->lock, flags); in pdom_attach_iommu()
2185 pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index); in pdom_attach_iommu()
2187 pdom_iommu_info->refcnt++; in pdom_attach_iommu()
2193 ret = -ENOMEM; in pdom_attach_iommu()
2197 pdom_iommu_info->iommu = iommu; in pdom_attach_iommu()
2198 pdom_iommu_info->refcnt = 1; in pdom_attach_iommu()
2200 curr = xa_cmpxchg(&pdom->iommu_array, iommu->index, in pdom_attach_iommu()
2204 ret = -ENOSPC; in pdom_attach_iommu()
2209 spin_unlock_irqrestore(&pdom->lock, flags); in pdom_attach_iommu()
2219 spin_lock_irqsave(&pdom->lock, flags); in pdom_detach_iommu()
2221 pdom_iommu_info = xa_load(&pdom->iommu_array, iommu->index); in pdom_detach_iommu()
2223 spin_unlock_irqrestore(&pdom->lock, flags); in pdom_detach_iommu()
2227 pdom_iommu_info->refcnt--; in pdom_detach_iommu()
2228 if (pdom_iommu_info->refcnt == 0) { in pdom_detach_iommu()
2229 xa_erase(&pdom->iommu_array, iommu->index); in pdom_detach_iommu()
2233 spin_unlock_irqrestore(&pdom->lock, flags); in pdom_detach_iommu()
2249 mutex_lock(&dev_data->mutex); in attach_device()
2251 if (dev_data->domain != NULL) { in attach_device()
2252 ret = -EBUSY; in attach_device()
2270 pdev = dev_is_pci(dev_data->dev) ? to_pci_dev(dev_data->dev) : NULL; in attach_device()
2286 dev_data->domain = domain; in attach_device()
2287 spin_lock_irqsave(&domain->lock, flags); in attach_device()
2288 list_add(&dev_data->list, &domain->dev_list); in attach_device()
2289 spin_unlock_irqrestore(&domain->lock, flags); in attach_device()
2295 mutex_unlock(&dev_data->mutex); in attach_device()
2307 struct protection_domain *domain = dev_data->domain; in detach_device()
2310 mutex_lock(&dev_data->mutex); in detach_device()
2318 if (WARN_ON(!dev_data->domain)) in detach_device()
2322 if (dev_data->ppr) { in detach_device()
2330 /* Clear DTE and flush the entry */ in detach_device()
2334 spin_lock_irqsave(&domain->lock, flags); in detach_device()
2336 list_del(&dev_data->list); in detach_device()
2337 spin_unlock_irqrestore(&domain->lock, flags); in detach_device()
2344 dev_data->domain = NULL; in detach_device()
2346 /* decrease reference counters - needs to happen after the flushes */ in detach_device()
2350 mutex_unlock(&dev_data->mutex); in detach_device()
2361 return ERR_PTR(-ENODEV); in amd_iommu_probe_device()
2365 return ERR_PTR(-ENODEV); in amd_iommu_probe_device()
2368 if (!iommu->iommu.ops) in amd_iommu_probe_device()
2369 return ERR_PTR(-ENODEV); in amd_iommu_probe_device()
2372 return &iommu->iommu; in amd_iommu_probe_device()
2376 dev_err(dev, "Failed to initialize - trying to proceed anyway\n"); in amd_iommu_probe_device()
2383 iommu_dev = &iommu->iommu; in amd_iommu_probe_device()
2392 dev_data->max_pasids = min_t(u32, iommu->iommu.max_pasids, in amd_iommu_probe_device()
2409 WARN_ON(dev_data->domain); in amd_iommu_release_device()
2413 * device is re-plugged - not doing so would introduce a ton of races. in amd_iommu_release_device()
2437 WARN_ON(!list_empty(&domain->dev_list)); in protection_domain_free()
2438 if (domain->domain.type & __IOMMU_DOMAIN_PAGING) in protection_domain_free()
2439 free_io_pgtable_ops(&domain->iop.pgtbl.ops); in protection_domain_free()
2440 pdom_id_free(domain->id); in protection_domain_free()
2446 spin_lock_init(&domain->lock); in protection_domain_init()
2447 INIT_LIST_HEAD(&domain->dev_list); in protection_domain_init()
2448 INIT_LIST_HEAD(&domain->dev_data_list); in protection_domain_init()
2449 xa_init(&domain->iommu_array); in protection_domain_init()
2466 domain->id = domid; in protection_domain_alloc()
2479 switch (domain->pd_mode) { in pdom_setup_pgtable()
2488 domain->iop.pgtbl.cfg.amd.nid = dev_to_node(dev); in pdom_setup_pgtable()
2489 pgtbl_ops = alloc_io_pgtable_ops(fmt, &domain->iop.pgtbl.cfg, domain); in pdom_setup_pgtable()
2491 return -ENOMEM; in pdom_setup_pgtable()
2502 return ((1ULL << PM_LEVEL_SHIFT(amd_iommu_gpt_level)) - 1); in dma_max_address()
2507 return iommu && (iommu->features & FEATURE_HDSUP); in amd_iommu_hd_support()
2521 return ERR_PTR(-ENOMEM); in do_iommu_domain_alloc()
2523 domain->pd_mode = pgtable; in do_iommu_domain_alloc()
2526 pdom_id_free(domain->id); in do_iommu_domain_alloc()
2531 domain->domain.geometry.aperture_start = 0; in do_iommu_domain_alloc()
2532 domain->domain.geometry.aperture_end = dma_max_address(pgtable); in do_iommu_domain_alloc()
2533 domain->domain.geometry.force_aperture = true; in do_iommu_domain_alloc()
2534 domain->domain.pgsize_bitmap = domain->iop.pgtbl.cfg.pgsize_bitmap; in do_iommu_domain_alloc()
2536 domain->domain.type = IOMMU_DOMAIN_UNMANAGED; in do_iommu_domain_alloc()
2537 domain->domain.ops = iommu->iommu.ops->default_domain_ops; in do_iommu_domain_alloc()
2540 domain->domain.dirty_ops = &amd_dirty_ops; in do_iommu_domain_alloc()
2542 return &domain->domain; in do_iommu_domain_alloc()
2555 return ERR_PTR(-EOPNOTSUPP); in amd_iommu_domain_alloc_paging_flags()
2574 return ERR_PTR(-EOPNOTSUPP); in amd_iommu_domain_alloc_paging_flags()
2589 if (dev_data->domain) in blocked_domain_attach_device()
2592 /* Clear DTE and flush the entry */ in blocked_domain_attach_device()
2593 mutex_lock(&dev_data->mutex); in blocked_domain_attach_device()
2595 mutex_unlock(&dev_data->mutex); in blocked_domain_attach_device()
2626 domain->type = IOMMU_DOMAIN_IDENTITY; in amd_iommu_init_identity_domain()
2627 domain->ops = &identity_domain_ops; in amd_iommu_init_identity_domain()
2628 domain->owner = &amd_iommu_ops; in amd_iommu_init_identity_domain()
2635 /* Same as blocked domain except it supports only ops->attach_dev() */
2655 if (dev_data->domain == domain) in amd_iommu_attach_device()
2658 dev_data->defer_attach = false; in amd_iommu_attach_device()
2664 if (dom->dirty_ops && !amd_iommu_hd_support(iommu)) in amd_iommu_attach_device()
2665 return -EINVAL; in amd_iommu_attach_device()
2667 if (dev_data->domain) in amd_iommu_attach_device()
2674 if (dom->type == IOMMU_DOMAIN_UNMANAGED) in amd_iommu_attach_device()
2675 dev_data->use_vapic = 1; in amd_iommu_attach_device()
2677 dev_data->use_vapic = 0; in amd_iommu_attach_device()
2688 struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops; in amd_iommu_iotlb_sync_map()
2690 if (ops->map_pages) in amd_iommu_iotlb_sync_map()
2700 struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops; in amd_iommu_map_pages()
2702 int ret = -EINVAL; in amd_iommu_map_pages()
2704 if ((domain->pd_mode == PD_MODE_V1) && in amd_iommu_map_pages()
2705 (domain->iop.mode == PAGE_MODE_NONE)) in amd_iommu_map_pages()
2706 return -EINVAL; in amd_iommu_map_pages()
2713 if (ops->map_pages) { in amd_iommu_map_pages()
2714 ret = ops->map_pages(ops, iova, paddr, pgsize, in amd_iommu_map_pages()
2728 * to whether "non-present cache" is on, it is probably best to prefer in amd_iommu_iotlb_gather_add_page()
2732 * the guest, and the trade-off is different: unnecessary TLB flushes in amd_iommu_iotlb_gather_add_page()
2747 struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops; in amd_iommu_unmap_pages()
2750 if ((domain->pd_mode == PD_MODE_V1) && in amd_iommu_unmap_pages()
2751 (domain->iop.mode == PAGE_MODE_NONE)) in amd_iommu_unmap_pages()
2754 r = (ops->unmap_pages) ? ops->unmap_pages(ops, iova, pgsize, pgcount, NULL) : 0; in amd_iommu_unmap_pages()
2766 struct io_pgtable_ops *ops = &domain->iop.pgtbl.ops; in amd_iommu_iova_to_phys()
2768 return ops->iova_to_phys(ops, iova); in amd_iommu_iova_to_phys()
2800 struct dev_table_entry *dte; in amd_iommu_set_dirty_tracking() local
2807 spin_lock_irqsave(&pdomain->lock, flags); in amd_iommu_set_dirty_tracking()
2808 if (!(pdomain->dirty_tracking ^ enable)) { in amd_iommu_set_dirty_tracking()
2809 spin_unlock_irqrestore(&pdomain->lock, flags); in amd_iommu_set_dirty_tracking()
2813 list_for_each_entry(dev_data, &pdomain->dev_list, list) { in amd_iommu_set_dirty_tracking()
2814 spin_lock(&dev_data->dte_lock); in amd_iommu_set_dirty_tracking()
2816 dte = &get_dev_table(iommu)[dev_data->devid]; in amd_iommu_set_dirty_tracking()
2817 new = dte->data[0]; in amd_iommu_set_dirty_tracking()
2819 dte->data[0] = new; in amd_iommu_set_dirty_tracking()
2820 spin_unlock(&dev_data->dte_lock); in amd_iommu_set_dirty_tracking()
2822 /* Flush device DTE */ in amd_iommu_set_dirty_tracking()
2831 pdomain->dirty_tracking = enable; in amd_iommu_set_dirty_tracking()
2832 spin_unlock_irqrestore(&pdomain->lock, flags); in amd_iommu_set_dirty_tracking()
2843 struct io_pgtable_ops *ops = &pdomain->iop.pgtbl.ops; in amd_iommu_read_and_clear_dirty()
2846 if (!ops || !ops->read_and_clear_dirty) in amd_iommu_read_and_clear_dirty()
2847 return -EOPNOTSUPP; in amd_iommu_read_and_clear_dirty()
2849 spin_lock_irqsave(&pdomain->lock, lflags); in amd_iommu_read_and_clear_dirty()
2850 if (!pdomain->dirty_tracking && dirty->bitmap) { in amd_iommu_read_and_clear_dirty()
2851 spin_unlock_irqrestore(&pdomain->lock, lflags); in amd_iommu_read_and_clear_dirty()
2852 return -EINVAL; in amd_iommu_read_and_clear_dirty()
2854 spin_unlock_irqrestore(&pdomain->lock, lflags); in amd_iommu_read_and_clear_dirty()
2856 return ops->read_and_clear_dirty(ops, iova, size, flags, dirty); in amd_iommu_read_and_clear_dirty()
2874 pci_seg = iommu->pci_seg; in amd_iommu_get_resv_regions()
2876 list_for_each_entry(entry, &pci_seg->unity_map, list) { in amd_iommu_get_resv_regions()
2880 if (devid < entry->devid_start || devid > entry->devid_end) in amd_iommu_get_resv_regions()
2884 length = entry->address_end - entry->address_start; in amd_iommu_get_resv_regions()
2885 if (entry->prot & IOMMU_PROT_IR) in amd_iommu_get_resv_regions()
2887 if (entry->prot & IOMMU_PROT_IW) in amd_iommu_get_resv_regions()
2889 if (entry->prot & IOMMU_UNITY_MAP_FLAG_EXCL_RANGE) in amd_iommu_get_resv_regions()
2893 region = iommu_alloc_resv_region(entry->address_start, in amd_iommu_get_resv_regions()
2897 dev_err(dev, "Out of memory allocating dm-regions\n"); in amd_iommu_get_resv_regions()
2900 list_add_tail(®ion->list, head); in amd_iommu_get_resv_regions()
2904 MSI_RANGE_END - MSI_RANGE_START + 1, in amd_iommu_get_resv_regions()
2908 list_add_tail(®ion->list, head); in amd_iommu_get_resv_regions()
2911 HT_RANGE_END - HT_RANGE_START + 1, in amd_iommu_get_resv_regions()
2915 list_add_tail(®ion->list, head); in amd_iommu_get_resv_regions()
2922 return dev_data->defer_attach; in amd_iommu_is_attach_deferred()
2930 spin_lock_irqsave(&dom->lock, flags); in amd_iommu_flush_iotlb_all()
2932 spin_unlock_irqrestore(&dom->lock, flags); in amd_iommu_flush_iotlb_all()
2941 spin_lock_irqsave(&dom->lock, flags); in amd_iommu_iotlb_sync()
2942 amd_iommu_domain_flush_pages(dom, gather->start, in amd_iommu_iotlb_sync()
2943 gather->end - gather->start + 1); in amd_iommu_iotlb_sync()
2944 spin_unlock_irqrestore(&dom->lock, flags); in amd_iommu_iotlb_sync()
2956 if (dev_is_pci(dev) && to_pci_dev(dev)->untrusted) in amd_iommu_def_domain_type()
2961 * - memory encryption is active, because some of those devices in amd_iommu_def_domain_type()
2962 * (AMD GPUs) don't have the encryption bit in their DMA-mask in amd_iommu_def_domain_type()
2964 * - SNP is enabled, because it prohibits DTE[Mode]=0. in amd_iommu_def_domain_type()
2996 ret = -EINVAL; in amd_iommu_dev_enable_feature()
3012 ret = -EINVAL; in amd_iommu_dev_disable_feature()
3065 if (iommu->irtcachedis_enabled) in iommu_flush_irt_and_complete()
3069 data = atomic64_inc_return(&iommu->cmd_sem_val); in iommu_flush_irt_and_complete()
3072 raw_spin_lock_irqsave(&iommu->lock, flags); in iommu_flush_irt_and_complete()
3081 raw_spin_unlock_irqrestore(&iommu->lock, flags); in iommu_flush_irt_and_complete()
3088 struct dev_table_entry *dte = &get_dev_table(iommu)[devid]; in set_dte_irq_entry() local
3092 spin_lock(&dev_data->dte_lock); in set_dte_irq_entry()
3094 new = READ_ONCE(dte->data[2]); in set_dte_irq_entry()
3096 new |= iommu_virt_to_phys(table->table); in set_dte_irq_entry()
3100 WRITE_ONCE(dte->data[2], new); in set_dte_irq_entry()
3103 spin_unlock(&dev_data->dte_lock); in set_dte_irq_entry()
3109 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; in get_irq_table()
3111 if (WARN_ONCE(!pci_seg->rlookup_table[devid], in get_irq_table()
3113 __func__, pci_seg->id, devid)) in get_irq_table()
3116 table = pci_seg->irq_lookup_table[devid]; in get_irq_table()
3118 __func__, pci_seg->id, devid)) in get_irq_table()
3132 table->table = kmem_cache_alloc(amd_iommu_irq_cache, GFP_KERNEL); in __alloc_irq_table()
3133 if (!table->table) { in __alloc_irq_table()
3137 raw_spin_lock_init(&table->lock); in __alloc_irq_table()
3140 memset(table->table, 0, in __alloc_irq_table()
3143 memset(table->table, 0, in __alloc_irq_table()
3151 struct amd_iommu_pci_seg *pci_seg = iommu->pci_seg; in set_remap_table_entry()
3153 pci_seg->irq_lookup_table[devid] = table; in set_remap_table_entry()
3163 struct amd_iommu *iommu = rlookup_amd_iommu(&pdev->dev); in set_remap_table_entry_alias()
3166 return -EINVAL; in set_remap_table_entry_alias()
3168 pci_seg = iommu->pci_seg; in set_remap_table_entry_alias()
3169 pci_seg->irq_lookup_table[alias] = table; in set_remap_table_entry_alias()
3171 iommu_flush_dte(pci_seg->rlookup_table[alias], alias); in set_remap_table_entry_alias()
3187 pci_seg = iommu->pci_seg; in alloc_irq_table()
3188 table = pci_seg->irq_lookup_table[devid]; in alloc_irq_table()
3192 alias = pci_seg->alias_table[devid]; in alloc_irq_table()
3193 table = pci_seg->irq_lookup_table[alias]; in alloc_irq_table()
3207 table = pci_seg->irq_lookup_table[devid]; in alloc_irq_table()
3211 table = pci_seg->irq_lookup_table[alias]; in alloc_irq_table()
3236 kmem_cache_free(amd_iommu_irq_cache, new_table->table); in alloc_irq_table()
3251 return -ENODEV; in alloc_irq_index()
3256 raw_spin_lock_irqsave(&table->lock, flags); in alloc_irq_index()
3259 for (index = ALIGN(table->min_index, alignment), c = 0; in alloc_irq_index()
3261 if (!iommu->irte_ops->is_allocated(table, index)) { in alloc_irq_index()
3270 for (; c != 0; --c) in alloc_irq_index()
3271 iommu->irte_ops->set_allocated(table, index - c + 1); in alloc_irq_index()
3273 index -= count - 1; in alloc_irq_index()
3280 index = -ENOSPC; in alloc_irq_index()
3283 raw_spin_unlock_irqrestore(&table->lock, flags); in alloc_irq_index()
3298 return -ENOMEM; in __modify_irte_ga()
3300 raw_spin_lock_irqsave(&table->lock, flags); in __modify_irte_ga()
3302 entry = (struct irte_ga *)table->table; in __modify_irte_ga()
3306 * We use cmpxchg16 to atomically update the 128-bit IRTE, in __modify_irte_ga()
3311 old = entry->irte; in __modify_irte_ga()
3312 WARN_ON(!try_cmpxchg128(&entry->irte, &old, irte->irte)); in __modify_irte_ga()
3314 raw_spin_unlock_irqrestore(&table->lock, flags); in __modify_irte_ga()
3341 return -ENOMEM; in modify_irte()
3343 raw_spin_lock_irqsave(&table->lock, flags); in modify_irte()
3344 table->table[index] = irte->val; in modify_irte()
3345 raw_spin_unlock_irqrestore(&table->lock, flags); in modify_irte()
3361 raw_spin_lock_irqsave(&table->lock, flags); in free_irte()
3362 iommu->irte_ops->clear_allocated(table, index); in free_irte()
3363 raw_spin_unlock_irqrestore(&table->lock, flags); in free_irte()
3374 irte->val = 0; in irte_prepare()
3375 irte->fields.vector = vector; in irte_prepare()
3376 irte->fields.int_type = delivery_mode; in irte_prepare()
3377 irte->fields.destination = dest_apicid; in irte_prepare()
3378 irte->fields.dm = dest_mode; in irte_prepare()
3379 irte->fields.valid = 1; in irte_prepare()
3388 irte->lo.val = 0; in irte_ga_prepare()
3389 irte->hi.val = 0; in irte_ga_prepare()
3390 irte->lo.fields_remap.int_type = delivery_mode; in irte_ga_prepare()
3391 irte->lo.fields_remap.dm = dest_mode; in irte_ga_prepare()
3392 irte->hi.fields.vector = vector; in irte_ga_prepare()
3393 irte->lo.fields_remap.destination = APICID_TO_IRTE_DEST_LO(dest_apicid); in irte_ga_prepare()
3394 irte->hi.fields.destination = APICID_TO_IRTE_DEST_HI(dest_apicid); in irte_ga_prepare()
3395 irte->lo.fields_remap.valid = 1; in irte_ga_prepare()
3402 irte->fields.valid = 1; in irte_activate()
3410 irte->lo.fields_remap.valid = 1; in irte_ga_activate()
3418 irte->fields.valid = 0; in irte_deactivate()
3426 irte->lo.fields_remap.valid = 0; in irte_ga_deactivate()
3435 irte->fields.vector = vector; in irte_set_affinity()
3436 irte->fields.destination = dest_apicid; in irte_set_affinity()
3445 if (!irte->lo.fields_remap.guest_mode) { in irte_ga_set_affinity()
3446 irte->hi.fields.vector = vector; in irte_ga_set_affinity()
3447 irte->lo.fields_remap.destination = in irte_ga_set_affinity()
3449 irte->hi.fields.destination = in irte_ga_set_affinity()
3458 table->table[index] = IRTE_ALLOCATED; in irte_set_allocated()
3463 struct irte_ga *ptr = (struct irte_ga *)table->table; in irte_ga_set_allocated()
3466 memset(&irte->lo.val, 0, sizeof(u64)); in irte_ga_set_allocated()
3467 memset(&irte->hi.val, 0, sizeof(u64)); in irte_ga_set_allocated()
3468 irte->hi.fields.vector = 0xff; in irte_ga_set_allocated()
3473 union irte *ptr = (union irte *)table->table; in irte_is_allocated()
3476 return irte->val != 0; in irte_is_allocated()
3481 struct irte_ga *ptr = (struct irte_ga *)table->table; in irte_ga_is_allocated()
3484 return irte->hi.fields.vector != 0; in irte_ga_is_allocated()
3489 table->table[index] = 0; in irte_clear_allocated()
3494 struct irte_ga *ptr = (struct irte_ga *)table->table; in irte_ga_clear_allocated()
3497 memset(&irte->lo.val, 0, sizeof(u64)); in irte_ga_clear_allocated()
3498 memset(&irte->hi.val, 0, sizeof(u64)); in irte_ga_clear_allocated()
3503 switch (info->type) { in get_devid()
3505 return get_ioapic_devid(info->devid); in get_devid()
3507 return get_hpet_devid(info->devid); in get_devid()
3510 return get_device_sbdf_id(msi_desc_to_dev(info->desc)); in get_devid()
3513 return -1; in get_devid()
3527 msg->data = index; in fill_msi_msg()
3528 msg->address_lo = 0; in fill_msi_msg()
3529 msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW; in fill_msi_msg()
3530 msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH; in fill_msi_msg()
3538 struct irq_2_irte *irte_info = &data->irq_2_irte; in irq_remapping_prepare_irte()
3539 struct amd_iommu *iommu = data->iommu; in irq_remapping_prepare_irte()
3544 data->irq_2_irte.devid = devid; in irq_remapping_prepare_irte()
3545 data->irq_2_irte.index = index + sub_handle; in irq_remapping_prepare_irte()
3546 iommu->irte_ops->prepare(data->entry, APIC_DELIVERY_MODE_FIXED, in irq_remapping_prepare_irte()
3547 apic->dest_mode_logical, irq_cfg->vector, in irq_remapping_prepare_irte()
3548 irq_cfg->dest_apicid, devid); in irq_remapping_prepare_irte()
3550 switch (info->type) { in irq_remapping_prepare_irte()
3555 fill_msi_msg(&data->msi_entry, irte_info->index); in irq_remapping_prepare_irte()
3596 return -EINVAL; in irq_remapping_alloc()
3597 if (nr_irqs > 1 && info->type != X86_IRQ_ALLOC_TYPE_PCI_MSI) in irq_remapping_alloc()
3598 return -EINVAL; in irq_remapping_alloc()
3602 return -EINVAL; in irq_remapping_alloc()
3608 return -EINVAL; in irq_remapping_alloc()
3614 if (info->type == X86_IRQ_ALLOC_TYPE_IOAPIC) { in irq_remapping_alloc()
3619 if (!table->min_index) { in irq_remapping_alloc()
3624 table->min_index = 32; in irq_remapping_alloc()
3626 iommu->irte_ops->set_allocated(table, i); in irq_remapping_alloc()
3628 WARN_ON(table->min_index != 32); in irq_remapping_alloc()
3629 index = info->ioapic.pin; in irq_remapping_alloc()
3631 index = -ENOMEM; in irq_remapping_alloc()
3633 } else if (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI || in irq_remapping_alloc()
3634 info->type == X86_IRQ_ALLOC_TYPE_PCI_MSIX) { in irq_remapping_alloc()
3635 bool align = (info->type == X86_IRQ_ALLOC_TYPE_PCI_MSI); in irq_remapping_alloc()
3638 msi_desc_to_pci_dev(info->desc)); in irq_remapping_alloc()
3653 ret = -EINVAL; in irq_remapping_alloc()
3657 ret = -ENOMEM; in irq_remapping_alloc()
3663 data->entry = kzalloc(sizeof(union irte), GFP_KERNEL); in irq_remapping_alloc()
3665 data->entry = kzalloc(sizeof(struct irte_ga), in irq_remapping_alloc()
3667 if (!data->entry) { in irq_remapping_alloc()
3672 data->iommu = iommu; in irq_remapping_alloc()
3673 irq_data->hwirq = (devid << 16) + i; in irq_remapping_alloc()
3674 irq_data->chip_data = data; in irq_remapping_alloc()
3675 irq_data->chip = &amd_ir_chip; in irq_remapping_alloc()
3682 for (i--; i >= 0; i--) { in irq_remapping_alloc()
3685 kfree(irq_data->chip_data); in irq_remapping_alloc()
3704 if (irq_data && irq_data->chip_data) { in irq_remapping_free()
3705 data = irq_data->chip_data; in irq_remapping_free()
3706 irte_info = &data->irq_2_irte; in irq_remapping_free()
3707 free_irte(data->iommu, irte_info->devid, irte_info->index); in irq_remapping_free()
3708 kfree(data->entry); in irq_remapping_free()
3723 struct amd_ir_data *data = irq_data->chip_data; in irq_remapping_activate()
3724 struct irq_2_irte *irte_info = &data->irq_2_irte; in irq_remapping_activate()
3725 struct amd_iommu *iommu = data->iommu; in irq_remapping_activate()
3731 iommu->irte_ops->activate(iommu, data->entry, irte_info->devid, in irq_remapping_activate()
3732 irte_info->index); in irq_remapping_activate()
3740 struct amd_ir_data *data = irq_data->chip_data; in irq_remapping_deactivate()
3741 struct irq_2_irte *irte_info = &data->irq_2_irte; in irq_remapping_deactivate()
3742 struct amd_iommu *iommu = data->iommu; in irq_remapping_deactivate()
3745 iommu->irte_ops->deactivate(iommu, data->entry, irte_info->devid, in irq_remapping_deactivate()
3746 irte_info->index); in irq_remapping_deactivate()
3753 int devid = -1; in irq_remapping_select()
3759 devid = get_ioapic_devid(fwspec->param[0]); in irq_remapping_select()
3761 devid = get_hpet_devid(fwspec->param[0]); in irq_remapping_select()
3767 return iommu && iommu->ir_domain == d; in irq_remapping_select()
3781 struct irte_ga *entry = (struct irte_ga *) ir_data->entry; in amd_iommu_activate_guest_mode()
3787 valid = entry->lo.fields_vapic.valid; in amd_iommu_activate_guest_mode()
3789 entry->lo.val = 0; in amd_iommu_activate_guest_mode()
3790 entry->hi.val = 0; in amd_iommu_activate_guest_mode()
3792 entry->lo.fields_vapic.valid = valid; in amd_iommu_activate_guest_mode()
3793 entry->lo.fields_vapic.guest_mode = 1; in amd_iommu_activate_guest_mode()
3794 entry->lo.fields_vapic.ga_log_intr = 1; in amd_iommu_activate_guest_mode()
3795 entry->hi.fields.ga_root_ptr = ir_data->ga_root_ptr; in amd_iommu_activate_guest_mode()
3796 entry->hi.fields.vector = ir_data->ga_vector; in amd_iommu_activate_guest_mode()
3797 entry->lo.fields_vapic.ga_tag = ir_data->ga_tag; in amd_iommu_activate_guest_mode()
3799 return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, in amd_iommu_activate_guest_mode()
3800 ir_data->irq_2_irte.index, entry); in amd_iommu_activate_guest_mode()
3807 struct irte_ga *entry = (struct irte_ga *) ir_data->entry; in amd_iommu_deactivate_guest_mode()
3808 struct irq_cfg *cfg = ir_data->cfg; in amd_iommu_deactivate_guest_mode()
3812 !entry || !entry->lo.fields_vapic.guest_mode) in amd_iommu_deactivate_guest_mode()
3815 valid = entry->lo.fields_remap.valid; in amd_iommu_deactivate_guest_mode()
3817 entry->lo.val = 0; in amd_iommu_deactivate_guest_mode()
3818 entry->hi.val = 0; in amd_iommu_deactivate_guest_mode()
3820 entry->lo.fields_remap.valid = valid; in amd_iommu_deactivate_guest_mode()
3821 entry->lo.fields_remap.dm = apic->dest_mode_logical; in amd_iommu_deactivate_guest_mode()
3822 entry->lo.fields_remap.int_type = APIC_DELIVERY_MODE_FIXED; in amd_iommu_deactivate_guest_mode()
3823 entry->hi.fields.vector = cfg->vector; in amd_iommu_deactivate_guest_mode()
3824 entry->lo.fields_remap.destination = in amd_iommu_deactivate_guest_mode()
3825 APICID_TO_IRTE_DEST_LO(cfg->dest_apicid); in amd_iommu_deactivate_guest_mode()
3826 entry->hi.fields.destination = in amd_iommu_deactivate_guest_mode()
3827 APICID_TO_IRTE_DEST_HI(cfg->dest_apicid); in amd_iommu_deactivate_guest_mode()
3829 return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, in amd_iommu_deactivate_guest_mode()
3830 ir_data->irq_2_irte.index, entry); in amd_iommu_deactivate_guest_mode()
3838 struct vcpu_data *vcpu_pi_info = pi_data->vcpu_data; in amd_ir_set_vcpu_affinity()
3839 struct amd_ir_data *ir_data = data->chip_data; in amd_ir_set_vcpu_affinity()
3840 struct irq_2_irte *irte_info = &ir_data->irq_2_irte; in amd_ir_set_vcpu_affinity()
3843 if (ir_data->iommu == NULL) in amd_ir_set_vcpu_affinity()
3844 return -EINVAL; in amd_ir_set_vcpu_affinity()
3846 dev_data = search_dev_data(ir_data->iommu, irte_info->devid); in amd_ir_set_vcpu_affinity()
3849 * This device has never been set up for guest mode. in amd_ir_set_vcpu_affinity()
3852 if (!dev_data || !dev_data->use_vapic) in amd_ir_set_vcpu_affinity()
3855 ir_data->cfg = irqd_cfg(data); in amd_ir_set_vcpu_affinity()
3856 pi_data->ir_data = ir_data; in amd_ir_set_vcpu_affinity()
3859 * SVM tries to set up for VAPIC mode, but we are in in amd_ir_set_vcpu_affinity()
3860 * legacy mode. So, we force legacy mode instead. in amd_ir_set_vcpu_affinity()
3865 pi_data->is_guest_mode = false; in amd_ir_set_vcpu_affinity()
3868 pi_data->prev_ga_tag = ir_data->cached_ga_tag; in amd_ir_set_vcpu_affinity()
3869 if (pi_data->is_guest_mode) { in amd_ir_set_vcpu_affinity()
3870 ir_data->ga_root_ptr = (pi_data->base >> 12); in amd_ir_set_vcpu_affinity()
3871 ir_data->ga_vector = vcpu_pi_info->vector; in amd_ir_set_vcpu_affinity()
3872 ir_data->ga_tag = pi_data->ga_tag; in amd_ir_set_vcpu_affinity()
3875 ir_data->cached_ga_tag = pi_data->ga_tag; in amd_ir_set_vcpu_affinity()
3884 ir_data->cached_ga_tag = 0; in amd_ir_set_vcpu_affinity()
3901 iommu->irte_ops->set_affinity(iommu, ir_data->entry, irte_info->devid, in amd_ir_update_irte()
3902 irte_info->index, cfg->vector, in amd_ir_update_irte()
3903 cfg->dest_apicid); in amd_ir_update_irte()
3909 struct amd_ir_data *ir_data = data->chip_data; in amd_ir_set_affinity()
3910 struct irq_2_irte *irte_info = &ir_data->irq_2_irte; in amd_ir_set_affinity()
3912 struct irq_data *parent = data->parent_data; in amd_ir_set_affinity()
3913 struct amd_iommu *iommu = ir_data->iommu; in amd_ir_set_affinity()
3917 return -ENODEV; in amd_ir_set_affinity()
3919 ret = parent->chip->irq_set_affinity(parent, mask, force); in amd_ir_set_affinity()
3936 struct amd_ir_data *ir_data = irq_data->chip_data; in ir_compose_msi_msg()
3938 *msg = ir_data->msi_entry; in ir_compose_msi_msg()
3942 .name = "AMD-IR",
3951 .prefix = "IR-",
3959 fn = irq_domain_alloc_named_id_fwnode("AMD-IR", iommu->index); in amd_iommu_create_irq_domain()
3961 return -ENOMEM; in amd_iommu_create_irq_domain()
3962 iommu->ir_domain = irq_domain_create_hierarchy(arch_get_ir_parent_domain(), 0, 0, in amd_iommu_create_irq_domain()
3964 if (!iommu->ir_domain) { in amd_iommu_create_irq_domain()
3966 return -ENOMEM; in amd_iommu_create_irq_domain()
3969 irq_domain_update_bus_token(iommu->ir_domain, DOMAIN_BUS_AMDVI); in amd_iommu_create_irq_domain()
3970 iommu->ir_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT | in amd_iommu_create_irq_domain()
3972 iommu->ir_domain->msi_parent_ops = &amdvi_msi_parent_ops; in amd_iommu_create_irq_domain()
3980 struct irte_ga *entry = (struct irte_ga *) ir_data->entry; in amd_iommu_update_ga()
3983 !entry || !entry->lo.fields_vapic.guest_mode) in amd_iommu_update_ga()
3986 if (!ir_data->iommu) in amd_iommu_update_ga()
3987 return -ENODEV; in amd_iommu_update_ga()
3990 entry->lo.fields_vapic.destination = in amd_iommu_update_ga()
3992 entry->hi.fields.destination = in amd_iommu_update_ga()
3995 entry->lo.fields_vapic.is_run = is_run; in amd_iommu_update_ga()
3997 return __modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid, in amd_iommu_update_ga()
3998 ir_data->irq_2_irte.index, entry); in amd_iommu_update_ga()