Lines Matching +full:iommu +full:- +full:ctx
1 // SPDX-License-Identifier: GPL-2.0
9 #include <linux/io-64-nonatomic-lo-hi.h>
13 #include <linux/iommu.h>
55 static void idxd_xa_pasid_remove(struct idxd_user_context *ctx);
66 struct idxd_user_context *ctx = dev_to_uctx(dev); in cr_faults_show() local
68 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULTS]); in cr_faults_show()
75 struct idxd_user_context *ctx = dev_to_uctx(dev); in cr_fault_failures_show() local
77 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULT_FAILS]); in cr_fault_failures_show()
83 struct idxd_user_context *ctx = dev_to_uctx(dev); in pid_show() local
85 return sysfs_emit(buf, "%u\n", ctx->pid); in pid_show()
99 struct idxd_user_context *ctx = dev_to_uctx(dev); in cdev_file_attr_visible() local
100 struct idxd_wq *wq = ctx->wq; in cdev_file_attr_visible()
105 return a->mode; in cdev_file_attr_visible()
120 struct idxd_user_context *ctx = dev_to_uctx(dev); in idxd_file_dev_release() local
121 struct idxd_wq *wq = ctx->wq; in idxd_file_dev_release()
122 struct idxd_device *idxd = wq->idxd; in idxd_file_dev_release()
125 ida_free(&file_ida, ctx->id); in idxd_file_dev_release()
127 /* Wait for in-flight operations to complete. */ in idxd_file_dev_release()
129 idxd_device_drain_pasid(idxd, ctx->pasid); in idxd_file_dev_release()
141 if (ctx->sva) { in idxd_file_dev_release()
142 idxd_cdev_evl_drain_pasid(wq, ctx->pasid); in idxd_file_dev_release()
143 iommu_sva_unbind_device(ctx->sva); in idxd_file_dev_release()
144 idxd_xa_pasid_remove(ctx); in idxd_file_dev_release()
146 kfree(ctx); in idxd_file_dev_release()
147 mutex_lock(&wq->wq_lock); in idxd_file_dev_release()
149 mutex_unlock(&wq->wq_lock); in idxd_file_dev_release()
162 struct idxd_wq *wq = idxd_cdev->wq; in idxd_cdev_dev_release()
164 cdev_ctx = &ictx[wq->idxd->data->type]; in idxd_cdev_dev_release()
165 ida_free(&cdev_ctx->minor_ida, idxd_cdev->minor); in idxd_cdev_dev_release()
176 struct cdev *cdev = inode->i_cdev; in inode_idxd_cdev()
185 return idxd_cdev->wq; in inode_wq()
188 static void idxd_xa_pasid_remove(struct idxd_user_context *ctx) in idxd_xa_pasid_remove() argument
190 struct idxd_wq *wq = ctx->wq; in idxd_xa_pasid_remove()
193 mutex_lock(&wq->uc_lock); in idxd_xa_pasid_remove()
194 ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL); in idxd_xa_pasid_remove()
195 if (ptr != (void *)ctx) in idxd_xa_pasid_remove()
196 dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n", in idxd_xa_pasid_remove()
197 ctx->pasid); in idxd_xa_pasid_remove()
198 mutex_unlock(&wq->uc_lock); in idxd_xa_pasid_remove()
203 struct idxd_user_context *ctx; in idxd_user_counter_increment() local
208 mutex_lock(&wq->uc_lock); in idxd_user_counter_increment()
209 ctx = xa_load(&wq->upasid_xa, pasid); in idxd_user_counter_increment()
210 if (!ctx) { in idxd_user_counter_increment()
211 mutex_unlock(&wq->uc_lock); in idxd_user_counter_increment()
214 ctx->counters[index]++; in idxd_user_counter_increment()
215 mutex_unlock(&wq->uc_lock); in idxd_user_counter_increment()
220 struct idxd_user_context *ctx; in idxd_cdev_open() local
230 idxd = wq->idxd; in idxd_cdev_open()
231 dev = &idxd->pdev->dev; in idxd_cdev_open()
235 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); in idxd_cdev_open()
236 if (!ctx) in idxd_cdev_open()
237 return -ENOMEM; in idxd_cdev_open()
239 mutex_lock(&wq->wq_lock); in idxd_cdev_open()
242 rc = -EBUSY; in idxd_cdev_open()
246 ctx->wq = wq; in idxd_cdev_open()
247 filp->private_data = ctx; in idxd_cdev_open()
248 ctx->pid = current->pid; in idxd_cdev_open()
251 sva = iommu_sva_bind_device(dev, current->mm); in idxd_cdev_open()
260 rc = -EINVAL; in idxd_cdev_open()
264 ctx->sva = sva; in idxd_cdev_open()
265 ctx->pasid = pasid; in idxd_cdev_open()
266 ctx->mm = current->mm; in idxd_cdev_open()
268 mutex_lock(&wq->uc_lock); in idxd_cdev_open()
269 rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL); in idxd_cdev_open()
270 mutex_unlock(&wq->uc_lock); in idxd_cdev_open()
283 idxd_cdev = wq->idxd_cdev; in idxd_cdev_open()
284 ctx->id = ida_alloc(&file_ida, GFP_KERNEL); in idxd_cdev_open()
285 if (ctx->id < 0) { in idxd_cdev_open()
289 ctx->idxd_dev.type = IDXD_DEV_CDEV_FILE; in idxd_cdev_open()
290 fdev = user_ctx_dev(ctx); in idxd_cdev_open()
292 fdev->parent = cdev_dev(idxd_cdev); in idxd_cdev_open()
293 fdev->bus = &dsa_bus_type; in idxd_cdev_open()
294 fdev->type = &idxd_cdev_file_type; in idxd_cdev_open()
296 rc = dev_set_name(fdev, "file%d", ctx->id); in idxd_cdev_open()
309 mutex_unlock(&wq->wq_lock); in idxd_cdev_open()
318 idxd_xa_pasid_remove(ctx); in idxd_cdev_open()
323 mutex_unlock(&wq->wq_lock); in idxd_cdev_open()
324 kfree(ctx); in idxd_cdev_open()
330 struct idxd_device *idxd = wq->idxd; in idxd_cdev_evl_drain_pasid()
331 struct idxd_evl *evl = idxd->evl; in idxd_cdev_evl_drain_pasid()
340 mutex_lock(&evl->lock); in idxd_cdev_evl_drain_pasid()
341 status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET); in idxd_cdev_evl_drain_pasid()
344 size = evl->size; in idxd_cdev_evl_drain_pasid()
347 entry_head = (struct __evl_entry *)(evl->log + (h * ent_size)); in idxd_cdev_evl_drain_pasid()
348 if (entry_head->pasid == pasid && entry_head->wq_idx == wq->id) in idxd_cdev_evl_drain_pasid()
349 set_bit(h, evl->bmap); in idxd_cdev_evl_drain_pasid()
352 drain_workqueue(wq->wq); in idxd_cdev_evl_drain_pasid()
353 mutex_unlock(&evl->lock); in idxd_cdev_evl_drain_pasid()
358 struct idxd_user_context *ctx = filep->private_data; in idxd_cdev_release() local
359 struct idxd_wq *wq = ctx->wq; in idxd_cdev_release()
360 struct idxd_device *idxd = wq->idxd; in idxd_cdev_release()
361 struct device *dev = &idxd->pdev->dev; in idxd_cdev_release()
364 filep->private_data = NULL; in idxd_cdev_release()
366 device_unregister(user_ctx_dev(ctx)); in idxd_cdev_release()
374 struct device *dev = &wq->idxd->pdev->dev; in check_vma()
376 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { in check_vma()
379 current->comm, func, in check_vma()
380 vma->vm_end - vma->vm_start); in check_vma()
381 return -EINVAL; in check_vma()
389 struct idxd_user_context *ctx = filp->private_data; in idxd_cdev_mmap() local
390 struct idxd_wq *wq = ctx->wq; in idxd_cdev_mmap()
391 struct idxd_device *idxd = wq->idxd; in idxd_cdev_mmap()
392 struct pci_dev *pdev = idxd->pdev; in idxd_cdev_mmap()
397 dev_dbg(&pdev->dev, "%s called\n", __func__); in idxd_cdev_mmap()
402 * (See the INTEL-SA-01084 security advisory) in idxd_cdev_mmap()
407 if (!idxd->user_submission_safe && !capable(CAP_SYS_RAWIO)) in idxd_cdev_mmap()
408 return -EPERM; in idxd_cdev_mmap()
415 pfn = (base + idxd_get_wq_portal_full_offset(wq->id, in idxd_cdev_mmap()
417 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); in idxd_cdev_mmap()
418 vma->vm_private_data = ctx; in idxd_cdev_mmap()
420 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, in idxd_cdev_mmap()
421 vma->vm_page_prot); in idxd_cdev_mmap()
424 static int idxd_submit_user_descriptor(struct idxd_user_context *ctx, in idxd_submit_user_descriptor() argument
427 struct idxd_wq *wq = ctx->wq; in idxd_submit_user_descriptor()
428 struct idxd_dev *idxd_dev = &wq->idxd->idxd_dev; in idxd_submit_user_descriptor()
436 return -EFAULT; in idxd_submit_user_descriptor()
445 !wq->idxd->user_submission_safe) in idxd_submit_user_descriptor()
446 return -EINVAL; in idxd_submit_user_descriptor()
453 return -EINVAL; in idxd_submit_user_descriptor()
459 descriptor.pasid = ctx->pasid; in idxd_submit_user_descriptor()
472 struct idxd_user_context *ctx = filp->private_data; in idxd_cdev_write() local
477 int rc = idxd_submit_user_descriptor(ctx, udesc + i); in idxd_cdev_write()
491 struct idxd_user_context *ctx = filp->private_data; in idxd_cdev_poll() local
492 struct idxd_wq *wq = ctx->wq; in idxd_cdev_poll()
493 struct idxd_device *idxd = wq->idxd; in idxd_cdev_poll()
496 poll_wait(filp, &wq->err_queue, wait); in idxd_cdev_poll()
497 spin_lock(&idxd->dev_lock); in idxd_cdev_poll()
498 if (idxd->sw_err.valid) in idxd_cdev_poll()
500 spin_unlock(&idxd->dev_lock); in idxd_cdev_poll()
516 return MAJOR(ictx[idxd->data->type].devt); in idxd_cdev_get_major()
521 struct idxd_device *idxd = wq->idxd; in idxd_wq_add_cdev()
530 return -ENOMEM; in idxd_wq_add_cdev()
532 idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV; in idxd_wq_add_cdev()
533 idxd_cdev->wq = wq; in idxd_wq_add_cdev()
534 cdev = &idxd_cdev->cdev; in idxd_wq_add_cdev()
536 cdev_ctx = &ictx[wq->idxd->data->type]; in idxd_wq_add_cdev()
537 minor = ida_alloc_max(&cdev_ctx->minor_ida, MINORMASK, GFP_KERNEL); in idxd_wq_add_cdev()
542 idxd_cdev->minor = minor; in idxd_wq_add_cdev()
545 dev->parent = wq_confdev(wq); in idxd_wq_add_cdev()
546 dev->bus = &dsa_bus_type; in idxd_wq_add_cdev()
547 dev->type = &idxd_cdev_device_type; in idxd_wq_add_cdev()
548 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); in idxd_wq_add_cdev()
550 rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id); in idxd_wq_add_cdev()
554 wq->idxd_cdev = idxd_cdev; in idxd_wq_add_cdev()
558 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc); in idxd_wq_add_cdev()
566 wq->idxd_cdev = NULL; in idxd_wq_add_cdev()
574 idxd_cdev = wq->idxd_cdev; in idxd_wq_del_cdev()
575 wq->idxd_cdev = NULL; in idxd_wq_del_cdev()
576 cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev)); in idxd_wq_del_cdev()
582 struct device *dev = &idxd_dev->conf_dev; in idxd_user_drv_probe()
584 struct idxd_device *idxd = wq->idxd; in idxd_user_drv_probe()
587 if (idxd->state != IDXD_DEV_ENABLED) in idxd_user_drv_probe()
588 return -ENXIO; in idxd_user_drv_probe()
590 mutex_lock(&wq->wq_lock); in idxd_user_drv_probe()
593 idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME; in idxd_user_drv_probe()
594 rc = -ENODEV; in idxd_user_drv_probe()
600 * - If no IOMMU or IOMMU Passthrough without SVA, userspace in idxd_user_drv_probe()
602 * - The IDXD cdev driver does not provide any ways to pin in idxd_user_drv_probe()
604 * PA without IOMMU SVA. Therefore the application has no way in idxd_user_drv_probe()
609 idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU; in idxd_user_drv_probe()
610 dev_dbg(&idxd->pdev->dev, in idxd_user_drv_probe()
613 rc = -EOPNOTSUPP; in idxd_user_drv_probe()
617 wq->wq = create_workqueue(dev_name(wq_confdev(wq))); in idxd_user_drv_probe()
618 if (!wq->wq) { in idxd_user_drv_probe()
619 rc = -ENOMEM; in idxd_user_drv_probe()
623 wq->type = IDXD_WQT_USER; in idxd_user_drv_probe()
630 idxd->cmd_status = IDXD_SCMD_CDEV_ERR; in idxd_user_drv_probe()
634 idxd->cmd_status = 0; in idxd_user_drv_probe()
635 mutex_unlock(&wq->wq_lock); in idxd_user_drv_probe()
641 destroy_workqueue(wq->wq); in idxd_user_drv_probe()
642 wq->type = IDXD_WQT_NONE; in idxd_user_drv_probe()
644 mutex_unlock(&wq->wq_lock); in idxd_user_drv_probe()
652 mutex_lock(&wq->wq_lock); in idxd_user_drv_remove()
655 wq->type = IDXD_WQT_NONE; in idxd_user_drv_remove()
656 destroy_workqueue(wq->wq); in idxd_user_drv_remove()
657 wq->wq = NULL; in idxd_user_drv_remove()
658 mutex_unlock(&wq->wq_lock); in idxd_user_drv_remove()
689 for (i--; i >= 0; i--) in idxd_cdev_register()
706 * idxd_copy_cr - copy completion record to user address space found by wq and
721 struct device *dev = &wq->idxd->pdev->dev; in idxd_copy_cr()
723 struct idxd_user_context *ctx; in idxd_copy_cr() local
726 mutex_lock(&wq->uc_lock); in idxd_copy_cr()
728 ctx = xa_load(&wq->upasid_xa, pasid); in idxd_copy_cr()
729 if (!ctx) { in idxd_copy_cr()
734 mm = ctx->mm; in idxd_copy_cr()
742 len - status_size); in idxd_copy_cr()
746 * when a non-zero status is polled. in idxd_copy_cr()
755 * record information once polling for a non-zero status. in idxd_copy_cr()
767 mutex_unlock(&wq->uc_lock); in idxd_copy_cr()
769 return len - left; in idxd_copy_cr()