Lines Matching +full:grant +full:- +full:dma
1 // SPDX-License-Identifier: GPL-2.0
4 * Xen dma-buf functionality for gntdev.
6 * DMA buffer implementation is based on drivers/gpu/drm/drm_prime.c.
13 #include <linux/dma-buf.h>
14 #include <linux/dma-direct.h>
23 #include "gntdev-common.h"
24 #include "gntdev-dmabuf.h"
45 /* Scatter-gather table of the imported buffer. */
47 /* dma-buf attachment of the imported buffer. */
54 /* Pages of this buffer (only for dma-buf export). */
70 /* List of exported DMA buffers. */
74 /* List of imported DMA buffers. */
79 * We reference this file while exporting dma-bufs, so
80 * the grant device context is not destroyed while there are
86 /* DMA buffer export support. */
88 /* Implementation of wait for exported DMA buffer to be released. */
100 return ERR_PTR(-ENOMEM); in dmabuf_exp_wait_obj_new()
102 init_completion(&obj->completion); in dmabuf_exp_wait_obj_new()
103 obj->gntdev_dmabuf = gntdev_dmabuf; in dmabuf_exp_wait_obj_new()
105 mutex_lock(&priv->lock); in dmabuf_exp_wait_obj_new()
106 list_add(&obj->next, &priv->exp_wait_list); in dmabuf_exp_wait_obj_new()
108 kref_put(&gntdev_dmabuf->u.exp.refcount, dmabuf_exp_release); in dmabuf_exp_wait_obj_new()
109 mutex_unlock(&priv->lock); in dmabuf_exp_wait_obj_new()
116 mutex_lock(&priv->lock); in dmabuf_exp_wait_obj_free()
117 list_del(&obj->next); in dmabuf_exp_wait_obj_free()
118 mutex_unlock(&priv->lock); in dmabuf_exp_wait_obj_free()
125 if (wait_for_completion_timeout(&obj->completion, in dmabuf_exp_wait_obj_wait()
127 return -ETIMEDOUT; in dmabuf_exp_wait_obj_wait()
137 list_for_each_entry(obj, &priv->exp_wait_list, next) in dmabuf_exp_wait_obj_signal()
138 if (obj->gntdev_dmabuf == gntdev_dmabuf) { in dmabuf_exp_wait_obj_signal()
140 complete_all(&obj->completion); in dmabuf_exp_wait_obj_signal()
148 struct gntdev_dmabuf *gntdev_dmabuf, *ret = ERR_PTR(-ENOENT); in dmabuf_exp_wait_obj_get_dmabuf()
150 mutex_lock(&priv->lock); in dmabuf_exp_wait_obj_get_dmabuf()
151 list_for_each_entry(gntdev_dmabuf, &priv->exp_list, next) in dmabuf_exp_wait_obj_get_dmabuf()
152 if (gntdev_dmabuf->fd == fd) { in dmabuf_exp_wait_obj_get_dmabuf()
154 kref_get(&gntdev_dmabuf->u.exp.refcount); in dmabuf_exp_wait_obj_get_dmabuf()
158 mutex_unlock(&priv->lock); in dmabuf_exp_wait_obj_get_dmabuf()
169 pr_debug("Will wait for dma-buf with fd %d\n", fd); in dmabuf_exp_wait_released()
171 * Try to find the DMA buffer: if not found means that in dmabuf_exp_wait_released()
193 /* DMA buffer export support. */
203 ret = -ENOMEM; in dmabuf_pages_to_sgt()
228 return -ENOMEM; in dmabuf_exp_ops_attach()
230 gntdev_dmabuf_attach->dir = DMA_NONE; in dmabuf_exp_ops_attach()
231 attach->priv = gntdev_dmabuf_attach; in dmabuf_exp_ops_attach()
238 struct gntdev_dmabuf_attachment *gntdev_dmabuf_attach = attach->priv; in dmabuf_exp_ops_detach()
241 struct sg_table *sgt = gntdev_dmabuf_attach->sgt; in dmabuf_exp_ops_detach()
244 if (gntdev_dmabuf_attach->dir != DMA_NONE) in dmabuf_exp_ops_detach()
245 dma_unmap_sgtable(attach->dev, sgt, in dmabuf_exp_ops_detach()
246 gntdev_dmabuf_attach->dir, in dmabuf_exp_ops_detach()
253 attach->priv = NULL; in dmabuf_exp_ops_detach()
261 struct gntdev_dmabuf_attachment *gntdev_dmabuf_attach = attach->priv; in dmabuf_exp_ops_map_dma_buf()
262 struct gntdev_dmabuf *gntdev_dmabuf = attach->dmabuf->priv; in dmabuf_exp_ops_map_dma_buf()
265 pr_debug("Mapping %d pages for dev %p\n", gntdev_dmabuf->nr_pages, in dmabuf_exp_ops_map_dma_buf()
266 attach->dev); in dmabuf_exp_ops_map_dma_buf()
269 return ERR_PTR(-EINVAL); in dmabuf_exp_ops_map_dma_buf()
272 if (gntdev_dmabuf_attach->dir == dir) in dmabuf_exp_ops_map_dma_buf()
273 return gntdev_dmabuf_attach->sgt; in dmabuf_exp_ops_map_dma_buf()
279 if (gntdev_dmabuf_attach->dir != DMA_NONE) in dmabuf_exp_ops_map_dma_buf()
280 return ERR_PTR(-EBUSY); in dmabuf_exp_ops_map_dma_buf()
282 sgt = dmabuf_pages_to_sgt(gntdev_dmabuf->pages, in dmabuf_exp_ops_map_dma_buf()
283 gntdev_dmabuf->nr_pages); in dmabuf_exp_ops_map_dma_buf()
285 if (dma_map_sgtable(attach->dev, sgt, dir, in dmabuf_exp_ops_map_dma_buf()
289 sgt = ERR_PTR(-ENOMEM); in dmabuf_exp_ops_map_dma_buf()
291 gntdev_dmabuf_attach->sgt = sgt; in dmabuf_exp_ops_map_dma_buf()
292 gntdev_dmabuf_attach->dir = dir; in dmabuf_exp_ops_map_dma_buf()
296 pr_debug("Failed to map sg table for dev %p\n", attach->dev); in dmabuf_exp_ops_map_dma_buf()
312 dmabuf_exp_wait_obj_signal(gntdev_dmabuf->priv, gntdev_dmabuf); in dmabuf_exp_release()
313 list_del(&gntdev_dmabuf->next); in dmabuf_exp_release()
314 fput(gntdev_dmabuf->priv->filp); in dmabuf_exp_release()
321 mutex_lock(&priv->lock); in dmabuf_exp_remove_map()
322 list_del(&map->next); in dmabuf_exp_remove_map()
324 mutex_unlock(&priv->lock); in dmabuf_exp_remove_map()
329 struct gntdev_dmabuf *gntdev_dmabuf = dma_buf->priv; in dmabuf_exp_ops_release()
330 struct gntdev_dmabuf_priv *priv = gntdev_dmabuf->priv; in dmabuf_exp_ops_release()
332 dmabuf_exp_remove_map(gntdev_dmabuf->u.exp.priv, in dmabuf_exp_ops_release()
333 gntdev_dmabuf->u.exp.map); in dmabuf_exp_ops_release()
334 mutex_lock(&priv->lock); in dmabuf_exp_ops_release()
335 kref_put(&gntdev_dmabuf->u.exp.refcount, dmabuf_exp_release); in dmabuf_exp_ops_release()
336 mutex_unlock(&priv->lock); in dmabuf_exp_ops_release()
365 return -ENOMEM; in dmabuf_exp_from_pages()
367 kref_init(&gntdev_dmabuf->u.exp.refcount); in dmabuf_exp_from_pages()
369 gntdev_dmabuf->priv = args->dmabuf_priv; in dmabuf_exp_from_pages()
370 gntdev_dmabuf->nr_pages = args->count; in dmabuf_exp_from_pages()
371 gntdev_dmabuf->pages = args->pages; in dmabuf_exp_from_pages()
372 gntdev_dmabuf->u.exp.priv = args->priv; in dmabuf_exp_from_pages()
373 gntdev_dmabuf->u.exp.map = args->map; in dmabuf_exp_from_pages()
376 if (args->dev->driver && args->dev->driver->owner) in dmabuf_exp_from_pages()
377 exp_info.owner = args->dev->driver->owner; in dmabuf_exp_from_pages()
381 exp_info.size = args->count << PAGE_SHIFT; in dmabuf_exp_from_pages()
385 gntdev_dmabuf->dmabuf = dma_buf_export(&exp_info); in dmabuf_exp_from_pages()
386 if (IS_ERR(gntdev_dmabuf->dmabuf)) { in dmabuf_exp_from_pages()
387 ret = PTR_ERR(gntdev_dmabuf->dmabuf); in dmabuf_exp_from_pages()
388 gntdev_dmabuf->dmabuf = NULL; in dmabuf_exp_from_pages()
392 ret = dma_buf_fd(gntdev_dmabuf->dmabuf, O_CLOEXEC); in dmabuf_exp_from_pages()
396 gntdev_dmabuf->fd = ret; in dmabuf_exp_from_pages()
397 args->fd = ret; in dmabuf_exp_from_pages()
399 pr_debug("Exporting DMA buffer with fd %d\n", ret); in dmabuf_exp_from_pages()
401 mutex_lock(&args->dmabuf_priv->lock); in dmabuf_exp_from_pages()
402 list_add(&gntdev_dmabuf->next, &args->dmabuf_priv->exp_list); in dmabuf_exp_from_pages()
403 mutex_unlock(&args->dmabuf_priv->lock); in dmabuf_exp_from_pages()
404 get_file(gntdev_dmabuf->priv->filp); in dmabuf_exp_from_pages()
408 if (gntdev_dmabuf->dmabuf) in dmabuf_exp_from_pages()
409 dma_buf_put(gntdev_dmabuf->dmabuf); in dmabuf_exp_from_pages()
421 return ERR_PTR(-EINVAL); in dmabuf_exp_alloc_backing_storage()
425 pr_debug("Wrong dma-buf flags: 0x%x\n", dmabuf_flags); in dmabuf_exp_alloc_backing_storage()
426 return ERR_PTR(-EINVAL); in dmabuf_exp_alloc_backing_storage()
431 return ERR_PTR(-ENOMEM); in dmabuf_exp_alloc_backing_storage()
448 map->grants[i].domid = domid; in dmabuf_exp_from_refs()
449 map->grants[i].ref = refs[i]; in dmabuf_exp_from_refs()
452 mutex_lock(&priv->lock); in dmabuf_exp_from_refs()
454 mutex_unlock(&priv->lock); in dmabuf_exp_from_refs()
456 map->flags |= GNTMAP_host_map; in dmabuf_exp_from_refs()
458 map->flags |= GNTMAP_device_map; in dmabuf_exp_from_refs()
467 args.dev = priv->dma_dev; in dmabuf_exp_from_refs()
468 args.dmabuf_priv = priv->dmabuf_priv; in dmabuf_exp_from_refs()
469 args.count = map->count; in dmabuf_exp_from_refs()
470 args.pages = map->pages; in dmabuf_exp_from_refs()
471 args.fd = -1; /* Shut up unnecessary gcc warning for i386 */ in dmabuf_exp_from_refs()
485 /* DMA buffer import support. */
496 pr_debug("Cannot allocate grant references, ret %d\n", ret); in dmabuf_imp_grant_foreign_access()
506 pr_debug("Cannot claim grant reference, ret %d\n", ret); in dmabuf_imp_grant_foreign_access()
533 kfree(gntdev_dmabuf->u.imp.refs); in dmabuf_imp_free_storage()
546 gntdev_dmabuf->u.imp.refs = kcalloc(count, in dmabuf_imp_alloc_storage()
547 sizeof(gntdev_dmabuf->u.imp.refs[0]), in dmabuf_imp_alloc_storage()
549 if (!gntdev_dmabuf->u.imp.refs) in dmabuf_imp_alloc_storage()
552 gntdev_dmabuf->nr_pages = count; in dmabuf_imp_alloc_storage()
555 gntdev_dmabuf->u.imp.refs[i] = INVALID_GRANT_REF; in dmabuf_imp_alloc_storage()
562 return ERR_PTR(-ENOMEM); in dmabuf_imp_alloc_storage()
587 gntdev_dmabuf->priv = priv; in dmabuf_imp_to_refs()
588 gntdev_dmabuf->fd = fd; in dmabuf_imp_to_refs()
596 gntdev_dmabuf->u.imp.attach = attach; in dmabuf_imp_to_refs()
605 if (sgt->sgl->offset) { in dmabuf_imp_to_refs()
606 ret = ERR_PTR(-EINVAL); in dmabuf_imp_to_refs()
607 pr_debug("DMA buffer has %d bytes offset, user-space expects 0\n", in dmabuf_imp_to_refs()
608 sgt->sgl->offset); in dmabuf_imp_to_refs()
613 if (attach->dmabuf->size != gntdev_dmabuf->nr_pages << PAGE_SHIFT) { in dmabuf_imp_to_refs()
614 ret = ERR_PTR(-EINVAL); in dmabuf_imp_to_refs()
615 pr_debug("DMA buffer has %zu pages, user-space expects %d\n", in dmabuf_imp_to_refs()
616 attach->dmabuf->size, gntdev_dmabuf->nr_pages); in dmabuf_imp_to_refs()
620 gntdev_dmabuf->u.imp.sgt = sgt; in dmabuf_imp_to_refs()
624 ret = ERR_PTR(-ENOMEM); in dmabuf_imp_to_refs()
631 * exported by DMA-buf, but since we deal with special Xen dma device here in dmabuf_imp_to_refs()
632 * (not a normal physical one) look at the dma addresses in the sg table in dmabuf_imp_to_refs()
644 gntdev_dmabuf->u.imp.refs, in dmabuf_imp_to_refs()
650 pr_debug("Imported DMA buffer with fd %d\n", fd); in dmabuf_imp_to_refs()
652 mutex_lock(&priv->lock); in dmabuf_imp_to_refs()
653 list_add(&gntdev_dmabuf->next, &priv->imp_list); in dmabuf_imp_to_refs()
654 mutex_unlock(&priv->lock); in dmabuf_imp_to_refs()
659 dmabuf_imp_end_foreign_access(gntdev_dmabuf->u.imp.refs, count); in dmabuf_imp_to_refs()
672 * Find the hyper dma-buf by its file descriptor and remove
678 struct gntdev_dmabuf *q, *gntdev_dmabuf, *ret = ERR_PTR(-ENOENT); in dmabuf_imp_find_unlink()
680 mutex_lock(&priv->lock); in dmabuf_imp_find_unlink()
681 list_for_each_entry_safe(gntdev_dmabuf, q, &priv->imp_list, next) { in dmabuf_imp_find_unlink()
682 if (gntdev_dmabuf->fd == fd) { in dmabuf_imp_find_unlink()
685 list_del(&gntdev_dmabuf->next); in dmabuf_imp_find_unlink()
689 mutex_unlock(&priv->lock); in dmabuf_imp_find_unlink()
703 pr_debug("Releasing DMA buffer with fd %d\n", fd); in dmabuf_imp_release()
705 dmabuf_imp_end_foreign_access(gntdev_dmabuf->u.imp.refs, in dmabuf_imp_release()
706 gntdev_dmabuf->nr_pages); in dmabuf_imp_release()
708 attach = gntdev_dmabuf->u.imp.attach; in dmabuf_imp_release()
710 if (gntdev_dmabuf->u.imp.sgt) in dmabuf_imp_release()
711 dma_buf_unmap_attachment_unlocked(attach, gntdev_dmabuf->u.imp.sgt, in dmabuf_imp_release()
713 dma_buf = attach->dmabuf; in dmabuf_imp_release()
714 dma_buf_detach(attach->dmabuf, attach); in dmabuf_imp_release()
725 list_for_each_entry_safe(gntdev_dmabuf, q, &priv->imp_list, next) in dmabuf_imp_release_all()
726 dmabuf_imp_release(priv, gntdev_dmabuf->fd); in dmabuf_imp_release_all()
729 /* DMA buffer IOCTL support. */
739 pr_debug("Cannot provide dma-buf: use_ptemode %d\n", in gntdev_ioctl_dmabuf_exp_from_refs()
741 return -EINVAL; in gntdev_ioctl_dmabuf_exp_from_refs()
745 return -EFAULT; in gntdev_ioctl_dmabuf_exp_from_refs()
748 return -EINVAL; in gntdev_ioctl_dmabuf_exp_from_refs()
752 return -ENOMEM; in gntdev_ioctl_dmabuf_exp_from_refs()
754 if (copy_from_user(refs, u->refs, sizeof(*refs) * op.count) != 0) { in gntdev_ioctl_dmabuf_exp_from_refs()
755 ret = -EFAULT; in gntdev_ioctl_dmabuf_exp_from_refs()
765 ret = -EFAULT; in gntdev_ioctl_dmabuf_exp_from_refs()
778 return -EFAULT; in gntdev_ioctl_dmabuf_exp_wait_released()
780 return dmabuf_exp_wait_released(priv->dmabuf_priv, op.fd, in gntdev_ioctl_dmabuf_exp_wait_released()
792 return -EFAULT; in gntdev_ioctl_dmabuf_imp_to_refs()
795 return -EINVAL; in gntdev_ioctl_dmabuf_imp_to_refs()
797 gntdev_dmabuf = dmabuf_imp_to_refs(priv->dmabuf_priv, in gntdev_ioctl_dmabuf_imp_to_refs()
798 priv->dma_dev, op.fd, in gntdev_ioctl_dmabuf_imp_to_refs()
803 if (copy_to_user(u->refs, gntdev_dmabuf->u.imp.refs, in gntdev_ioctl_dmabuf_imp_to_refs()
804 sizeof(*u->refs) * op.count) != 0) { in gntdev_ioctl_dmabuf_imp_to_refs()
805 ret = -EFAULT; in gntdev_ioctl_dmabuf_imp_to_refs()
811 dmabuf_imp_release(priv->dmabuf_priv, op.fd); in gntdev_ioctl_dmabuf_imp_to_refs()
821 return -EFAULT; in gntdev_ioctl_dmabuf_imp_release()
823 return dmabuf_imp_release(priv->dmabuf_priv, op.fd); in gntdev_ioctl_dmabuf_imp_release()
832 return ERR_PTR(-ENOMEM); in gntdev_dmabuf_init()
834 mutex_init(&priv->lock); in gntdev_dmabuf_init()
835 INIT_LIST_HEAD(&priv->exp_list); in gntdev_dmabuf_init()
836 INIT_LIST_HEAD(&priv->exp_wait_list); in gntdev_dmabuf_init()
837 INIT_LIST_HEAD(&priv->imp_list); in gntdev_dmabuf_init()
839 priv->filp = filp; in gntdev_dmabuf_init()