Lines Matching +full:per +full:- +full:device

1 // SPDX-License-Identifier: GPL-2.0
3 * Coherent per-device memory handling.
10 #include <linux/dma-direct.h>
11 #include <linux/dma-map-ops.h>
23 static inline struct dma_coherent_mem *dev_get_coherent_memory(struct device *dev) in dev_get_coherent_memory()
25 if (dev && dev->dma_mem) in dev_get_coherent_memory()
26 return dev->dma_mem; in dev_get_coherent_memory()
30 static inline dma_addr_t dma_get_device_base(struct device *dev, in dma_get_device_base()
33 if (mem->use_dev_dma_pfn_offset) in dma_get_device_base()
34 return phys_to_dma(dev, PFN_PHYS(mem->pfn_base)); in dma_get_device_base()
35 return mem->device_base; in dma_get_device_base()
46 return ERR_PTR(-EINVAL); in dma_init_coherent_memory()
50 return ERR_PTR(-EINVAL); in dma_init_coherent_memory()
55 dma_mem->bitmap = bitmap_zalloc(pages, GFP_KERNEL); in dma_init_coherent_memory()
56 if (!dma_mem->bitmap) in dma_init_coherent_memory()
59 dma_mem->virt_base = mem_base; in dma_init_coherent_memory()
60 dma_mem->device_base = device_addr; in dma_init_coherent_memory()
61 dma_mem->pfn_base = PFN_DOWN(phys_addr); in dma_init_coherent_memory()
62 dma_mem->size = pages; in dma_init_coherent_memory()
63 dma_mem->use_dev_dma_pfn_offset = use_dma_pfn_offset; in dma_init_coherent_memory()
64 spin_lock_init(&dma_mem->spinlock); in dma_init_coherent_memory()
74 return ERR_PTR(-ENOMEM); in dma_init_coherent_memory()
82 memunmap(mem->virt_base); in _dma_release_coherent_memory()
83 bitmap_free(mem->bitmap); in _dma_release_coherent_memory()
87 static int dma_assign_coherent_memory(struct device *dev, in dma_assign_coherent_memory()
91 return -ENODEV; in dma_assign_coherent_memory()
93 if (dev->dma_mem) in dma_assign_coherent_memory()
94 return -EBUSY; in dma_assign_coherent_memory()
96 dev->dma_mem = mem; in dma_assign_coherent_memory()
102 * is asked for coherent memory for this device. This shall only be used
103 * from platform code, usually based on the device tree description.
108 * device_addr is the DMA address the device needs to be programmed with to
115 * be declared per device.
117 int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, in dma_declare_coherent_memory()
133 void dma_release_coherent_memory(struct device *dev) in dma_release_coherent_memory()
136 _dma_release_coherent_memory(dev->dma_mem); in dma_release_coherent_memory()
137 dev->dma_mem = NULL; in dma_release_coherent_memory()
141 static void *__dma_alloc_from_coherent(struct device *dev, in __dma_alloc_from_coherent()
150 spin_lock_irqsave(&mem->spinlock, flags); in __dma_alloc_from_coherent()
152 if (unlikely(size > ((dma_addr_t)mem->size << PAGE_SHIFT))) in __dma_alloc_from_coherent()
155 pageno = bitmap_find_free_region(mem->bitmap, mem->size, order); in __dma_alloc_from_coherent()
164 ret = mem->virt_base + ((dma_addr_t)pageno << PAGE_SHIFT); in __dma_alloc_from_coherent()
165 spin_unlock_irqrestore(&mem->spinlock, flags); in __dma_alloc_from_coherent()
169 spin_unlock_irqrestore(&mem->spinlock, flags); in __dma_alloc_from_coherent()
174 * dma_alloc_from_dev_coherent() - allocate memory from device coherent pool
175 * @dev: device from which we allocate memory
181 * This function should be only called from per-arch dma_alloc_coherent()
182 * to support allocation from per-device coherent memory pools.
187 int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size, in dma_alloc_from_dev_coherent()
202 if (mem && vaddr >= mem->virt_base && vaddr < in __dma_release_from_coherent()
203 (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) { in __dma_release_from_coherent()
204 int page = (vaddr - mem->virt_base) >> PAGE_SHIFT; in __dma_release_from_coherent()
207 spin_lock_irqsave(&mem->spinlock, flags); in __dma_release_from_coherent()
208 bitmap_release_region(mem->bitmap, page, order); in __dma_release_from_coherent()
209 spin_unlock_irqrestore(&mem->spinlock, flags); in __dma_release_from_coherent()
216 * dma_release_from_dev_coherent() - free memory to device coherent memory pool
217 * @dev: device from which the memory was allocated
221 * This checks whether the memory was allocated from the per-device
227 int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr) in dma_release_from_dev_coherent()
237 if (mem && vaddr >= mem->virt_base && vaddr + size <= in __dma_mmap_from_coherent()
238 (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) { in __dma_mmap_from_coherent()
239 unsigned long off = vma->vm_pgoff; in __dma_mmap_from_coherent()
240 int start = (vaddr - mem->virt_base) >> PAGE_SHIFT; in __dma_mmap_from_coherent()
244 *ret = -ENXIO; in __dma_mmap_from_coherent()
245 if (off < count && user_count <= count - off) { in __dma_mmap_from_coherent()
246 unsigned long pfn = mem->pfn_base + start + off; in __dma_mmap_from_coherent()
247 *ret = remap_pfn_range(vma, vma->vm_start, pfn, in __dma_mmap_from_coherent()
249 vma->vm_page_prot); in __dma_mmap_from_coherent()
257 * dma_mmap_from_dev_coherent() - mmap memory from the device coherent pool
258 * @dev: device from which the memory was allocated
264 * This checks whether the memory was allocated from the per-device
267 * Returns 1 if @vaddr belongs to the device coherent pool and the caller
271 int dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma, in dma_mmap_from_dev_coherent()
282 void *dma_alloc_from_global_coherent(struct device *dev, ssize_t size, in dma_alloc_from_global_coherent()
325 * Support for reserved memory regions defined in device tree
337 static int rmem_dma_device_init(struct reserved_mem *rmem, struct device *dev) in rmem_dma_device_init()
339 if (!rmem->priv) { in rmem_dma_device_init()
342 mem = dma_init_coherent_memory(rmem->base, rmem->base, in rmem_dma_device_init()
343 rmem->size, true); in rmem_dma_device_init()
346 rmem->priv = mem; in rmem_dma_device_init()
348 dma_assign_coherent_memory(dev, rmem->priv); in rmem_dma_device_init()
353 struct device *dev) in rmem_dma_device_release()
356 dev->dma_mem = NULL; in rmem_dma_device_release()
366 unsigned long node = rmem->fdt_node; in rmem_dma_setup()
369 return -EINVAL; in rmem_dma_setup()
372 if (!of_get_flat_dt_prop(node, "no-map", NULL)) { in rmem_dma_setup()
373 pr_err("Reserved memory: regions without no-map are not yet supported\n"); in rmem_dma_setup()
374 return -EINVAL; in rmem_dma_setup()
379 if (of_get_flat_dt_prop(node, "linux,dma-default", NULL)) { in rmem_dma_setup()
382 dma_reserved_default_memory_base = rmem->base; in rmem_dma_setup()
383 dma_reserved_default_memory_size = rmem->size; in rmem_dma_setup()
387 rmem->ops = &rmem_dma_ops; in rmem_dma_setup()
389 &rmem->base, (unsigned long)rmem->size / SZ_1M); in rmem_dma_setup()
397 return -ENOMEM; in dma_init_reserved_memory()
404 RESERVEDMEM_OF_DECLARE(dma, "shared-dma-pool", rmem_dma_setup);