30  * This library provides helpers for GEM objects backed by shmem buffers
55 	struct drm_gem_shmem_object *shmem;  in __drm_gem_shmem_create()  local
65 		shmem = to_drm_gem_shmem_obj(obj);  in __drm_gem_shmem_create()
67 		shmem = kzalloc(sizeof(*shmem), GFP_KERNEL);  in __drm_gem_shmem_create()
68 		if (!shmem)  in __drm_gem_shmem_create()
70 		obj = &shmem->base;  in __drm_gem_shmem_create()
78 		shmem->map_wc = false; /* dma-buf mappings use always writecombine */  in __drm_gem_shmem_create()
91 	INIT_LIST_HEAD(&shmem->madv_list);  in __drm_gem_shmem_create()
105 	return shmem;  in __drm_gem_shmem_create()
119  * This function creates a shmem GEM object.
138  * This function creates a shmem GEM object in a given tmpfs mountpoint.
153  * drm_gem_shmem_free - Free resources associated with a shmem GEM object
154  * @shmem: shmem GEM object to free
159 void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_free()  argument
161 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_free()
164 		drm_prime_gem_destroy(obj, shmem->sgt);  in drm_gem_shmem_free()
166 		dma_resv_lock(shmem->base.resv, NULL);  in drm_gem_shmem_free()
168 		drm_WARN_ON(obj->dev, shmem->vmap_use_count);  in drm_gem_shmem_free()
170 		if (shmem->sgt) {  in drm_gem_shmem_free()
171 			dma_unmap_sgtable(obj->dev->dev, shmem->sgt,  in drm_gem_shmem_free()
173 			sg_free_table(shmem->sgt);  in drm_gem_shmem_free()
174 			kfree(shmem->sgt);  in drm_gem_shmem_free()
176 		if (shmem->pages)  in drm_gem_shmem_free()
177 			drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_free()
179 		drm_WARN_ON(obj->dev, shmem->pages_use_count);  in drm_gem_shmem_free()
181 		dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_free()
185 	kfree(shmem);  in drm_gem_shmem_free()
189 static int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_get_pages()  argument
191 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_get_pages()
194 	dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_get_pages()
196 	if (shmem->pages_use_count++ > 0)  in drm_gem_shmem_get_pages()
203 		shmem->pages_use_count = 0;  in drm_gem_shmem_get_pages()
213 	if (shmem->map_wc)  in drm_gem_shmem_get_pages()
217 	shmem->pages = pages;  in drm_gem_shmem_get_pages()
223  * drm_gem_shmem_put_pages - Decrease use count on the backing pages for a shmem GEM object
224  * @shmem: shmem GEM object
228 void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_put_pages()  argument
230 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_put_pages()
232 	dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_put_pages()
234 	if (drm_WARN_ON_ONCE(obj->dev, !shmem->pages_use_count))  in drm_gem_shmem_put_pages()
237 	if (--shmem->pages_use_count > 0)  in drm_gem_shmem_put_pages()
241 	if (shmem->map_wc)  in drm_gem_shmem_put_pages()
242 		set_pages_array_wb(shmem->pages, obj->size >> PAGE_SHIFT);  in drm_gem_shmem_put_pages()
245 	drm_gem_put_pages(obj, shmem->pages,  in drm_gem_shmem_put_pages()
246 			  shmem->pages_mark_dirty_on_put,  in drm_gem_shmem_put_pages()
247 			  shmem->pages_mark_accessed_on_put);  in drm_gem_shmem_put_pages()
248 	shmem->pages = NULL;  in drm_gem_shmem_put_pages()
252 int drm_gem_shmem_pin_locked(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_pin_locked()  argument
256 	dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_pin_locked()
258 	drm_WARN_ON(shmem->base.dev, shmem->base.import_attach);  in drm_gem_shmem_pin_locked()
260 	ret = drm_gem_shmem_get_pages(shmem);  in drm_gem_shmem_pin_locked()
266 void drm_gem_shmem_unpin_locked(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_unpin_locked()  argument
268 	dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_unpin_locked()
270 	drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_unpin_locked()
275  * drm_gem_shmem_pin - Pin backing pages for a shmem GEM object
276  * @shmem: shmem GEM object
284 int drm_gem_shmem_pin(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_pin()  argument
286 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_pin()
291 	ret = dma_resv_lock_interruptible(shmem->base.resv, NULL);  in drm_gem_shmem_pin()
294 	ret = drm_gem_shmem_pin_locked(shmem);  in drm_gem_shmem_pin()
295 	dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_pin()
302  * drm_gem_shmem_unpin - Unpin backing pages for a shmem GEM object
303  * @shmem: shmem GEM object
308 void drm_gem_shmem_unpin(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_unpin()  argument
310 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_unpin()
314 	dma_resv_lock(shmem->base.resv, NULL);  in drm_gem_shmem_unpin()
315 	drm_gem_shmem_unpin_locked(shmem);  in drm_gem_shmem_unpin()
316 	dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_unpin()
321  * drm_gem_shmem_vmap - Create a virtual mapping for a shmem GEM object
322  * @shmem: shmem GEM object
323  * @map: Returns the kernel virtual address of the SHMEM GEM object's backing
327  * exists for the buffer backing the shmem GEM object. It hides the differences
335 int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem,  in drm_gem_shmem_vmap()  argument
338 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_vmap()
352 		dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_vmap()
354 		if (shmem->vmap_use_count++ > 0) {  in drm_gem_shmem_vmap()
355 			iosys_map_set_vaddr(map, shmem->vaddr);  in drm_gem_shmem_vmap()
359 		ret = drm_gem_shmem_get_pages(shmem);  in drm_gem_shmem_vmap()
363 		if (shmem->map_wc)  in drm_gem_shmem_vmap()
365 		shmem->vaddr = vmap(shmem->pages, obj->size >> PAGE_SHIFT,  in drm_gem_shmem_vmap()
367 		if (!shmem->vaddr)  in drm_gem_shmem_vmap()
370 			iosys_map_set_vaddr(map, shmem->vaddr);  in drm_gem_shmem_vmap()
382 		drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_vmap()
384 	shmem->vmap_use_count = 0;  in drm_gem_shmem_vmap()
391  * drm_gem_shmem_vunmap - Unmap a virtual mapping for a shmem GEM object
392  * @shmem: shmem GEM object
393  * @map: Kernel virtual address where the SHMEM GEM object was mapped
402 void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem,  in drm_gem_shmem_vunmap()  argument
405 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_vunmap()
410 		dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_vunmap()
412 		if (drm_WARN_ON_ONCE(obj->dev, !shmem->vmap_use_count))  in drm_gem_shmem_vunmap()
415 		if (--shmem->vmap_use_count > 0)  in drm_gem_shmem_vunmap()
418 		vunmap(shmem->vaddr);  in drm_gem_shmem_vunmap()
419 		drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_vunmap()
422 	shmem->vaddr = NULL;  in drm_gem_shmem_vunmap()
431 	struct drm_gem_shmem_object *shmem;  in drm_gem_shmem_create_with_handle()  local
434 	shmem = drm_gem_shmem_create(dev, size);  in drm_gem_shmem_create_with_handle()
435 	if (IS_ERR(shmem))  in drm_gem_shmem_create_with_handle()
436 		return PTR_ERR(shmem);  in drm_gem_shmem_create_with_handle()
442 	ret = drm_gem_handle_create(file_priv, &shmem->base, handle);  in drm_gem_shmem_create_with_handle()
444 	drm_gem_object_put(&shmem->base);  in drm_gem_shmem_create_with_handle()
452 int drm_gem_shmem_madvise(struct drm_gem_shmem_object *shmem, int madv)  in drm_gem_shmem_madvise()  argument
454 	dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_madvise()
456 	if (shmem->madv >= 0)  in drm_gem_shmem_madvise()
457 		shmem->madv = madv;  in drm_gem_shmem_madvise()
459 	madv = shmem->madv;  in drm_gem_shmem_madvise()
465 void drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_purge()  argument
467 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_purge()
470 	dma_resv_assert_held(shmem->base.resv);  in drm_gem_shmem_purge()
472 	drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem));  in drm_gem_shmem_purge()
474 	dma_unmap_sgtable(dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0);  in drm_gem_shmem_purge()
475 	sg_free_table(shmem->sgt);  in drm_gem_shmem_purge()
476 	kfree(shmem->sgt);  in drm_gem_shmem_purge()
477 	shmem->sgt = NULL;  in drm_gem_shmem_purge()
479 	drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_purge()
481 	shmem->madv = -1;  in drm_gem_shmem_purge()
498  * drm_gem_shmem_dumb_create - Create a dumb shmem buffer object
538 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_fault()  local
547 	dma_resv_lock(shmem->base.resv, NULL);  in drm_gem_shmem_fault()
550 	    drm_WARN_ON_ONCE(obj->dev, !shmem->pages) ||  in drm_gem_shmem_fault()
551 	    shmem->madv < 0) {  in drm_gem_shmem_fault()
554 		page = shmem->pages[page_offset];  in drm_gem_shmem_fault()
559 	dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_fault()
567 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_vm_open()  local
571 	dma_resv_lock(shmem->base.resv, NULL);  in drm_gem_shmem_vm_open()
578 	if (!drm_WARN_ON_ONCE(obj->dev, !shmem->pages_use_count))  in drm_gem_shmem_vm_open()
579 		shmem->pages_use_count++;  in drm_gem_shmem_vm_open()
581 	dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_vm_open()
589 	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);  in drm_gem_shmem_vm_close()  local
591 	dma_resv_lock(shmem->base.resv, NULL);  in drm_gem_shmem_vm_close()
592 	drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_vm_close()
593 	dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_vm_close()
606  * drm_gem_shmem_mmap - Memory-map a shmem GEM object
607  * @shmem: shmem GEM object
611  * operation for shmem objects.
616 int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct *vma)  in drm_gem_shmem_mmap()  argument
618 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_mmap()
641 	dma_resv_lock(shmem->base.resv, NULL);  in drm_gem_shmem_mmap()
642 	ret = drm_gem_shmem_get_pages(shmem);  in drm_gem_shmem_mmap()
643 	dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_mmap()
650 	if (shmem->map_wc)  in drm_gem_shmem_mmap()
659  * @shmem: shmem GEM object
663 void drm_gem_shmem_print_info(const struct drm_gem_shmem_object *shmem,  in drm_gem_shmem_print_info()  argument
666 	if (shmem->base.import_attach)  in drm_gem_shmem_print_info()
669 	drm_printf_indent(p, indent, "pages_use_count=%u\n", shmem->pages_use_count);  in drm_gem_shmem_print_info()
670 	drm_printf_indent(p, indent, "vmap_use_count=%u\n", shmem->vmap_use_count);  in drm_gem_shmem_print_info()
671 	drm_printf_indent(p, indent, "vaddr=%p\n", shmem->vaddr);  in drm_gem_shmem_print_info()
677  *                              pages for a shmem GEM object
678  * @shmem: shmem GEM object
689 struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_get_sg_table()  argument
691 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_get_sg_table()
695 	return drm_prime_pages_to_sg(obj->dev, shmem->pages, obj->size >> PAGE_SHIFT);  in drm_gem_shmem_get_sg_table()
699 static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_get_pages_sgt_locked()  argument
701 	struct drm_gem_object *obj = &shmem->base;  in drm_gem_shmem_get_pages_sgt_locked()
705 	if (shmem->sgt)  in drm_gem_shmem_get_pages_sgt_locked()
706 		return shmem->sgt;  in drm_gem_shmem_get_pages_sgt_locked()
710 	ret = drm_gem_shmem_get_pages(shmem);  in drm_gem_shmem_get_pages_sgt_locked()
714 	sgt = drm_gem_shmem_get_sg_table(shmem);  in drm_gem_shmem_get_pages_sgt_locked()
724 	shmem->sgt = sgt;  in drm_gem_shmem_get_pages_sgt_locked()
732 	drm_gem_shmem_put_pages(shmem);  in drm_gem_shmem_get_pages_sgt_locked()
738  *				 scatter/gather table for a shmem GEM object.
739  * @shmem: shmem GEM object
752 struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem)  in drm_gem_shmem_get_pages_sgt()  argument
757 	ret = dma_resv_lock_interruptible(shmem->base.resv, NULL);  in drm_gem_shmem_get_pages_sgt()
760 	sgt = drm_gem_shmem_get_pages_sgt_locked(shmem);  in drm_gem_shmem_get_pages_sgt()
761 	dma_resv_unlock(shmem->base.resv);  in drm_gem_shmem_get_pages_sgt()
768  * drm_gem_shmem_prime_import_sg_table - Produce a shmem GEM object from
775  * another driver. Drivers that use the shmem helpers should set this as their
788 	struct drm_gem_shmem_object *shmem;  in drm_gem_shmem_prime_import_sg_table()  local
790 	shmem = __drm_gem_shmem_create(dev, size, true, NULL);  in drm_gem_shmem_prime_import_sg_table()
791 	if (IS_ERR(shmem))  in drm_gem_shmem_prime_import_sg_table()
792 		return ERR_CAST(shmem);  in drm_gem_shmem_prime_import_sg_table()
794 	shmem->sgt = sgt;  in drm_gem_shmem_prime_import_sg_table()
798 	return &shmem->base;  in drm_gem_shmem_prime_import_sg_table()
802 MODULE_DESCRIPTION("DRM SHMEM memory-management helpers");