1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2024, Advanced Micro Devices, Inc.
4  */
5 
6 #include <drm/amdxdna_accel.h>
7 #include <drm/drm_cache.h>
8 #include <drm/drm_device.h>
9 #include <drm/drm_gem.h>
10 #include <drm/drm_gem_shmem_helper.h>
11 #include <drm/gpu_scheduler.h>
12 #include <linux/iosys-map.h>
13 #include <linux/vmalloc.h>
14 
15 #include "amdxdna_ctx.h"
16 #include "amdxdna_gem.h"
17 #include "amdxdna_pci_drv.h"
18 
19 #define XDNA_MAX_CMD_BO_SIZE	SZ_32K
20 
21 static int
amdxdna_gem_insert_node_locked(struct amdxdna_gem_obj * abo,bool use_vmap)22 amdxdna_gem_insert_node_locked(struct amdxdna_gem_obj *abo, bool use_vmap)
23 {
24 	struct amdxdna_client *client = abo->client;
25 	struct amdxdna_dev *xdna = client->xdna;
26 	struct amdxdna_mem *mem = &abo->mem;
27 	u64 offset;
28 	u32 align;
29 	int ret;
30 
31 	align = 1 << max(PAGE_SHIFT, xdna->dev_info->dev_mem_buf_shift);
32 	ret = drm_mm_insert_node_generic(&abo->dev_heap->mm, &abo->mm_node,
33 					 mem->size, align,
34 					 0, DRM_MM_INSERT_BEST);
35 	if (ret) {
36 		XDNA_ERR(xdna, "Failed to alloc dev bo memory, ret %d", ret);
37 		return ret;
38 	}
39 
40 	mem->dev_addr = abo->mm_node.start;
41 	offset = mem->dev_addr - abo->dev_heap->mem.dev_addr;
42 	mem->userptr = abo->dev_heap->mem.userptr + offset;
43 	mem->pages = &abo->dev_heap->base.pages[offset >> PAGE_SHIFT];
44 	mem->nr_pages = mem->size >> PAGE_SHIFT;
45 
46 	if (use_vmap) {
47 		mem->kva = vmap(mem->pages, mem->nr_pages, VM_MAP, PAGE_KERNEL);
48 		if (!mem->kva) {
49 			XDNA_ERR(xdna, "Failed to vmap");
50 			drm_mm_remove_node(&abo->mm_node);
51 			return -EFAULT;
52 		}
53 	}
54 
55 	return 0;
56 }
57 
amdxdna_gem_obj_free(struct drm_gem_object * gobj)58 static void amdxdna_gem_obj_free(struct drm_gem_object *gobj)
59 {
60 	struct amdxdna_dev *xdna = to_xdna_dev(gobj->dev);
61 	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
62 	struct iosys_map map = IOSYS_MAP_INIT_VADDR(abo->mem.kva);
63 
64 	XDNA_DBG(xdna, "BO type %d xdna_addr 0x%llx", abo->type, abo->mem.dev_addr);
65 	if (abo->pinned)
66 		amdxdna_gem_unpin(abo);
67 
68 	if (abo->type == AMDXDNA_BO_DEV) {
69 		mutex_lock(&abo->client->mm_lock);
70 		drm_mm_remove_node(&abo->mm_node);
71 		mutex_unlock(&abo->client->mm_lock);
72 
73 		vunmap(abo->mem.kva);
74 		drm_gem_object_put(to_gobj(abo->dev_heap));
75 		drm_gem_object_release(gobj);
76 		mutex_destroy(&abo->lock);
77 		kfree(abo);
78 		return;
79 	}
80 
81 	if (abo->type == AMDXDNA_BO_DEV_HEAP)
82 		drm_mm_takedown(&abo->mm);
83 
84 	drm_gem_vunmap_unlocked(gobj, &map);
85 	mutex_destroy(&abo->lock);
86 	drm_gem_shmem_free(&abo->base);
87 }
88 
89 static const struct drm_gem_object_funcs amdxdna_gem_dev_obj_funcs = {
90 	.free = amdxdna_gem_obj_free,
91 };
92 
amdxdna_hmm_invalidate(struct mmu_interval_notifier * mni,const struct mmu_notifier_range * range,unsigned long cur_seq)93 static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni,
94 				   const struct mmu_notifier_range *range,
95 				   unsigned long cur_seq)
96 {
97 	struct amdxdna_gem_obj *abo = container_of(mni, struct amdxdna_gem_obj,
98 						   mem.notifier);
99 	struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
100 
101 	XDNA_DBG(xdna, "Invalid range 0x%llx, 0x%lx, type %d",
102 		 abo->mem.userptr, abo->mem.size, abo->type);
103 
104 	if (!mmu_notifier_range_blockable(range))
105 		return false;
106 
107 	xdna->dev_info->ops->hmm_invalidate(abo, cur_seq);
108 
109 	return true;
110 }
111 
112 static const struct mmu_interval_notifier_ops amdxdna_hmm_ops = {
113 	.invalidate = amdxdna_hmm_invalidate,
114 };
115 
amdxdna_hmm_unregister(struct amdxdna_gem_obj * abo)116 static void amdxdna_hmm_unregister(struct amdxdna_gem_obj *abo)
117 {
118 	struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
119 
120 	if (!xdna->dev_info->ops->hmm_invalidate)
121 		return;
122 
123 	mmu_interval_notifier_remove(&abo->mem.notifier);
124 	kvfree(abo->mem.pfns);
125 	abo->mem.pfns = NULL;
126 }
127 
amdxdna_hmm_register(struct amdxdna_gem_obj * abo,unsigned long addr,size_t len)128 static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo, unsigned long addr,
129 				size_t len)
130 {
131 	struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
132 	u32 nr_pages;
133 	int ret;
134 
135 	if (!xdna->dev_info->ops->hmm_invalidate)
136 		return 0;
137 
138 	if (abo->mem.pfns)
139 		return -EEXIST;
140 
141 	nr_pages = (PAGE_ALIGN(addr + len) - (addr & PAGE_MASK)) >> PAGE_SHIFT;
142 	abo->mem.pfns = kvcalloc(nr_pages, sizeof(*abo->mem.pfns),
143 				 GFP_KERNEL);
144 	if (!abo->mem.pfns)
145 		return -ENOMEM;
146 
147 	ret = mmu_interval_notifier_insert_locked(&abo->mem.notifier,
148 						  current->mm,
149 						  addr,
150 						  len,
151 						  &amdxdna_hmm_ops);
152 	if (ret) {
153 		XDNA_ERR(xdna, "Insert mmu notifier failed, ret %d", ret);
154 		kvfree(abo->mem.pfns);
155 	}
156 	abo->mem.userptr = addr;
157 
158 	return ret;
159 }
160 
amdxdna_gem_obj_mmap(struct drm_gem_object * gobj,struct vm_area_struct * vma)161 static int amdxdna_gem_obj_mmap(struct drm_gem_object *gobj,
162 				struct vm_area_struct *vma)
163 {
164 	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
165 	unsigned long num_pages;
166 	int ret;
167 
168 	ret = amdxdna_hmm_register(abo, vma->vm_start, gobj->size);
169 	if (ret)
170 		return ret;
171 
172 	ret = drm_gem_shmem_mmap(&abo->base, vma);
173 	if (ret)
174 		goto hmm_unreg;
175 
176 	num_pages = gobj->size >> PAGE_SHIFT;
177 	/* Try to insert the pages */
178 	vm_flags_mod(vma, VM_MIXEDMAP, VM_PFNMAP);
179 	ret = vm_insert_pages(vma, vma->vm_start, abo->base.pages, &num_pages);
180 	if (ret)
181 		XDNA_ERR(abo->client->xdna, "Failed insert pages, ret %d", ret);
182 
183 	return 0;
184 
185 hmm_unreg:
186 	amdxdna_hmm_unregister(abo);
187 	return ret;
188 }
189 
amdxdna_gem_vm_fault(struct vm_fault * vmf)190 static vm_fault_t amdxdna_gem_vm_fault(struct vm_fault *vmf)
191 {
192 	return drm_gem_shmem_vm_ops.fault(vmf);
193 }
194 
amdxdna_gem_vm_open(struct vm_area_struct * vma)195 static void amdxdna_gem_vm_open(struct vm_area_struct *vma)
196 {
197 	drm_gem_shmem_vm_ops.open(vma);
198 }
199 
amdxdna_gem_vm_close(struct vm_area_struct * vma)200 static void amdxdna_gem_vm_close(struct vm_area_struct *vma)
201 {
202 	struct drm_gem_object *gobj = vma->vm_private_data;
203 
204 	amdxdna_hmm_unregister(to_xdna_obj(gobj));
205 	drm_gem_shmem_vm_ops.close(vma);
206 }
207 
208 static const struct vm_operations_struct amdxdna_gem_vm_ops = {
209 	.fault = amdxdna_gem_vm_fault,
210 	.open = amdxdna_gem_vm_open,
211 	.close = amdxdna_gem_vm_close,
212 };
213 
214 static const struct drm_gem_object_funcs amdxdna_gem_shmem_funcs = {
215 	.free = amdxdna_gem_obj_free,
216 	.print_info = drm_gem_shmem_object_print_info,
217 	.pin = drm_gem_shmem_object_pin,
218 	.unpin = drm_gem_shmem_object_unpin,
219 	.get_sg_table = drm_gem_shmem_object_get_sg_table,
220 	.vmap = drm_gem_shmem_object_vmap,
221 	.vunmap = drm_gem_shmem_object_vunmap,
222 	.mmap = amdxdna_gem_obj_mmap,
223 	.vm_ops = &amdxdna_gem_vm_ops,
224 };
225 
226 static struct amdxdna_gem_obj *
amdxdna_gem_create_obj(struct drm_device * dev,size_t size)227 amdxdna_gem_create_obj(struct drm_device *dev, size_t size)
228 {
229 	struct amdxdna_gem_obj *abo;
230 
231 	abo = kzalloc(sizeof(*abo), GFP_KERNEL);
232 	if (!abo)
233 		return ERR_PTR(-ENOMEM);
234 
235 	abo->pinned = false;
236 	abo->assigned_hwctx = AMDXDNA_INVALID_CTX_HANDLE;
237 	mutex_init(&abo->lock);
238 
239 	abo->mem.userptr = AMDXDNA_INVALID_ADDR;
240 	abo->mem.dev_addr = AMDXDNA_INVALID_ADDR;
241 	abo->mem.size = size;
242 
243 	return abo;
244 }
245 
246 /* For drm_driver->gem_create_object callback */
247 struct drm_gem_object *
amdxdna_gem_create_object_cb(struct drm_device * dev,size_t size)248 amdxdna_gem_create_object_cb(struct drm_device *dev, size_t size)
249 {
250 	struct amdxdna_gem_obj *abo;
251 
252 	abo = amdxdna_gem_create_obj(dev, size);
253 	if (IS_ERR(abo))
254 		return ERR_CAST(abo);
255 
256 	to_gobj(abo)->funcs = &amdxdna_gem_shmem_funcs;
257 
258 	return to_gobj(abo);
259 }
260 
261 static struct amdxdna_gem_obj *
amdxdna_drm_alloc_shmem(struct drm_device * dev,struct amdxdna_drm_create_bo * args,struct drm_file * filp)262 amdxdna_drm_alloc_shmem(struct drm_device *dev,
263 			struct amdxdna_drm_create_bo *args,
264 			struct drm_file *filp)
265 {
266 	struct amdxdna_client *client = filp->driver_priv;
267 	struct drm_gem_shmem_object *shmem;
268 	struct amdxdna_gem_obj *abo;
269 
270 	shmem = drm_gem_shmem_create(dev, args->size);
271 	if (IS_ERR(shmem))
272 		return ERR_CAST(shmem);
273 
274 	shmem->map_wc = false;
275 
276 	abo = to_xdna_obj(&shmem->base);
277 	abo->client = client;
278 	abo->type = AMDXDNA_BO_SHMEM;
279 
280 	return abo;
281 }
282 
283 static struct amdxdna_gem_obj *
amdxdna_drm_create_dev_heap(struct drm_device * dev,struct amdxdna_drm_create_bo * args,struct drm_file * filp)284 amdxdna_drm_create_dev_heap(struct drm_device *dev,
285 			    struct amdxdna_drm_create_bo *args,
286 			    struct drm_file *filp)
287 {
288 	struct amdxdna_client *client = filp->driver_priv;
289 	struct amdxdna_dev *xdna = to_xdna_dev(dev);
290 	struct drm_gem_shmem_object *shmem;
291 	struct amdxdna_gem_obj *abo;
292 	int ret;
293 
294 	if (args->size > xdna->dev_info->dev_mem_size) {
295 		XDNA_DBG(xdna, "Invalid dev heap size 0x%llx, limit 0x%lx",
296 			 args->size, xdna->dev_info->dev_mem_size);
297 		return ERR_PTR(-EINVAL);
298 	}
299 
300 	mutex_lock(&client->mm_lock);
301 	if (client->dev_heap) {
302 		XDNA_DBG(client->xdna, "dev heap is already created");
303 		ret = -EBUSY;
304 		goto mm_unlock;
305 	}
306 
307 	shmem = drm_gem_shmem_create(dev, args->size);
308 	if (IS_ERR(shmem)) {
309 		ret = PTR_ERR(shmem);
310 		goto mm_unlock;
311 	}
312 
313 	shmem->map_wc = false;
314 	abo = to_xdna_obj(&shmem->base);
315 
316 	abo->type = AMDXDNA_BO_DEV_HEAP;
317 	abo->client = client;
318 	abo->mem.dev_addr = client->xdna->dev_info->dev_mem_base;
319 	drm_mm_init(&abo->mm, abo->mem.dev_addr, abo->mem.size);
320 
321 	client->dev_heap = abo;
322 	drm_gem_object_get(to_gobj(abo));
323 	mutex_unlock(&client->mm_lock);
324 
325 	return abo;
326 
327 mm_unlock:
328 	mutex_unlock(&client->mm_lock);
329 	return ERR_PTR(ret);
330 }
331 
332 struct amdxdna_gem_obj *
amdxdna_drm_alloc_dev_bo(struct drm_device * dev,struct amdxdna_drm_create_bo * args,struct drm_file * filp,bool use_vmap)333 amdxdna_drm_alloc_dev_bo(struct drm_device *dev,
334 			 struct amdxdna_drm_create_bo *args,
335 			 struct drm_file *filp, bool use_vmap)
336 {
337 	struct amdxdna_client *client = filp->driver_priv;
338 	struct amdxdna_dev *xdna = to_xdna_dev(dev);
339 	size_t aligned_sz = PAGE_ALIGN(args->size);
340 	struct amdxdna_gem_obj *abo, *heap;
341 	int ret;
342 
343 	mutex_lock(&client->mm_lock);
344 	heap = client->dev_heap;
345 	if (!heap) {
346 		ret = -EINVAL;
347 		goto mm_unlock;
348 	}
349 
350 	if (heap->mem.userptr == AMDXDNA_INVALID_ADDR) {
351 		XDNA_ERR(xdna, "Invalid dev heap userptr");
352 		ret = -EINVAL;
353 		goto mm_unlock;
354 	}
355 
356 	if (args->size > heap->mem.size) {
357 		XDNA_ERR(xdna, "Invalid dev bo size 0x%llx, limit 0x%lx",
358 			 args->size, heap->mem.size);
359 		ret = -EINVAL;
360 		goto mm_unlock;
361 	}
362 
363 	abo = amdxdna_gem_create_obj(&xdna->ddev, aligned_sz);
364 	if (IS_ERR(abo)) {
365 		ret = PTR_ERR(abo);
366 		goto mm_unlock;
367 	}
368 	to_gobj(abo)->funcs = &amdxdna_gem_dev_obj_funcs;
369 	abo->type = AMDXDNA_BO_DEV;
370 	abo->client = client;
371 	abo->dev_heap = heap;
372 	ret = amdxdna_gem_insert_node_locked(abo, use_vmap);
373 	if (ret) {
374 		XDNA_ERR(xdna, "Failed to alloc dev bo memory, ret %d", ret);
375 		goto mm_unlock;
376 	}
377 
378 	drm_gem_object_get(to_gobj(heap));
379 	drm_gem_private_object_init(&xdna->ddev, to_gobj(abo), aligned_sz);
380 
381 	mutex_unlock(&client->mm_lock);
382 	return abo;
383 
384 mm_unlock:
385 	mutex_unlock(&client->mm_lock);
386 	return ERR_PTR(ret);
387 }
388 
389 static struct amdxdna_gem_obj *
amdxdna_drm_create_cmd_bo(struct drm_device * dev,struct amdxdna_drm_create_bo * args,struct drm_file * filp)390 amdxdna_drm_create_cmd_bo(struct drm_device *dev,
391 			  struct amdxdna_drm_create_bo *args,
392 			  struct drm_file *filp)
393 {
394 	struct amdxdna_dev *xdna = to_xdna_dev(dev);
395 	struct drm_gem_shmem_object *shmem;
396 	struct amdxdna_gem_obj *abo;
397 	struct iosys_map map;
398 	int ret;
399 
400 	if (args->size > XDNA_MAX_CMD_BO_SIZE) {
401 		XDNA_ERR(xdna, "Command bo size 0x%llx too large", args->size);
402 		return ERR_PTR(-EINVAL);
403 	}
404 
405 	if (args->size < sizeof(struct amdxdna_cmd)) {
406 		XDNA_DBG(xdna, "Command BO size 0x%llx too small", args->size);
407 		return ERR_PTR(-EINVAL);
408 	}
409 
410 	shmem = drm_gem_shmem_create(dev, args->size);
411 	if (IS_ERR(shmem))
412 		return ERR_CAST(shmem);
413 
414 	shmem->map_wc = false;
415 	abo = to_xdna_obj(&shmem->base);
416 
417 	abo->type = AMDXDNA_BO_CMD;
418 	abo->client = filp->driver_priv;
419 
420 	ret = drm_gem_vmap_unlocked(to_gobj(abo), &map);
421 	if (ret) {
422 		XDNA_ERR(xdna, "Vmap cmd bo failed, ret %d", ret);
423 		goto release_obj;
424 	}
425 	abo->mem.kva = map.vaddr;
426 
427 	return abo;
428 
429 release_obj:
430 	drm_gem_shmem_free(shmem);
431 	return ERR_PTR(ret);
432 }
433 
amdxdna_drm_create_bo_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)434 int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
435 {
436 	struct amdxdna_dev *xdna = to_xdna_dev(dev);
437 	struct amdxdna_drm_create_bo *args = data;
438 	struct amdxdna_gem_obj *abo;
439 	int ret;
440 
441 	if (args->flags || args->vaddr || !args->size)
442 		return -EINVAL;
443 
444 	XDNA_DBG(xdna, "BO arg type %d vaddr 0x%llx size 0x%llx flags 0x%llx",
445 		 args->type, args->vaddr, args->size, args->flags);
446 	switch (args->type) {
447 	case AMDXDNA_BO_SHMEM:
448 		abo = amdxdna_drm_alloc_shmem(dev, args, filp);
449 		break;
450 	case AMDXDNA_BO_DEV_HEAP:
451 		abo = amdxdna_drm_create_dev_heap(dev, args, filp);
452 		break;
453 	case AMDXDNA_BO_DEV:
454 		abo = amdxdna_drm_alloc_dev_bo(dev, args, filp, false);
455 		break;
456 	case AMDXDNA_BO_CMD:
457 		abo = amdxdna_drm_create_cmd_bo(dev, args, filp);
458 		break;
459 	default:
460 		return -EINVAL;
461 	}
462 	if (IS_ERR(abo))
463 		return PTR_ERR(abo);
464 
465 	/* ready to publish object to userspace */
466 	ret = drm_gem_handle_create(filp, to_gobj(abo), &args->handle);
467 	if (ret) {
468 		XDNA_ERR(xdna, "Create handle failed");
469 		goto put_obj;
470 	}
471 
472 	XDNA_DBG(xdna, "BO hdl %d type %d userptr 0x%llx xdna_addr 0x%llx size 0x%lx",
473 		 args->handle, args->type, abo->mem.userptr,
474 		 abo->mem.dev_addr, abo->mem.size);
475 put_obj:
476 	/* Dereference object reference. Handle holds it now. */
477 	drm_gem_object_put(to_gobj(abo));
478 	return ret;
479 }
480 
amdxdna_gem_pin_nolock(struct amdxdna_gem_obj * abo)481 int amdxdna_gem_pin_nolock(struct amdxdna_gem_obj *abo)
482 {
483 	struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
484 	int ret;
485 
486 	switch (abo->type) {
487 	case AMDXDNA_BO_SHMEM:
488 	case AMDXDNA_BO_DEV_HEAP:
489 		ret = drm_gem_shmem_pin(&abo->base);
490 		break;
491 	case AMDXDNA_BO_DEV:
492 		ret = drm_gem_shmem_pin(&abo->dev_heap->base);
493 		break;
494 	default:
495 		ret = -EOPNOTSUPP;
496 	}
497 
498 	XDNA_DBG(xdna, "BO type %d ret %d", abo->type, ret);
499 	return ret;
500 }
501 
amdxdna_gem_pin(struct amdxdna_gem_obj * abo)502 int amdxdna_gem_pin(struct amdxdna_gem_obj *abo)
503 {
504 	int ret;
505 
506 	if (abo->type == AMDXDNA_BO_DEV)
507 		abo = abo->dev_heap;
508 
509 	mutex_lock(&abo->lock);
510 	ret = amdxdna_gem_pin_nolock(abo);
511 	mutex_unlock(&abo->lock);
512 
513 	return ret;
514 }
515 
amdxdna_gem_unpin(struct amdxdna_gem_obj * abo)516 void amdxdna_gem_unpin(struct amdxdna_gem_obj *abo)
517 {
518 	if (abo->type == AMDXDNA_BO_DEV)
519 		abo = abo->dev_heap;
520 
521 	mutex_lock(&abo->lock);
522 	drm_gem_shmem_unpin(&abo->base);
523 	mutex_unlock(&abo->lock);
524 }
525 
amdxdna_gem_get_obj(struct amdxdna_client * client,u32 bo_hdl,u8 bo_type)526 struct amdxdna_gem_obj *amdxdna_gem_get_obj(struct amdxdna_client *client,
527 					    u32 bo_hdl, u8 bo_type)
528 {
529 	struct amdxdna_dev *xdna = client->xdna;
530 	struct amdxdna_gem_obj *abo;
531 	struct drm_gem_object *gobj;
532 
533 	gobj = drm_gem_object_lookup(client->filp, bo_hdl);
534 	if (!gobj) {
535 		XDNA_DBG(xdna, "Can not find bo %d", bo_hdl);
536 		return NULL;
537 	}
538 
539 	abo = to_xdna_obj(gobj);
540 	if (bo_type == AMDXDNA_BO_INVALID || abo->type == bo_type)
541 		return abo;
542 
543 	drm_gem_object_put(gobj);
544 	return NULL;
545 }
546 
amdxdna_drm_get_bo_info_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)547 int amdxdna_drm_get_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
548 {
549 	struct amdxdna_drm_get_bo_info *args = data;
550 	struct amdxdna_dev *xdna = to_xdna_dev(dev);
551 	struct amdxdna_gem_obj *abo;
552 	struct drm_gem_object *gobj;
553 	int ret = 0;
554 
555 	if (args->ext || args->ext_flags || args->pad)
556 		return -EINVAL;
557 
558 	gobj = drm_gem_object_lookup(filp, args->handle);
559 	if (!gobj) {
560 		XDNA_DBG(xdna, "Lookup GEM object %d failed", args->handle);
561 		return -ENOENT;
562 	}
563 
564 	abo = to_xdna_obj(gobj);
565 	args->vaddr = abo->mem.userptr;
566 	args->xdna_addr = abo->mem.dev_addr;
567 
568 	if (abo->type != AMDXDNA_BO_DEV)
569 		args->map_offset = drm_vma_node_offset_addr(&gobj->vma_node);
570 	else
571 		args->map_offset = AMDXDNA_INVALID_ADDR;
572 
573 	XDNA_DBG(xdna, "BO hdl %d map_offset 0x%llx vaddr 0x%llx xdna_addr 0x%llx",
574 		 args->handle, args->map_offset, args->vaddr, args->xdna_addr);
575 
576 	drm_gem_object_put(gobj);
577 	return ret;
578 }
579 
580 /*
581  * The sync bo ioctl is to make sure the CPU cache is in sync with memory.
582  * This is required because NPU is not cache coherent device. CPU cache
583  * flushing/invalidation is expensive so it is best to handle this outside
584  * of the command submission path. This ioctl allows explicit cache
585  * flushing/invalidation outside of the critical path.
586  */
amdxdna_drm_sync_bo_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)587 int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev,
588 			      void *data, struct drm_file *filp)
589 {
590 	struct amdxdna_dev *xdna = to_xdna_dev(dev);
591 	struct amdxdna_drm_sync_bo *args = data;
592 	struct amdxdna_gem_obj *abo;
593 	struct drm_gem_object *gobj;
594 	int ret;
595 
596 	gobj = drm_gem_object_lookup(filp, args->handle);
597 	if (!gobj) {
598 		XDNA_ERR(xdna, "Lookup GEM object failed");
599 		return -ENOENT;
600 	}
601 	abo = to_xdna_obj(gobj);
602 
603 	ret = amdxdna_gem_pin(abo);
604 	if (ret) {
605 		XDNA_ERR(xdna, "Pin BO %d failed, ret %d", args->handle, ret);
606 		goto put_obj;
607 	}
608 
609 	if (abo->type == AMDXDNA_BO_DEV)
610 		drm_clflush_pages(abo->mem.pages, abo->mem.nr_pages);
611 	else
612 		drm_clflush_pages(abo->base.pages, gobj->size >> PAGE_SHIFT);
613 
614 	amdxdna_gem_unpin(abo);
615 
616 	XDNA_DBG(xdna, "Sync bo %d offset 0x%llx, size 0x%llx\n",
617 		 args->handle, args->offset, args->size);
618 
619 put_obj:
620 	drm_gem_object_put(gobj);
621 	return ret;
622 }
623