1 /*
2 * Copyright © 2022 Imagination Technologies Ltd.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a copy
5 * of this software and associated documentation files (the "Software"), to deal
6 * in the Software without restriction, including without limitation the rights
7 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 * copies of the Software, and to permit persons to whom the Software is
9 * furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include <assert.h>
25 #include <fcntl.h>
26 #include <stdbool.h>
27 #include <stddef.h>
28 #include <stdint.h>
29 #include <sys/mman.h>
30 #include <sys/types.h>
31 #include <vulkan/vulkan.h>
32 #include <unistd.h>
33 #include <xf86drm.h>
34
35 #include "drm-uapi/pvr_drm.h"
36 #include "pvr_drm.h"
37 #include "pvr_drm_bo.h"
38 #include "pvr_private.h"
39 #include "pvr_winsys_helper.h"
40 #include "util/bitscan.h"
41 #include "util/macros.h"
42 #include "vk_log.h"
43
pvr_drm_create_gem_bo(struct pvr_drm_winsys * drm_ws,uint32_t drm_flags,uint64_t size,uint32_t * const handle_out)44 static VkResult pvr_drm_create_gem_bo(struct pvr_drm_winsys *drm_ws,
45 uint32_t drm_flags,
46 uint64_t size,
47 uint32_t *const handle_out)
48 {
49 struct drm_pvr_ioctl_create_bo_args args = {
50 .size = size,
51 .flags = drm_flags,
52 };
53 VkResult result;
54
55 result = pvr_ioctlf(drm_ws->base.render_fd,
56 DRM_IOCTL_PVR_CREATE_BO,
57 &args,
58 VK_ERROR_OUT_OF_DEVICE_MEMORY,
59 "Failed to create gem bo");
60 if (result != VK_SUCCESS)
61 return result;
62
63 *handle_out = args.handle;
64
65 return VK_SUCCESS;
66 }
67
pvr_drm_destroy_gem_bo(struct pvr_drm_winsys * drm_ws,uint32_t handle)68 static VkResult pvr_drm_destroy_gem_bo(struct pvr_drm_winsys *drm_ws,
69 uint32_t handle)
70 {
71 struct drm_gem_close args = {
72 .handle = handle,
73 };
74
75 /* The kernel driver doesn't have a corresponding DRM_IOCTL_PVR_DESTROY_BO
76 * IOCTL as DRM provides a common IOCTL for doing this.
77 */
78 return pvr_ioctlf(drm_ws->base.render_fd,
79 DRM_IOCTL_GEM_CLOSE,
80 &args,
81 VK_ERROR_UNKNOWN,
82 "Failed to destroy gem bo");
83 }
84
pvr_drm_get_bo_mmap_offset(struct pvr_drm_winsys * drm_ws,uint32_t handle,uint64_t * const offset_out)85 static VkResult pvr_drm_get_bo_mmap_offset(struct pvr_drm_winsys *drm_ws,
86 uint32_t handle,
87 uint64_t *const offset_out)
88 {
89 struct drm_pvr_ioctl_get_bo_mmap_offset_args args = {
90 .handle = handle,
91 };
92 VkResult result;
93
94 result = pvr_ioctl(drm_ws->base.render_fd,
95 DRM_IOCTL_PVR_GET_BO_MMAP_OFFSET,
96 &args,
97 VK_ERROR_MEMORY_MAP_FAILED);
98 if (result != VK_SUCCESS)
99 return result;
100
101 *offset_out = args.offset;
102
103 return VK_SUCCESS;
104 }
105
pvr_drm_buffer_acquire(struct pvr_drm_winsys_bo * drm_bo)106 static void pvr_drm_buffer_acquire(struct pvr_drm_winsys_bo *drm_bo)
107 {
108 p_atomic_inc(&drm_bo->ref_count);
109 }
110
pvr_drm_buffer_release(struct pvr_drm_winsys_bo * drm_bo)111 static void pvr_drm_buffer_release(struct pvr_drm_winsys_bo *drm_bo)
112 {
113 if (p_atomic_dec_return(&drm_bo->ref_count) == 0) {
114 struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(drm_bo->base.ws);
115
116 pvr_drm_destroy_gem_bo(drm_ws, drm_bo->handle);
117
118 vk_free(drm_ws->base.alloc, drm_bo);
119 }
120 }
121
122 static VkResult
pvr_drm_display_buffer_create(struct pvr_drm_winsys * drm_ws,uint64_t size,struct pvr_winsys_bo ** const bo_out)123 pvr_drm_display_buffer_create(struct pvr_drm_winsys *drm_ws,
124 uint64_t size,
125 struct pvr_winsys_bo **const bo_out)
126 {
127 uint32_t handle;
128 VkResult result;
129 int ret;
130 int fd;
131
132 result =
133 pvr_winsys_helper_display_buffer_create(&drm_ws->base, size, &handle);
134 if (result != VK_SUCCESS)
135 return result;
136
137 ret = drmPrimeHandleToFD(drm_ws->base.display_fd, handle, DRM_CLOEXEC, &fd);
138 pvr_winsys_helper_display_buffer_destroy(&drm_ws->base, handle);
139 if (ret)
140 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
141
142 result = pvr_drm_winsys_buffer_create_from_fd(&drm_ws->base, fd, bo_out);
143 close(fd);
144 if (result != VK_SUCCESS)
145 return result;
146
147 assert((*bo_out)->size >= size);
148
149 return VK_SUCCESS;
150 }
151
pvr_drm_get_alloc_flags(uint32_t ws_flags)152 static uint64_t pvr_drm_get_alloc_flags(uint32_t ws_flags)
153 {
154 uint64_t drm_flags = 0U;
155
156 if (ws_flags & PVR_WINSYS_BO_FLAG_GPU_UNCACHED)
157 drm_flags |= DRM_PVR_BO_BYPASS_DEVICE_CACHE;
158
159 if (ws_flags & PVR_WINSYS_BO_FLAG_PM_FW_PROTECT)
160 drm_flags |= DRM_PVR_BO_PM_FW_PROTECT;
161
162 if (ws_flags & PVR_WINSYS_BO_FLAG_CPU_ACCESS)
163 drm_flags |= DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS;
164
165 return drm_flags;
166 }
167
pvr_drm_winsys_buffer_create(struct pvr_winsys * ws,uint64_t size,uint64_t alignment,enum pvr_winsys_bo_type type,uint32_t ws_flags,struct pvr_winsys_bo ** const bo_out)168 VkResult pvr_drm_winsys_buffer_create(struct pvr_winsys *ws,
169 uint64_t size,
170 uint64_t alignment,
171 enum pvr_winsys_bo_type type,
172 uint32_t ws_flags,
173 struct pvr_winsys_bo **const bo_out)
174 {
175 const uint64_t drm_flags = pvr_drm_get_alloc_flags(ws_flags);
176 struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(ws);
177 struct pvr_drm_winsys_bo *drm_bo;
178 uint32_t handle = 0;
179 VkResult result;
180
181 assert(util_is_power_of_two_nonzero64(alignment));
182 size = ALIGN_POT(size, alignment);
183 size = ALIGN_POT(size, ws->page_size);
184
185 if (type == PVR_WINSYS_BO_TYPE_DISPLAY)
186 return pvr_drm_display_buffer_create(drm_ws, size, bo_out);
187
188 drm_bo = vk_zalloc(ws->alloc,
189 sizeof(*drm_bo),
190 8,
191 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
192 if (!drm_bo)
193 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
194
195 result = pvr_drm_create_gem_bo(drm_ws, drm_flags, size, &handle);
196 if (result != VK_SUCCESS)
197 goto err_vk_free_drm_bo;
198
199 drm_bo->base.size = size;
200 drm_bo->base.ws = ws;
201 drm_bo->handle = handle;
202 drm_bo->flags = drm_flags;
203
204 p_atomic_set(&drm_bo->ref_count, 1);
205
206 *bo_out = &drm_bo->base;
207
208 return VK_SUCCESS;
209
210 err_vk_free_drm_bo:
211 vk_free(ws->alloc, drm_bo);
212
213 return result;
214 }
215
216 VkResult
pvr_drm_winsys_buffer_create_from_fd(struct pvr_winsys * ws,int fd,struct pvr_winsys_bo ** const bo_out)217 pvr_drm_winsys_buffer_create_from_fd(struct pvr_winsys *ws,
218 int fd,
219 struct pvr_winsys_bo **const bo_out)
220 {
221 struct pvr_drm_winsys_bo *drm_bo;
222 uint32_t handle;
223 VkResult result;
224 off_t size;
225 int ret;
226
227 drm_bo = vk_zalloc(ws->alloc,
228 sizeof(*drm_bo),
229 8,
230 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
231 if (!drm_bo)
232 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
233
234 size = lseek(fd, 0, SEEK_END);
235 if (size == (off_t)-1) {
236 result = vk_error(NULL, VK_ERROR_INVALID_EXTERNAL_HANDLE);
237 goto err_vk_free_drm_bo;
238 }
239
240 ret = drmPrimeFDToHandle(ws->render_fd, fd, &handle);
241 if (ret) {
242 result = vk_error(NULL, VK_ERROR_INVALID_EXTERNAL_HANDLE);
243 goto err_vk_free_drm_bo;
244 }
245
246 drm_bo->base.ws = ws;
247 drm_bo->base.size = (uint64_t)size;
248 drm_bo->base.is_imported = true;
249 drm_bo->handle = handle;
250
251 p_atomic_set(&drm_bo->ref_count, 1);
252
253 *bo_out = &drm_bo->base;
254
255 return VK_SUCCESS;
256
257 err_vk_free_drm_bo:
258 vk_free(ws->alloc, drm_bo);
259
260 return result;
261 }
262
pvr_drm_winsys_buffer_destroy(struct pvr_winsys_bo * bo)263 void pvr_drm_winsys_buffer_destroy(struct pvr_winsys_bo *bo)
264 {
265 struct pvr_drm_winsys_bo *drm_bo = to_pvr_drm_winsys_bo(bo);
266
267 pvr_drm_buffer_release(drm_bo);
268 }
269
pvr_drm_winsys_buffer_get_fd(struct pvr_winsys_bo * bo,int * const fd_out)270 VkResult pvr_drm_winsys_buffer_get_fd(struct pvr_winsys_bo *bo,
271 int *const fd_out)
272 {
273 struct pvr_drm_winsys_bo *drm_bo = to_pvr_drm_winsys_bo(bo);
274 struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(bo->ws);
275 int ret;
276
277 ret = drmPrimeHandleToFD(drm_ws->base.render_fd,
278 drm_bo->handle,
279 DRM_CLOEXEC,
280 fd_out);
281 if (ret)
282 return vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
283
284 return VK_SUCCESS;
285 }
286
pvr_drm_winsys_buffer_map(struct pvr_winsys_bo * bo)287 VkResult pvr_drm_winsys_buffer_map(struct pvr_winsys_bo *bo)
288 {
289 struct pvr_drm_winsys_bo *drm_bo = to_pvr_drm_winsys_bo(bo);
290 struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(bo->ws);
291 uint64_t offset = 0;
292 void *map = NULL;
293 VkResult result;
294
295 assert(!bo->map);
296
297 result = pvr_drm_get_bo_mmap_offset(drm_ws, drm_bo->handle, &offset);
298 if (result != VK_SUCCESS)
299 goto err_out;
300
301 result = pvr_mmap(bo->size,
302 PROT_READ | PROT_WRITE,
303 MAP_SHARED,
304 drm_ws->base.render_fd,
305 offset,
306 &map);
307 if (result != VK_SUCCESS)
308 goto err_out;
309
310 VG(VALGRIND_MALLOCLIKE_BLOCK(map, bo->size, 0, true));
311
312 pvr_drm_buffer_acquire(drm_bo);
313 bo->map = map;
314
315 return VK_SUCCESS;
316
317 err_out:
318 return result;
319 }
320
pvr_drm_winsys_buffer_unmap(struct pvr_winsys_bo * bo)321 void pvr_drm_winsys_buffer_unmap(struct pvr_winsys_bo *bo)
322 {
323 struct pvr_drm_winsys_bo *drm_bo = to_pvr_drm_winsys_bo(bo);
324
325 assert(bo->map);
326
327 pvr_munmap(bo->map, bo->size);
328
329 VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0));
330
331 bo->map = NULL;
332
333 pvr_drm_buffer_release(drm_bo);
334 }
335
336 /* This function must be used to allocate from a heap carveout and must only be
337 * used within the winsys code. This also means whoever is using it, must know
338 * what they are doing.
339 */
pvr_drm_heap_alloc_carveout(struct pvr_winsys_heap * const heap,const pvr_dev_addr_t carveout_dev_addr,uint64_t size,uint64_t alignment,struct pvr_winsys_vma ** const vma_out)340 VkResult pvr_drm_heap_alloc_carveout(struct pvr_winsys_heap *const heap,
341 const pvr_dev_addr_t carveout_dev_addr,
342 uint64_t size,
343 uint64_t alignment,
344 struct pvr_winsys_vma **const vma_out)
345 {
346 const struct pvr_drm_winsys *const drm_ws = to_pvr_drm_winsys(heap->ws);
347 struct pvr_drm_winsys_vma *drm_vma;
348 VkResult result;
349
350 assert(util_is_power_of_two_nonzero64(alignment));
351
352 drm_vma = vk_zalloc(drm_ws->base.alloc,
353 sizeof(*drm_vma),
354 8,
355 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
356 if (!drm_vma) {
357 result = vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
358 goto err_out;
359 }
360
361 /* The powervr kernel mode driver returns a page aligned size when
362 * allocating buffers.
363 */
364 alignment = MAX2(alignment, heap->page_size);
365 size = ALIGN_POT(size, alignment);
366
367 /* TODO: Should we keep track of the allocations in the carveout? */
368
369 drm_vma->base.dev_addr = carveout_dev_addr;
370 drm_vma->base.heap = heap;
371 drm_vma->base.size = size;
372
373 p_atomic_inc(&heap->ref_count);
374
375 *vma_out = &drm_vma->base;
376
377 return VK_SUCCESS;
378
379 err_out:
380 return result;
381 }
382
pvr_drm_winsys_heap_alloc(struct pvr_winsys_heap * heap,uint64_t size,uint64_t alignment,struct pvr_winsys_vma ** const vma_out)383 VkResult pvr_drm_winsys_heap_alloc(struct pvr_winsys_heap *heap,
384 uint64_t size,
385 uint64_t alignment,
386 struct pvr_winsys_vma **const vma_out)
387 {
388 const struct pvr_drm_winsys *const drm_ws = to_pvr_drm_winsys(heap->ws);
389 struct pvr_drm_winsys_vma *drm_vma;
390 VkResult result;
391
392 drm_vma = vk_alloc(drm_ws->base.alloc,
393 sizeof(*drm_vma),
394 8,
395 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
396 if (!drm_vma) {
397 result = vk_error(NULL, VK_ERROR_OUT_OF_HOST_MEMORY);
398 goto err_out;
399 }
400
401 result = pvr_winsys_helper_heap_alloc(heap, size, alignment, &drm_vma->base);
402 if (result != VK_SUCCESS)
403 goto err_free_vma;
404
405 *vma_out = &drm_vma->base;
406
407 return VK_SUCCESS;
408
409 err_free_vma:
410 vk_free(drm_ws->base.alloc, drm_vma);
411
412 err_out:
413 return result;
414 }
415
pvr_drm_winsys_heap_free(struct pvr_winsys_vma * vma)416 void pvr_drm_winsys_heap_free(struct pvr_winsys_vma *vma)
417 {
418 struct pvr_drm_winsys *drm_ws = to_pvr_drm_winsys(vma->heap->ws);
419 struct pvr_drm_winsys_vma *drm_vma = to_pvr_drm_winsys_vma(vma);
420 const uint64_t carveout_addr = vma->heap->static_data_carveout_addr.addr;
421
422 /* A vma with an existing device mapping should not be freed. */
423 assert(!drm_vma->base.bo);
424
425 /* Check if we are dealing with carveout address range. */
426 if (vma->dev_addr.addr >= carveout_addr &&
427 vma->dev_addr.addr <
428 (carveout_addr + vma->heap->static_data_carveout_size)) {
429 /* For the carveout addresses just decrement the reference count. */
430 p_atomic_dec(&vma->heap->ref_count);
431 } else {
432 /* Free allocated virtual space. */
433 pvr_winsys_helper_heap_free(vma);
434 }
435
436 vk_free(drm_ws->base.alloc, drm_vma);
437 }
438
pvr_drm_winsys_vma_map(struct pvr_winsys_vma * vma,struct pvr_winsys_bo * bo,uint64_t offset,uint64_t size,pvr_dev_addr_t * const dev_addr_out)439 VkResult pvr_drm_winsys_vma_map(struct pvr_winsys_vma *vma,
440 struct pvr_winsys_bo *bo,
441 uint64_t offset,
442 uint64_t size,
443 pvr_dev_addr_t *const dev_addr_out)
444 {
445 struct pvr_drm_winsys_bo *const drm_bo = to_pvr_drm_winsys_bo(bo);
446 struct pvr_drm_winsys *const drm_ws = to_pvr_drm_winsys(bo->ws);
447 const uint32_t virt_offset = offset & (vma->heap->page_size - 1);
448 const uint64_t aligned_virt_size =
449 ALIGN_POT(virt_offset + size, vma->heap->page_size);
450 const uint32_t phys_page_offset = offset - virt_offset;
451
452 struct drm_pvr_ioctl_vm_map_args args = { .device_addr = vma->dev_addr.addr,
453 .flags = 0U,
454 .handle = drm_bo->handle,
455 .offset = phys_page_offset,
456 .size = aligned_virt_size,
457 .vm_context_handle =
458 drm_ws->vm_context };
459
460 VkResult result;
461
462 /* Address should not be mapped already. */
463 assert(!vma->bo);
464
465 /* Check if bo and vma can accommodate the given size and offset. */
466 if (ALIGN_POT(offset + size, vma->heap->page_size) > bo->size ||
467 aligned_virt_size > vma->size) {
468 return vk_error(NULL, VK_ERROR_MEMORY_MAP_FAILED);
469 }
470
471 result = pvr_ioctl(drm_ws->base.render_fd,
472 DRM_IOCTL_PVR_VM_MAP,
473 &args,
474 VK_ERROR_MEMORY_MAP_FAILED);
475 if (result != VK_SUCCESS)
476 return result;
477
478 pvr_drm_buffer_acquire(drm_bo);
479
480 vma->bo = &drm_bo->base;
481 vma->bo_offset = offset;
482 vma->mapped_size = aligned_virt_size;
483
484 if (dev_addr_out)
485 *dev_addr_out = PVR_DEV_ADDR_OFFSET(vma->dev_addr, virt_offset);
486
487 return VK_SUCCESS;
488 }
489
pvr_drm_winsys_vma_unmap(struct pvr_winsys_vma * vma)490 void pvr_drm_winsys_vma_unmap(struct pvr_winsys_vma *vma)
491 {
492 struct pvr_drm_winsys_bo *const drm_bo = to_pvr_drm_winsys_bo(vma->bo);
493 struct pvr_drm_winsys *const drm_ws = to_pvr_drm_winsys(vma->bo->ws);
494
495 struct drm_pvr_ioctl_vm_unmap_args args = {
496 .vm_context_handle = drm_ws->vm_context,
497 .device_addr = vma->dev_addr.addr,
498 .size = vma->mapped_size,
499 };
500
501 /* Address should be mapped. */
502 assert(vma->bo);
503
504 pvr_ioctlf(drm_ws->base.render_fd,
505 DRM_IOCTL_PVR_VM_UNMAP,
506 &args,
507 VK_ERROR_UNKNOWN,
508 "Unmap failed");
509
510 pvr_drm_buffer_release(drm_bo);
511
512 vma->bo = NULL;
513 }
514