1 /*
2 * Copyright © 2024 Collabora Ltd. and Red Hat Inc.
3 * SPDX-License-Identifier: MIT
4 */
5 #ifndef NVKMD_H
6 #define NVKMD_H 1
7
8 #include "nv_device_info.h"
9 #include "util/simple_mtx.h"
10 #include "util/u_atomic.h"
11
12 #include "../nvk_debug.h"
13 #include "vulkan/vulkan_core.h"
14
15 #include <assert.h>
16 #include <stdbool.h>
17 #include <stdbool.h>
18 #include <sys/types.h>
19
20 struct nvkmd_ctx;
21 struct nvkmd_dev;
22 struct nvkmd_mem;
23 struct nvkmd_pdev;
24 struct nvkmd_va;
25
26 struct _drmDevice;
27 struct vk_object_base;
28 struct vk_sync_wait;
29 struct vk_sync_signal;
30
31 /*
32 * Enums
33 */
34
35 enum nvkmd_mem_flags {
36 /** Place memory as local as possible.
37 *
38 * This should be the default for most memory allocations. On discrete
39 * GPUs, it will default to be placed in VRAM but may be paged out to GART,
40 * depending on system memory pressure.
41 */
42 NVKMD_MEM_LOCAL = 1 << 0,
43
44 /** Place the memory in GART */
45 NVKMD_MEM_GART = 1 << 1,
46
47 /** Place the memory in VRAM */
48 NVKMD_MEM_VRAM = 1 << 2,
49
50 /** This memory object may be mapped */
51 NVKMD_MEM_CAN_MAP = 1 << 3,
52
53 /** This memory object may be shared with other processes */
54 NVKMD_MEM_SHARED = 1 << 4,
55 };
56
57 enum nvkmd_mem_map_flags {
58 NVKMD_MEM_MAP_RD = 1 << 0,
59 NVKMD_MEM_MAP_WR = 1 << 1,
60 NVKMD_MEM_MAP_RDWR = NVKMD_MEM_MAP_RD | NVKMD_MEM_MAP_WR,
61
62 /** Create a client mapping
63 *
64 * This sets nvkmd_mem::client_map instead of nvkmd_mem::map. These
65 * mappings may be different from internal mappings and have different
66 * rules. Only one client mapping may exist at a time but internal
67 * mappings are reference counted. Only client mappings can be used with
68 * MAP_FIXED or unmapped with nvkmd_mem_overmap().
69 */
70 NVKMD_MEM_MAP_CLIENT = 1 << 2,
71
72 NVKMD_MEM_MAP_FIXED = 1 << 3,
73 };
74
75 enum nvkmd_va_flags {
76 /** This VA should be configured for sparse (soft faults) */
77 NVKMD_VA_SPARSE = 1 << 0,
78
79 /** This VA should come from the capture/replay pool */
80 NVKMD_VA_REPLAY = 1 << 1,
81
82 /** Attempt to place this VA at the requested address and fail otherwise */
83 NVKMD_VA_ALLOC_FIXED = 1 << 2,
84
85 /** This VA will only be used with GART.
86 *
87 * Alignment requirements for GART-only are lower.
88 */
89 NVKMD_VA_GART = 1 << 3,
90 };
91
92 enum nvkmd_engines {
93 NVKMD_ENGINE_COPY = 1 << 0,
94 NVKMD_ENGINE_2D = 1 << 1,
95 NVKMD_ENGINE_3D = 1 << 2,
96 NVKMD_ENGINE_M2MF = 1 << 3,
97 NVKMD_ENGINE_COMPUTE = 1 << 4,
98 NVKMD_ENGINE_BIND = 1 << 5,
99 };
100
101 enum nvkmd_bind_op {
102 NVKMD_BIND_OP_BIND,
103 NVKMD_BIND_OP_UNBIND,
104 };
105
106 /*
107 * Structs
108 */
109
110 struct nvkmd_info {
111 bool has_dma_buf;
112 bool has_get_vram_used;
113 bool has_alloc_tiled;
114 bool has_map_fixed;
115 bool has_overmap;
116 };
117
118 struct nvkmd_pdev_ops {
119 void (*destroy)(struct nvkmd_pdev *pdev);
120
121 uint64_t (*get_vram_used)(struct nvkmd_pdev *pdev);
122
123 int (*get_drm_primary_fd)(struct nvkmd_pdev *pdev);
124
125 VkResult (*create_dev)(struct nvkmd_pdev *pdev,
126 struct vk_object_base *log_obj,
127 struct nvkmd_dev **dev_out);
128 };
129
130 struct nvkmd_pdev {
131 const struct nvkmd_pdev_ops *ops;
132
133 enum nvk_debug debug_flags;
134
135 struct nv_device_info dev_info;
136 struct nvkmd_info kmd_info;
137
138 /** Device-wide GPU memory bind alignment
139 *
140 * Individual BOs may have a smaller alignment if, for instance, we know
141 * they are pinned to system ram.
142 */
143 uint32_t bind_align_B;
144
145 struct {
146 dev_t render_dev;
147 dev_t primary_dev;
148 } drm;
149
150 const struct vk_sync_type *const *sync_types;
151 };
152
153 struct nvkmd_dev_ops {
154 void (*destroy)(struct nvkmd_dev *dev);
155
156 uint64_t (*get_gpu_timestamp)(struct nvkmd_dev *dev);
157
158 int (*get_drm_fd)(struct nvkmd_dev *dev);
159
160 VkResult (*alloc_mem)(struct nvkmd_dev *dev,
161 struct vk_object_base *log_obj,
162 uint64_t size_B, uint64_t align_B,
163 enum nvkmd_mem_flags flags,
164 struct nvkmd_mem **mem_out);
165
166 VkResult (*alloc_tiled_mem)(struct nvkmd_dev *dev,
167 struct vk_object_base *log_obj,
168 uint64_t size_B, uint64_t align_B,
169 uint8_t pte_kind, uint16_t tile_mode,
170 enum nvkmd_mem_flags flags,
171 struct nvkmd_mem **mem_out);
172
173 VkResult (*import_dma_buf)(struct nvkmd_dev *dev,
174 struct vk_object_base *log_obj,
175 int fd, struct nvkmd_mem **mem_out);
176
177 VkResult (*alloc_va)(struct nvkmd_dev *dev,
178 struct vk_object_base *log_obj,
179 enum nvkmd_va_flags flags, uint8_t pte_kind,
180 uint64_t size_B, uint64_t align_B,
181 uint64_t fixed_addr, struct nvkmd_va **va_out);
182
183 VkResult (*create_ctx)(struct nvkmd_dev *dev,
184 struct vk_object_base *log_obj,
185 enum nvkmd_engines engines,
186 struct nvkmd_ctx **ctx_out);
187 };
188
189 struct nvkmd_dev {
190 const struct nvkmd_dev_ops *ops;
191 struct nvkmd_pdev *pdev;
192
193 /* Start and end of the usable VA space. All nvkmd_va objects will be
194 * allocated within this range.
195 */
196 uint64_t va_start, va_end;
197 };
198
199 struct nvkmd_mem_ops {
200 void (*free)(struct nvkmd_mem *mem);
201
202 VkResult (*map)(struct nvkmd_mem *mem,
203 struct vk_object_base *log_obj,
204 enum nvkmd_mem_map_flags flags,
205 void *fixed_addr,
206 void **map_out);
207
208 void (*unmap)(struct nvkmd_mem *mem,
209 enum nvkmd_mem_map_flags flags,
210 void *map);
211
212 VkResult (*overmap)(struct nvkmd_mem *mem,
213 struct vk_object_base *log_obj,
214 enum nvkmd_mem_map_flags flags,
215 void *map);
216
217 VkResult (*export_dma_buf)(struct nvkmd_mem *mem,
218 struct vk_object_base *log_obj,
219 int *fd_out);
220
221 /** Handle to use for NVK_DEBUG_VM logging */
222 uint32_t (*log_handle)(struct nvkmd_mem *mem);
223 };
224
225 struct nvkmd_mem {
226 const struct nvkmd_mem_ops *ops;
227 struct nvkmd_dev *dev;
228
229 uint32_t refcnt;
230
231 enum nvkmd_mem_flags flags;
232 uint32_t bind_align_B;
233
234 uint64_t size_B;
235 struct nvkmd_va *va;
236
237 simple_mtx_t map_mutex;
238 uint32_t map_cnt;
239 void *map;
240
241 void *client_map;
242 };
243
244 void nvkmd_mem_init(struct nvkmd_dev *dev,
245 struct nvkmd_mem *mem,
246 const struct nvkmd_mem_ops *ops,
247 enum nvkmd_mem_flags flags,
248 uint64_t size_B,
249 uint32_t bind_align_B);
250
251 struct nvkmd_va_ops {
252 void (*free)(struct nvkmd_va *va);
253
254 VkResult (*bind_mem)(struct nvkmd_va *va,
255 struct vk_object_base *log_obj,
256 uint64_t va_offset_B,
257 struct nvkmd_mem *mem,
258 uint64_t mem_offset_B,
259 uint64_t range_B);
260
261 VkResult (*unbind)(struct nvkmd_va *va,
262 struct vk_object_base *log_obj,
263 uint64_t va_offset_B,
264 uint64_t range_B);
265 };
266
267 struct nvkmd_va {
268 const struct nvkmd_va_ops *ops;
269 struct nvkmd_dev *dev;
270
271 enum nvkmd_va_flags flags;
272 uint8_t pte_kind;
273 uint64_t addr;
274 uint64_t size_B;
275 };
276
277 struct nvkmd_ctx_exec {
278 uint64_t addr;
279 uint32_t size_B;
280 bool no_prefetch;
281 };
282
283 struct nvkmd_ctx_bind {
284 enum nvkmd_bind_op op;
285
286 struct nvkmd_va *va;
287 uint64_t va_offset_B;
288
289 /* Ignored if op != NVK_BIND_OP_UNBIND */
290 struct nvkmd_mem *mem;
291 uint64_t mem_offset_B;
292
293 uint64_t range_B;
294 };
295
296 struct nvkmd_ctx_ops {
297 void (*destroy)(struct nvkmd_ctx *ctx);
298
299 VkResult (*wait)(struct nvkmd_ctx *ctx,
300 struct vk_object_base *log_obj,
301 uint32_t wait_count,
302 const struct vk_sync_wait *waits);
303
304 VkResult (*exec)(struct nvkmd_ctx *ctx,
305 struct vk_object_base *log_obj,
306 uint32_t exec_count,
307 const struct nvkmd_ctx_exec *execs);
308
309 VkResult (*bind)(struct nvkmd_ctx *ctx,
310 struct vk_object_base *log_obj,
311 uint32_t bind_count,
312 const struct nvkmd_ctx_bind *binds);
313
314 /* Implies flush() */
315 VkResult (*signal)(struct nvkmd_ctx *ctx,
316 struct vk_object_base *log_obj,
317 uint32_t signal_count,
318 const struct vk_sync_signal *signals);
319
320 VkResult (*flush)(struct nvkmd_ctx *ctx,
321 struct vk_object_base *log_obj);
322
323 /* Implies flush() */
324 VkResult (*sync)(struct nvkmd_ctx *ctx,
325 struct vk_object_base *log_obj);
326 };
327
328 struct nvkmd_ctx {
329 const struct nvkmd_ctx_ops *ops;
330 struct nvkmd_dev *dev;
331 };
332
333 /*
334 * Macros
335 *
336 * All subclassed structs must be named nvkmd_<subcls>_<strct> where the
337 * original struct is named nvkmd_<strct>
338 */
339
340 #define NVKMD_DECL_SUBCLASS(strct, subcls) \
341 extern const struct nvkmd_##strct##_ops nvkmd_##subcls##_##strct##_ops; \
342 static inline struct nvkmd_##subcls##_##strct * \
343 nvkmd_##subcls##_##strct(struct nvkmd_##strct *nvkmd) \
344 { \
345 assert(nvkmd->ops == &nvkmd_##subcls##_##strct##_ops); \
346 return container_of(nvkmd, struct nvkmd_##subcls##_##strct, base); \
347 }
348
349 /*
350 * Methods
351 *
352 * Even though everything goes through a function pointer table, we always add
353 * an inline wrapper in case we want to move something into "core" NVKMD.
354 */
355
356 VkResult MUST_CHECK
357 nvkmd_try_create_pdev_for_drm(struct _drmDevice *drm_device,
358 struct vk_object_base *log_obj,
359 enum nvk_debug debug_flags,
360 struct nvkmd_pdev **pdev_out);
361
362 static inline void
nvkmd_pdev_destroy(struct nvkmd_pdev * pdev)363 nvkmd_pdev_destroy(struct nvkmd_pdev *pdev)
364 {
365 pdev->ops->destroy(pdev);
366 }
367
368 static inline uint64_t
nvkmd_pdev_get_vram_used(struct nvkmd_pdev * pdev)369 nvkmd_pdev_get_vram_used(struct nvkmd_pdev *pdev)
370 {
371 return pdev->ops->get_vram_used(pdev);
372 }
373
374 static inline int
nvkmd_pdev_get_drm_primary_fd(struct nvkmd_pdev * pdev)375 nvkmd_pdev_get_drm_primary_fd(struct nvkmd_pdev *pdev)
376 {
377 if (pdev->ops->get_drm_primary_fd == NULL)
378 return -1;
379
380 return pdev->ops->get_drm_primary_fd(pdev);
381 }
382
383 static inline VkResult MUST_CHECK
nvkmd_pdev_create_dev(struct nvkmd_pdev * pdev,struct vk_object_base * log_obj,struct nvkmd_dev ** dev_out)384 nvkmd_pdev_create_dev(struct nvkmd_pdev *pdev,
385 struct vk_object_base *log_obj,
386 struct nvkmd_dev **dev_out)
387 {
388 return pdev->ops->create_dev(pdev, log_obj, dev_out);
389 }
390
391 static inline void
nvkmd_dev_destroy(struct nvkmd_dev * dev)392 nvkmd_dev_destroy(struct nvkmd_dev *dev)
393 {
394 dev->ops->destroy(dev);
395 }
396
397 static inline uint64_t
nvkmd_dev_get_gpu_timestamp(struct nvkmd_dev * dev)398 nvkmd_dev_get_gpu_timestamp(struct nvkmd_dev *dev)
399 {
400 return dev->ops->get_gpu_timestamp(dev);
401 }
402
403 static inline int
nvkmd_dev_get_drm_fd(struct nvkmd_dev * dev)404 nvkmd_dev_get_drm_fd(struct nvkmd_dev *dev)
405 {
406 if (dev->ops->get_drm_fd == NULL)
407 return -1;
408
409 return dev->ops->get_drm_fd(dev);
410 }
411
412 static inline VkResult MUST_CHECK
nvkmd_dev_alloc_mem(struct nvkmd_dev * dev,struct vk_object_base * log_obj,uint64_t size_B,uint64_t align_B,enum nvkmd_mem_flags flags,struct nvkmd_mem ** mem_out)413 nvkmd_dev_alloc_mem(struct nvkmd_dev *dev,
414 struct vk_object_base *log_obj,
415 uint64_t size_B, uint64_t align_B,
416 enum nvkmd_mem_flags flags,
417 struct nvkmd_mem **mem_out)
418 {
419 return dev->ops->alloc_mem(dev, log_obj, size_B, align_B, flags, mem_out);
420 }
421
422 static inline VkResult MUST_CHECK
nvkmd_dev_alloc_tiled_mem(struct nvkmd_dev * dev,struct vk_object_base * log_obj,uint64_t size_B,uint64_t align_B,uint8_t pte_kind,uint16_t tile_mode,enum nvkmd_mem_flags flags,struct nvkmd_mem ** mem_out)423 nvkmd_dev_alloc_tiled_mem(struct nvkmd_dev *dev,
424 struct vk_object_base *log_obj,
425 uint64_t size_B, uint64_t align_B,
426 uint8_t pte_kind, uint16_t tile_mode,
427 enum nvkmd_mem_flags flags,
428 struct nvkmd_mem **mem_out)
429 {
430 return dev->ops->alloc_tiled_mem(dev, log_obj, size_B, align_B,
431 pte_kind, tile_mode, flags, mem_out);
432 }
433
434 /* Implies NVKMD_MEM_CAN_MAP */
435 VkResult MUST_CHECK
436 nvkmd_dev_alloc_mapped_mem(struct nvkmd_dev *dev,
437 struct vk_object_base *log_obj,
438 uint64_t size_B, uint64_t align_B,
439 enum nvkmd_mem_flags flags,
440 enum nvkmd_mem_map_flags map_flags,
441 struct nvkmd_mem **mem_out);
442
443 static inline VkResult MUST_CHECK
nvkmd_dev_import_dma_buf(struct nvkmd_dev * dev,struct vk_object_base * log_obj,int fd,struct nvkmd_mem ** mem_out)444 nvkmd_dev_import_dma_buf(struct nvkmd_dev *dev,
445 struct vk_object_base *log_obj,
446 int fd, struct nvkmd_mem **mem_out)
447 {
448 return dev->ops->import_dma_buf(dev, log_obj, fd, mem_out);
449 }
450
451 VkResult MUST_CHECK
452 nvkmd_dev_alloc_va(struct nvkmd_dev *dev,
453 struct vk_object_base *log_obj,
454 enum nvkmd_va_flags flags, uint8_t pte_kind,
455 uint64_t size_B, uint64_t align_B,
456 uint64_t fixed_addr, struct nvkmd_va **va_out);
457
458 static inline VkResult MUST_CHECK
nvkmd_dev_create_ctx(struct nvkmd_dev * dev,struct vk_object_base * log_obj,enum nvkmd_engines engines,struct nvkmd_ctx ** ctx_out)459 nvkmd_dev_create_ctx(struct nvkmd_dev *dev,
460 struct vk_object_base *log_obj,
461 enum nvkmd_engines engines,
462 struct nvkmd_ctx **ctx_out)
463 {
464 return dev->ops->create_ctx(dev, log_obj, engines, ctx_out);
465 }
466
467 static inline struct nvkmd_mem *
nvkmd_mem_ref(struct nvkmd_mem * mem)468 nvkmd_mem_ref(struct nvkmd_mem *mem)
469 {
470 p_atomic_inc(&mem->refcnt);
471 return mem;
472 }
473
474 void nvkmd_mem_unref(struct nvkmd_mem *mem);
475
476 VkResult MUST_CHECK
477 nvkmd_mem_map(struct nvkmd_mem *mem, struct vk_object_base *log_obj,
478 enum nvkmd_mem_map_flags flags, void *fixed_addr,
479 void **map_out);
480
481 void nvkmd_mem_unmap(struct nvkmd_mem *mem, enum nvkmd_mem_map_flags flags);
482
483 static inline VkResult MUST_CHECK
nvkmd_mem_overmap(struct nvkmd_mem * mem,struct vk_object_base * log_obj,enum nvkmd_mem_map_flags flags)484 nvkmd_mem_overmap(struct nvkmd_mem *mem, struct vk_object_base *log_obj,
485 enum nvkmd_mem_map_flags flags)
486 {
487 assert(flags & NVKMD_MEM_MAP_CLIENT);
488 assert(mem->client_map != NULL);
489
490 VkResult result = mem->ops->overmap(mem, log_obj, flags, mem->client_map);
491 if (result == VK_SUCCESS)
492 mem->client_map = NULL;
493
494 return result;
495 }
496
497 static inline VkResult MUST_CHECK
nvkmd_mem_export_dma_buf(struct nvkmd_mem * mem,struct vk_object_base * log_obj,int * fd_out)498 nvkmd_mem_export_dma_buf(struct nvkmd_mem *mem,
499 struct vk_object_base *log_obj,
500 int *fd_out)
501 {
502 assert(mem->flags & NVKMD_MEM_SHARED);
503
504 return mem->ops->export_dma_buf(mem, log_obj, fd_out);
505 }
506
507 void
508 nvkmd_va_free(struct nvkmd_va *va);
509
510 VkResult MUST_CHECK
511 nvkmd_va_bind_mem(struct nvkmd_va *va,
512 struct vk_object_base *log_obj,
513 uint64_t va_offset_B,
514 struct nvkmd_mem *mem,
515 uint64_t mem_offset_B,
516 uint64_t range_B);
517
518 VkResult MUST_CHECK
519 nvkmd_va_unbind(struct nvkmd_va *va,
520 struct vk_object_base *log_obj,
521 uint64_t va_offset_B,
522 uint64_t range_B);
523
524 static inline void
nvkmd_ctx_destroy(struct nvkmd_ctx * ctx)525 nvkmd_ctx_destroy(struct nvkmd_ctx *ctx)
526 {
527 ctx->ops->destroy(ctx);
528 }
529
530 static inline VkResult MUST_CHECK
nvkmd_ctx_wait(struct nvkmd_ctx * ctx,struct vk_object_base * log_obj,uint32_t wait_count,const struct vk_sync_wait * waits)531 nvkmd_ctx_wait(struct nvkmd_ctx *ctx,
532 struct vk_object_base *log_obj,
533 uint32_t wait_count,
534 const struct vk_sync_wait *waits)
535 {
536 return ctx->ops->wait(ctx, log_obj, wait_count, waits);
537 }
538
539 static inline VkResult MUST_CHECK
nvkmd_ctx_exec(struct nvkmd_ctx * ctx,struct vk_object_base * log_obj,uint32_t exec_count,const struct nvkmd_ctx_exec * execs)540 nvkmd_ctx_exec(struct nvkmd_ctx *ctx,
541 struct vk_object_base *log_obj,
542 uint32_t exec_count,
543 const struct nvkmd_ctx_exec *execs)
544 {
545 return ctx->ops->exec(ctx, log_obj, exec_count, execs);
546 }
547
548 VkResult MUST_CHECK
549 nvkmd_ctx_bind(struct nvkmd_ctx *ctx,
550 struct vk_object_base *log_obj,
551 uint32_t bind_count,
552 const struct nvkmd_ctx_bind *binds);
553
554 static inline VkResult MUST_CHECK
nvkmd_ctx_signal(struct nvkmd_ctx * ctx,struct vk_object_base * log_obj,uint32_t signal_count,const struct vk_sync_signal * signals)555 nvkmd_ctx_signal(struct nvkmd_ctx *ctx,
556 struct vk_object_base *log_obj,
557 uint32_t signal_count,
558 const struct vk_sync_signal *signals)
559 {
560 return ctx->ops->signal(ctx, log_obj, signal_count, signals);
561 }
562
563 static inline VkResult MUST_CHECK
nvkmd_ctx_flush(struct nvkmd_ctx * ctx,struct vk_object_base * log_obj)564 nvkmd_ctx_flush(struct nvkmd_ctx *ctx,
565 struct vk_object_base *log_obj)
566 {
567 return ctx->ops->flush(ctx, log_obj);
568 }
569
570 static inline VkResult MUST_CHECK
nvkmd_ctx_sync(struct nvkmd_ctx * ctx,struct vk_object_base * log_obj)571 nvkmd_ctx_sync(struct nvkmd_ctx *ctx,
572 struct vk_object_base *log_obj)
573 {
574 return ctx->ops->sync(ctx, log_obj);
575 }
576
577 #endif /* NVKMD_H */
578