1 /*
2 * Copyright 2021 Alyssa Rosenzweig
3 * SPDX-License-Identifier: MIT
4 */
5
6 #pragma once
7
8 #include <stdint.h>
9 #include <xf86drm.h>
10 #include "util/ralloc.h"
11 #include "util/simple_mtx.h"
12 #include "util/sparse_array.h"
13 #include "util/timespec.h"
14 #include "util/vma.h"
15 #include "agx_bo.h"
16 #include "decode.h"
17 #include "layout.h"
18 #include "unstable_asahi_drm.h"
19
20 // TODO: this is a lie right now
21 static const uint64_t AGX_SUPPORTED_INCOMPAT_FEATURES =
22 DRM_ASAHI_FEAT_MANDATORY_ZS_COMPRESSION;
23
24 enum agx_dbg {
25 AGX_DBG_TRACE = BITFIELD_BIT(0),
26 /* bit 1 unused */
27 AGX_DBG_NO16 = BITFIELD_BIT(2),
28 AGX_DBG_DIRTY = BITFIELD_BIT(3),
29 AGX_DBG_PRECOMPILE = BITFIELD_BIT(4),
30 AGX_DBG_PERF = BITFIELD_BIT(5),
31 AGX_DBG_NOCOMPRESS = BITFIELD_BIT(6),
32 AGX_DBG_NOCLUSTER = BITFIELD_BIT(7),
33 AGX_DBG_SYNC = BITFIELD_BIT(8),
34 AGX_DBG_STATS = BITFIELD_BIT(9),
35 AGX_DBG_RESOURCE = BITFIELD_BIT(10),
36 AGX_DBG_BATCH = BITFIELD_BIT(11),
37 AGX_DBG_NOWC = BITFIELD_BIT(12),
38 AGX_DBG_SYNCTVB = BITFIELD_BIT(13),
39 AGX_DBG_SMALLTILE = BITFIELD_BIT(14),
40 AGX_DBG_NOMSAA = BITFIELD_BIT(15),
41 AGX_DBG_NOSHADOW = BITFIELD_BIT(16),
42 /* bit 17 unused */
43 AGX_DBG_SCRATCH = BITFIELD_BIT(18),
44 /* bit 19 unused */
45 AGX_DBG_FEEDBACK = BITFIELD_BIT(20),
46 AGX_DBG_1QUEUE = BITFIELD_BIT(21),
47 };
48
49 /* How many power-of-two levels in the BO cache do we want? 2^14 minimum chosen
50 * as it is the page size that all allocations are rounded to
51 */
52 #define MIN_BO_CACHE_BUCKET (14) /* 2^14 = 16KB */
53 #define MAX_BO_CACHE_BUCKET (22) /* 2^22 = 4MB */
54
55 /* Fencepost problem, hence the off-by-one */
56 #define NR_BO_CACHE_BUCKETS (MAX_BO_CACHE_BUCKET - MIN_BO_CACHE_BUCKET + 1)
57
58 /* Forward decl only, do not pull in all of NIR */
59 struct nir_shader;
60
61 #define BARRIER_RENDER (1 << DRM_ASAHI_SUBQUEUE_RENDER)
62 #define BARRIER_COMPUTE (1 << DRM_ASAHI_SUBQUEUE_COMPUTE)
63
64 typedef struct {
65 struct agx_bo *(*bo_alloc)(struct agx_device *dev, size_t size, size_t align,
66 enum agx_bo_flags flags);
67 int (*bo_bind)(struct agx_device *dev, struct agx_bo *bo, uint64_t addr,
68 size_t size_B, uint64_t offset_B, uint32_t flags,
69 bool unbind);
70 void (*bo_mmap)(struct agx_device *dev, struct agx_bo *bo);
71 ssize_t (*get_params)(struct agx_device *dev, void *buf, size_t size);
72 int (*submit)(struct agx_device *dev, struct drm_asahi_submit *submit,
73 uint32_t vbo_res_id);
74 } agx_device_ops_t;
75
76 struct agx_device {
77 uint32_t debug;
78
79 /* NIR library of AGX helpers/shaders. Immutable once created. */
80 const struct nir_shader *libagx;
81
82 char name[64];
83 struct drm_asahi_params_global params;
84 uint64_t next_global_id, last_global_id;
85 bool is_virtio;
86 agx_device_ops_t ops;
87
88 /* vdrm device */
89 struct vdrm_device *vdrm;
90 uint32_t next_blob_id;
91
92 /* Device handle */
93 int fd;
94
95 /* VM handle */
96 uint32_t vm_id;
97
98 /* Global queue handle */
99 uint32_t queue_id;
100
101 /* VMA heaps */
102 simple_mtx_t vma_lock;
103 uint64_t shader_base;
104 struct util_vma_heap main_heap;
105 struct util_vma_heap usc_heap;
106 uint64_t guard_size;
107
108 struct renderonly *ro;
109
110 pthread_mutex_t bo_map_lock;
111 struct util_sparse_array bo_map;
112 uint32_t max_handle;
113
114 struct {
115 simple_mtx_t lock;
116
117 /* List containing all cached BOs sorted in LRU (Least Recently Used)
118 * order so we can quickly evict BOs that are more than 1 second old.
119 */
120 struct list_head lru;
121
122 /* The BO cache is a set of buckets with power-of-two sizes. Each bucket
123 * is a linked list of free panfrost_bo objects.
124 */
125 struct list_head buckets[NR_BO_CACHE_BUCKETS];
126
127 /* Current size of the BO cache in bytes (sum of sizes of cached BOs) */
128 size_t size;
129
130 /* Number of hits/misses for the BO cache */
131 uint64_t hits, misses;
132 } bo_cache;
133
134 struct agx_bo *helper;
135
136 struct agxdecode_ctx *agxdecode;
137 };
138
139 static inline bool
agx_has_soft_fault(struct agx_device * dev)140 agx_has_soft_fault(struct agx_device *dev)
141 {
142 return dev->params.feat_compat & DRM_ASAHI_FEAT_SOFT_FAULTS;
143 }
144
145 static uint32_t
agx_usc_addr(struct agx_device * dev,uint64_t addr)146 agx_usc_addr(struct agx_device *dev, uint64_t addr)
147 {
148 assert(addr >= dev->shader_base);
149 assert((addr - dev->shader_base) <= UINT32_MAX);
150
151 return addr - dev->shader_base;
152 }
153
154 bool agx_open_device(void *memctx, struct agx_device *dev);
155
156 void agx_close_device(struct agx_device *dev);
157
158 static inline struct agx_bo *
agx_lookup_bo(struct agx_device * dev,uint32_t handle)159 agx_lookup_bo(struct agx_device *dev, uint32_t handle)
160 {
161 return util_sparse_array_get(&dev->bo_map, handle);
162 }
163
164 uint64_t agx_get_global_id(struct agx_device *dev);
165
166 uint32_t agx_create_command_queue(struct agx_device *dev, uint32_t caps,
167 uint32_t priority);
168 int agx_destroy_command_queue(struct agx_device *dev, uint32_t queue_id);
169
170 int agx_import_sync_file(struct agx_device *dev, struct agx_bo *bo, int fd);
171 int agx_export_sync_file(struct agx_device *dev, struct agx_bo *bo);
172
173 void agx_debug_fault(struct agx_device *dev, uint64_t addr);
174
175 uint64_t agx_get_gpu_timestamp(struct agx_device *dev);
176
177 static inline uint64_t
agx_gpu_time_to_ns(struct agx_device * dev,uint64_t gpu_time)178 agx_gpu_time_to_ns(struct agx_device *dev, uint64_t gpu_time)
179 {
180 return (gpu_time * NSEC_PER_SEC) / dev->params.timer_frequency_hz;
181 }
182
183 void agx_get_device_uuid(const struct agx_device *dev, void *uuid);
184 void agx_get_driver_uuid(void *uuid);
185
186 struct agx_va *agx_va_alloc(struct agx_device *dev, uint32_t size_B,
187 uint32_t align_B, enum agx_va_flags flags,
188 uint64_t fixed_va);
189 void agx_va_free(struct agx_device *dev, struct agx_va *va);
190