1 /*
2 * Copyright © 2024 Collabora Ltd.
3 * SPDX-License-Identifier: MIT
4 */
5
6 #ifndef PANVK_CMD_BUFFER_H
7 #define PANVK_CMD_BUFFER_H
8
9 #ifndef PAN_ARCH
10 #error "PAN_ARCH must be defined"
11 #endif
12
13 #include <stdint.h>
14
15 #include "genxml/cs_builder.h"
16
17 #include "panvk_cmd_desc_state.h"
18 #include "panvk_cmd_push_constant.h"
19 #include "panvk_queue.h"
20
21 #include "vk_command_buffer.h"
22
23 #include "util/list.h"
24
25 #define MAX_VBS 16
26 #define MAX_RTS 8
27
28 struct panvk_cs_sync32 {
29 uint32_t seqno;
30 uint32_t error;
31 };
32
33 struct panvk_cs_sync64 {
34 uint64_t seqno;
35 uint32_t error;
36 uint32_t pad;
37 };
38
39 struct panvk_cs_desc_ringbuf {
40 uint64_t syncobj;
41 uint64_t ptr;
42 uint32_t pos;
43 uint32_t pad;
44 };
45
46 /* 512k of render descriptors that can be used when
47 * VK_COMMAND_BUFFER_USAGE_SIMULTANEOUS_USE_BIT is set on the command buffer. */
48 #define RENDER_DESC_RINGBUF_SIZE (512 * 1024)
49
50 struct panvk_cs_subqueue_context {
51 uint64_t syncobjs;
52 uint32_t iter_sb;
53 uint32_t pad;
54 struct {
55 struct panvk_cs_desc_ringbuf desc_ringbuf;
56 uint64_t tiler_heap;
57 uint64_t geom_buf;
58 } render;
59 uint64_t debug_syncobjs;
60 } __attribute__((aligned(64)));
61
62 struct panvk_cache_flush_info {
63 enum mali_cs_flush_mode l2;
64 enum mali_cs_flush_mode lsc;
65 bool others;
66 };
67
68 struct panvk_cs_deps {
69 bool needs_draw_flush;
70 struct {
71 uint32_t wait_sb_mask;
72 struct panvk_cache_flush_info cache_flush;
73 } src[PANVK_SUBQUEUE_COUNT];
74
75 struct {
76 uint32_t wait_subqueue_mask;
77 } dst[PANVK_SUBQUEUE_COUNT];
78 };
79
80 enum panvk_sb_ids {
81 PANVK_SB_LS = 0,
82 PANVK_SB_IMM_FLUSH = 0,
83 PANVK_SB_DEFERRED_SYNC = 1,
84 PANVK_SB_DEFERRED_FLUSH = 2,
85 PANVK_SB_ITER_START = 3,
86 PANVK_SB_ITER_COUNT = 5,
87 };
88
89 #define SB_IMM_MASK 0
90 #define SB_MASK(nm) BITFIELD_BIT(PANVK_SB_##nm)
91 #define SB_ID(nm) PANVK_SB_##nm
92 #define SB_ITER(x) (PANVK_SB_ITER_START + (x))
93 #define SB_WAIT_ITER(x) BITFIELD_BIT(PANVK_SB_ITER_START + (x))
94 #define SB_ALL_MASK BITFIELD_MASK(8)
95
96 static inline uint32_t
next_iter_sb(uint32_t sb)97 next_iter_sb(uint32_t sb)
98 {
99 return sb + 1 < PANVK_SB_ITER_COUNT ? sb + 1 : 0;
100 }
101
102 enum panvk_cs_regs {
103 /* RUN_IDVS staging regs. */
104 PANVK_CS_REG_RUN_IDVS_SR_START = 0,
105 PANVK_CS_REG_RUN_IDVS_SR_END = 60,
106
107 /* RUN_FRAGMENT staging regs. */
108 PANVK_CS_REG_RUN_FRAGMENT_SR_START = 40,
109 PANVK_CS_REG_RUN_FRAGMENT_SR_END = 46,
110
111 /* RUN_COMPUTE staging regs. */
112 PANVK_CS_REG_RUN_COMPUTE_SR_START = 0,
113 PANVK_CS_REG_RUN_COMPUTE_SR_END = 39,
114
115 /* Range of registers that can be used to store temporary data on
116 * all queues. Note that some queues have extra space they can use
117 * as scratch space.*/
118 PANVK_CS_REG_SCRATCH_START = 66,
119 PANVK_CS_REG_SCRATCH_END = 83,
120
121 /* Driver context. */
122 PANVK_CS_REG_PROGRESS_SEQNO_START = 84,
123 PANVK_CS_REG_PROGRESS_SEQNO_END = 89,
124 PANVK_CS_REG_SUBQUEUE_CTX_START = 90,
125 PANVK_CS_REG_SUBQUEUE_CTX_END = 91,
126 };
127
128 static inline struct cs_index
cs_scratch_reg_tuple(struct cs_builder * b,unsigned start,unsigned count)129 cs_scratch_reg_tuple(struct cs_builder *b, unsigned start, unsigned count)
130 {
131 assert(PANVK_CS_REG_SCRATCH_START + start + count - 1 <=
132 PANVK_CS_REG_SCRATCH_END);
133 return cs_reg_tuple(b, PANVK_CS_REG_SCRATCH_START + start, count);
134 }
135
136 static inline struct cs_index
cs_scratch_reg32(struct cs_builder * b,unsigned reg)137 cs_scratch_reg32(struct cs_builder *b, unsigned reg)
138 {
139 return cs_scratch_reg_tuple(b, reg, 1);
140 }
141
142 static inline struct cs_index
cs_scratch_reg64(struct cs_builder * b,unsigned reg)143 cs_scratch_reg64(struct cs_builder *b, unsigned reg)
144 {
145 assert(reg % 2 == 0);
146 return cs_scratch_reg_tuple(b, reg, 2);
147 }
148
149 static inline struct cs_index
cs_sr_reg_tuple(struct cs_builder * b,unsigned start,unsigned count)150 cs_sr_reg_tuple(struct cs_builder *b, unsigned start, unsigned count)
151 {
152 assert(start + count - 1 < PANVK_CS_REG_SCRATCH_START);
153 return cs_reg_tuple(b, start, count);
154 }
155
156 static inline struct cs_index
cs_sr_reg32(struct cs_builder * b,unsigned reg)157 cs_sr_reg32(struct cs_builder *b, unsigned reg)
158 {
159 return cs_sr_reg_tuple(b, reg, 1);
160 }
161
162 static inline struct cs_index
cs_sr_reg64(struct cs_builder * b,unsigned reg)163 cs_sr_reg64(struct cs_builder *b, unsigned reg)
164 {
165 assert(reg % 2 == 0);
166 return cs_sr_reg_tuple(b, reg, 2);
167 }
168
169 static inline struct cs_index
cs_subqueue_ctx_reg(struct cs_builder * b)170 cs_subqueue_ctx_reg(struct cs_builder *b)
171 {
172 return cs_reg64(b, PANVK_CS_REG_SUBQUEUE_CTX_START);
173 }
174
175 static inline struct cs_index
cs_progress_seqno_reg(struct cs_builder * b,enum panvk_subqueue_id subqueue)176 cs_progress_seqno_reg(struct cs_builder *b, enum panvk_subqueue_id subqueue)
177 {
178 assert(PANVK_CS_REG_PROGRESS_SEQNO_START + (subqueue * 2) <
179 PANVK_CS_REG_PROGRESS_SEQNO_END);
180 return cs_reg64(b, PANVK_CS_REG_PROGRESS_SEQNO_START + (subqueue * 2));
181 }
182
183 struct panvk_cs_reg_upd_context {
184 reg_perm_cb_t reg_perm;
185 struct panvk_cs_reg_upd_context *next;
186 };
187
188 struct panvk_cs_state {
189 struct cs_builder builder;
190
191 struct cs_load_store_tracker ls_tracker;
192
193 /* Used to debug register writes in invalid contexts. */
194 struct {
195 struct panvk_cs_reg_upd_context *upd_ctx_stack;
196 reg_perm_cb_t base_perm;
197 } reg_access;
198
199 /* Sync point relative to the beginning of the command buffer.
200 * Needs to be offset with the subqueue sync point. */
201 int32_t relative_sync_point;
202 };
203
204 static inline struct panvk_cs_reg_upd_context *
panvk_cs_reg_ctx_push(struct cs_builder * b,struct panvk_cs_reg_upd_context * ctx,reg_perm_cb_t reg_perm)205 panvk_cs_reg_ctx_push(struct cs_builder *b,
206 struct panvk_cs_reg_upd_context *ctx,
207 reg_perm_cb_t reg_perm)
208 {
209 struct panvk_cs_state *cs_state =
210 container_of(b, struct panvk_cs_state, builder);
211
212 ctx->reg_perm = reg_perm;
213 ctx->next = cs_state->reg_access.upd_ctx_stack;
214 cs_state->reg_access.upd_ctx_stack = ctx;
215 return ctx;
216 }
217
218 static inline void
panvk_cs_reg_ctx_pop(struct cs_builder * b,struct panvk_cs_reg_upd_context * ctx)219 panvk_cs_reg_ctx_pop(struct cs_builder *b, struct panvk_cs_reg_upd_context *ctx)
220 {
221 struct panvk_cs_state *cs_state =
222 container_of(b, struct panvk_cs_state, builder);
223
224 assert(cs_state->reg_access.upd_ctx_stack == ctx);
225
226 cs_state->reg_access.upd_ctx_stack = ctx->next;
227 }
228
229 struct panvk_cs_reg_range {
230 unsigned start;
231 unsigned end;
232 };
233
234 #define PANVK_CS_REG_RANGE(__name) \
235 { \
236 .start = PANVK_CS_REG_##__name##_START, \
237 .end = PANVK_CS_REG_##__name##_END, \
238 }
239
240 #define panvk_cs_reg_blacklist(__name, ...) \
241 static inline enum cs_reg_perm panvk_cs_##__name##_reg_perm( \
242 struct cs_builder *b, unsigned reg) \
243 { \
244 const struct panvk_cs_reg_range ranges[] = { \
245 __VA_ARGS__, \
246 }; \
247 \
248 for (unsigned i = 0; i < ARRAY_SIZE(ranges); i++) { \
249 if (reg >= ranges[i].start && reg <= ranges[i].end) \
250 return CS_REG_RD; \
251 } \
252 \
253 return CS_REG_RW; \
254 }
255
256 panvk_cs_reg_blacklist(vt, PANVK_CS_REG_RANGE(RUN_IDVS_SR),
257 PANVK_CS_REG_RANGE(PROGRESS_SEQNO),
258 PANVK_CS_REG_RANGE(SUBQUEUE_CTX));
259 panvk_cs_reg_blacklist(frag, PANVK_CS_REG_RANGE(RUN_FRAGMENT_SR),
260 PANVK_CS_REG_RANGE(PROGRESS_SEQNO),
261 PANVK_CS_REG_RANGE(SUBQUEUE_CTX));
262 panvk_cs_reg_blacklist(compute, PANVK_CS_REG_RANGE(RUN_COMPUTE_SR),
263 PANVK_CS_REG_RANGE(PROGRESS_SEQNO),
264 PANVK_CS_REG_RANGE(SUBQUEUE_CTX));
265
266 #define panvk_cs_reg_whitelist(__name, ...) \
267 static inline enum cs_reg_perm panvk_cs_##__name##_reg_perm( \
268 struct cs_builder *b, unsigned reg) \
269 { \
270 const struct panvk_cs_reg_range ranges[] = { \
271 __VA_ARGS__, \
272 }; \
273 \
274 for (unsigned i = 0; i < ARRAY_SIZE(ranges); i++) { \
275 if (reg >= ranges[i].start && reg <= ranges[i].end) \
276 return CS_REG_RW; \
277 } \
278 \
279 return CS_REG_RD; \
280 }
281
282 #define panvk_cs_reg_upd_ctx(__b, __name) \
283 for (struct panvk_cs_reg_upd_context __reg_upd_ctx, \
284 *reg_upd_ctxp = panvk_cs_reg_ctx_push(__b, &__reg_upd_ctx, \
285 panvk_cs_##__name##_reg_perm); \
286 reg_upd_ctxp; \
287 panvk_cs_reg_ctx_pop(__b, &__reg_upd_ctx), reg_upd_ctxp = NULL)
288
289 panvk_cs_reg_whitelist(progress_seqno, PANVK_CS_REG_RANGE(PROGRESS_SEQNO));
290 #define cs_update_progress_seqno(__b) panvk_cs_reg_upd_ctx(__b, progress_seqno)
291
292 panvk_cs_reg_whitelist(compute_ctx, PANVK_CS_REG_RANGE(RUN_COMPUTE_SR));
293 #define cs_update_compute_ctx(__b) panvk_cs_reg_upd_ctx(__b, compute_ctx)
294
295 panvk_cs_reg_whitelist(frag_ctx, PANVK_CS_REG_RANGE(RUN_FRAGMENT_SR));
296 #define cs_update_frag_ctx(__b) panvk_cs_reg_upd_ctx(__b, frag_ctx)
297
298 panvk_cs_reg_whitelist(vt_ctx, PANVK_CS_REG_RANGE(RUN_IDVS_SR));
299 #define cs_update_vt_ctx(__b) panvk_cs_reg_upd_ctx(__b, vt_ctx)
300
301 struct panvk_tls_state {
302 struct panfrost_ptr desc;
303 struct pan_tls_info info;
304 unsigned max_wg_count;
305 };
306
307 struct panvk_cmd_compute_state {
308 struct panvk_descriptor_state desc_state;
309 const struct panvk_shader *shader;
310 struct panvk_compute_sysvals sysvals;
311 mali_ptr push_uniforms;
312 struct {
313 struct panvk_shader_desc_state desc;
314 } cs;
315 };
316
317 struct panvk_attrib_buf {
318 mali_ptr address;
319 unsigned size;
320 };
321
322 struct panvk_resolve_attachment {
323 VkResolveModeFlagBits mode;
324 struct panvk_image_view *src_iview;
325 struct panvk_image_view *dst_iview;
326 };
327
328 struct panvk_cmd_graphics_state {
329 struct panvk_descriptor_state desc_state;
330
331 struct {
332 struct vk_vertex_input_state vi;
333 struct vk_sample_locations_state sl;
334 } dynamic;
335
336 struct panvk_graphics_sysvals sysvals;
337
338 struct panvk_shader_link link;
339 bool linked;
340
341 struct {
342 const struct panvk_shader *shader;
343 struct panvk_shader_desc_state desc;
344 mali_ptr spd;
345 } fs;
346
347 struct {
348 const struct panvk_shader *shader;
349 struct panvk_shader_desc_state desc;
350 struct {
351 mali_ptr pos, var;
352 } spds;
353 } vs;
354
355 struct {
356 struct panvk_attrib_buf bufs[MAX_VBS];
357 unsigned count;
358 bool dirty;
359 } vb;
360
361 /* Index buffer */
362 struct {
363 struct panvk_buffer *buffer;
364 uint64_t offset;
365 uint8_t index_size;
366 uint32_t first_vertex, base_vertex, base_instance;
367 bool dirty;
368 } ib;
369
370 struct {
371 struct panvk_blend_info info;
372 } cb;
373
374 struct {
375 VkRenderingFlags flags;
376 uint32_t layer_count;
377
378 enum vk_rp_attachment_flags bound_attachments;
379 struct {
380 VkFormat fmts[MAX_RTS];
381 uint8_t samples[MAX_RTS];
382 struct panvk_resolve_attachment resolve[MAX_RTS];
383 } color_attachments;
384
385 struct pan_image_view zs_pview;
386
387 struct {
388 struct panvk_resolve_attachment resolve;
389 } z_attachment, s_attachment;
390
391 struct {
392 struct pan_fb_info info;
393 bool crc_valid[MAX_RTS];
394 } fb;
395
396 struct panfrost_ptr fbds;
397 mali_ptr tiler;
398 bool dirty;
399 } render;
400
401 mali_ptr push_uniforms;
402 mali_ptr tsd;
403 };
404
405 struct panvk_cmd_buffer {
406 struct vk_command_buffer vk;
407 VkCommandBufferUsageFlags flags;
408 struct panvk_pool cs_pool;
409 struct panvk_pool desc_pool;
410 struct panvk_pool tls_pool;
411 struct list_head push_sets;
412
413 uint32_t flush_id;
414
415 struct {
416 struct panvk_cmd_graphics_state gfx;
417 struct panvk_cmd_compute_state compute;
418 struct panvk_push_constant_state push_constants;
419 struct panvk_cs_state cs[PANVK_SUBQUEUE_COUNT];
420 struct panvk_tls_state tls;
421 } state;
422 };
423
424 VK_DEFINE_HANDLE_CASTS(panvk_cmd_buffer, vk.base, VkCommandBuffer,
425 VK_OBJECT_TYPE_COMMAND_BUFFER)
426
427 static inline struct cs_builder *
panvk_get_cs_builder(struct panvk_cmd_buffer * cmdbuf,uint32_t subqueue)428 panvk_get_cs_builder(struct panvk_cmd_buffer *cmdbuf, uint32_t subqueue)
429 {
430 return &cmdbuf->state.cs[subqueue].builder;
431 }
432
433 static inline struct panvk_descriptor_state *
panvk_cmd_get_desc_state(struct panvk_cmd_buffer * cmdbuf,VkPipelineBindPoint bindpoint)434 panvk_cmd_get_desc_state(struct panvk_cmd_buffer *cmdbuf,
435 VkPipelineBindPoint bindpoint)
436 {
437 switch (bindpoint) {
438 case VK_PIPELINE_BIND_POINT_GRAPHICS:
439 return &cmdbuf->state.gfx.desc_state;
440
441 case VK_PIPELINE_BIND_POINT_COMPUTE:
442 return &cmdbuf->state.compute.desc_state;
443
444 default:
445 assert(!"Unsupported bind point");
446 return NULL;
447 }
448 }
449
450 extern const struct vk_command_buffer_ops panvk_per_arch(cmd_buffer_ops);
451
452 void panvk_per_arch(cmd_flush_draws)(struct panvk_cmd_buffer *cmdbuf);
453
454 void panvk_per_arch(cs_pick_iter_sb)(struct panvk_cmd_buffer *cmdbuf,
455 enum panvk_subqueue_id subqueue);
456
457 void panvk_per_arch(get_cs_deps)(struct panvk_cmd_buffer *cmdbuf,
458 const VkDependencyInfo *in,
459 struct panvk_cs_deps *out);
460
461 #endif /* PANVK_CMD_BUFFER_H */
462