1 /*
2 * Copyright © 2021 Ilia Mirkin
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 * SOFTWARE.
22 */
23
24 #include "ir3/ir3_compiler.h"
25
26 #include "util/u_math.h"
27 #include "util/u_queue.h"
28 #include "util/half_float.h"
29
30 #include "adreno_pm4.xml.h"
31 #include "adreno_common.xml.h"
32 #include "a4xx.xml.h"
33
34 #include "ir3_asm.h"
35 #include "main.h"
36
37 struct a4xx_backend {
38 struct backend base;
39
40 struct ir3_compiler *compiler;
41 struct fd_device *dev;
42 };
43 define_cast(backend, a4xx_backend);
44
45 /*
46 * Backend implementation:
47 */
48
49 static struct kernel *
a4xx_assemble(struct backend * b,FILE * in)50 a4xx_assemble(struct backend *b, FILE *in)
51 {
52 struct a4xx_backend *a4xx_backend = to_a4xx_backend(b);
53 struct ir3_kernel *ir3_kernel = ir3_asm_assemble(a4xx_backend->compiler, in);
54 ir3_kernel->backend = b;
55 return &ir3_kernel->base;
56 }
57
58 static void
a4xx_disassemble(struct kernel * kernel,FILE * out)59 a4xx_disassemble(struct kernel *kernel, FILE *out)
60 {
61 ir3_asm_disassemble(to_ir3_kernel(kernel), out);
62 }
63
64 static void
cs_program_emit(struct fd_ringbuffer * ring,struct kernel * kernel)65 cs_program_emit(struct fd_ringbuffer *ring, struct kernel *kernel)
66 {
67 struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
68 struct ir3_shader_variant *v = ir3_kernel->v;
69 const struct ir3_info *i = &v->info;
70 enum a3xx_threadsize thrsz = i->double_threadsize ? FOUR_QUADS : TWO_QUADS;
71
72 OUT_PKT0(ring, REG_A4XX_UCHE_INVALIDATE0, 2);
73 OUT_RING(ring, 0x00000000);
74 OUT_RING(ring, 0x00000012);
75
76 OUT_WFI(ring);
77
78 OUT_PKT0(ring, REG_A4XX_SP_MODE_CONTROL, 1);
79 OUT_RING(ring, 0x0000001e);
80
81 OUT_PKT0(ring, REG_A4XX_TPL1_TP_MODE_CONTROL, 1);
82 OUT_RING(ring, 0x00000038);
83
84 OUT_PKT0(ring, REG_A4XX_TPL1_TP_FS_TEX_COUNT, 1);
85 OUT_RING(ring, 0x00000000);
86
87 OUT_WFI(ring);
88
89 OUT_PKT0(ring, REG_A4XX_HLSQ_MODE_CONTROL, 1);
90 OUT_RING(ring, 0x00000003);
91
92 OUT_PKT0(ring, REG_A4XX_HLSQ_CONTROL_0_REG, 1);
93 OUT_RING(ring, 0x080005f0);
94
95 OUT_PKT0(ring, REG_A4XX_HLSQ_UPDATE_CONTROL, 1);
96 OUT_RING(ring, 0x00000038);
97
98 OUT_PKT0(ring, REG_A4XX_SP_SP_CTRL_REG, 1);
99 OUT_RING(ring, 0x00860010);
100 // OUT_RING(ring, 0x00920000);
101
102 OUT_PKT0(ring, REG_A4XX_SP_INSTR_CACHE_CTRL, 1);
103 OUT_RING(ring, 0x000004ff);
104 // OUT_RING(ring, 0x00000260);
105
106 OUT_PKT0(ring, REG_A4XX_SP_FS_CTRL_REG1, 1);
107 OUT_RING(ring, 0x80000000);
108
109 OUT_PKT0(ring, REG_A4XX_SP_CS_CTRL_REG0, 1);
110 OUT_RING(ring,
111 A4XX_SP_CS_CTRL_REG0_THREADSIZE(thrsz) |
112 A4XX_SP_CS_CTRL_REG0_SUPERTHREADMODE |
113 A4XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT(i->max_half_reg + 1) |
114 A4XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT(i->max_reg + 1));
115
116 OUT_PKT0(ring, REG_A4XX_HLSQ_CS_CONTROL_REG, 1);
117 OUT_RING(ring, A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET(0) |
118 A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET(0) |
119 A4XX_HLSQ_CS_CONTROL_REG_ENABLED |
120 A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH(1) |
121 COND(v->has_ssbo, A4XX_HLSQ_CS_CONTROL_REG_SSBO_ENABLE) |
122 A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH(v->constlen / 4));
123
124 OUT_PKT0(ring, REG_A4XX_SP_CS_OBJ_START, 1);
125 OUT_RELOC(ring, v->bo, 0, 0, 0); /* SP_CS_OBJ_START */
126
127 OUT_PKT0(ring, REG_A4XX_SP_CS_LENGTH_REG, 1);
128 OUT_RING(ring, v->instrlen);
129
130 uint32_t local_invocation_id, work_group_id, num_wg_id;
131 local_invocation_id =
132 ir3_find_sysval_regid(v, SYSTEM_VALUE_LOCAL_INVOCATION_ID);
133 work_group_id = ir3_kernel->info.wgid;
134 num_wg_id = ir3_kernel->info.numwg;
135
136 OUT_PKT0(ring, REG_A4XX_HLSQ_CL_CONTROL_0, 2);
137 OUT_RING(ring, A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID(work_group_id) |
138 A4XX_HLSQ_CL_CONTROL_0_KERNELDIMCONSTID(regid(63, 0)) |
139 A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID(local_invocation_id));
140 OUT_RING(ring, A4XX_HLSQ_CL_CONTROL_1_UNK0CONSTID(regid(63, 0)) |
141 A4XX_HLSQ_CL_CONTROL_1_WORKGROUPSIZECONSTID(regid(63, 0)));
142
143 OUT_PKT0(ring, REG_A4XX_HLSQ_CL_KERNEL_CONST, 1);
144 OUT_RING(ring, A4XX_HLSQ_CL_KERNEL_CONST_UNK0CONSTID(regid(63, 0)) |
145 A4XX_HLSQ_CL_KERNEL_CONST_NUMWGCONSTID(num_wg_id));
146
147 OUT_PKT0(ring, REG_A4XX_HLSQ_CL_WG_OFFSET, 1);
148 OUT_RING(ring, A4XX_HLSQ_CL_WG_OFFSET_UNK0CONSTID(regid(63, 0)));
149
150 OUT_PKT3(ring, CP_LOAD_STATE4, 2);
151 OUT_RING(ring, CP_LOAD_STATE4_0_DST_OFF(0) |
152 CP_LOAD_STATE4_0_STATE_SRC(SS4_INDIRECT) |
153 CP_LOAD_STATE4_0_STATE_BLOCK(SB4_CS_SHADER) |
154 CP_LOAD_STATE4_0_NUM_UNIT(v->instrlen));
155 OUT_RELOC(ring, v->bo, 0, CP_LOAD_STATE4_1_STATE_TYPE(ST4_SHADER), 0);
156 }
157
158 static void
emit_const(struct fd_ringbuffer * ring,struct kernel * kernel,uint32_t constid,uint32_t sizedwords,const uint32_t * dwords)159 emit_const(struct fd_ringbuffer *ring, struct kernel *kernel, uint32_t constid, uint32_t sizedwords,
160 const uint32_t *dwords)
161 {
162 uint32_t align_sz;
163
164 assert((constid % 4) == 0);
165
166 /* Overwrite appropriate entries with buffer addresses */
167 struct fd_bo **replacements =
168 (struct fd_bo **)calloc(sizedwords, sizeof(struct fd_bo *));
169 for (int i = 0; i < MAX_BUFS; i++) {
170 if (kernel->buf_addr_regs[i] != INVALID_REG) {
171 int idx = kernel->buf_addr_regs[i];
172 assert(idx < sizedwords);
173
174 replacements[idx] = kernel->bufs[i];
175 }
176 }
177
178 align_sz = align(sizedwords, 4);
179
180 OUT_PKT3(ring, CP_LOAD_STATE4, 2 + align_sz);
181 OUT_RING(ring, CP_LOAD_STATE4_0_DST_OFF(constid / 4) |
182 CP_LOAD_STATE4_0_STATE_SRC(SS4_DIRECT) |
183 CP_LOAD_STATE4_0_STATE_BLOCK(SB4_CS_SHADER) |
184 CP_LOAD_STATE4_0_NUM_UNIT(DIV_ROUND_UP(sizedwords, 4)));
185 OUT_RING(ring, CP_LOAD_STATE4_1_EXT_SRC_ADDR(0) |
186 CP_LOAD_STATE4_1_STATE_TYPE(ST4_CONSTANTS));
187 for (unsigned i = 0; i < sizedwords; i++) {
188 if (replacements[i])
189 OUT_RELOC(ring, replacements[i], 0, 0, 0);
190 else
191 OUT_RING(ring, dwords[i]);
192 }
193
194 /* Zero-pad to multiple of 4 dwords */
195 for (uint32_t i = sizedwords; i < align_sz; i++) {
196 OUT_RING(ring, 0);
197 }
198
199 free(replacements);
200 }
201
202 static void
cs_const_emit(struct fd_ringbuffer * ring,struct kernel * kernel,uint32_t grid[3])203 cs_const_emit(struct fd_ringbuffer *ring, struct kernel *kernel,
204 uint32_t grid[3])
205 {
206 struct ir3_kernel *ir3_kernel = to_ir3_kernel(kernel);
207 struct ir3_shader_variant *v = ir3_kernel->v;
208
209 const struct ir3_const_state *const_state = ir3_const_state(v);
210 uint32_t base = const_state->offsets.immediate;
211 int size = DIV_ROUND_UP(const_state->immediates_count, 4);
212
213 /* truncate size to avoid writing constants that shader
214 * does not use:
215 */
216 size = MIN2(size + base, v->constlen) - base;
217
218 /* convert out of vec4: */
219 base *= 4;
220 size *= 4;
221
222 if (size > 0) {
223 emit_const(ring, kernel, base, size, const_state->immediates);
224 }
225 }
226
227 static void
cs_ibo_emit(struct fd_ringbuffer * ring,struct fd_submit * submit,struct kernel * kernel)228 cs_ibo_emit(struct fd_ringbuffer *ring, struct fd_submit *submit,
229 struct kernel *kernel)
230 {
231 OUT_PKT3(ring, CP_LOAD_STATE4, 2 + (4 * kernel->num_bufs));
232 OUT_RING(ring, CP_LOAD_STATE4_0_DST_OFF(0) |
233 CP_LOAD_STATE4_0_STATE_SRC(SS4_DIRECT) |
234 CP_LOAD_STATE4_0_STATE_BLOCK(SB4_CS_SSBO) |
235 CP_LOAD_STATE4_0_NUM_UNIT(kernel->num_bufs));
236 OUT_RING(ring, CP_LOAD_STATE4_1_STATE_TYPE(ST4_SHADER) |
237 CP_LOAD_STATE4_1_EXT_SRC_ADDR(0));
238 for (unsigned i = 0; i < kernel->num_bufs; i++) {
239 OUT_RELOC(ring, kernel->bufs[i], 0, 0, 0);
240 #if 1
241 OUT_RING(ring, 0);
242 OUT_RING(ring, 0);
243 OUT_RING(ring, 0);
244 #else
245 OUT_RING(ring, kernel->buf_sizes[i]);
246 OUT_RING(ring, kernel->buf_sizes[i]);
247 OUT_RING(ring, 0x00000004);
248 #endif
249 }
250
251 OUT_PKT3(ring, CP_LOAD_STATE4, 2 + (2 * kernel->num_bufs));
252 OUT_RING(ring, CP_LOAD_STATE4_0_DST_OFF(0) |
253 CP_LOAD_STATE4_0_STATE_SRC(SS4_DIRECT) |
254 CP_LOAD_STATE4_0_STATE_BLOCK(SB4_CS_SSBO) |
255 CP_LOAD_STATE4_0_NUM_UNIT(kernel->num_bufs));
256 OUT_RING(ring, CP_LOAD_STATE4_1_STATE_TYPE(ST4_CONSTANTS) |
257 CP_LOAD_STATE4_1_EXT_SRC_ADDR(0));
258 for (unsigned i = 0; i < kernel->num_bufs; i++) {
259 unsigned sz = kernel->buf_sizes[i];
260
261 /* width is in dwords, overflows into height: */
262 sz /= 4;
263
264 #if 1
265 OUT_RING(ring, A4XX_SSBO_1_0_WIDTH(sz));
266 OUT_RING(ring, A4XX_SSBO_1_1_HEIGHT(sz >> 16));
267 #else
268 OUT_RING(ring, A4XX_SSBO_1_0_WIDTH(sz) |
269 A4XX_SSBO_1_0_FMT(RB4_R32_UINT) |
270 A4XX_SSBO_1_0_CPP(4));
271 OUT_RING(ring, A4XX_SSBO_1_1_HEIGHT(DIV_ROUND_UP(sz, 1 << 16)) |
272 A4XX_SSBO_1_1_DEPTH(1));
273 #endif
274 }
275 }
276
277 static void
a4xx_emit_grid(struct kernel * kernel,uint32_t grid[3],struct fd_submit * submit)278 a4xx_emit_grid(struct kernel *kernel, uint32_t grid[3],
279 struct fd_submit *submit)
280 {
281 struct fd_ringbuffer *ring = fd_submit_new_ringbuffer(
282 submit, 0,
283 (enum fd_ringbuffer_flags)(FD_RINGBUFFER_PRIMARY | FD_RINGBUFFER_GROWABLE));
284
285 cs_program_emit(ring, kernel);
286 cs_const_emit(ring, kernel, grid);
287 cs_ibo_emit(ring, submit, kernel);
288
289 const unsigned *local_size = kernel->local_size;
290 const unsigned *num_groups = grid;
291
292 unsigned work_dim = 0;
293 for (int i = 0; i < 3; i++) {
294 if (!grid[i])
295 break;
296 work_dim++;
297 }
298
299 OUT_PKT0(ring, REG_A4XX_HLSQ_CL_NDRANGE_0, 7);
300 OUT_RING(ring, A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM(work_dim) |
301 A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX(local_size[0] - 1) |
302 A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY(local_size[1] - 1) |
303 A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ(local_size[2] - 1));
304 OUT_RING(ring,
305 A4XX_HLSQ_CL_NDRANGE_1_SIZE_X(local_size[0] * num_groups[0]));
306 OUT_RING(ring, 0); /* HLSQ_CL_NDRANGE_2_GLOBALOFF_X */
307 OUT_RING(ring,
308 A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y(local_size[1] * num_groups[1]));
309 OUT_RING(ring, 0); /* HLSQ_CL_NDRANGE_4_GLOBALOFF_Y */
310 OUT_RING(ring,
311 A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z(local_size[2] * num_groups[2]));
312 OUT_RING(ring, 0); /* HLSQ_CL_NDRANGE_6_GLOBALOFF_Z */
313
314 #if 1
315 OUT_PKT3(ring, CP_EXEC_CS, 4);
316 OUT_RING(ring, 0x00000000);
317 OUT_RING(ring, CP_EXEC_CS_1_NGROUPS_X(grid[0]));
318 OUT_RING(ring, CP_EXEC_CS_2_NGROUPS_Y(grid[1]));
319 OUT_RING(ring, CP_EXEC_CS_3_NGROUPS_Z(grid[2]));
320 #else
321 OUT_PKT0(ring, REG_A4XX_HLSQ_CL_KERNEL_GROUP_X, 3);
322 OUT_RING(ring, grid[0]); /* HLSQ_CL_KERNEL_GROUP_X */
323 OUT_RING(ring, grid[1]); /* HLSQ_CL_KERNEL_GROUP_Y */
324 OUT_RING(ring, grid[2]); /* HLSQ_CL_KERNEL_GROUP_Z */
325
326 OUT_PKT3(ring, CP_RUN_OPENCL, 1);
327 OUT_RING(ring, 0);
328 #endif
329
330 OUT_WFI(ring);
331
332 /* TODO: cache_flush */
333 }
334
335 struct backend *
a4xx_init(struct fd_device * dev,const struct fd_dev_id * dev_id)336 a4xx_init(struct fd_device *dev, const struct fd_dev_id *dev_id)
337 {
338 struct a4xx_backend *a4xx_backend =
339 (struct a4xx_backend *)calloc(1, sizeof(*a4xx_backend));
340
341 a4xx_backend->base = (struct backend){
342 .assemble = a4xx_assemble,
343 .disassemble = a4xx_disassemble,
344 .emit_grid = a4xx_emit_grid,
345 };
346
347 struct ir3_compiler_options compiler_options = {};
348 a4xx_backend->compiler =
349 ir3_compiler_create(dev, dev_id, fd_dev_info_raw(dev_id), &compiler_options);
350 a4xx_backend->dev = dev;
351
352 return &a4xx_backend->base;
353 }
354