1 /*
2 * Copyright © 2017-2018 Rob Clark <[email protected]>
3 * SPDX-License-Identifier: MIT
4 *
5 * Authors:
6 * Rob Clark <[email protected]>
7 */
8
9 /* 500 gets us LDIB but doesn't change any other a4xx instructions */
10 #define GPU 500
11
12 #include "ir3_context.h"
13 #include "ir3_image.h"
14
15 /* SSBO data is available at this CB address, addressed like regular consts
16 * containing the following data in each vec4:
17 *
18 * [ base address, pitch, array_pitch, cpp ]
19 *
20 * These mirror the values uploaded to A4XX_SSBO_0 state. For A5XX, these are
21 * uploaded manually by the driver.
22 */
23 #define A4XX_SSBO_CB_BASE(i) (0x700 + ((i) << 2))
24
25 /*
26 * Handlers for instructions changed/added in a4xx:
27 */
28
29 /* Convert byte offset to address of appropriate width for GPU */
30 static struct ir3_instruction *
byte_offset_to_address(struct ir3_context * ctx,nir_src * ssbo,struct ir3_instruction * byte_offset)31 byte_offset_to_address(struct ir3_context *ctx,
32 nir_src *ssbo,
33 struct ir3_instruction *byte_offset)
34 {
35 struct ir3_block *b = ctx->block;
36
37 if (ctx->compiler->gen == 4) {
38 uint32_t index = nir_src_as_uint(*ssbo);
39 unsigned cb = A4XX_SSBO_CB_BASE(index);
40 byte_offset = ir3_ADD_U(b, create_uniform(b, cb), 0, byte_offset, 0);
41 }
42
43 if (ctx->compiler->is_64bit) {
44 return ir3_collect(b, byte_offset, create_immed(b, 0));
45 } else {
46 return byte_offset;
47 }
48 }
49
50 /* src[] = { buffer_index, offset }. No const_index */
51 static void
emit_intrinsic_load_ssbo(struct ir3_context * ctx,nir_intrinsic_instr * intr,struct ir3_instruction ** dst)52 emit_intrinsic_load_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr,
53 struct ir3_instruction **dst)
54 {
55 struct ir3_block *b = ctx->block;
56 struct ir3_instruction *ldgb, *src0, *src1, *byte_offset, *offset;
57
58 struct ir3_instruction *ssbo = ir3_ssbo_to_ibo(ctx, intr->src[0]);
59
60 byte_offset = ir3_get_src(ctx, &intr->src[1])[0];
61 offset = ir3_get_src(ctx, &intr->src[2])[0];
62
63 /* src0 is uvec2(offset*4, 0), src1 is offset.. nir already *= 4: */
64 src0 = byte_offset_to_address(ctx, &intr->src[0], byte_offset);
65 src1 = offset;
66
67 ldgb = ir3_LDGB(b, ssbo, 0, src0, 0, src1, 0);
68 ldgb->dsts[0]->wrmask = MASK(intr->num_components);
69 ldgb->cat6.iim_val = intr->num_components;
70 ldgb->cat6.d = 4;
71 ldgb->cat6.type = TYPE_U32;
72 ldgb->barrier_class = IR3_BARRIER_BUFFER_R;
73 ldgb->barrier_conflict = IR3_BARRIER_BUFFER_W;
74
75 ir3_split_dest(b, dst, ldgb, 0, intr->num_components);
76 }
77
78 /* src[] = { value, block_index, offset }. const_index[] = { write_mask } */
79 static void
emit_intrinsic_store_ssbo(struct ir3_context * ctx,nir_intrinsic_instr * intr)80 emit_intrinsic_store_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
81 {
82 struct ir3_block *b = ctx->block;
83 struct ir3_instruction *stgb, *src0, *src1, *src2, *byte_offset, *offset;
84 unsigned wrmask = nir_intrinsic_write_mask(intr);
85 unsigned ncomp = ffs(~wrmask) - 1;
86
87 assert(wrmask == BITFIELD_MASK(intr->num_components));
88
89 struct ir3_instruction *ssbo = ir3_ssbo_to_ibo(ctx, intr->src[1]);
90
91 byte_offset = ir3_get_src(ctx, &intr->src[2])[0];
92 offset = ir3_get_src(ctx, &intr->src[3])[0];
93
94 /* src0 is value, src1 is offset, src2 is uvec2(offset*4, 0)..
95 * nir already *= 4:
96 */
97 src0 = ir3_create_collect(b, ir3_get_src(ctx, &intr->src[0]), ncomp);
98 src1 = offset;
99 src2 = byte_offset_to_address(ctx, &intr->src[1], byte_offset);
100
101 stgb = ir3_STGB(b, ssbo, 0, src0, 0, src1, 0, src2, 0);
102 stgb->cat6.iim_val = ncomp;
103 stgb->cat6.d = 4;
104 stgb->cat6.type = TYPE_U32;
105 stgb->barrier_class = IR3_BARRIER_BUFFER_W;
106 stgb->barrier_conflict = IR3_BARRIER_BUFFER_R | IR3_BARRIER_BUFFER_W;
107
108 array_insert(b, b->keeps, stgb);
109 }
110
111 static struct ir3_instruction *
emit_atomic(struct ir3_block * b,nir_atomic_op op,struct ir3_instruction * bo,struct ir3_instruction * data,struct ir3_instruction * offset,struct ir3_instruction * byte_offset)112 emit_atomic(struct ir3_block *b,
113 nir_atomic_op op,
114 struct ir3_instruction *bo,
115 struct ir3_instruction *data,
116 struct ir3_instruction *offset,
117 struct ir3_instruction *byte_offset)
118 {
119 switch (op) {
120 case nir_atomic_op_iadd:
121 return ir3_ATOMIC_S_ADD(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
122 case nir_atomic_op_imin:
123 return ir3_ATOMIC_S_MIN(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
124 case nir_atomic_op_umin:
125 return ir3_ATOMIC_S_MIN(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
126 case nir_atomic_op_imax:
127 return ir3_ATOMIC_S_MAX(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
128 case nir_atomic_op_umax:
129 return ir3_ATOMIC_S_MAX(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
130 case nir_atomic_op_iand:
131 return ir3_ATOMIC_S_AND(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
132 case nir_atomic_op_ior:
133 return ir3_ATOMIC_S_OR(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
134 case nir_atomic_op_ixor:
135 return ir3_ATOMIC_S_XOR(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
136 case nir_atomic_op_xchg:
137 return ir3_ATOMIC_S_XCHG(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
138 case nir_atomic_op_cmpxchg:
139 return ir3_ATOMIC_S_CMPXCHG(b, bo, 0, data, 0, offset, 0, byte_offset, 0);
140 default:
141 unreachable("boo");
142 }
143 }
144
145 /*
146 * SSBO atomic intrinsics
147 *
148 * All of the SSBO atomic memory operations read a value from memory,
149 * compute a new value using one of the operations below, write the new
150 * value to memory, and return the original value read.
151 *
152 * All operations take 3 sources except CompSwap that takes 4. These
153 * sources represent:
154 *
155 * 0: The SSBO buffer index.
156 * 1: The byte offset into the SSBO buffer of the variable that the atomic
157 * operation will operate on.
158 * 2: The data parameter to the atomic function (i.e. the value to add
159 * in, etc).
160 * 3: CompSwap: the second data parameter.
161 * Non-CompSwap: The dword offset into the SSBO buffer variable.
162 * 4: CompSwap: The dword offset into the SSBO buffer variable.
163 *
164 * We use custom ssbo_*_ir3 intrinsics generated by ir3_nir_lower_io_offsets()
165 * so we can have the dword offset generated in NIR.
166 */
167 static struct ir3_instruction *
emit_intrinsic_atomic_ssbo(struct ir3_context * ctx,nir_intrinsic_instr * intr)168 emit_intrinsic_atomic_ssbo(struct ir3_context *ctx, nir_intrinsic_instr *intr)
169 {
170 struct ir3_block *b = ctx->block;
171 nir_atomic_op op = nir_intrinsic_atomic_op(intr);
172 type_t type = nir_atomic_op_type(op) == nir_type_int ? TYPE_S32 : TYPE_U32;
173
174 struct ir3_instruction *ssbo = ir3_ssbo_to_ibo(ctx, intr->src[0]);
175
176 struct ir3_instruction *data = ir3_get_src(ctx, &intr->src[2])[0];
177 /* 64b byte offset */
178 struct ir3_instruction *byte_offset =
179 byte_offset_to_address(ctx, &intr->src[0], ir3_get_src(ctx, &intr->src[1])[0]);
180 /* dword offset for everything but cmpxchg */
181 struct ir3_instruction *src3 = ir3_get_src(ctx, &intr->src[3])[0];
182
183 if (op == nir_atomic_op_cmpxchg) {
184 /* for cmpxchg, src0 is [ui]vec2(data, compare): */
185 data = ir3_collect(b, src3, data);
186 src3 = ir3_get_src(ctx, &intr->src[4])[0];
187 }
188
189 struct ir3_instruction *atomic =
190 emit_atomic(b, op, ssbo, data, src3, byte_offset);
191
192 atomic->cat6.iim_val = 1;
193 atomic->cat6.d = 4;
194 atomic->cat6.type = type;
195 atomic->barrier_class = IR3_BARRIER_BUFFER_W;
196 atomic->barrier_conflict = IR3_BARRIER_BUFFER_R | IR3_BARRIER_BUFFER_W;
197
198 /* even if nothing consume the result, we can't DCE the instruction: */
199 array_insert(b, b->keeps, atomic);
200
201 return atomic;
202 }
203
204 static struct ir3_instruction *
get_image_offset(struct ir3_context * ctx,const nir_intrinsic_instr * instr,struct ir3_instruction * const * coords,bool byteoff)205 get_image_offset(struct ir3_context *ctx, const nir_intrinsic_instr *instr,
206 struct ir3_instruction *const *coords, bool byteoff)
207 {
208 struct ir3_block *b = ctx->block;
209 struct ir3_instruction *offset;
210 unsigned index = nir_src_as_uint(instr->src[0]);
211 unsigned ncoords = ir3_get_image_coords(instr, NULL);
212
213 /* to calculate the byte offset (yes, uggg) we need (up to) three
214 * const values to know the bytes per pixel, and y and z stride:
215 */
216 unsigned cb;
217 if (ctx->compiler->gen > 4) {
218 const struct ir3_const_state *const_state = ir3_const_state(ctx->so);
219 assert(const_state->image_dims.mask & (1 << index));
220
221 cb = regid(const_state->offsets.image_dims, 0) +
222 const_state->image_dims.off[index];
223 } else {
224 index += ctx->s->info.num_ssbos;
225 cb = A4XX_SSBO_CB_BASE(index);
226 }
227
228 /* offset = coords.x * bytes_per_pixel: */
229 if (ctx->compiler->gen == 4)
230 offset = ir3_MUL_S24(b, coords[0], 0, create_uniform(b, cb + 3), 0);
231 else
232 offset = ir3_MUL_S24(b, coords[0], 0, create_uniform(b, cb + 0), 0);
233 if (ncoords > 1) {
234 /* offset += coords.y * y_pitch: */
235 offset =
236 ir3_MAD_S24(b, create_uniform(b, cb + 1), 0, coords[1], 0, offset, 0);
237 }
238 if (ncoords > 2) {
239 /* offset += coords.z * z_pitch: */
240 offset =
241 ir3_MAD_S24(b, create_uniform(b, cb + 2), 0, coords[2], 0, offset, 0);
242 }
243
244 /* a4xx: must add in the base address: */
245 if (ctx->compiler->gen == 4)
246 offset = ir3_ADD_U(b, offset, 0, create_uniform(b, cb + 0), 0);
247
248 if (!byteoff) {
249 /* Some cases, like atomics, seem to use dword offset instead
250 * of byte offsets.. blob just puts an extra shr.b in there
251 * in those cases:
252 */
253 offset = ir3_SHR_B(b, offset, 0, create_immed(b, 2), 0);
254 }
255
256 if (ctx->compiler->is_64bit)
257 return ir3_collect(b, offset, create_immed(b, 0));
258 else
259 return offset;
260 }
261
262 /* src[] = { deref, coord, sample_index }. const_index[] = {} */
263 static void
emit_intrinsic_load_image(struct ir3_context * ctx,nir_intrinsic_instr * intr,struct ir3_instruction ** dst)264 emit_intrinsic_load_image(struct ir3_context *ctx, nir_intrinsic_instr *intr,
265 struct ir3_instruction **dst)
266 {
267 struct ir3_block *b = ctx->block;
268 struct ir3_instruction *const *coords = ir3_get_src(ctx, &intr->src[1]);
269 struct ir3_instruction *ibo = ir3_image_to_ibo(ctx, intr->src[0]);
270 struct ir3_instruction *offset = get_image_offset(ctx, intr, coords, true);
271 unsigned ncoords = ir3_get_image_coords(intr, NULL);
272 unsigned ncomp =
273 ir3_get_num_components_for_image_format(nir_intrinsic_format(intr));
274
275 struct ir3_instruction *ldib;
276 /* At least A420 does not have LDIB. Use LDGB and perform conversion
277 * ourselves.
278 *
279 * TODO: Actually do the conversion. ES 3.1 only requires this for
280 * single-component 32-bit types anyways.
281 */
282 if (ctx->compiler->gen > 4) {
283 ldib = ir3_LDIB(
284 b, ibo, 0, offset, 0, ir3_create_collect(b, coords, ncoords), 0);
285 } else {
286 ldib = ir3_LDGB(
287 b, ibo, 0, offset, 0, ir3_create_collect(b, coords, ncoords), 0);
288 switch (nir_intrinsic_format(intr)) {
289 case PIPE_FORMAT_R32_UINT:
290 case PIPE_FORMAT_R32_SINT:
291 case PIPE_FORMAT_R32_FLOAT:
292 break;
293 default:
294 /* For some reason even more 32-bit components don't work. */
295 assert(0);
296 break;
297 }
298 }
299 ldib->dsts[0]->wrmask = MASK(intr->num_components);
300 ldib->cat6.iim_val = ncomp;
301 ldib->cat6.d = ncoords;
302 ldib->cat6.type = ir3_get_type_for_image_intrinsic(intr);
303 ldib->cat6.typed = true;
304 ldib->barrier_class = IR3_BARRIER_IMAGE_R;
305 ldib->barrier_conflict = IR3_BARRIER_IMAGE_W;
306
307 ir3_split_dest(b, dst, ldib, 0, intr->num_components);
308 }
309
310 /* src[] = { index, coord, sample_index, value }. const_index[] = {} */
311 static void
emit_intrinsic_store_image(struct ir3_context * ctx,nir_intrinsic_instr * intr)312 emit_intrinsic_store_image(struct ir3_context *ctx, nir_intrinsic_instr *intr)
313 {
314 struct ir3_block *b = ctx->block;
315 struct ir3_instruction *stib, *offset;
316 struct ir3_instruction *const *value = ir3_get_src(ctx, &intr->src[3]);
317 struct ir3_instruction *const *coords = ir3_get_src(ctx, &intr->src[1]);
318 struct ir3_instruction *ibo = ir3_image_to_ibo(ctx, intr->src[0]);
319 unsigned ncoords = ir3_get_image_coords(intr, NULL);
320 unsigned ncomp =
321 ir3_get_num_components_for_image_format(nir_intrinsic_format(intr));
322
323 /* src0 is value
324 * src1 is coords
325 * src2 is 64b byte offset
326 */
327
328 offset = get_image_offset(ctx, intr, coords, true);
329
330 /* NOTE: stib seems to take byte offset, but stgb.typed can be used
331 * too and takes a dword offset.. not quite sure yet why blob uses
332 * one over the other in various cases.
333 */
334
335 stib = ir3_STIB(b, ibo, 0, ir3_create_collect(b, value, ncomp), 0,
336 ir3_create_collect(b, coords, ncoords), 0, offset, 0);
337 stib->cat6.iim_val = ncomp;
338 stib->cat6.d = ncoords;
339 stib->cat6.type = ir3_get_type_for_image_intrinsic(intr);
340 stib->cat6.typed = true;
341 stib->barrier_class = IR3_BARRIER_IMAGE_W;
342 stib->barrier_conflict = IR3_BARRIER_IMAGE_R | IR3_BARRIER_IMAGE_W;
343
344 array_insert(b, b->keeps, stib);
345 }
346
347 /* src[] = { deref, coord, sample_index, value, compare }. const_index[] = {} */
348 static struct ir3_instruction *
emit_intrinsic_atomic_image(struct ir3_context * ctx,nir_intrinsic_instr * intr)349 emit_intrinsic_atomic_image(struct ir3_context *ctx, nir_intrinsic_instr *intr)
350 {
351 struct ir3_block *b = ctx->block;
352 struct ir3_instruction *atomic, *src0, *src1, *src2;
353 struct ir3_instruction *const *coords = ir3_get_src(ctx, &intr->src[1]);
354 struct ir3_instruction *image = ir3_image_to_ibo(ctx, intr->src[0]);
355 unsigned ncoords = ir3_get_image_coords(intr, NULL);
356 nir_atomic_op op = nir_intrinsic_atomic_op(intr);
357
358 /* src0 is value (or uvec2(value, compare))
359 * src1 is coords
360 * src2 is 64b byte offset
361 */
362 src0 = ir3_get_src(ctx, &intr->src[3])[0];
363 src1 = ir3_create_collect(b, coords, ncoords);
364 src2 = get_image_offset(ctx, intr, coords, ctx->compiler->gen == 4);
365
366 if (op == nir_atomic_op_cmpxchg)
367 src0 = ir3_collect(b, ir3_get_src(ctx, &intr->src[4])[0], src0);
368
369 atomic = emit_atomic(b, op, image, src0, src1, src2);
370 atomic->cat6.iim_val = 1;
371 atomic->cat6.d = ncoords;
372 atomic->cat6.type = ir3_get_type_for_image_intrinsic(intr);
373 atomic->cat6.typed = ctx->compiler->gen == 5;
374 atomic->barrier_class = IR3_BARRIER_IMAGE_W;
375 atomic->barrier_conflict = IR3_BARRIER_IMAGE_R | IR3_BARRIER_IMAGE_W;
376
377 /* even if nothing consume the result, we can't DCE the instruction: */
378 array_insert(b, b->keeps, atomic);
379
380 return atomic;
381 }
382
383 static struct ir3_instruction *
emit_intrinsic_atomic_global(struct ir3_context * ctx,nir_intrinsic_instr * intr)384 emit_intrinsic_atomic_global(struct ir3_context *ctx, nir_intrinsic_instr *intr)
385 {
386 unreachable("Global atomic are unimplemented on A5xx");
387 }
388
389 const struct ir3_context_funcs ir3_a4xx_funcs = {
390 .emit_intrinsic_load_ssbo = emit_intrinsic_load_ssbo,
391 .emit_intrinsic_store_ssbo = emit_intrinsic_store_ssbo,
392 .emit_intrinsic_atomic_ssbo = emit_intrinsic_atomic_ssbo,
393 .emit_intrinsic_load_image = emit_intrinsic_load_image,
394 .emit_intrinsic_store_image = emit_intrinsic_store_image,
395 .emit_intrinsic_atomic_image = emit_intrinsic_atomic_image,
396 .emit_intrinsic_image_size = emit_intrinsic_image_size_tex,
397 .emit_intrinsic_load_global_ir3 = NULL,
398 .emit_intrinsic_store_global_ir3 = NULL,
399 .emit_intrinsic_atomic_global = emit_intrinsic_atomic_global,
400 };
401