1 /*
2 * Copyright © 2018 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "isl/isl.h"
25
26 #include "brw_nir.h"
27 #include "compiler/nir/nir_builder.h"
28 #include "compiler/nir/nir_format_convert.h"
29
30 struct format_info {
31 const struct isl_format_layout *fmtl;
32 unsigned chans;
33 unsigned bits[4];
34 };
35
36 static struct format_info
get_format_info(enum isl_format fmt)37 get_format_info(enum isl_format fmt)
38 {
39 const struct isl_format_layout *fmtl = isl_format_get_layout(fmt);
40
41 return (struct format_info) {
42 .fmtl = fmtl,
43 .chans = isl_format_get_num_channels(fmt),
44 .bits = {
45 fmtl->channels.r.bits,
46 fmtl->channels.g.bits,
47 fmtl->channels.b.bits,
48 fmtl->channels.a.bits
49 },
50 };
51 }
52
53 static nir_def *
convert_color_for_load(nir_builder * b,const struct intel_device_info * devinfo,nir_def * color,enum isl_format image_fmt,enum isl_format lower_fmt,unsigned dest_components)54 convert_color_for_load(nir_builder *b, const struct intel_device_info *devinfo,
55 nir_def *color,
56 enum isl_format image_fmt, enum isl_format lower_fmt,
57 unsigned dest_components)
58 {
59 if (image_fmt == lower_fmt)
60 goto expand_vec;
61
62 if (image_fmt == ISL_FORMAT_R11G11B10_FLOAT) {
63 assert(lower_fmt == ISL_FORMAT_R32_UINT);
64 color = nir_format_unpack_11f11f10f(b, color);
65 goto expand_vec;
66 }
67
68 struct format_info image = get_format_info(image_fmt);
69 struct format_info lower = get_format_info(lower_fmt);
70
71 const bool needs_sign_extension =
72 isl_format_has_snorm_channel(image_fmt) ||
73 isl_format_has_sint_channel(image_fmt);
74
75 /* We only check the red channel to detect if we need to pack/unpack */
76 assert(image.bits[0] != lower.bits[0] ||
77 memcmp(image.bits, lower.bits, sizeof(image.bits)) == 0);
78
79 if (image.bits[0] != lower.bits[0] && lower_fmt == ISL_FORMAT_R32_UINT) {
80 if (needs_sign_extension)
81 color = nir_format_unpack_sint(b, color, image.bits, image.chans);
82 else
83 color = nir_format_unpack_uint(b, color, image.bits, image.chans);
84 } else {
85 /* All these formats are homogeneous */
86 for (unsigned i = 1; i < image.chans; i++)
87 assert(image.bits[i] == image.bits[0]);
88
89 if (image.bits[0] != lower.bits[0]) {
90 color = nir_format_bitcast_uvec_unmasked(b, color, lower.bits[0],
91 image.bits[0]);
92 }
93
94 if (needs_sign_extension)
95 color = nir_format_sign_extend_ivec(b, color, image.bits);
96 }
97
98 switch (image.fmtl->channels.r.type) {
99 case ISL_UNORM:
100 assert(isl_format_has_uint_channel(lower_fmt));
101 color = nir_format_unorm_to_float(b, color, image.bits);
102 break;
103
104 case ISL_SNORM:
105 assert(isl_format_has_uint_channel(lower_fmt));
106 color = nir_format_snorm_to_float(b, color, image.bits);
107 break;
108
109 case ISL_SFLOAT:
110 if (image.bits[0] == 16)
111 color = nir_unpack_half_2x16_split_x(b, color);
112 break;
113
114 case ISL_UINT:
115 case ISL_SINT:
116 break;
117
118 default:
119 unreachable("Invalid image channel type");
120 }
121
122 expand_vec:
123 assert(dest_components == 1 || dest_components == 4);
124 assert(color->num_components <= dest_components);
125 if (color->num_components == dest_components)
126 return color;
127
128 nir_def *comps[4];
129 for (unsigned i = 0; i < color->num_components; i++)
130 comps[i] = nir_channel(b, color, i);
131
132 for (unsigned i = color->num_components; i < 3; i++)
133 comps[i] = nir_imm_int(b, 0);
134
135 if (color->num_components < 4) {
136 if (isl_format_has_int_channel(image_fmt))
137 comps[3] = nir_imm_int(b, 1);
138 else
139 comps[3] = nir_imm_float(b, 1);
140 }
141
142 return nir_vec(b, comps, dest_components);
143 }
144
145 static bool
lower_image_load_instr(nir_builder * b,const struct intel_device_info * devinfo,nir_intrinsic_instr * intrin,bool sparse)146 lower_image_load_instr(nir_builder *b,
147 const struct intel_device_info *devinfo,
148 nir_intrinsic_instr *intrin,
149 bool sparse)
150 {
151 nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
152 nir_variable *var = nir_deref_instr_get_variable(deref);
153
154 if (var->data.image.format == PIPE_FORMAT_NONE)
155 return false;
156
157 const enum isl_format image_fmt =
158 isl_format_for_pipe_format(var->data.image.format);
159
160 assert(isl_has_matching_typed_storage_image_format(devinfo, image_fmt));
161 const enum isl_format lower_fmt =
162 isl_lower_storage_image_format(devinfo, image_fmt);
163 const unsigned dest_components =
164 sparse ? (intrin->num_components - 1) : intrin->num_components;
165
166 /* Use an undef to hold the uses of the load while we do the color
167 * conversion.
168 */
169 nir_def *placeholder = nir_undef(b, 4, 32);
170 nir_def_rewrite_uses(&intrin->def, placeholder);
171
172 intrin->num_components = isl_format_get_num_channels(lower_fmt);
173 intrin->def.num_components = intrin->num_components;
174
175 b->cursor = nir_after_instr(&intrin->instr);
176
177 nir_def *color = convert_color_for_load(b, devinfo, &intrin->def, image_fmt, lower_fmt,
178 dest_components);
179
180 if (sparse) {
181 /* Put the sparse component back on the original instruction */
182 intrin->num_components++;
183 intrin->def.num_components = intrin->num_components;
184
185 /* Carry over the sparse component without modifying it with the
186 * converted color.
187 */
188 nir_def *sparse_color[NIR_MAX_VEC_COMPONENTS];
189 for (unsigned i = 0; i < dest_components; i++)
190 sparse_color[i] = nir_channel(b, color, i);
191 sparse_color[dest_components] =
192 nir_channel(b, &intrin->def, intrin->num_components - 1);
193 color = nir_vec(b, sparse_color, dest_components + 1);
194 }
195
196 nir_def_rewrite_uses(placeholder, color);
197 nir_instr_remove(placeholder->parent_instr);
198
199 return true;
200 }
201
202 static nir_def *
convert_color_for_store(nir_builder * b,const struct intel_device_info * devinfo,nir_def * color,enum isl_format image_fmt,enum isl_format lower_fmt)203 convert_color_for_store(nir_builder *b, const struct intel_device_info *devinfo,
204 nir_def *color,
205 enum isl_format image_fmt, enum isl_format lower_fmt)
206 {
207 struct format_info image = get_format_info(image_fmt);
208 struct format_info lower = get_format_info(lower_fmt);
209
210 color = nir_trim_vector(b, color, image.chans);
211
212 if (image_fmt == lower_fmt)
213 return color;
214
215 if (image_fmt == ISL_FORMAT_R11G11B10_FLOAT) {
216 assert(lower_fmt == ISL_FORMAT_R32_UINT);
217 return nir_format_pack_11f11f10f(b, color);
218 }
219
220 switch (image.fmtl->channels.r.type) {
221 case ISL_UNORM:
222 assert(isl_format_has_uint_channel(lower_fmt));
223 color = nir_format_float_to_unorm(b, color, image.bits);
224 break;
225
226 case ISL_SNORM:
227 assert(isl_format_has_uint_channel(lower_fmt));
228 color = nir_format_float_to_snorm(b, color, image.bits);
229 break;
230
231 case ISL_SFLOAT:
232 if (image.bits[0] == 16)
233 color = nir_format_float_to_half(b, color);
234 break;
235
236 case ISL_UINT:
237 color = nir_format_clamp_uint(b, color, image.bits);
238 break;
239
240 case ISL_SINT:
241 color = nir_format_clamp_sint(b, color, image.bits);
242 break;
243
244 default:
245 unreachable("Invalid image channel type");
246 }
247
248 if (image.bits[0] < 32 &&
249 (isl_format_has_snorm_channel(image_fmt) ||
250 isl_format_has_sint_channel(image_fmt)))
251 color = nir_format_mask_uvec(b, color, image.bits);
252
253 if (image.bits[0] != lower.bits[0] && lower_fmt == ISL_FORMAT_R32_UINT) {
254 color = nir_format_pack_uint(b, color, image.bits, image.chans);
255 } else {
256 /* All these formats are homogeneous */
257 for (unsigned i = 1; i < image.chans; i++)
258 assert(image.bits[i] == image.bits[0]);
259
260 if (image.bits[0] != lower.bits[0]) {
261 color = nir_format_bitcast_uvec_unmasked(b, color, image.bits[0],
262 lower.bits[0]);
263 }
264 }
265
266 return color;
267 }
268
269 static bool
lower_image_store_instr(nir_builder * b,const struct intel_device_info * devinfo,nir_intrinsic_instr * intrin)270 lower_image_store_instr(nir_builder *b,
271 const struct intel_device_info *devinfo,
272 nir_intrinsic_instr *intrin)
273 {
274 nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
275 nir_variable *var = nir_deref_instr_get_variable(deref);
276
277 /* For write-only surfaces, we trust that the hardware can just do the
278 * conversion for us.
279 */
280 if (var->data.access & ACCESS_NON_READABLE)
281 return false;
282
283 if (var->data.image.format == PIPE_FORMAT_NONE)
284 return false;
285
286 const enum isl_format image_fmt =
287 isl_format_for_pipe_format(var->data.image.format);
288
289 assert(isl_has_matching_typed_storage_image_format(devinfo, image_fmt));
290 const enum isl_format lower_fmt =
291 isl_lower_storage_image_format(devinfo, image_fmt);
292
293 /* Color conversion goes before the store */
294 b->cursor = nir_before_instr(&intrin->instr);
295
296 nir_def *color = convert_color_for_store(b, devinfo,
297 intrin->src[3].ssa,
298 image_fmt, lower_fmt);
299 intrin->num_components = isl_format_get_num_channels(lower_fmt);
300 nir_src_rewrite(&intrin->src[3], color);
301
302 return true;
303 }
304
305 static bool
brw_nir_lower_storage_image_instr(nir_builder * b,nir_instr * instr,void * cb_data)306 brw_nir_lower_storage_image_instr(nir_builder *b,
307 nir_instr *instr,
308 void *cb_data)
309 {
310 if (instr->type != nir_instr_type_intrinsic)
311 return false;
312 const struct brw_nir_lower_storage_image_opts *opts = cb_data;
313
314 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
315 switch (intrin->intrinsic) {
316 case nir_intrinsic_image_deref_load:
317 if (opts->lower_loads)
318 return lower_image_load_instr(b, opts->devinfo, intrin, false);
319 return false;
320
321 case nir_intrinsic_image_deref_sparse_load:
322 if (opts->lower_loads)
323 return lower_image_load_instr(b, opts->devinfo, intrin, true);
324 return false;
325
326 case nir_intrinsic_image_deref_store:
327 if (opts->lower_stores)
328 return lower_image_store_instr(b, opts->devinfo, intrin);
329 return false;
330
331 default:
332 /* Nothing to do */
333 return false;
334 }
335 }
336
337 bool
brw_nir_lower_storage_image(nir_shader * shader,const struct brw_nir_lower_storage_image_opts * opts)338 brw_nir_lower_storage_image(nir_shader *shader,
339 const struct brw_nir_lower_storage_image_opts *opts)
340 {
341 bool progress = false;
342
343 const nir_lower_image_options image_options = {
344 .lower_cube_size = true,
345 .lower_image_samples_to_one = true,
346 };
347
348 progress |= nir_lower_image(shader, &image_options);
349
350 progress |= nir_shader_instructions_pass(shader,
351 brw_nir_lower_storage_image_instr,
352 nir_metadata_none,
353 (void *)opts);
354
355 return progress;
356 }
357