1 /*
2 * Copyright © 2014 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Connor Abbott ([email protected])
25 *
26 */
27
28 #include "compiler/nir/nir.h"
29 #include "compiler/nir/nir_builder.h"
30 #include "gl_nir.h"
31 #include "ir_uniform.h"
32 #include "main/config.h"
33 #include "main/shader_types.h"
34 #include <assert.h>
35
36 /*
37 * replace atomic counter intrinsics that use a variable with intrinsics
38 * that directly store the buffer index and byte offset
39 */
40
41 static bool
lower_deref_instr(nir_builder * b,nir_intrinsic_instr * instr,const struct gl_shader_program * shader_program,nir_shader * shader,bool use_binding_as_idx)42 lower_deref_instr(nir_builder *b, nir_intrinsic_instr *instr,
43 const struct gl_shader_program *shader_program,
44 nir_shader *shader, bool use_binding_as_idx)
45 {
46 nir_intrinsic_op op;
47 switch (instr->intrinsic) {
48 case nir_intrinsic_atomic_counter_read_deref:
49 op = nir_intrinsic_atomic_counter_read;
50 break;
51
52 case nir_intrinsic_atomic_counter_inc_deref:
53 op = nir_intrinsic_atomic_counter_inc;
54 break;
55
56 case nir_intrinsic_atomic_counter_pre_dec_deref:
57 op = nir_intrinsic_atomic_counter_pre_dec;
58 break;
59
60 case nir_intrinsic_atomic_counter_post_dec_deref:
61 op = nir_intrinsic_atomic_counter_post_dec;
62 break;
63
64 case nir_intrinsic_atomic_counter_add_deref:
65 op = nir_intrinsic_atomic_counter_add;
66 break;
67
68 case nir_intrinsic_atomic_counter_min_deref:
69 op = nir_intrinsic_atomic_counter_min;
70 break;
71
72 case nir_intrinsic_atomic_counter_max_deref:
73 op = nir_intrinsic_atomic_counter_max;
74 break;
75
76 case nir_intrinsic_atomic_counter_and_deref:
77 op = nir_intrinsic_atomic_counter_and;
78 break;
79
80 case nir_intrinsic_atomic_counter_or_deref:
81 op = nir_intrinsic_atomic_counter_or;
82 break;
83
84 case nir_intrinsic_atomic_counter_xor_deref:
85 op = nir_intrinsic_atomic_counter_xor;
86 break;
87
88 case nir_intrinsic_atomic_counter_exchange_deref:
89 op = nir_intrinsic_atomic_counter_exchange;
90 break;
91
92 case nir_intrinsic_atomic_counter_comp_swap_deref:
93 op = nir_intrinsic_atomic_counter_comp_swap;
94 break;
95
96 default:
97 return false;
98 }
99
100 nir_deref_instr *deref = nir_src_as_deref(instr->src[0]);
101 nir_variable *var = nir_deref_instr_get_variable(deref);
102
103 if (var->data.mode != nir_var_uniform &&
104 var->data.mode != nir_var_mem_ssbo &&
105 var->data.mode != nir_var_mem_shared)
106 return false; /* atomics passed as function arguments can't be lowered */
107
108 const unsigned uniform_loc = var->data.location;
109 const unsigned idx = use_binding_as_idx ? var->data.binding :
110 shader_program->data->UniformStorage[uniform_loc].opaque[shader->info.stage].index;
111
112 b->cursor = nir_before_instr(&instr->instr);
113
114 int offset_value = 0;
115 int range_base = 0;
116 if (!b->shader->options->lower_atomic_offset_to_range_base)
117 offset_value = var->data.offset;
118 else
119 range_base = var->data.offset;
120
121 nir_def *offset = nir_imm_int(b, offset_value);
122 for (nir_deref_instr *d = deref; d->deref_type != nir_deref_type_var;
123 d = nir_deref_instr_parent(d)) {
124 assert(d->deref_type == nir_deref_type_array);
125
126 unsigned array_stride = ATOMIC_COUNTER_SIZE;
127 if (glsl_type_is_array(d->type))
128 array_stride *= glsl_get_aoa_size(d->type);
129
130 offset = nir_iadd(b, offset, nir_imul(b, d->arr.index.ssa,
131 nir_imm_int(b, array_stride)));
132 }
133
134 /* Since the first source is a deref and the first source in the lowered
135 * instruction is the offset, we can just swap it out and change the
136 * opcode.
137 */
138 instr->intrinsic = op;
139 nir_intrinsic_set_range_base(instr, range_base);
140
141 nir_src_rewrite(&instr->src[0], offset);
142 nir_intrinsic_set_base(instr, idx);
143
144 nir_deref_instr_remove_if_unused(deref);
145
146 return true;
147 }
148
149 struct lower_atomics_data {
150 bool use_binding_as_idx;
151 nir_shader *shader;
152 const struct gl_shader_program *shader_program;
153 };
154
155 static bool
gl_nir_lower_atomics_instr(nir_builder * b,nir_instr * instr,void * cb_data)156 gl_nir_lower_atomics_instr(nir_builder *b, nir_instr *instr, void *cb_data)
157 {
158 if (instr->type != nir_instr_type_intrinsic)
159 return false;
160
161 struct lower_atomics_data *data = cb_data;
162
163 return lower_deref_instr(b,
164 nir_instr_as_intrinsic(instr),
165 data->shader_program,
166 data->shader,
167 data->use_binding_as_idx);
168 }
169
170 bool
gl_nir_lower_atomics(nir_shader * shader,const struct gl_shader_program * shader_program,bool use_binding_as_idx)171 gl_nir_lower_atomics(nir_shader *shader,
172 const struct gl_shader_program *shader_program,
173 bool use_binding_as_idx)
174 {
175 struct lower_atomics_data data = {
176 .use_binding_as_idx = use_binding_as_idx,
177 .shader = shader,
178 .shader_program = shader_program,
179 };
180
181 return nir_shader_instructions_pass(shader, gl_nir_lower_atomics_instr,
182 nir_metadata_control_flow,
183 &data);
184 }
185