1 /*
2 * Copyright © 2019 Valve Corporation
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 #include "aco_builder.h"
8 #include "aco_ir.h"
9
10 #include "util/u_math.h"
11
12 #include <set>
13 #include <vector>
14
15 namespace aco {
16
17 namespace {
18
19 enum WQMState : uint8_t {
20 Unspecified = 0,
21 Exact = 1 << 0,
22 WQM = 1 << 1, /* with control flow applied */
23 };
24
25 enum mask_type : uint8_t {
26 mask_type_global = 1 << 0,
27 mask_type_exact = 1 << 1,
28 mask_type_wqm = 1 << 2,
29 mask_type_loop = 1 << 3, /* active lanes of a loop */
30 };
31
32 struct loop_info {
33 Block* loop_header;
34 uint16_t num_exec_masks;
35 bool has_divergent_break;
36 bool has_divergent_continue;
37 bool has_discard; /* has a discard or demote */
loop_infoaco::__anon9ac258bc0111::loop_info38 loop_info(Block* b, uint16_t num, bool breaks, bool cont, bool discard)
39 : loop_header(b), num_exec_masks(num), has_divergent_break(breaks),
40 has_divergent_continue(cont), has_discard(discard)
41 {}
42 };
43
44 struct block_info {
45 std::vector<std::pair<Operand, uint8_t>>
46 exec; /* Vector of exec masks. Either a temporary or const -1. */
47 };
48
49 struct exec_ctx {
50 Program* program;
51 std::vector<block_info> info;
52 std::vector<loop_info> loop;
53 bool handle_wqm = false;
exec_ctxaco::__anon9ac258bc0111::exec_ctx54 exec_ctx(Program* program_) : program(program_), info(program->blocks.size()) {}
55 };
56
57 bool
needs_exact(aco_ptr<Instruction> & instr)58 needs_exact(aco_ptr<Instruction>& instr)
59 {
60 if (instr->isMUBUF()) {
61 return instr->mubuf().disable_wqm;
62 } else if (instr->isMTBUF()) {
63 return instr->mtbuf().disable_wqm;
64 } else if (instr->isMIMG()) {
65 return instr->mimg().disable_wqm;
66 } else if (instr->isFlatLike()) {
67 return instr->flatlike().disable_wqm;
68 } else {
69 /* Require Exact for p_jump_to_epilog because if p_exit_early_if is
70 * emitted inside the same block, the main FS will always jump to the PS
71 * epilog without considering the exec mask.
72 */
73 return instr->isEXP() || instr->opcode == aco_opcode::p_jump_to_epilog ||
74 instr->opcode == aco_opcode::p_dual_src_export_gfx11;
75 }
76 }
77
78 WQMState
get_instr_needs(aco_ptr<Instruction> & instr)79 get_instr_needs(aco_ptr<Instruction>& instr)
80 {
81 if (needs_exact(instr))
82 return Exact;
83
84 bool pred_by_exec = needs_exec_mask(instr.get()) || instr->opcode == aco_opcode::p_logical_end ||
85 instr->isBranch();
86
87 return pred_by_exec ? WQM : Unspecified;
88 }
89
90 Operand
get_exec_op(Operand t)91 get_exec_op(Operand t)
92 {
93 if (t.isUndefined())
94 return Operand(exec, t.regClass());
95 else
96 return t;
97 }
98
99 void
transition_to_WQM(exec_ctx & ctx,Builder bld,unsigned idx)100 transition_to_WQM(exec_ctx& ctx, Builder bld, unsigned idx)
101 {
102 if (ctx.info[idx].exec.back().second & mask_type_wqm)
103 return;
104 if (ctx.info[idx].exec.back().second & mask_type_global) {
105 Operand exec_mask = ctx.info[idx].exec.back().first;
106 if (exec_mask.isUndefined())
107 ctx.info[idx].exec.back().first = bld.copy(bld.def(bld.lm), Operand(exec, bld.lm));
108
109 exec_mask = bld.sop1(Builder::s_wqm, Definition(exec, bld.lm), bld.def(s1, scc),
110 get_exec_op(exec_mask));
111 ctx.info[idx].exec.emplace_back(exec_mask, mask_type_global | mask_type_wqm);
112 return;
113 }
114 /* otherwise, the WQM mask should be one below the current mask */
115 ctx.info[idx].exec.pop_back();
116 assert(ctx.info[idx].exec.back().second & mask_type_wqm);
117 assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
118 assert(ctx.info[idx].exec.back().first.isTemp());
119 ctx.info[idx].exec.back().first =
120 bld.copy(Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
121 }
122
123 void
transition_to_Exact(exec_ctx & ctx,Builder bld,unsigned idx)124 transition_to_Exact(exec_ctx& ctx, Builder bld, unsigned idx)
125 {
126 if (ctx.info[idx].exec.back().second & mask_type_exact)
127 return;
128 /* We can't remove the loop exec mask, because that can cause exec.size() to
129 * be less than num_exec_masks. The loop exec mask also needs to be kept
130 * around for various uses. */
131 if ((ctx.info[idx].exec.back().second & mask_type_global) &&
132 !(ctx.info[idx].exec.back().second & mask_type_loop)) {
133 ctx.info[idx].exec.pop_back();
134 assert(ctx.info[idx].exec.back().second & mask_type_exact);
135 assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
136 assert(ctx.info[idx].exec.back().first.isTemp());
137 ctx.info[idx].exec.back().first =
138 bld.copy(Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
139 return;
140 }
141 /* otherwise, we create an exact mask and push to the stack */
142 Operand wqm = ctx.info[idx].exec.back().first;
143 if (wqm.isUndefined()) {
144 wqm = bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.def(s1, scc),
145 Definition(exec, bld.lm), ctx.info[idx].exec[0].first, Operand(exec, bld.lm));
146 } else {
147 bld.sop2(Builder::s_and, Definition(exec, bld.lm), bld.def(s1, scc),
148 ctx.info[idx].exec[0].first, wqm);
149 }
150 ctx.info[idx].exec.back().first = Operand(wqm);
151 ctx.info[idx].exec.emplace_back(Operand(bld.lm), mask_type_exact);
152 }
153
154 unsigned
add_coupling_code(exec_ctx & ctx,Block * block,std::vector<aco_ptr<Instruction>> & instructions)155 add_coupling_code(exec_ctx& ctx, Block* block, std::vector<aco_ptr<Instruction>>& instructions)
156 {
157 unsigned idx = block->index;
158 Builder bld(ctx.program, &instructions);
159 Block::edge_vec& preds = block->linear_preds;
160 bool restore_exec = false;
161
162 /* start block */
163 if (preds.empty()) {
164 aco_ptr<Instruction>& startpgm = block->instructions[0];
165 assert(startpgm->opcode == aco_opcode::p_startpgm);
166 bld.insert(std::move(startpgm));
167
168 unsigned count = 1;
169 while (block->instructions[count]->opcode == aco_opcode::p_init_scratch ||
170 block->instructions[count]->opcode == aco_opcode::s_setprio) {
171 bld.insert(std::move(block->instructions[count]));
172 count++;
173 }
174
175 Operand start_exec(bld.lm);
176
177 /* exec seems to need to be manually initialized with combined shaders */
178 if (ctx.program->stage.num_sw_stages() > 1 ||
179 ctx.program->stage.hw == AC_HW_NEXT_GEN_GEOMETRY_SHADER ||
180 (ctx.program->stage.sw == SWStage::VS &&
181 (ctx.program->stage.hw == AC_HW_HULL_SHADER ||
182 ctx.program->stage.hw == AC_HW_LEGACY_GEOMETRY_SHADER)) ||
183 (ctx.program->stage.sw == SWStage::TES &&
184 ctx.program->stage.hw == AC_HW_LEGACY_GEOMETRY_SHADER)) {
185 start_exec = Operand::c32_or_c64(-1u, bld.lm == s2);
186 bld.copy(Definition(exec, bld.lm), start_exec);
187 }
188
189 /* EXEC is automatically initialized by the HW for compute shaders.
190 * We know for sure exec is initially -1 when the shader always has full subgroups.
191 */
192 if (ctx.program->stage == compute_cs && ctx.program->info.cs.uses_full_subgroups)
193 start_exec = Operand::c32_or_c64(-1u, bld.lm == s2);
194
195 if (ctx.handle_wqm) {
196 ctx.info[idx].exec.emplace_back(start_exec, mask_type_global | mask_type_exact);
197 /* Initialize WQM already */
198 transition_to_WQM(ctx, bld, idx);
199 } else {
200 uint8_t mask = mask_type_global;
201 if (ctx.program->needs_wqm) {
202 bld.sop1(Builder::s_wqm, Definition(exec, bld.lm), bld.def(s1, scc),
203 Operand(exec, bld.lm));
204 mask |= mask_type_wqm;
205 } else {
206 mask |= mask_type_exact;
207 }
208 ctx.info[idx].exec.emplace_back(start_exec, mask);
209 }
210
211 return count;
212 }
213
214 /* loop entry block */
215 if (block->kind & block_kind_loop_header) {
216 assert(preds[0] == idx - 1);
217 ctx.info[idx].exec = ctx.info[idx - 1].exec;
218 loop_info& info = ctx.loop.back();
219 assert(ctx.info[idx].exec.size() == info.num_exec_masks);
220
221 /* create ssa names for outer exec masks */
222 if (info.has_discard && preds.size() > 1) {
223 aco_ptr<Instruction> phi;
224 for (int i = 0; i < info.num_exec_masks - 1; i++) {
225 phi.reset(
226 create_instruction(aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1));
227 phi->definitions[0] = bld.def(bld.lm);
228 phi->operands[0] = get_exec_op(ctx.info[preds[0]].exec[i].first);
229 ctx.info[idx].exec[i].first = bld.insert(std::move(phi));
230 }
231 }
232
233 ctx.info[idx].exec.back().second |= mask_type_loop;
234
235 if (info.has_divergent_continue) {
236 /* create ssa name for loop active mask */
237 aco_ptr<Instruction> phi{
238 create_instruction(aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
239 phi->definitions[0] = bld.def(bld.lm);
240 phi->operands[0] = get_exec_op(ctx.info[preds[0]].exec.back().first);
241 ctx.info[idx].exec.back().first = bld.insert(std::move(phi));
242
243 restore_exec = true;
244 uint8_t mask_type = ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact);
245 ctx.info[idx].exec.emplace_back(ctx.info[idx].exec.back().first, mask_type);
246 }
247
248 } else if (block->kind & block_kind_loop_exit) {
249 Block* header = ctx.loop.back().loop_header;
250 loop_info& info = ctx.loop.back();
251
252 for (ASSERTED unsigned pred : preds)
253 assert(ctx.info[pred].exec.size() >= info.num_exec_masks);
254
255 /* fill the loop header phis */
256 Block::edge_vec& header_preds = header->linear_preds;
257 int instr_idx = 0;
258 if (info.has_discard && header_preds.size() > 1) {
259 while (instr_idx < info.num_exec_masks - 1) {
260 aco_ptr<Instruction>& phi = header->instructions[instr_idx];
261 assert(phi->opcode == aco_opcode::p_linear_phi);
262 for (unsigned i = 1; i < phi->operands.size(); i++)
263 phi->operands[i] = get_exec_op(ctx.info[header_preds[i]].exec[instr_idx].first);
264 instr_idx++;
265 }
266 }
267
268 if (info.has_divergent_continue) {
269 aco_ptr<Instruction>& phi = header->instructions[instr_idx++];
270 assert(phi->opcode == aco_opcode::p_linear_phi);
271 for (unsigned i = 1; i < phi->operands.size(); i++)
272 phi->operands[i] =
273 get_exec_op(ctx.info[header_preds[i]].exec[info.num_exec_masks - 1].first);
274 restore_exec = true;
275 }
276
277 if (info.has_divergent_break) {
278 restore_exec = true;
279 /* Drop the loop active mask. */
280 info.num_exec_masks--;
281 }
282 assert(!(block->kind & block_kind_top_level) || info.num_exec_masks <= 2);
283
284 /* create the loop exit phis if not trivial */
285 for (unsigned exec_idx = 0; exec_idx < info.num_exec_masks; exec_idx++) {
286 Operand same = ctx.info[preds[0]].exec[exec_idx].first;
287 uint8_t type = ctx.info[header_preds[0]].exec[exec_idx].second;
288 bool trivial = true;
289
290 for (unsigned i = 1; i < preds.size() && trivial; i++) {
291 if (ctx.info[preds[i]].exec[exec_idx].first != same)
292 trivial = false;
293 }
294
295 if (trivial) {
296 ctx.info[idx].exec.emplace_back(same, type);
297 } else {
298 /* create phi for loop footer */
299 aco_ptr<Instruction> phi{
300 create_instruction(aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
301 phi->definitions[0] = bld.def(bld.lm);
302 for (unsigned i = 0; i < phi->operands.size(); i++)
303 phi->operands[i] = get_exec_op(ctx.info[preds[i]].exec[exec_idx].first);
304 ctx.info[idx].exec.emplace_back(bld.insert(std::move(phi)), type);
305 }
306 }
307
308 assert(ctx.info[idx].exec.size() == info.num_exec_masks);
309 ctx.loop.pop_back();
310
311 } else if (preds.size() == 1) {
312 ctx.info[idx].exec = ctx.info[preds[0]].exec;
313 } else {
314 assert(preds.size() == 2);
315 /* if one of the predecessors ends in exact mask, we pop it from stack */
316 unsigned num_exec_masks =
317 std::min(ctx.info[preds[0]].exec.size(), ctx.info[preds[1]].exec.size());
318
319 if (block->kind & block_kind_merge) {
320 restore_exec = true;
321 num_exec_masks--;
322 }
323 if (block->kind & block_kind_top_level)
324 num_exec_masks = std::min(num_exec_masks, 2u);
325
326 /* create phis for diverged exec masks */
327 for (unsigned i = 0; i < num_exec_masks; i++) {
328 /* skip trivial phis */
329 if (ctx.info[preds[0]].exec[i].first == ctx.info[preds[1]].exec[i].first) {
330 Operand t = ctx.info[preds[0]].exec[i].first;
331 /* discard/demote can change the state of the current exec mask */
332 assert(!t.isTemp() ||
333 ctx.info[preds[0]].exec[i].second == ctx.info[preds[1]].exec[i].second);
334 uint8_t mask = ctx.info[preds[0]].exec[i].second & ctx.info[preds[1]].exec[i].second;
335 ctx.info[idx].exec.emplace_back(t, mask);
336 continue;
337 }
338
339 Temp phi = bld.pseudo(aco_opcode::p_linear_phi, bld.def(bld.lm),
340 get_exec_op(ctx.info[preds[0]].exec[i].first),
341 get_exec_op(ctx.info[preds[1]].exec[i].first));
342 uint8_t mask_type = ctx.info[preds[0]].exec[i].second & ctx.info[preds[1]].exec[i].second;
343 ctx.info[idx].exec.emplace_back(phi, mask_type);
344 }
345 }
346
347 unsigned i = 0;
348 while (block->instructions[i]->opcode == aco_opcode::p_phi ||
349 block->instructions[i]->opcode == aco_opcode::p_linear_phi) {
350 bld.insert(std::move(block->instructions[i]));
351 i++;
352 }
353
354 if (ctx.handle_wqm) {
355 /* End WQM handling if not needed anymore */
356 if (block->kind & block_kind_top_level && ctx.info[idx].exec.size() == 2) {
357 if (block->instructions[i]->opcode == aco_opcode::p_end_wqm) {
358 ctx.info[idx].exec.back().second |= mask_type_global;
359 transition_to_Exact(ctx, bld, idx);
360 ctx.handle_wqm = false;
361 restore_exec = false;
362 i++;
363 }
364 }
365 }
366
367 /* restore exec mask after divergent control flow */
368 if (restore_exec) {
369 Operand restore = get_exec_op(ctx.info[idx].exec.back().first);
370 assert(restore.size() == bld.lm.size());
371 bld.copy(Definition(exec, bld.lm), restore);
372 if (!restore.isConstant())
373 ctx.info[idx].exec.back().first = Operand(bld.lm);
374 }
375
376 return i;
377 }
378
379 /* Avoid live-range splits in Exact mode:
380 * Because the data register of atomic VMEM instructions
381 * is shared between src and dst, it might be necessary
382 * to create live-range splits during RA.
383 * Make the live-range splits explicit in WQM mode.
384 */
385 void
handle_atomic_data(exec_ctx & ctx,Builder & bld,unsigned block_idx,aco_ptr<Instruction> & instr)386 handle_atomic_data(exec_ctx& ctx, Builder& bld, unsigned block_idx, aco_ptr<Instruction>& instr)
387 {
388 /* check if this is an atomic VMEM instruction */
389 int idx = -1;
390 if (!instr->isVMEM() || instr->definitions.empty())
391 return;
392 else if (instr->isMIMG())
393 idx = instr->operands[2].isTemp() ? 2 : -1;
394 else if (instr->operands.size() == 4)
395 idx = 3;
396
397 if (idx != -1) {
398 /* insert explicit copy of atomic data in WQM-mode */
399 transition_to_WQM(ctx, bld, block_idx);
400 Temp data = instr->operands[idx].getTemp();
401 data = bld.copy(bld.def(data.regClass()), data);
402 instr->operands[idx].setTemp(data);
403 }
404 }
405
406 void
process_instructions(exec_ctx & ctx,Block * block,std::vector<aco_ptr<Instruction>> & instructions,unsigned idx)407 process_instructions(exec_ctx& ctx, Block* block, std::vector<aco_ptr<Instruction>>& instructions,
408 unsigned idx)
409 {
410 block_info& info = ctx.info[block->index];
411 WQMState state;
412 if (info.exec.back().second & mask_type_wqm) {
413 state = WQM;
414 } else {
415 assert(!ctx.handle_wqm || info.exec.back().second & mask_type_exact);
416 state = Exact;
417 }
418
419 Builder bld(ctx.program, &instructions);
420
421 for (; idx < block->instructions.size(); idx++) {
422 aco_ptr<Instruction> instr = std::move(block->instructions[idx]);
423
424 WQMState needs = ctx.handle_wqm ? get_instr_needs(instr) : Unspecified;
425
426 if (needs == WQM && state != WQM) {
427 transition_to_WQM(ctx, bld, block->index);
428 state = WQM;
429 } else if (needs == Exact) {
430 if (ctx.handle_wqm)
431 handle_atomic_data(ctx, bld, block->index, instr);
432 transition_to_Exact(ctx, bld, block->index);
433 state = Exact;
434 }
435
436 if (instr->opcode == aco_opcode::p_discard_if) {
437 Operand current_exec = Operand(exec, bld.lm);
438
439 if (block->instructions[idx + 1]->opcode == aco_opcode::p_end_wqm) {
440 /* Transition to Exact without extra instruction. */
441 info.exec.resize(1);
442 assert(info.exec[0].second == (mask_type_exact | mask_type_global));
443 current_exec = get_exec_op(info.exec[0].first);
444 info.exec[0].first = Operand(bld.lm);
445 state = Exact;
446 } else if (info.exec.size() >= 2 && ctx.handle_wqm) {
447 /* Preserve the WQM mask */
448 info.exec[1].second &= ~mask_type_global;
449 }
450
451 Temp cond, exit_cond;
452 if (instr->operands[0].isConstant()) {
453 assert(instr->operands[0].constantValue() == -1u);
454 /* save condition and set exec to zero */
455 exit_cond = bld.tmp(s1);
456 cond =
457 bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.scc(Definition(exit_cond)),
458 Definition(exec, bld.lm), Operand::zero(), Operand(exec, bld.lm));
459 } else {
460 cond = instr->operands[0].getTemp();
461 /* discard from current exec */
462 exit_cond = bld.sop2(Builder::s_andn2, Definition(exec, bld.lm), bld.def(s1, scc),
463 current_exec, cond)
464 .def(1)
465 .getTemp();
466 }
467
468 /* discard from inner to outer exec mask on stack */
469 int num = info.exec.size() - 2;
470 for (int i = num; i >= 0; i--) {
471 Instruction* andn2 = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc),
472 info.exec[i].first, cond);
473 info.exec[i].first = Operand(andn2->definitions[0].getTemp());
474 exit_cond = andn2->definitions[1].getTemp();
475 }
476
477 instr->opcode = aco_opcode::p_exit_early_if;
478 instr->operands[0] = bld.scc(exit_cond);
479 assert(!ctx.handle_wqm || (info.exec[0].second & mask_type_wqm) == 0);
480
481 } else if (instr->opcode == aco_opcode::p_is_helper) {
482 Definition dst = instr->definitions[0];
483 assert(dst.size() == bld.lm.size());
484 if (state == Exact) {
485 instr.reset(create_instruction(bld.w64or32(Builder::s_mov), Format::SOP1, 1, 1));
486 instr->operands[0] = Operand::zero();
487 instr->definitions[0] = dst;
488 } else {
489 std::pair<Operand, uint8_t>& exact_mask = info.exec[0];
490 assert(exact_mask.second & mask_type_exact);
491
492 instr.reset(create_instruction(bld.w64or32(Builder::s_andn2), Format::SOP2, 2, 2));
493 instr->operands[0] = Operand(exec, bld.lm); /* current exec */
494 instr->operands[1] = Operand(exact_mask.first);
495 instr->definitions[0] = dst;
496 instr->definitions[1] = bld.def(s1, scc);
497 }
498 } else if (instr->opcode == aco_opcode::p_demote_to_helper) {
499 assert((info.exec[0].second & mask_type_exact) &&
500 (info.exec[0].second & mask_type_global));
501
502 const bool nested_cf = !(info.exec.back().second & mask_type_global);
503 if (ctx.handle_wqm && state == Exact && nested_cf) {
504 /* Transition back to WQM without extra instruction. */
505 info.exec.pop_back();
506 state = WQM;
507 } else if (block->instructions[idx + 1]->opcode == aco_opcode::p_end_wqm) {
508 /* Transition to Exact without extra instruction. */
509 info.exec.resize(1);
510 state = Exact;
511 } else if (nested_cf) {
512 /* Save curent exec temporarily. */
513 info.exec.back().first = bld.copy(bld.def(bld.lm), Operand(exec, bld.lm));
514 }
515
516 /* Remove invocations from global exact mask. */
517 Definition def = state == Exact ? Definition(exec, bld.lm) : bld.def(bld.lm);
518 Operand src = instr->operands[0].isConstant() ? Operand(exec, bld.lm) : instr->operands[0];
519
520 Definition exit_cond =
521 bld.sop2(Builder::s_andn2, def, bld.def(s1, scc), get_exec_op(info.exec[0].first), src)
522 .def(1);
523 info.exec[0].first = Operand(def.getTemp());
524
525 /* Update global WQM mask and store in exec. */
526 if (state == WQM) {
527 assert(info.exec.size() > 1);
528 exit_cond =
529 bld.sop1(Builder::s_wqm, Definition(exec, bld.lm), bld.def(s1, scc), def.getTemp())
530 .def(1);
531 }
532
533 /* End shader if global mask is zero. */
534 instr->opcode = aco_opcode::p_exit_early_if;
535 instr->operands[0] = bld.scc(exit_cond.getTemp());
536 bld.insert(std::move(instr));
537
538 /* Update all other exec masks. */
539 if (nested_cf) {
540 const unsigned global_idx = state == WQM ? 1 : 0;
541 for (unsigned i = global_idx + 1; i < info.exec.size() - 1; i++) {
542 info.exec[i].first =
543 bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc),
544 get_exec_op(info.exec[i].first), Operand(exec, bld.lm));
545 }
546 /* Update current exec and save WQM mask. */
547 info.exec[global_idx].first =
548 bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.def(s1, scc),
549 Definition(exec, bld.lm), info.exec.back().first, Operand(exec, bld.lm));
550 info.exec.back().first = Operand(bld.lm);
551 }
552 continue;
553
554 } else if (instr->opcode == aco_opcode::p_elect) {
555 bool all_lanes_enabled = info.exec.back().first.constantEquals(-1u);
556 Definition dst = instr->definitions[0];
557
558 if (all_lanes_enabled) {
559 bld.copy(Definition(dst), Operand::c32_or_c64(1u, dst.size() == 2));
560 } else {
561 Temp first_lane_idx = bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm));
562 bld.sop2(Builder::s_lshl, Definition(dst), bld.def(s1, scc),
563 Operand::c32_or_c64(1u, dst.size() == 2), Operand(first_lane_idx));
564 }
565 continue;
566 } else if (instr->opcode == aco_opcode::p_end_wqm) {
567 assert(block->kind & block_kind_top_level);
568 assert(info.exec.size() <= 2);
569 /* This instruction indicates the end of WQM mode. */
570 info.exec.back().second |= mask_type_global;
571 transition_to_Exact(ctx, bld, block->index);
572 state = Exact;
573 ctx.handle_wqm = false;
574 continue;
575 }
576
577 bld.insert(std::move(instr));
578 }
579 }
580
581 void
add_branch_code(exec_ctx & ctx,Block * block)582 add_branch_code(exec_ctx& ctx, Block* block)
583 {
584 unsigned idx = block->index;
585 Builder bld(ctx.program, block);
586
587 if (block->linear_succs.empty())
588 return;
589
590 if (block->kind & block_kind_loop_preheader) {
591 /* collect information about the succeeding loop */
592 bool has_divergent_break = false;
593 bool has_divergent_continue = false;
594 bool has_discard = false;
595 unsigned loop_nest_depth = ctx.program->blocks[idx + 1].loop_nest_depth;
596
597 for (unsigned i = idx + 1; ctx.program->blocks[i].loop_nest_depth >= loop_nest_depth; i++) {
598 Block& loop_block = ctx.program->blocks[i];
599
600 if (loop_block.kind & block_kind_uses_discard)
601 has_discard = true;
602 if (loop_block.loop_nest_depth != loop_nest_depth)
603 continue;
604
605 if (loop_block.kind & block_kind_uniform)
606 continue;
607 else if (loop_block.kind & block_kind_break)
608 has_divergent_break = true;
609 else if (loop_block.kind & block_kind_continue)
610 has_divergent_continue = true;
611 }
612
613 if (has_divergent_break) {
614 /* save restore exec mask */
615 uint8_t mask = ctx.info[idx].exec.back().second;
616 if (ctx.info[idx].exec.back().first.constantEquals(-1u)) {
617 ctx.info[idx].exec.emplace_back(Operand(exec, bld.lm), mask);
618 } else {
619 bld.reset(bld.instructions, std::prev(bld.instructions->end()));
620 Operand restore = bld.copy(bld.def(bld.lm), Operand(exec, bld.lm));
621 ctx.info[idx].exec.emplace(std::prev(ctx.info[idx].exec.end()), restore, mask);
622 bld.reset(bld.instructions);
623 }
624 ctx.info[idx].exec.back().second &= (mask_type_wqm | mask_type_exact);
625 }
626 unsigned num_exec_masks = ctx.info[idx].exec.size();
627
628 ctx.loop.emplace_back(&ctx.program->blocks[block->linear_succs[0]], num_exec_masks,
629 has_divergent_break, has_divergent_continue, has_discard);
630 }
631
632 /* For normal breaks, this is the exec mask. For discard+break, it's the
633 * old exec mask before it was zero'd.
634 */
635 Operand break_cond = Operand(exec, bld.lm);
636
637 if (block->kind & block_kind_continue_or_break) {
638 assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[1]].linear_succs[0]].kind &
639 block_kind_loop_header);
640 assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[0]].linear_succs[0]].kind &
641 block_kind_loop_exit);
642 assert(block->instructions.back()->opcode == aco_opcode::p_branch);
643 block->instructions.pop_back();
644
645 bool need_parallelcopy = false;
646 while (!(ctx.info[idx].exec.back().second & mask_type_loop)) {
647 ctx.info[idx].exec.pop_back();
648 need_parallelcopy = true;
649 }
650
651 if (need_parallelcopy)
652 ctx.info[idx].exec.back().first =
653 bld.copy(Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
654 bld.branch(aco_opcode::p_cbranch_nz, bld.def(s2), Operand(exec, bld.lm),
655 block->linear_succs[1], block->linear_succs[0]);
656 return;
657 }
658
659 if (block->kind & block_kind_uniform) {
660 Pseudo_branch_instruction& branch = block->instructions.back()->branch();
661 if (branch.opcode == aco_opcode::p_branch) {
662 branch.target[0] = block->linear_succs[0];
663 } else {
664 branch.target[0] = block->linear_succs[1];
665 branch.target[1] = block->linear_succs[0];
666 }
667 return;
668 }
669
670 if (block->kind & block_kind_branch) {
671 // orig = s_and_saveexec_b64
672 assert(block->linear_succs.size() == 2);
673 assert(block->instructions.back()->opcode == aco_opcode::p_cbranch_z);
674 Temp cond = block->instructions.back()->operands[0].getTemp();
675 aco_ptr<Instruction> branch = std::move(block->instructions.back());
676 block->instructions.pop_back();
677
678 uint8_t mask_type = ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact);
679 if (ctx.info[idx].exec.back().first.constantEquals(-1u)) {
680 bld.copy(Definition(exec, bld.lm), cond);
681 } else {
682 Temp old_exec = bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.def(s1, scc),
683 Definition(exec, bld.lm), cond, Operand(exec, bld.lm));
684
685 ctx.info[idx].exec.back().first = Operand(old_exec);
686 }
687
688 /* add next current exec to the stack */
689 ctx.info[idx].exec.emplace_back(Operand(bld.lm), mask_type);
690
691 Builder::Result r = bld.branch(aco_opcode::p_cbranch_z, bld.def(s2), Operand(exec, bld.lm),
692 block->linear_succs[1], block->linear_succs[0]);
693 r->branch().rarely_taken = branch->branch().rarely_taken;
694 r->branch().never_taken = branch->branch().never_taken;
695 return;
696 }
697
698 if (block->kind & block_kind_invert) {
699 // exec = s_andn2_b64 (original_exec, exec)
700 assert(block->instructions.back()->opcode == aco_opcode::p_branch);
701 aco_ptr<Instruction> branch = std::move(block->instructions.back());
702 block->instructions.pop_back();
703 assert(ctx.info[idx].exec.size() >= 2);
704 Operand orig_exec = ctx.info[idx].exec[ctx.info[idx].exec.size() - 2].first;
705 bld.sop2(Builder::s_andn2, Definition(exec, bld.lm), bld.def(s1, scc), orig_exec,
706 Operand(exec, bld.lm));
707
708 Builder::Result r = bld.branch(aco_opcode::p_cbranch_z, bld.def(s2), Operand(exec, bld.lm),
709 block->linear_succs[1], block->linear_succs[0]);
710 r->branch().rarely_taken = branch->branch().rarely_taken;
711 r->branch().never_taken = branch->branch().never_taken;
712 return;
713 }
714
715 if (block->kind & block_kind_break) {
716 // loop_mask = s_andn2_b64 (loop_mask, exec)
717 assert(block->instructions.back()->opcode == aco_opcode::p_branch);
718 block->instructions.pop_back();
719
720 Temp cond = Temp();
721 for (int exec_idx = ctx.info[idx].exec.size() - 2; exec_idx >= 0; exec_idx--) {
722 cond = bld.tmp(s1);
723 Operand exec_mask = ctx.info[idx].exec[exec_idx].first;
724 exec_mask = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.scc(Definition(cond)),
725 exec_mask, break_cond);
726 ctx.info[idx].exec[exec_idx].first = exec_mask;
727 if (ctx.info[idx].exec[exec_idx].second & mask_type_loop)
728 break;
729 }
730
731 /* check if the successor is the merge block, otherwise set exec to 0 */
732 // TODO: this could be done better by directly branching to the merge block
733 unsigned succ_idx = ctx.program->blocks[block->linear_succs[1]].linear_succs[0];
734 Block& succ = ctx.program->blocks[succ_idx];
735 if (!(succ.kind & block_kind_invert || succ.kind & block_kind_merge)) {
736 bld.copy(Definition(exec, bld.lm), Operand::zero(bld.lm.bytes()));
737 }
738
739 bld.branch(aco_opcode::p_cbranch_nz, bld.def(s2), bld.scc(cond), block->linear_succs[1],
740 block->linear_succs[0]);
741 return;
742 }
743
744 if (block->kind & block_kind_continue) {
745 assert(block->instructions.back()->opcode == aco_opcode::p_branch);
746 block->instructions.pop_back();
747
748 Temp cond = Temp();
749 for (int exec_idx = ctx.info[idx].exec.size() - 2; exec_idx >= 0; exec_idx--) {
750 if (ctx.info[idx].exec[exec_idx].second & mask_type_loop)
751 break;
752 cond = bld.tmp(s1);
753 Operand exec_mask = ctx.info[idx].exec[exec_idx].first;
754 exec_mask = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.scc(Definition(cond)),
755 exec_mask, Operand(exec, bld.lm));
756 ctx.info[idx].exec[exec_idx].first = exec_mask;
757 }
758 assert(cond != Temp());
759
760 /* check if the successor is the merge block, otherwise set exec to 0 */
761 // TODO: this could be done better by directly branching to the merge block
762 unsigned succ_idx = ctx.program->blocks[block->linear_succs[1]].linear_succs[0];
763 Block& succ = ctx.program->blocks[succ_idx];
764 if (!(succ.kind & block_kind_invert || succ.kind & block_kind_merge)) {
765 bld.copy(Definition(exec, bld.lm), Operand::zero(bld.lm.bytes()));
766 }
767
768 bld.branch(aco_opcode::p_cbranch_nz, bld.def(s2), bld.scc(cond), block->linear_succs[1],
769 block->linear_succs[0]);
770 return;
771 }
772 }
773
774 void
process_block(exec_ctx & ctx,Block * block)775 process_block(exec_ctx& ctx, Block* block)
776 {
777 std::vector<aco_ptr<Instruction>> instructions;
778 instructions.reserve(block->instructions.size());
779
780 unsigned idx = add_coupling_code(ctx, block, instructions);
781
782 assert(!block->linear_succs.empty() || ctx.info[block->index].exec.size() <= 2);
783
784 process_instructions(ctx, block, instructions, idx);
785
786 block->instructions = std::move(instructions);
787
788 add_branch_code(ctx, block);
789 }
790
791 } /* end namespace */
792
793 void
insert_exec_mask(Program * program)794 insert_exec_mask(Program* program)
795 {
796 exec_ctx ctx(program);
797
798 if (program->needs_wqm && program->needs_exact)
799 ctx.handle_wqm = true;
800
801 for (Block& block : program->blocks)
802 process_block(ctx, &block);
803 }
804
805 } // namespace aco
806