1*61046927SAndroid Build Coastguard Worker /*
2*61046927SAndroid Build Coastguard Worker * Copyright © 2018 Google
3*61046927SAndroid Build Coastguard Worker *
4*61046927SAndroid Build Coastguard Worker * SPDX-License-Identifier: MIT
5*61046927SAndroid Build Coastguard Worker */
6*61046927SAndroid Build Coastguard Worker
7*61046927SAndroid Build Coastguard Worker #include "aco_interface.h"
8*61046927SAndroid Build Coastguard Worker
9*61046927SAndroid Build Coastguard Worker #include "aco_ir.h"
10*61046927SAndroid Build Coastguard Worker
11*61046927SAndroid Build Coastguard Worker #include "util/memstream.h"
12*61046927SAndroid Build Coastguard Worker
13*61046927SAndroid Build Coastguard Worker #include "ac_gpu_info.h"
14*61046927SAndroid Build Coastguard Worker #include <array>
15*61046927SAndroid Build Coastguard Worker #include <iostream>
16*61046927SAndroid Build Coastguard Worker #include <vector>
17*61046927SAndroid Build Coastguard Worker
18*61046927SAndroid Build Coastguard Worker using namespace aco;
19*61046927SAndroid Build Coastguard Worker
20*61046927SAndroid Build Coastguard Worker namespace {
21*61046927SAndroid Build Coastguard Worker
22*61046927SAndroid Build Coastguard Worker static const std::array<aco_compiler_statistic_info, aco_num_statistics> statistic_infos = []()
__anon47907a690202() 23*61046927SAndroid Build Coastguard Worker {
24*61046927SAndroid Build Coastguard Worker std::array<aco_compiler_statistic_info, aco_num_statistics> ret{};
25*61046927SAndroid Build Coastguard Worker ret[aco_statistic_hash] =
26*61046927SAndroid Build Coastguard Worker aco_compiler_statistic_info{"Hash", "CRC32 hash of code and constant data"};
27*61046927SAndroid Build Coastguard Worker ret[aco_statistic_instructions] =
28*61046927SAndroid Build Coastguard Worker aco_compiler_statistic_info{"Instructions", "Instruction count"};
29*61046927SAndroid Build Coastguard Worker ret[aco_statistic_copies] =
30*61046927SAndroid Build Coastguard Worker aco_compiler_statistic_info{"Copies", "Copy instructions created for pseudo-instructions"};
31*61046927SAndroid Build Coastguard Worker ret[aco_statistic_branches] = aco_compiler_statistic_info{"Branches", "Branch instructions"};
32*61046927SAndroid Build Coastguard Worker ret[aco_statistic_latency] =
33*61046927SAndroid Build Coastguard Worker aco_compiler_statistic_info{"Latency", "Issue cycles plus stall cycles"};
34*61046927SAndroid Build Coastguard Worker ret[aco_statistic_inv_throughput] = aco_compiler_statistic_info{
35*61046927SAndroid Build Coastguard Worker "Inverse Throughput", "Estimated busy cycles to execute one wave"};
36*61046927SAndroid Build Coastguard Worker ret[aco_statistic_vmem_clauses] = aco_compiler_statistic_info{
37*61046927SAndroid Build Coastguard Worker "VMEM Clause", "Number of VMEM clauses (includes 1-sized clauses)"};
38*61046927SAndroid Build Coastguard Worker ret[aco_statistic_smem_clauses] = aco_compiler_statistic_info{
39*61046927SAndroid Build Coastguard Worker "SMEM Clause", "Number of SMEM clauses (includes 1-sized clauses)"};
40*61046927SAndroid Build Coastguard Worker ret[aco_statistic_sgpr_presched] =
41*61046927SAndroid Build Coastguard Worker aco_compiler_statistic_info{"Pre-Sched SGPRs", "SGPR usage before scheduling"};
42*61046927SAndroid Build Coastguard Worker ret[aco_statistic_vgpr_presched] =
43*61046927SAndroid Build Coastguard Worker aco_compiler_statistic_info{"Pre-Sched VGPRs", "VGPR usage before scheduling"};
44*61046927SAndroid Build Coastguard Worker ret[aco_statistic_valu] = aco_compiler_statistic_info{"VALU", "Number of VALU instructions"};
45*61046927SAndroid Build Coastguard Worker ret[aco_statistic_salu] = aco_compiler_statistic_info{"SALU", "Number of SALU instructions"};
46*61046927SAndroid Build Coastguard Worker ret[aco_statistic_vmem] = aco_compiler_statistic_info{"VMEM", "Number of VMEM instructions"};
47*61046927SAndroid Build Coastguard Worker ret[aco_statistic_smem] = aco_compiler_statistic_info{"SMEM", "Number of SMEM instructions"};
48*61046927SAndroid Build Coastguard Worker ret[aco_statistic_vopd] = aco_compiler_statistic_info{"VOPD", "Number of VOPD instructions"};
49*61046927SAndroid Build Coastguard Worker return ret;
50*61046927SAndroid Build Coastguard Worker }();
51*61046927SAndroid Build Coastguard Worker
52*61046927SAndroid Build Coastguard Worker static void
validate(Program * program)53*61046927SAndroid Build Coastguard Worker validate(Program* program)
54*61046927SAndroid Build Coastguard Worker {
55*61046927SAndroid Build Coastguard Worker if (!(debug_flags & DEBUG_VALIDATE_IR))
56*61046927SAndroid Build Coastguard Worker return;
57*61046927SAndroid Build Coastguard Worker
58*61046927SAndroid Build Coastguard Worker ASSERTED bool is_valid = validate_ir(program);
59*61046927SAndroid Build Coastguard Worker assert(is_valid);
60*61046927SAndroid Build Coastguard Worker }
61*61046927SAndroid Build Coastguard Worker
62*61046927SAndroid Build Coastguard Worker static std::string
get_disasm_string(Program * program,std::vector<uint32_t> & code,unsigned exec_size)63*61046927SAndroid Build Coastguard Worker get_disasm_string(Program* program, std::vector<uint32_t>& code, unsigned exec_size)
64*61046927SAndroid Build Coastguard Worker {
65*61046927SAndroid Build Coastguard Worker std::string disasm;
66*61046927SAndroid Build Coastguard Worker
67*61046927SAndroid Build Coastguard Worker char* data = NULL;
68*61046927SAndroid Build Coastguard Worker size_t disasm_size = 0;
69*61046927SAndroid Build Coastguard Worker struct u_memstream mem;
70*61046927SAndroid Build Coastguard Worker if (u_memstream_open(&mem, &data, &disasm_size)) {
71*61046927SAndroid Build Coastguard Worker FILE* const memf = u_memstream_get(&mem);
72*61046927SAndroid Build Coastguard Worker if (check_print_asm_support(program)) {
73*61046927SAndroid Build Coastguard Worker print_asm(program, code, exec_size / 4u, memf);
74*61046927SAndroid Build Coastguard Worker } else {
75*61046927SAndroid Build Coastguard Worker fprintf(memf, "Shader disassembly is not supported in the current configuration"
76*61046927SAndroid Build Coastguard Worker #if !AMD_LLVM_AVAILABLE
77*61046927SAndroid Build Coastguard Worker " (LLVM not available)"
78*61046927SAndroid Build Coastguard Worker #endif
79*61046927SAndroid Build Coastguard Worker ", falling back to print_program.\n\n");
80*61046927SAndroid Build Coastguard Worker aco_print_program(program, memf);
81*61046927SAndroid Build Coastguard Worker }
82*61046927SAndroid Build Coastguard Worker fputc(0, memf);
83*61046927SAndroid Build Coastguard Worker u_memstream_close(&mem);
84*61046927SAndroid Build Coastguard Worker disasm = std::string(data, data + disasm_size);
85*61046927SAndroid Build Coastguard Worker free(data);
86*61046927SAndroid Build Coastguard Worker }
87*61046927SAndroid Build Coastguard Worker
88*61046927SAndroid Build Coastguard Worker return disasm;
89*61046927SAndroid Build Coastguard Worker }
90*61046927SAndroid Build Coastguard Worker
91*61046927SAndroid Build Coastguard Worker static std::string
aco_postprocess_shader(const struct aco_compiler_options * options,const struct aco_shader_info * info,std::unique_ptr<Program> & program)92*61046927SAndroid Build Coastguard Worker aco_postprocess_shader(const struct aco_compiler_options* options,
93*61046927SAndroid Build Coastguard Worker const struct aco_shader_info* info, std::unique_ptr<Program>& program)
94*61046927SAndroid Build Coastguard Worker {
95*61046927SAndroid Build Coastguard Worker std::string llvm_ir;
96*61046927SAndroid Build Coastguard Worker
97*61046927SAndroid Build Coastguard Worker if (options->dump_preoptir)
98*61046927SAndroid Build Coastguard Worker aco_print_program(program.get(), stderr);
99*61046927SAndroid Build Coastguard Worker
100*61046927SAndroid Build Coastguard Worker ASSERTED bool is_valid = validate_cfg(program.get());
101*61046927SAndroid Build Coastguard Worker assert(is_valid);
102*61046927SAndroid Build Coastguard Worker
103*61046927SAndroid Build Coastguard Worker if (!info->is_trap_handler_shader) {
104*61046927SAndroid Build Coastguard Worker dominator_tree(program.get());
105*61046927SAndroid Build Coastguard Worker lower_phis(program.get());
106*61046927SAndroid Build Coastguard Worker
107*61046927SAndroid Build Coastguard Worker if (program->gfx_level <= GFX7)
108*61046927SAndroid Build Coastguard Worker lower_subdword(program.get());
109*61046927SAndroid Build Coastguard Worker
110*61046927SAndroid Build Coastguard Worker validate(program.get());
111*61046927SAndroid Build Coastguard Worker
112*61046927SAndroid Build Coastguard Worker /* Optimization */
113*61046927SAndroid Build Coastguard Worker if (!options->optimisations_disabled) {
114*61046927SAndroid Build Coastguard Worker if (!(debug_flags & DEBUG_NO_VN))
115*61046927SAndroid Build Coastguard Worker value_numbering(program.get());
116*61046927SAndroid Build Coastguard Worker if (!(debug_flags & DEBUG_NO_OPT))
117*61046927SAndroid Build Coastguard Worker optimize(program.get());
118*61046927SAndroid Build Coastguard Worker }
119*61046927SAndroid Build Coastguard Worker
120*61046927SAndroid Build Coastguard Worker /* cleanup and exec mask handling */
121*61046927SAndroid Build Coastguard Worker setup_reduce_temp(program.get());
122*61046927SAndroid Build Coastguard Worker insert_exec_mask(program.get());
123*61046927SAndroid Build Coastguard Worker validate(program.get());
124*61046927SAndroid Build Coastguard Worker
125*61046927SAndroid Build Coastguard Worker /* spilling and scheduling */
126*61046927SAndroid Build Coastguard Worker live_var_analysis(program.get());
127*61046927SAndroid Build Coastguard Worker if (program->collect_statistics)
128*61046927SAndroid Build Coastguard Worker collect_presched_stats(program.get());
129*61046927SAndroid Build Coastguard Worker spill(program.get());
130*61046927SAndroid Build Coastguard Worker }
131*61046927SAndroid Build Coastguard Worker
132*61046927SAndroid Build Coastguard Worker if (options->record_ir) {
133*61046927SAndroid Build Coastguard Worker char* data = NULL;
134*61046927SAndroid Build Coastguard Worker size_t size = 0;
135*61046927SAndroid Build Coastguard Worker u_memstream mem;
136*61046927SAndroid Build Coastguard Worker if (u_memstream_open(&mem, &data, &size)) {
137*61046927SAndroid Build Coastguard Worker FILE* const memf = u_memstream_get(&mem);
138*61046927SAndroid Build Coastguard Worker aco_print_program(program.get(), memf);
139*61046927SAndroid Build Coastguard Worker fputc(0, memf);
140*61046927SAndroid Build Coastguard Worker u_memstream_close(&mem);
141*61046927SAndroid Build Coastguard Worker }
142*61046927SAndroid Build Coastguard Worker
143*61046927SAndroid Build Coastguard Worker llvm_ir = std::string(data, data + size);
144*61046927SAndroid Build Coastguard Worker free(data);
145*61046927SAndroid Build Coastguard Worker }
146*61046927SAndroid Build Coastguard Worker
147*61046927SAndroid Build Coastguard Worker if ((debug_flags & DEBUG_LIVE_INFO) && options->dump_shader)
148*61046927SAndroid Build Coastguard Worker aco_print_program(program.get(), stderr, print_live_vars | print_kill);
149*61046927SAndroid Build Coastguard Worker
150*61046927SAndroid Build Coastguard Worker if (!info->is_trap_handler_shader) {
151*61046927SAndroid Build Coastguard Worker if (!options->optimisations_disabled && !(debug_flags & DEBUG_NO_SCHED))
152*61046927SAndroid Build Coastguard Worker schedule_program(program.get());
153*61046927SAndroid Build Coastguard Worker validate(program.get());
154*61046927SAndroid Build Coastguard Worker
155*61046927SAndroid Build Coastguard Worker /* Register Allocation */
156*61046927SAndroid Build Coastguard Worker register_allocation(program.get());
157*61046927SAndroid Build Coastguard Worker
158*61046927SAndroid Build Coastguard Worker if (validate_ra(program.get())) {
159*61046927SAndroid Build Coastguard Worker aco_print_program(program.get(), stderr);
160*61046927SAndroid Build Coastguard Worker abort();
161*61046927SAndroid Build Coastguard Worker } else if (options->dump_shader) {
162*61046927SAndroid Build Coastguard Worker aco_print_program(program.get(), stderr);
163*61046927SAndroid Build Coastguard Worker }
164*61046927SAndroid Build Coastguard Worker
165*61046927SAndroid Build Coastguard Worker validate(program.get());
166*61046927SAndroid Build Coastguard Worker
167*61046927SAndroid Build Coastguard Worker /* Optimization */
168*61046927SAndroid Build Coastguard Worker if (!options->optimisations_disabled && !(debug_flags & DEBUG_NO_OPT)) {
169*61046927SAndroid Build Coastguard Worker optimize_postRA(program.get());
170*61046927SAndroid Build Coastguard Worker validate(program.get());
171*61046927SAndroid Build Coastguard Worker }
172*61046927SAndroid Build Coastguard Worker
173*61046927SAndroid Build Coastguard Worker ssa_elimination(program.get());
174*61046927SAndroid Build Coastguard Worker }
175*61046927SAndroid Build Coastguard Worker
176*61046927SAndroid Build Coastguard Worker /* Lower to HW Instructions */
177*61046927SAndroid Build Coastguard Worker lower_to_hw_instr(program.get());
178*61046927SAndroid Build Coastguard Worker validate(program.get());
179*61046927SAndroid Build Coastguard Worker
180*61046927SAndroid Build Coastguard Worker if (!options->optimisations_disabled && !(debug_flags & DEBUG_NO_SCHED_VOPD))
181*61046927SAndroid Build Coastguard Worker schedule_vopd(program.get());
182*61046927SAndroid Build Coastguard Worker
183*61046927SAndroid Build Coastguard Worker /* Schedule hardware instructions for ILP */
184*61046927SAndroid Build Coastguard Worker if (!options->optimisations_disabled && !(debug_flags & DEBUG_NO_SCHED_ILP))
185*61046927SAndroid Build Coastguard Worker schedule_ilp(program.get());
186*61046927SAndroid Build Coastguard Worker
187*61046927SAndroid Build Coastguard Worker insert_waitcnt(program.get());
188*61046927SAndroid Build Coastguard Worker insert_NOPs(program.get());
189*61046927SAndroid Build Coastguard Worker if (program->gfx_level >= GFX11)
190*61046927SAndroid Build Coastguard Worker insert_delay_alu(program.get());
191*61046927SAndroid Build Coastguard Worker
192*61046927SAndroid Build Coastguard Worker if (program->gfx_level >= GFX10)
193*61046927SAndroid Build Coastguard Worker form_hard_clauses(program.get());
194*61046927SAndroid Build Coastguard Worker
195*61046927SAndroid Build Coastguard Worker if (program->gfx_level >= GFX11)
196*61046927SAndroid Build Coastguard Worker combine_delay_alu(program.get());
197*61046927SAndroid Build Coastguard Worker
198*61046927SAndroid Build Coastguard Worker if (program->collect_statistics || (debug_flags & DEBUG_PERF_INFO))
199*61046927SAndroid Build Coastguard Worker collect_preasm_stats(program.get());
200*61046927SAndroid Build Coastguard Worker
201*61046927SAndroid Build Coastguard Worker return llvm_ir;
202*61046927SAndroid Build Coastguard Worker }
203*61046927SAndroid Build Coastguard Worker
204*61046927SAndroid Build Coastguard Worker typedef void(select_shader_part_callback)(Program* program, void* pinfo, ac_shader_config* config,
205*61046927SAndroid Build Coastguard Worker const struct aco_compiler_options* options,
206*61046927SAndroid Build Coastguard Worker const struct aco_shader_info* info,
207*61046927SAndroid Build Coastguard Worker const struct ac_shader_args* args);
208*61046927SAndroid Build Coastguard Worker
209*61046927SAndroid Build Coastguard Worker static void
aco_compile_shader_part(const struct aco_compiler_options * options,const struct aco_shader_info * info,const struct ac_shader_args * args,select_shader_part_callback select_shader_part,void * pinfo,aco_shader_part_callback * build_binary,void ** binary,bool is_prolog=false)210*61046927SAndroid Build Coastguard Worker aco_compile_shader_part(const struct aco_compiler_options* options,
211*61046927SAndroid Build Coastguard Worker const struct aco_shader_info* info, const struct ac_shader_args* args,
212*61046927SAndroid Build Coastguard Worker select_shader_part_callback select_shader_part, void* pinfo,
213*61046927SAndroid Build Coastguard Worker aco_shader_part_callback* build_binary, void** binary,
214*61046927SAndroid Build Coastguard Worker bool is_prolog = false)
215*61046927SAndroid Build Coastguard Worker {
216*61046927SAndroid Build Coastguard Worker init();
217*61046927SAndroid Build Coastguard Worker
218*61046927SAndroid Build Coastguard Worker ac_shader_config config = {0};
219*61046927SAndroid Build Coastguard Worker std::unique_ptr<Program> program{new Program};
220*61046927SAndroid Build Coastguard Worker
221*61046927SAndroid Build Coastguard Worker program->collect_statistics = options->record_stats;
222*61046927SAndroid Build Coastguard Worker if (program->collect_statistics)
223*61046927SAndroid Build Coastguard Worker memset(program->statistics, 0, sizeof(program->statistics));
224*61046927SAndroid Build Coastguard Worker
225*61046927SAndroid Build Coastguard Worker program->debug.func = options->debug.func;
226*61046927SAndroid Build Coastguard Worker program->debug.private_data = options->debug.private_data;
227*61046927SAndroid Build Coastguard Worker
228*61046927SAndroid Build Coastguard Worker program->is_prolog = is_prolog;
229*61046927SAndroid Build Coastguard Worker program->is_epilog = !is_prolog;
230*61046927SAndroid Build Coastguard Worker
231*61046927SAndroid Build Coastguard Worker /* Instruction selection */
232*61046927SAndroid Build Coastguard Worker select_shader_part(program.get(), pinfo, &config, options, info, args);
233*61046927SAndroid Build Coastguard Worker
234*61046927SAndroid Build Coastguard Worker aco_postprocess_shader(options, info, program);
235*61046927SAndroid Build Coastguard Worker
236*61046927SAndroid Build Coastguard Worker /* assembly */
237*61046927SAndroid Build Coastguard Worker std::vector<uint32_t> code;
238*61046927SAndroid Build Coastguard Worker bool append_endpgm = !(options->is_opengl && is_prolog);
239*61046927SAndroid Build Coastguard Worker unsigned exec_size = emit_program(program.get(), code, NULL, append_endpgm);
240*61046927SAndroid Build Coastguard Worker
241*61046927SAndroid Build Coastguard Worker bool get_disasm = options->dump_shader || options->record_ir;
242*61046927SAndroid Build Coastguard Worker
243*61046927SAndroid Build Coastguard Worker std::string disasm;
244*61046927SAndroid Build Coastguard Worker if (get_disasm)
245*61046927SAndroid Build Coastguard Worker disasm = get_disasm_string(program.get(), code, exec_size);
246*61046927SAndroid Build Coastguard Worker
247*61046927SAndroid Build Coastguard Worker (*build_binary)(binary, config.num_sgprs, config.num_vgprs, code.data(), code.size(),
248*61046927SAndroid Build Coastguard Worker disasm.data(), disasm.size());
249*61046927SAndroid Build Coastguard Worker }
250*61046927SAndroid Build Coastguard Worker
251*61046927SAndroid Build Coastguard Worker } /* end namespace */
252*61046927SAndroid Build Coastguard Worker
253*61046927SAndroid Build Coastguard Worker void
aco_compile_shader(const struct aco_compiler_options * options,const struct aco_shader_info * info,unsigned shader_count,struct nir_shader * const * shaders,const struct ac_shader_args * args,aco_callback * build_binary,void ** binary)254*61046927SAndroid Build Coastguard Worker aco_compile_shader(const struct aco_compiler_options* options, const struct aco_shader_info* info,
255*61046927SAndroid Build Coastguard Worker unsigned shader_count, struct nir_shader* const* shaders,
256*61046927SAndroid Build Coastguard Worker const struct ac_shader_args* args, aco_callback* build_binary, void** binary)
257*61046927SAndroid Build Coastguard Worker {
258*61046927SAndroid Build Coastguard Worker init();
259*61046927SAndroid Build Coastguard Worker
260*61046927SAndroid Build Coastguard Worker ac_shader_config config = {0};
261*61046927SAndroid Build Coastguard Worker std::unique_ptr<Program> program{new Program};
262*61046927SAndroid Build Coastguard Worker
263*61046927SAndroid Build Coastguard Worker program->collect_statistics = options->record_stats;
264*61046927SAndroid Build Coastguard Worker if (program->collect_statistics)
265*61046927SAndroid Build Coastguard Worker memset(program->statistics, 0, sizeof(program->statistics));
266*61046927SAndroid Build Coastguard Worker
267*61046927SAndroid Build Coastguard Worker program->debug.func = options->debug.func;
268*61046927SAndroid Build Coastguard Worker program->debug.private_data = options->debug.private_data;
269*61046927SAndroid Build Coastguard Worker
270*61046927SAndroid Build Coastguard Worker /* Instruction Selection */
271*61046927SAndroid Build Coastguard Worker if (info->is_trap_handler_shader)
272*61046927SAndroid Build Coastguard Worker select_trap_handler_shader(program.get(), shaders[0], &config, options, info, args);
273*61046927SAndroid Build Coastguard Worker else
274*61046927SAndroid Build Coastguard Worker select_program(program.get(), shader_count, shaders, &config, options, info, args);
275*61046927SAndroid Build Coastguard Worker
276*61046927SAndroid Build Coastguard Worker std::string llvm_ir = aco_postprocess_shader(options, info, program);
277*61046927SAndroid Build Coastguard Worker
278*61046927SAndroid Build Coastguard Worker /* assembly */
279*61046927SAndroid Build Coastguard Worker std::vector<uint32_t> code;
280*61046927SAndroid Build Coastguard Worker std::vector<struct aco_symbol> symbols;
281*61046927SAndroid Build Coastguard Worker /* OpenGL combine multi shader parts into one continous code block,
282*61046927SAndroid Build Coastguard Worker * so only last part need the s_endpgm instruction.
283*61046927SAndroid Build Coastguard Worker */
284*61046927SAndroid Build Coastguard Worker bool append_endpgm = !(options->is_opengl && info->ps.has_epilog);
285*61046927SAndroid Build Coastguard Worker unsigned exec_size = emit_program(program.get(), code, &symbols, append_endpgm);
286*61046927SAndroid Build Coastguard Worker
287*61046927SAndroid Build Coastguard Worker if (program->collect_statistics)
288*61046927SAndroid Build Coastguard Worker collect_postasm_stats(program.get(), code);
289*61046927SAndroid Build Coastguard Worker
290*61046927SAndroid Build Coastguard Worker bool get_disasm = options->dump_shader || options->record_ir;
291*61046927SAndroid Build Coastguard Worker
292*61046927SAndroid Build Coastguard Worker std::string disasm;
293*61046927SAndroid Build Coastguard Worker if (get_disasm)
294*61046927SAndroid Build Coastguard Worker disasm = get_disasm_string(program.get(), code, exec_size);
295*61046927SAndroid Build Coastguard Worker
296*61046927SAndroid Build Coastguard Worker size_t stats_size = 0;
297*61046927SAndroid Build Coastguard Worker if (program->collect_statistics)
298*61046927SAndroid Build Coastguard Worker stats_size = aco_num_statistics * sizeof(uint32_t);
299*61046927SAndroid Build Coastguard Worker
300*61046927SAndroid Build Coastguard Worker (*build_binary)(binary, &config, llvm_ir.c_str(), llvm_ir.size(), disasm.c_str(), disasm.size(),
301*61046927SAndroid Build Coastguard Worker program->statistics, stats_size, exec_size, code.data(), code.size(),
302*61046927SAndroid Build Coastguard Worker symbols.data(), symbols.size());
303*61046927SAndroid Build Coastguard Worker }
304*61046927SAndroid Build Coastguard Worker
305*61046927SAndroid Build Coastguard Worker void
aco_compile_rt_prolog(const struct aco_compiler_options * options,const struct aco_shader_info * info,const struct ac_shader_args * in_args,const struct ac_shader_args * out_args,aco_callback * build_prolog,void ** binary)306*61046927SAndroid Build Coastguard Worker aco_compile_rt_prolog(const struct aco_compiler_options* options,
307*61046927SAndroid Build Coastguard Worker const struct aco_shader_info* info, const struct ac_shader_args* in_args,
308*61046927SAndroid Build Coastguard Worker const struct ac_shader_args* out_args, aco_callback* build_prolog,
309*61046927SAndroid Build Coastguard Worker void** binary)
310*61046927SAndroid Build Coastguard Worker {
311*61046927SAndroid Build Coastguard Worker init();
312*61046927SAndroid Build Coastguard Worker
313*61046927SAndroid Build Coastguard Worker /* create program */
314*61046927SAndroid Build Coastguard Worker ac_shader_config config = {0};
315*61046927SAndroid Build Coastguard Worker std::unique_ptr<Program> program{new Program};
316*61046927SAndroid Build Coastguard Worker program->collect_statistics = false;
317*61046927SAndroid Build Coastguard Worker program->debug.func = NULL;
318*61046927SAndroid Build Coastguard Worker program->debug.private_data = NULL;
319*61046927SAndroid Build Coastguard Worker
320*61046927SAndroid Build Coastguard Worker select_rt_prolog(program.get(), &config, options, info, in_args, out_args);
321*61046927SAndroid Build Coastguard Worker validate(program.get());
322*61046927SAndroid Build Coastguard Worker insert_waitcnt(program.get());
323*61046927SAndroid Build Coastguard Worker insert_NOPs(program.get());
324*61046927SAndroid Build Coastguard Worker if (program->gfx_level >= GFX11)
325*61046927SAndroid Build Coastguard Worker insert_delay_alu(program.get());
326*61046927SAndroid Build Coastguard Worker if (program->gfx_level >= GFX10)
327*61046927SAndroid Build Coastguard Worker form_hard_clauses(program.get());
328*61046927SAndroid Build Coastguard Worker if (program->gfx_level >= GFX11)
329*61046927SAndroid Build Coastguard Worker combine_delay_alu(program.get());
330*61046927SAndroid Build Coastguard Worker
331*61046927SAndroid Build Coastguard Worker if (options->dump_shader)
332*61046927SAndroid Build Coastguard Worker aco_print_program(program.get(), stderr);
333*61046927SAndroid Build Coastguard Worker
334*61046927SAndroid Build Coastguard Worker /* assembly */
335*61046927SAndroid Build Coastguard Worker std::vector<uint32_t> code;
336*61046927SAndroid Build Coastguard Worker code.reserve(align(program->blocks[0].instructions.size() * 2, 16));
337*61046927SAndroid Build Coastguard Worker unsigned exec_size = emit_program(program.get(), code);
338*61046927SAndroid Build Coastguard Worker
339*61046927SAndroid Build Coastguard Worker bool get_disasm = options->dump_shader || options->record_ir;
340*61046927SAndroid Build Coastguard Worker
341*61046927SAndroid Build Coastguard Worker std::string disasm;
342*61046927SAndroid Build Coastguard Worker if (get_disasm)
343*61046927SAndroid Build Coastguard Worker disasm = get_disasm_string(program.get(), code, exec_size);
344*61046927SAndroid Build Coastguard Worker
345*61046927SAndroid Build Coastguard Worker (*build_prolog)(binary, &config, NULL, 0, disasm.c_str(), disasm.size(), program->statistics, 0,
346*61046927SAndroid Build Coastguard Worker exec_size, code.data(), code.size(), NULL, 0);
347*61046927SAndroid Build Coastguard Worker }
348*61046927SAndroid Build Coastguard Worker
349*61046927SAndroid Build Coastguard Worker void
aco_compile_vs_prolog(const struct aco_compiler_options * options,const struct aco_shader_info * info,const struct aco_vs_prolog_info * pinfo,const struct ac_shader_args * args,aco_shader_part_callback * build_prolog,void ** binary)350*61046927SAndroid Build Coastguard Worker aco_compile_vs_prolog(const struct aco_compiler_options* options,
351*61046927SAndroid Build Coastguard Worker const struct aco_shader_info* info, const struct aco_vs_prolog_info* pinfo,
352*61046927SAndroid Build Coastguard Worker const struct ac_shader_args* args, aco_shader_part_callback* build_prolog,
353*61046927SAndroid Build Coastguard Worker void** binary)
354*61046927SAndroid Build Coastguard Worker {
355*61046927SAndroid Build Coastguard Worker init();
356*61046927SAndroid Build Coastguard Worker
357*61046927SAndroid Build Coastguard Worker /* create program */
358*61046927SAndroid Build Coastguard Worker ac_shader_config config = {0};
359*61046927SAndroid Build Coastguard Worker std::unique_ptr<Program> program{new Program};
360*61046927SAndroid Build Coastguard Worker program->collect_statistics = false;
361*61046927SAndroid Build Coastguard Worker program->debug.func = NULL;
362*61046927SAndroid Build Coastguard Worker program->debug.private_data = NULL;
363*61046927SAndroid Build Coastguard Worker
364*61046927SAndroid Build Coastguard Worker /* create IR */
365*61046927SAndroid Build Coastguard Worker select_vs_prolog(program.get(), pinfo, &config, options, info, args);
366*61046927SAndroid Build Coastguard Worker validate(program.get());
367*61046927SAndroid Build Coastguard Worker insert_NOPs(program.get());
368*61046927SAndroid Build Coastguard Worker if (program->gfx_level >= GFX10)
369*61046927SAndroid Build Coastguard Worker form_hard_clauses(program.get());
370*61046927SAndroid Build Coastguard Worker
371*61046927SAndroid Build Coastguard Worker if (options->dump_shader)
372*61046927SAndroid Build Coastguard Worker aco_print_program(program.get(), stderr);
373*61046927SAndroid Build Coastguard Worker
374*61046927SAndroid Build Coastguard Worker /* assembly */
375*61046927SAndroid Build Coastguard Worker std::vector<uint32_t> code;
376*61046927SAndroid Build Coastguard Worker code.reserve(align(program->blocks[0].instructions.size() * 2, 16));
377*61046927SAndroid Build Coastguard Worker unsigned exec_size = emit_program(program.get(), code);
378*61046927SAndroid Build Coastguard Worker
379*61046927SAndroid Build Coastguard Worker bool get_disasm = options->dump_shader || options->record_ir;
380*61046927SAndroid Build Coastguard Worker
381*61046927SAndroid Build Coastguard Worker std::string disasm;
382*61046927SAndroid Build Coastguard Worker if (get_disasm)
383*61046927SAndroid Build Coastguard Worker disasm = get_disasm_string(program.get(), code, exec_size);
384*61046927SAndroid Build Coastguard Worker
385*61046927SAndroid Build Coastguard Worker (*build_prolog)(binary, config.num_sgprs, config.num_vgprs, code.data(), code.size(),
386*61046927SAndroid Build Coastguard Worker disasm.data(), disasm.size());
387*61046927SAndroid Build Coastguard Worker }
388*61046927SAndroid Build Coastguard Worker
389*61046927SAndroid Build Coastguard Worker void
aco_compile_ps_epilog(const struct aco_compiler_options * options,const struct aco_shader_info * info,const struct aco_ps_epilog_info * pinfo,const struct ac_shader_args * args,aco_shader_part_callback * build_epilog,void ** binary)390*61046927SAndroid Build Coastguard Worker aco_compile_ps_epilog(const struct aco_compiler_options* options,
391*61046927SAndroid Build Coastguard Worker const struct aco_shader_info* info, const struct aco_ps_epilog_info* pinfo,
392*61046927SAndroid Build Coastguard Worker const struct ac_shader_args* args, aco_shader_part_callback* build_epilog,
393*61046927SAndroid Build Coastguard Worker void** binary)
394*61046927SAndroid Build Coastguard Worker {
395*61046927SAndroid Build Coastguard Worker aco_compile_shader_part(options, info, args, select_ps_epilog, (void*)pinfo, build_epilog,
396*61046927SAndroid Build Coastguard Worker binary);
397*61046927SAndroid Build Coastguard Worker }
398*61046927SAndroid Build Coastguard Worker
399*61046927SAndroid Build Coastguard Worker void
aco_compile_ps_prolog(const struct aco_compiler_options * options,const struct aco_shader_info * info,const struct aco_ps_prolog_info * pinfo,const struct ac_shader_args * args,aco_shader_part_callback * build_prolog,void ** binary)400*61046927SAndroid Build Coastguard Worker aco_compile_ps_prolog(const struct aco_compiler_options* options,
401*61046927SAndroid Build Coastguard Worker const struct aco_shader_info* info, const struct aco_ps_prolog_info* pinfo,
402*61046927SAndroid Build Coastguard Worker const struct ac_shader_args* args, aco_shader_part_callback* build_prolog,
403*61046927SAndroid Build Coastguard Worker void** binary)
404*61046927SAndroid Build Coastguard Worker {
405*61046927SAndroid Build Coastguard Worker aco_compile_shader_part(options, info, args, select_ps_prolog, (void*)pinfo, build_prolog,
406*61046927SAndroid Build Coastguard Worker binary, true);
407*61046927SAndroid Build Coastguard Worker }
408*61046927SAndroid Build Coastguard Worker
409*61046927SAndroid Build Coastguard Worker uint64_t
aco_get_codegen_flags()410*61046927SAndroid Build Coastguard Worker aco_get_codegen_flags()
411*61046927SAndroid Build Coastguard Worker {
412*61046927SAndroid Build Coastguard Worker init();
413*61046927SAndroid Build Coastguard Worker /* Exclude flags which don't affect code generation. */
414*61046927SAndroid Build Coastguard Worker uint64_t exclude =
415*61046927SAndroid Build Coastguard Worker DEBUG_VALIDATE_IR | DEBUG_VALIDATE_RA | DEBUG_PERF_INFO | DEBUG_LIVE_INFO;
416*61046927SAndroid Build Coastguard Worker return debug_flags & ~exclude;
417*61046927SAndroid Build Coastguard Worker }
418*61046927SAndroid Build Coastguard Worker
419*61046927SAndroid Build Coastguard Worker bool
aco_is_gpu_supported(const struct radeon_info * info)420*61046927SAndroid Build Coastguard Worker aco_is_gpu_supported(const struct radeon_info* info)
421*61046927SAndroid Build Coastguard Worker {
422*61046927SAndroid Build Coastguard Worker switch (info->gfx_level) {
423*61046927SAndroid Build Coastguard Worker case GFX6:
424*61046927SAndroid Build Coastguard Worker case GFX7:
425*61046927SAndroid Build Coastguard Worker case GFX8:
426*61046927SAndroid Build Coastguard Worker return true;
427*61046927SAndroid Build Coastguard Worker case GFX9:
428*61046927SAndroid Build Coastguard Worker return info->has_graphics; /* no CDNA support */
429*61046927SAndroid Build Coastguard Worker case GFX10:
430*61046927SAndroid Build Coastguard Worker case GFX10_3:
431*61046927SAndroid Build Coastguard Worker case GFX11:
432*61046927SAndroid Build Coastguard Worker case GFX11_5:
433*61046927SAndroid Build Coastguard Worker return true;
434*61046927SAndroid Build Coastguard Worker default:
435*61046927SAndroid Build Coastguard Worker return false;
436*61046927SAndroid Build Coastguard Worker }
437*61046927SAndroid Build Coastguard Worker }
438*61046927SAndroid Build Coastguard Worker
439*61046927SAndroid Build Coastguard Worker bool
aco_nir_op_supports_packed_math_16bit(const nir_alu_instr * alu)440*61046927SAndroid Build Coastguard Worker aco_nir_op_supports_packed_math_16bit(const nir_alu_instr* alu)
441*61046927SAndroid Build Coastguard Worker {
442*61046927SAndroid Build Coastguard Worker switch (alu->op) {
443*61046927SAndroid Build Coastguard Worker case nir_op_f2f16: {
444*61046927SAndroid Build Coastguard Worker nir_shader* shader = nir_cf_node_get_function(&alu->instr.block->cf_node)->function->shader;
445*61046927SAndroid Build Coastguard Worker unsigned execution_mode = shader->info.float_controls_execution_mode;
446*61046927SAndroid Build Coastguard Worker return (shader->options->force_f2f16_rtz && !nir_is_rounding_mode_rtne(execution_mode, 16)) ||
447*61046927SAndroid Build Coastguard Worker nir_is_rounding_mode_rtz(execution_mode, 16);
448*61046927SAndroid Build Coastguard Worker }
449*61046927SAndroid Build Coastguard Worker case nir_op_fadd:
450*61046927SAndroid Build Coastguard Worker case nir_op_fsub:
451*61046927SAndroid Build Coastguard Worker case nir_op_fmul:
452*61046927SAndroid Build Coastguard Worker case nir_op_ffma:
453*61046927SAndroid Build Coastguard Worker case nir_op_fdiv:
454*61046927SAndroid Build Coastguard Worker case nir_op_flrp:
455*61046927SAndroid Build Coastguard Worker case nir_op_fabs:
456*61046927SAndroid Build Coastguard Worker case nir_op_fneg:
457*61046927SAndroid Build Coastguard Worker case nir_op_fsat:
458*61046927SAndroid Build Coastguard Worker case nir_op_fmin:
459*61046927SAndroid Build Coastguard Worker case nir_op_fmax:
460*61046927SAndroid Build Coastguard Worker case nir_op_f2f16_rtz:
461*61046927SAndroid Build Coastguard Worker case nir_op_iabs:
462*61046927SAndroid Build Coastguard Worker case nir_op_iadd:
463*61046927SAndroid Build Coastguard Worker case nir_op_iadd_sat:
464*61046927SAndroid Build Coastguard Worker case nir_op_uadd_sat:
465*61046927SAndroid Build Coastguard Worker case nir_op_isub:
466*61046927SAndroid Build Coastguard Worker case nir_op_isub_sat:
467*61046927SAndroid Build Coastguard Worker case nir_op_usub_sat:
468*61046927SAndroid Build Coastguard Worker case nir_op_ineg:
469*61046927SAndroid Build Coastguard Worker case nir_op_imul:
470*61046927SAndroid Build Coastguard Worker case nir_op_imin:
471*61046927SAndroid Build Coastguard Worker case nir_op_imax:
472*61046927SAndroid Build Coastguard Worker case nir_op_umin:
473*61046927SAndroid Build Coastguard Worker case nir_op_umax: return true;
474*61046927SAndroid Build Coastguard Worker case nir_op_ishl: /* TODO: in NIR, these have 32bit shift operands */
475*61046927SAndroid Build Coastguard Worker case nir_op_ishr: /* while Radeon needs 16bit operands when vectorized */
476*61046927SAndroid Build Coastguard Worker case nir_op_ushr:
477*61046927SAndroid Build Coastguard Worker default: return false;
478*61046927SAndroid Build Coastguard Worker }
479*61046927SAndroid Build Coastguard Worker }
480*61046927SAndroid Build Coastguard Worker
481*61046927SAndroid Build Coastguard Worker const aco_compiler_statistic_info* aco_statistic_infos = statistic_infos.data();
482*61046927SAndroid Build Coastguard Worker
483*61046927SAndroid Build Coastguard Worker void
aco_print_asm(const struct radeon_info * info,unsigned wave_size,uint32_t * binary,unsigned num_dw)484*61046927SAndroid Build Coastguard Worker aco_print_asm(const struct radeon_info *info, unsigned wave_size,
485*61046927SAndroid Build Coastguard Worker uint32_t *binary, unsigned num_dw)
486*61046927SAndroid Build Coastguard Worker {
487*61046927SAndroid Build Coastguard Worker std::vector<uint32_t> binarray(binary, binary + num_dw);
488*61046927SAndroid Build Coastguard Worker aco::Program prog;
489*61046927SAndroid Build Coastguard Worker
490*61046927SAndroid Build Coastguard Worker prog.gfx_level = info->gfx_level;
491*61046927SAndroid Build Coastguard Worker prog.family = info->family;
492*61046927SAndroid Build Coastguard Worker prog.wave_size = wave_size;
493*61046927SAndroid Build Coastguard Worker prog.blocks.push_back(aco::Block());
494*61046927SAndroid Build Coastguard Worker
495*61046927SAndroid Build Coastguard Worker aco::print_asm(&prog, binarray, num_dw, stderr);
496*61046927SAndroid Build Coastguard Worker }
497