1 // Copyright © 2022 Collabora, Ltd.
2 // SPDX-License-Identifier: MIT
3
4 use crate::from_nir::*;
5 use crate::ir::{ShaderInfo, ShaderIoInfo, ShaderModel, ShaderStageInfo};
6 use crate::sm50::ShaderModel50;
7 use crate::sm70::ShaderModel70;
8 use crate::sph;
9
10 use compiler::bindings::*;
11 use nak_bindings::*;
12
13 use std::cmp::max;
14 use std::env;
15 use std::ffi::{CStr, CString};
16 use std::fmt::Write;
17 use std::os::raw::c_void;
18 use std::sync::OnceLock;
19
20 #[repr(u8)]
21 enum DebugFlags {
22 Print,
23 Serial,
24 Spill,
25 Annotate,
26 NoUgpr,
27 }
28
29 pub struct Debug {
30 flags: u32,
31 }
32
33 impl Debug {
new() -> Debug34 fn new() -> Debug {
35 let debug_var = "NAK_DEBUG";
36 let debug_str = match env::var(debug_var) {
37 Ok(s) => s,
38 Err(_) => {
39 return Debug { flags: 0 };
40 }
41 };
42
43 let mut flags = 0;
44 for flag in debug_str.split(',') {
45 match flag.trim() {
46 "print" => flags |= 1 << DebugFlags::Print as u8,
47 "serial" => flags |= 1 << DebugFlags::Serial as u8,
48 "spill" => flags |= 1 << DebugFlags::Spill as u8,
49 "annotate" => flags |= 1 << DebugFlags::Annotate as u8,
50 "nougpr" => flags |= 1 << DebugFlags::NoUgpr as u8,
51 unk => eprintln!("Unknown NAK_DEBUG flag \"{}\"", unk),
52 }
53 }
54 Debug { flags: flags }
55 }
56 }
57
58 pub trait GetDebugFlags {
debug_flags(&self) -> u3259 fn debug_flags(&self) -> u32;
60
print(&self) -> bool61 fn print(&self) -> bool {
62 self.debug_flags() & (1 << DebugFlags::Print as u8) != 0
63 }
64
serial(&self) -> bool65 fn serial(&self) -> bool {
66 self.debug_flags() & (1 << DebugFlags::Serial as u8) != 0
67 }
68
spill(&self) -> bool69 fn spill(&self) -> bool {
70 self.debug_flags() & (1 << DebugFlags::Spill as u8) != 0
71 }
72
annotate(&self) -> bool73 fn annotate(&self) -> bool {
74 self.debug_flags() & (1 << DebugFlags::Annotate as u8) != 0
75 }
76
no_ugpr(&self) -> bool77 fn no_ugpr(&self) -> bool {
78 self.debug_flags() & (1 << DebugFlags::NoUgpr as u8) != 0
79 }
80 }
81
82 pub static DEBUG: OnceLock<Debug> = OnceLock::new();
83
84 impl GetDebugFlags for OnceLock<Debug> {
debug_flags(&self) -> u3285 fn debug_flags(&self) -> u32 {
86 self.get_or_init(Debug::new).flags
87 }
88 }
89
90 #[no_mangle]
nak_should_print_nir() -> bool91 pub extern "C" fn nak_should_print_nir() -> bool {
92 DEBUG.print()
93 }
94
nir_options(dev: &nv_device_info) -> nir_shader_compiler_options95 fn nir_options(dev: &nv_device_info) -> nir_shader_compiler_options {
96 let mut op: nir_shader_compiler_options = unsafe { std::mem::zeroed() };
97
98 op.lower_fdiv = true;
99 op.fuse_ffma16 = true;
100 op.fuse_ffma32 = true;
101 op.fuse_ffma64 = true;
102 op.lower_flrp16 = true;
103 op.lower_flrp32 = true;
104 op.lower_flrp64 = true;
105 op.lower_fsqrt = dev.sm < 52;
106 op.lower_bitfield_extract = dev.sm >= 70;
107 op.lower_bitfield_insert = true;
108 op.lower_pack_half_2x16 = true;
109 op.lower_pack_unorm_2x16 = true;
110 op.lower_pack_snorm_2x16 = true;
111 op.lower_pack_unorm_4x8 = true;
112 op.lower_pack_snorm_4x8 = true;
113 op.lower_unpack_half_2x16 = true;
114 op.lower_unpack_unorm_2x16 = true;
115 op.lower_unpack_snorm_2x16 = true;
116 op.lower_unpack_unorm_4x8 = true;
117 op.lower_unpack_snorm_4x8 = true;
118 op.lower_insert_byte = true;
119 op.lower_insert_word = true;
120 op.lower_cs_local_index_to_id = true;
121 op.lower_device_index_to_zero = true;
122 op.lower_isign = true;
123 op.lower_uadd_sat = dev.sm < 70;
124 op.lower_usub_sat = dev.sm < 70;
125 op.lower_iadd_sat = true; // TODO
126 op.use_interpolated_input_intrinsics = true;
127 op.lower_doubles_options = nir_lower_drcp
128 | nir_lower_dsqrt
129 | nir_lower_drsq
130 | nir_lower_dtrunc
131 | nir_lower_dfloor
132 | nir_lower_dceil
133 | nir_lower_dfract
134 | nir_lower_dround_even
135 | nir_lower_dsat;
136 if dev.sm >= 70 {
137 op.lower_doubles_options |= nir_lower_dminmax;
138 }
139 op.lower_int64_options = !(nir_lower_icmp64
140 | nir_lower_iadd64
141 | nir_lower_ineg64
142 | nir_lower_shift64
143 | nir_lower_imul_2x32_64
144 | nir_lower_conv64);
145 op.lower_ldexp = true;
146 op.lower_fmod = true;
147 op.lower_ffract = true;
148 op.lower_fpow = true;
149 op.lower_scmp = true;
150 op.lower_uadd_carry = true;
151 op.lower_usub_borrow = true;
152 op.has_iadd3 = dev.sm >= 70;
153 op.has_imad32 = dev.sm >= 70;
154 op.has_sdot_4x8 = dev.sm >= 70;
155 op.has_udot_4x8 = dev.sm >= 70;
156 op.has_sudot_4x8 = dev.sm >= 70;
157 // We set .ftz on f32 by default so we can support fmulz whenever the client
158 // doesn't explicitly request denorms.
159 op.has_fmulz_no_denorms = true;
160 op.has_find_msb_rev = true;
161 op.has_pack_half_2x16_rtz = true;
162 op.has_bfm = dev.sm >= 70;
163 op.discard_is_demote = true;
164
165 op.max_unroll_iterations = 32;
166 op.has_ddx_intrinsics = true;
167 op.scalarize_ddx = true;
168
169 op
170 }
171
172 #[no_mangle]
nak_compiler_create( dev: *const nv_device_info, ) -> *mut nak_compiler173 pub extern "C" fn nak_compiler_create(
174 dev: *const nv_device_info,
175 ) -> *mut nak_compiler {
176 assert!(!dev.is_null());
177 let dev = unsafe { &*dev };
178
179 let nak = Box::new(nak_compiler {
180 sm: dev.sm,
181 warps_per_sm: dev.max_warps_per_mp,
182 nir_options: nir_options(dev),
183 });
184
185 Box::into_raw(nak)
186 }
187
188 #[no_mangle]
nak_compiler_destroy(nak: *mut nak_compiler)189 pub extern "C" fn nak_compiler_destroy(nak: *mut nak_compiler) {
190 unsafe { drop(Box::from_raw(nak)) };
191 }
192
193 #[no_mangle]
nak_debug_flags(_nak: *const nak_compiler) -> u64194 pub extern "C" fn nak_debug_flags(_nak: *const nak_compiler) -> u64 {
195 DEBUG.debug_flags().into()
196 }
197
198 #[no_mangle]
nak_nir_options( nak: *const nak_compiler, ) -> *const nir_shader_compiler_options199 pub extern "C" fn nak_nir_options(
200 nak: *const nak_compiler,
201 ) -> *const nir_shader_compiler_options {
202 assert!(!nak.is_null());
203 let nak = unsafe { &*nak };
204 &nak.nir_options
205 }
206
207 #[repr(C)]
208 pub struct ShaderBin {
209 bin: nak_shader_bin,
210 code: Vec<u32>,
211 asm: CString,
212 }
213
214 impl ShaderBin {
new( sm: &dyn ShaderModel, info: &ShaderInfo, fs_key: Option<&nak_fs_key>, code: Vec<u32>, asm: &str, ) -> ShaderBin215 pub fn new(
216 sm: &dyn ShaderModel,
217 info: &ShaderInfo,
218 fs_key: Option<&nak_fs_key>,
219 code: Vec<u32>,
220 asm: &str,
221 ) -> ShaderBin {
222 let asm = CString::new(asm)
223 .expect("NAK assembly has unexpected null characters");
224
225 let c_info = nak_shader_info {
226 stage: match info.stage {
227 ShaderStageInfo::Compute(_) => MESA_SHADER_COMPUTE,
228 ShaderStageInfo::Vertex => MESA_SHADER_VERTEX,
229 ShaderStageInfo::Fragment(_) => MESA_SHADER_FRAGMENT,
230 ShaderStageInfo::Geometry(_) => MESA_SHADER_GEOMETRY,
231 ShaderStageInfo::TessellationInit(_) => MESA_SHADER_TESS_CTRL,
232 ShaderStageInfo::Tessellation(_) => MESA_SHADER_TESS_EVAL,
233 },
234 sm: sm.sm(),
235 num_gprs: if sm.sm() >= 70 {
236 max(4, info.num_gprs + 2)
237 } else {
238 max(4, info.num_gprs)
239 },
240 num_control_barriers: info.num_control_barriers,
241 _pad0: Default::default(),
242 num_instrs: info.num_instrs,
243 slm_size: info.slm_size,
244 crs_size: sm.crs_size(info.max_crs_depth),
245 __bindgen_anon_1: match &info.stage {
246 ShaderStageInfo::Compute(cs_info) => {
247 nak_shader_info__bindgen_ty_1 {
248 cs: nak_shader_info__bindgen_ty_1__bindgen_ty_1 {
249 local_size: [
250 cs_info.local_size[0],
251 cs_info.local_size[1],
252 cs_info.local_size[2],
253 ],
254 smem_size: cs_info.smem_size,
255 _pad: Default::default(),
256 },
257 }
258 }
259 ShaderStageInfo::Fragment(fs_info) => {
260 let fs_io_info = match &info.io {
261 ShaderIoInfo::Fragment(io) => io,
262 _ => unreachable!(),
263 };
264 nak_shader_info__bindgen_ty_1 {
265 fs: nak_shader_info__bindgen_ty_1__bindgen_ty_2 {
266 writes_depth: fs_io_info.writes_depth,
267 reads_sample_mask: fs_io_info.reads_sample_mask,
268 post_depth_coverage: fs_info.post_depth_coverage,
269 uses_sample_shading: fs_info.uses_sample_shading,
270 early_fragment_tests: fs_info.early_fragment_tests,
271 _pad: Default::default(),
272 },
273 }
274 }
275 ShaderStageInfo::Tessellation(ts_info) => {
276 nak_shader_info__bindgen_ty_1 {
277 ts: nak_shader_info__bindgen_ty_1__bindgen_ty_3 {
278 domain: ts_info.domain as u8,
279 spacing: ts_info.spacing as u8,
280 prims: ts_info.primitives as u8,
281 _pad: Default::default(),
282 },
283 }
284 }
285 _ => nak_shader_info__bindgen_ty_1 {
286 _pad: Default::default(),
287 },
288 },
289 vtg: match &info.io {
290 ShaderIoInfo::Vtg(io) => nak_shader_info__bindgen_ty_2 {
291 writes_layer: io.attr_written(NAK_ATTR_RT_ARRAY_INDEX),
292 writes_point_size: io.attr_written(NAK_ATTR_POINT_SIZE),
293 clip_enable: io.clip_enable.try_into().unwrap(),
294 cull_enable: io.cull_enable.try_into().unwrap(),
295 xfb: if let Some(xfb) = &io.xfb {
296 **xfb
297 } else {
298 unsafe { std::mem::zeroed() }
299 },
300 },
301 _ => unsafe { std::mem::zeroed() },
302 },
303 hdr: sph::encode_header(sm, &info, fs_key),
304 };
305
306 if DEBUG.print() {
307 let stage_name = unsafe {
308 let c_name = _mesa_shader_stage_to_string(c_info.stage as u32);
309 CStr::from_ptr(c_name).to_str().expect("Invalid UTF-8")
310 };
311
312 eprintln!("Stage: {}", stage_name);
313 eprintln!("Instruction count: {}", c_info.num_instrs);
314 eprintln!("Num GPRs: {}", c_info.num_gprs);
315 eprintln!("SLM size: {}", c_info.slm_size);
316
317 if c_info.stage != MESA_SHADER_COMPUTE {
318 eprint_hex("Header", &c_info.hdr);
319 }
320
321 eprint_hex("Encoded shader", &code);
322 }
323
324 let bin = nak_shader_bin {
325 info: c_info,
326 code_size: (code.len() * 4).try_into().unwrap(),
327 code: code.as_ptr() as *const c_void,
328 asm_str: if asm.is_empty() {
329 std::ptr::null()
330 } else {
331 asm.as_ptr()
332 },
333 };
334 ShaderBin {
335 bin: bin,
336 code: code,
337 asm: asm,
338 }
339 }
340 }
341
342 impl std::ops::Deref for ShaderBin {
343 type Target = nak_shader_bin;
344
deref(&self) -> &nak_shader_bin345 fn deref(&self) -> &nak_shader_bin {
346 &self.bin
347 }
348 }
349
350 #[no_mangle]
nak_shader_bin_destroy(bin: *mut nak_shader_bin)351 pub extern "C" fn nak_shader_bin_destroy(bin: *mut nak_shader_bin) {
352 unsafe {
353 _ = Box::from_raw(bin as *mut ShaderBin);
354 };
355 }
356
eprint_hex(label: &str, data: &[u32])357 fn eprint_hex(label: &str, data: &[u32]) {
358 eprint!("{}:", label);
359 for i in 0..data.len() {
360 if (i % 8) == 0 {
361 eprintln!("");
362 eprint!(" ");
363 }
364 eprint!(" {:08x}", data[i]);
365 }
366 eprintln!("");
367 }
368
369 macro_rules! pass {
370 ($s: expr, $pass: ident) => {
371 $s.$pass();
372 if DEBUG.print() {
373 eprintln!("NAK IR after {}:\n{}", stringify!($pass), $s);
374 }
375 };
376 }
377
378 #[no_mangle]
nak_compile_shader( nir: *mut nir_shader, dump_asm: bool, nak: *const nak_compiler, robust2_modes: nir_variable_mode, fs_key: *const nak_fs_key, ) -> *mut nak_shader_bin379 pub extern "C" fn nak_compile_shader(
380 nir: *mut nir_shader,
381 dump_asm: bool,
382 nak: *const nak_compiler,
383 robust2_modes: nir_variable_mode,
384 fs_key: *const nak_fs_key,
385 ) -> *mut nak_shader_bin {
386 unsafe { nak_postprocess_nir(nir, nak, robust2_modes, fs_key) };
387 let nak = unsafe { &*nak };
388 let nir = unsafe { &*nir };
389 let fs_key = if fs_key.is_null() {
390 None
391 } else {
392 Some(unsafe { &*fs_key })
393 };
394
395 let sm: Box<dyn ShaderModel> = if nak.sm >= 70 {
396 Box::new(ShaderModel70::new(nak.sm))
397 } else if nak.sm >= 50 {
398 Box::new(ShaderModel50::new(nak.sm))
399 } else {
400 panic!("Unsupported shader model");
401 };
402
403 let mut s = nak_shader_from_nir(nir, sm.as_ref());
404
405 if DEBUG.print() {
406 eprintln!("NAK IR:\n{}", &s);
407 }
408
409 pass!(s, opt_bar_prop);
410 pass!(s, opt_uniform_instrs);
411 pass!(s, opt_copy_prop);
412 pass!(s, opt_prmt);
413 pass!(s, opt_lop);
414 pass!(s, opt_copy_prop);
415 pass!(s, opt_dce);
416 pass!(s, opt_out);
417 pass!(s, legalize);
418 pass!(s, assign_regs);
419 pass!(s, lower_par_copies);
420 pass!(s, lower_copy_swap);
421 if nak.sm >= 70 {
422 pass!(s, opt_jump_thread);
423 } else {
424 pass!(s, opt_crs);
425 }
426 pass!(s, calc_instr_deps);
427
428 s.gather_info();
429
430 let mut asm = String::new();
431 if dump_asm {
432 write!(asm, "{}", s).expect("Failed to dump assembly");
433 }
434
435 s.remove_annotations();
436
437 let code = sm.encode_shader(&s);
438 let bin =
439 Box::new(ShaderBin::new(sm.as_ref(), &s.info, fs_key, code, &asm));
440 Box::into_raw(bin) as *mut nak_shader_bin
441 }
442