1 /*
2 * Copyright © 2020 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #ifndef BRW_RT_H
25 #define BRW_RT_H
26
27 #include <stdint.h>
28
29 #include "compiler/shader_enums.h"
30 #include "util/macros.h"
31
32 #ifdef __cplusplus
33 extern "C" {
34 #endif
35
36 /** Vulkan defines shaderGroupHandleSize = 32 */
37 #define BRW_RT_SBT_HANDLE_SIZE 32
38
39 /** RT_DISPATCH_GLOBALS size (see gen_rt.xml) */
40 #define BRW_RT_DISPATCH_GLOBALS_SIZE 80
41
42 /** Offset after the RT dispatch globals at which "push" constants live */
43 #define BRW_RT_PUSH_CONST_OFFSET 128
44
45 /** Stride of the resume SBT */
46 #define BRW_BTD_RESUME_SBT_STRIDE 8
47
48 /* Vulkan always uses exactly two levels of BVH: world and object. At the API
49 * level, these are referred to as top and bottom.
50 */
51 enum brw_rt_bvh_level {
52 BRW_RT_BVH_LEVEL_WORLD = 0,
53 BRW_RT_BVH_LEVEL_OBJECT = 1,
54 };
55 #define BRW_RT_MAX_BVH_LEVELS 2
56
57 enum brw_rt_bvh_node_type {
58 BRW_RT_BVH_NODE_TYPE_INTERNAL = 0,
59 BRW_RT_BVH_NODE_TYPE_INSTANCE = 1,
60 BRW_RT_BVH_NODE_TYPE_PROCEDURAL = 3,
61 BRW_RT_BVH_NODE_TYPE_QUAD = 4,
62 };
63
64 /** HitKind values returned for triangle geometry
65 *
66 * This enum must match the SPIR-V enum.
67 */
68 enum brw_rt_hit_kind {
69 BRW_RT_HIT_KIND_FRONT_FACE = 0xfe,
70 BRW_RT_HIT_KIND_BACK_FACE = 0xff,
71 };
72
73 /** Ray flags
74 *
75 * This enum must match the SPIR-V RayFlags enum.
76 */
77 enum brw_rt_ray_flags {
78 BRW_RT_RAY_FLAG_FORCE_OPAQUE = 0x01,
79 BRW_RT_RAY_FLAG_FORCE_NON_OPAQUE = 0x02,
80 BRW_RT_RAY_FLAG_TERMINATE_ON_FIRST_HIT = 0x04,
81 BRW_RT_RAY_FLAG_SKIP_CLOSEST_HIT_SHADER = 0x08,
82 BRW_RT_RAY_FLAG_CULL_BACK_FACING_TRIANGLES = 0x10,
83 BRW_RT_RAY_FLAG_CULL_FRONT_FACING_TRIANGLES = 0x20,
84 BRW_RT_RAY_FLAG_CULL_OPAQUE = 0x40,
85 BRW_RT_RAY_FLAG_CULL_NON_OPAQUE = 0x80,
86 BRW_RT_RAY_FLAG_SKIP_TRIANGLES = 0x100,
87 BRW_RT_RAY_FLAG_SKIP_AABBS = 0x200,
88 };
89
90 struct brw_rt_scratch_layout {
91 /** Number of stack IDs per DSS */
92 uint32_t stack_ids_per_dss;
93
94 /** Start offset (in bytes) of the hardware MemRay stack */
95 uint32_t ray_stack_start;
96
97 /** Stride (in bytes) of the hardware MemRay stack */
98 uint32_t ray_stack_stride;
99
100 /** Start offset (in bytes) of the SW stacks */
101 uint64_t sw_stack_start;
102
103 /** Size (in bytes) of the SW stack for a single shader invocation */
104 uint32_t sw_stack_size;
105
106 /** Total size (in bytes) of the RT scratch memory area */
107 uint64_t total_size;
108 };
109
110 /** Parameters passed to the raygen trampoline shader
111 *
112 * This struct is carefully construected to be 32B and must be passed to the
113 * raygen trampoline shader as as inline constant data.
114 */
115 struct brw_rt_raygen_trampoline_params {
116 /** The GPU address of the RT_DISPATCH_GLOBALS */
117 uint64_t rt_disp_globals_addr;
118
119 /** The GPU address of the BINDLESS_SHADER_RECORD for the raygen shader */
120 uint64_t raygen_bsr_addr;
121
122 /** 1 if this is an indirect dispatch, 0 otherwise */
123 uint8_t is_indirect;
124
125 /** The integer log2 of the local group size
126 *
127 * Ray-tracing shaders don't have a concept of local vs. global workgroup
128 * size. They only have a single 3D launch size. The raygen trampoline
129 * shader is always dispatched with a local workgroup size equal to the
130 * SIMD width but the shape of the local workgroup is determined at
131 * dispatch time based on the shape of the launch and passed to the
132 * trampoline via this field. (There's no sense having a Z dimension on
133 * the local workgroup if the launch is 2D.)
134 *
135 * We use the integer log2 of the size because there's no point in
136 * non-power-of-two sizes and shifts are cheaper than division.
137 */
138 uint8_t local_group_size_log2[3];
139
140 uint32_t pad[3];
141 };
142
143 /** Size of the "hot zone" in bytes
144 *
145 * The hot zone is a SW-defined data structure which is a single uvec4
146 * containing two bits of information:
147 *
148 * - hotzone.x: Stack offset (in bytes)
149 *
150 * This is the offset (in bytes) into the per-thread scratch space at which
151 * the current shader's stack starts. This is incremented by the calling
152 * shader prior to any shader call type instructions and gets decremented
153 * by the resume shader as part of completing the return operation.
154 *
155 *
156 * - hotzone.yzw: The launch ID associated with the current thread
157 *
158 * Inside a bindless shader, the only information we have is the DSS ID
159 * from the hardware EU and a per-DSS stack ID. In particular, the three-
160 * dimensional launch ID is lost the moment we leave the raygen trampoline.
161 */
162 #define BRW_RT_SIZEOF_HOTZONE 16
163
164 /* From the BSpec "Address Computation for Memory Based Data Structures:
165 * Ray and TraversalStack (Async Ray Tracing)":
166 *
167 * sizeof(Ray) = 64B, sizeof(HitInfo) = 32B, sizeof(TravStack) = 32B.
168 */
169 #define BRW_RT_SIZEOF_RAY 64
170 #define BRW_RT_SIZEOF_HIT_INFO 32
171 #define BRW_RT_SIZEOF_TRAV_STACK 32
172
173 /* From the BSpec:
174 *
175 * syncStackSize = (maxBVHLevels % 2 == 1) ?
176 * (sizeof(HitInfo) * 2 +
177 * (sizeof(Ray) + sizeof(TravStack)) * maxBVHLevels + 32B) :
178 * (sizeof(HitInfo) * 2 +
179 * (sizeof(Ray) + sizeof(TravStack)) * maxBVHLevels);
180 *
181 * The select is just to align to 64B.
182 */
183 #define BRW_RT_SIZEOF_RAY_QUERY \
184 (BRW_RT_SIZEOF_HIT_INFO * 2 + \
185 (BRW_RT_SIZEOF_RAY + BRW_RT_SIZEOF_TRAV_STACK) * BRW_RT_MAX_BVH_LEVELS + \
186 (BRW_RT_MAX_BVH_LEVELS % 2 ? 32 : 0))
187
188 #define BRW_RT_SIZEOF_SHADOW_RAY_QUERY \
189 (BRW_RT_SIZEOF_HIT_INFO * 2 + \
190 (BRW_RT_SIZEOF_RAY + BRW_RT_SIZEOF_TRAV_STACK) * BRW_RT_MAX_BVH_LEVELS)
191
192 #define BRW_RT_SIZEOF_HW_STACK \
193 (BRW_RT_SIZEOF_HIT_INFO * 2 + \
194 BRW_RT_SIZEOF_RAY * BRW_RT_MAX_BVH_LEVELS + \
195 BRW_RT_SIZEOF_TRAV_STACK * BRW_RT_MAX_BVH_LEVELS)
196
197 /* This is a mesa-defined region for hit attribute data */
198 #define BRW_RT_SIZEOF_HIT_ATTRIB_DATA 64
199 #define BRW_RT_OFFSETOF_HIT_ATTRIB_DATA BRW_RT_SIZEOF_HW_STACK
200
201 #define BRW_RT_ASYNC_STACK_STRIDE \
202 ALIGN_POT(BRW_RT_OFFSETOF_HIT_ATTRIB_DATA + \
203 BRW_RT_SIZEOF_HIT_ATTRIB_DATA, 64)
204
205 static inline void
brw_rt_compute_scratch_layout(struct brw_rt_scratch_layout * layout,const struct intel_device_info * devinfo,uint32_t stack_ids_per_dss,uint32_t sw_stack_size)206 brw_rt_compute_scratch_layout(struct brw_rt_scratch_layout *layout,
207 const struct intel_device_info *devinfo,
208 uint32_t stack_ids_per_dss,
209 uint32_t sw_stack_size)
210 {
211 layout->stack_ids_per_dss = stack_ids_per_dss;
212
213 const uint32_t dss_count = intel_device_info_dual_subslice_id_bound(devinfo);
214 const uint32_t num_stack_ids = dss_count * stack_ids_per_dss;
215
216 uint64_t size = 0;
217
218 /* The first thing in our scratch area is an array of "hot zones" which
219 * store the stack offset as well as the launch IDs for each active
220 * invocation.
221 */
222 size += BRW_RT_SIZEOF_HOTZONE * num_stack_ids;
223
224 /* Next, we place the HW ray stacks */
225 assert(size % 64 == 0); /* Cache-line aligned */
226 assert(size < UINT32_MAX);
227 layout->ray_stack_start = size;
228 layout->ray_stack_stride = BRW_RT_ASYNC_STACK_STRIDE;
229 size += num_stack_ids * layout->ray_stack_stride;
230
231 /* Finally, we place the SW stacks for the individual ray-tracing shader
232 * invocations. We align these to 64B to ensure that we don't have any
233 * shared cache lines which could hurt performance.
234 */
235 assert(size % 64 == 0);
236 layout->sw_stack_start = size;
237 layout->sw_stack_size = ALIGN(sw_stack_size, 64);
238
239 /* Currently it's always the case that sw_stack_size is a power of
240 * two, but power-of-two SW stack sizes are prone to causing
241 * collisions in the hashing function used by the L3 to map memory
242 * addresses to banks, which can cause stack accesses from most
243 * DSSes to bottleneck on a single L3 bank. Fix it by padding the
244 * SW stack by a single cacheline if it was a power of two.
245 */
246 if (layout->sw_stack_size > 64 &&
247 util_is_power_of_two_nonzero(layout->sw_stack_size))
248 layout->sw_stack_size += 64;
249
250 size += num_stack_ids * layout->sw_stack_size;
251
252 layout->total_size = size;
253 }
254
255 static inline uint32_t
brw_rt_ray_queries_hw_stacks_size(const struct intel_device_info * devinfo)256 brw_rt_ray_queries_hw_stacks_size(const struct intel_device_info *devinfo)
257 {
258 /* Maximum slice/subslice/EU ID can be computed from the max_scratch_ids
259 * which includes all the threads.
260 */
261 uint32_t max_eu_id = devinfo->max_scratch_ids[MESA_SHADER_COMPUTE];
262 uint32_t max_simd_size = 16; /* Cannot run in SIMD32 with ray queries */
263 return max_eu_id * max_simd_size * BRW_RT_SIZEOF_RAY_QUERY;
264 }
265
266 static inline uint32_t
brw_rt_ray_queries_shadow_stack_size(const struct intel_device_info * devinfo)267 brw_rt_ray_queries_shadow_stack_size(const struct intel_device_info *devinfo)
268 {
269 /* Maximum slice/subslice/EU ID can be computed from the max_scratch_ids
270 * which includes all the threads.
271 */
272 uint32_t max_eu_id = devinfo->max_scratch_ids[MESA_SHADER_COMPUTE];
273 uint32_t max_simd_size = 16; /* Cannot run in SIMD32 with ray queries */
274 return max_eu_id * max_simd_size * BRW_RT_SIZEOF_SHADOW_RAY_QUERY;
275 }
276
277 static inline uint32_t
brw_rt_ray_queries_shadow_stacks_size(const struct intel_device_info * devinfo,uint32_t ray_queries)278 brw_rt_ray_queries_shadow_stacks_size(const struct intel_device_info *devinfo,
279 uint32_t ray_queries)
280 {
281 /* Don't bother a shadow stack if we only have a single query. We can
282 * directly write in the HW buffer.
283 */
284 return (ray_queries > 1 ? ray_queries : 0) * brw_rt_ray_queries_shadow_stack_size(devinfo) +
285 ray_queries * 4; /* Ctrl + Level data */
286 }
287
288 #ifdef __cplusplus
289 }
290 #endif
291
292 #endif /* BRW_RT_H */
293