1 /*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23 #ifndef IRIS_CONTEXT_H
24 #define IRIS_CONTEXT_H
25
26 #include "pipe/p_context.h"
27 #include "pipe/p_state.h"
28 #include "util/perf/u_trace.h"
29 #include "util/set.h"
30 #include "util/slab.h"
31 #include "util/u_debug.h"
32 #include "util/macros.h"
33 #include "util/u_threaded_context.h"
34 #include "intel/blorp/blorp.h"
35 #include "intel/dev/intel_debug.h"
36 #include "intel/common/intel_l3_config.h"
37 #include "intel/compiler/intel_shader_enums.h"
38 #include "intel/ds/intel_driver_ds.h"
39 #include "iris_batch.h"
40 #include "iris_binder.h"
41 #include "iris_fence.h"
42 #include "iris_resource.h"
43 #include "iris_screen.h"
44
45 struct iris_bo;
46 struct iris_context;
47 struct blorp_batch;
48 struct blorp_params;
49
50 #define IRIS_MAX_DRAW_BUFFERS 8
51 #define IRIS_MAX_SOL_BINDINGS 64
52
53 #define IRIS_MAX_TEXTURE_BUFFER_SIZE (1 << 27)
54 /* IRIS_MAX_ABOS and IRIS_MAX_SSBOS must be the same. */
55 #define IRIS_MAX_ABOS 16
56 #define IRIS_MAX_SSBOS 16
57 #define IRIS_MAX_VIEWPORTS 16
58 #define IRIS_MAX_CLIP_PLANES 8
59 #define IRIS_MAX_GLOBAL_BINDINGS 128
60
61 enum {
62 DRI_CONF_BO_REUSE_DISABLED,
63 DRI_CONF_BO_REUSE_ALL
64 };
65
66 enum iris_param_domain {
67 ELK_PARAM_DOMAIN_BUILTIN = 0,
68 ELK_PARAM_DOMAIN_IMAGE,
69 };
70
71 #define ELK_PARAM(domain, val) (ELK_PARAM_DOMAIN_##domain << 24 | (val))
72 #define ELK_PARAM_DOMAIN(param) ((uint32_t)(param) >> 24)
73 #define ELK_PARAM_VALUE(param) ((uint32_t)(param) & 0x00ffffff)
74 #define ELK_PARAM_IMAGE(idx, offset) ELK_PARAM(IMAGE, ((idx) << 8) | (offset))
75 #define ELK_PARAM_IMAGE_IDX(value) (ELK_PARAM_VALUE(value) >> 8)
76 #define ELK_PARAM_IMAGE_OFFSET(value)(ELK_PARAM_VALUE(value) & 0xf)
77
78 /**
79 * Dirty flags. When state changes, we flag some combination of these
80 * to indicate that particular GPU commands need to be re-emitted.
81 *
82 * Each bit typically corresponds to a single 3DSTATE_* command packet, but
83 * in rare cases they map to a group of related packets that need to be
84 * emitted together.
85 *
86 * See iris_upload_render_state().
87 */
88 #define IRIS_DIRTY_COLOR_CALC_STATE (1ull << 0)
89 #define IRIS_DIRTY_POLYGON_STIPPLE (1ull << 1)
90 #define IRIS_DIRTY_SCISSOR_RECT (1ull << 2)
91 #define IRIS_DIRTY_WM_DEPTH_STENCIL (1ull << 3)
92 #define IRIS_DIRTY_CC_VIEWPORT (1ull << 4)
93 #define IRIS_DIRTY_SF_CL_VIEWPORT (1ull << 5)
94 #define IRIS_DIRTY_PS_BLEND (1ull << 6)
95 #define IRIS_DIRTY_BLEND_STATE (1ull << 7)
96 #define IRIS_DIRTY_RASTER (1ull << 8)
97 #define IRIS_DIRTY_CLIP (1ull << 9)
98 #define IRIS_DIRTY_SBE (1ull << 10)
99 #define IRIS_DIRTY_LINE_STIPPLE (1ull << 11)
100 #define IRIS_DIRTY_VERTEX_ELEMENTS (1ull << 12)
101 #define IRIS_DIRTY_MULTISAMPLE (1ull << 13)
102 #define IRIS_DIRTY_VERTEX_BUFFERS (1ull << 14)
103 #define IRIS_DIRTY_SAMPLE_MASK (1ull << 15)
104 #define IRIS_DIRTY_URB (1ull << 16)
105 #define IRIS_DIRTY_DEPTH_BUFFER (1ull << 17)
106 #define IRIS_DIRTY_WM (1ull << 18)
107 #define IRIS_DIRTY_SO_BUFFERS (1ull << 19)
108 #define IRIS_DIRTY_SO_DECL_LIST (1ull << 20)
109 #define IRIS_DIRTY_STREAMOUT (1ull << 21)
110 #define IRIS_DIRTY_VF_SGVS (1ull << 22)
111 #define IRIS_DIRTY_VF (1ull << 23)
112 #define IRIS_DIRTY_VF_TOPOLOGY (1ull << 24)
113 #define IRIS_DIRTY_RENDER_RESOLVES_AND_FLUSHES (1ull << 25)
114 #define IRIS_DIRTY_COMPUTE_RESOLVES_AND_FLUSHES (1ull << 26)
115 #define IRIS_DIRTY_VF_STATISTICS (1ull << 27)
116 #define IRIS_DIRTY_PMA_FIX (1ull << 28)
117 #define IRIS_DIRTY_DEPTH_BOUNDS (1ull << 29)
118 #define IRIS_DIRTY_RENDER_BUFFER (1ull << 30)
119 #define IRIS_DIRTY_STENCIL_REF (1ull << 31)
120 #define IRIS_DIRTY_VERTEX_BUFFER_FLUSHES (1ull << 32)
121 #define IRIS_DIRTY_RENDER_MISC_BUFFER_FLUSHES (1ull << 33)
122 #define IRIS_DIRTY_COMPUTE_MISC_BUFFER_FLUSHES (1ull << 34)
123 #define IRIS_DIRTY_VFG (1ull << 35)
124 #define IRIS_DIRTY_DS_WRITE_ENABLE (1ull << 36)
125
126 #define IRIS_ALL_DIRTY_FOR_COMPUTE (IRIS_DIRTY_COMPUTE_RESOLVES_AND_FLUSHES | \
127 IRIS_DIRTY_COMPUTE_MISC_BUFFER_FLUSHES)
128
129 #define IRIS_ALL_DIRTY_FOR_RENDER (~IRIS_ALL_DIRTY_FOR_COMPUTE)
130
131 /**
132 * Per-stage dirty flags. When state changes, we flag some combination of
133 * these to indicate that particular GPU commands need to be re-emitted.
134 * Unlike the IRIS_DIRTY_* flags these are shader stage-specific and can be
135 * indexed by shifting the mask by the shader stage index.
136 *
137 * See iris_upload_render_state().
138 */
139 #define IRIS_STAGE_DIRTY_SAMPLER_STATES_VS (1ull << 0)
140 #define IRIS_STAGE_DIRTY_SAMPLER_STATES_TCS (1ull << 1)
141 #define IRIS_STAGE_DIRTY_SAMPLER_STATES_TES (1ull << 2)
142 #define IRIS_STAGE_DIRTY_SAMPLER_STATES_GS (1ull << 3)
143 #define IRIS_STAGE_DIRTY_SAMPLER_STATES_PS (1ull << 4)
144 #define IRIS_STAGE_DIRTY_SAMPLER_STATES_CS (1ull << 5)
145 #define IRIS_STAGE_DIRTY_UNCOMPILED_VS (1ull << 6)
146 #define IRIS_STAGE_DIRTY_UNCOMPILED_TCS (1ull << 7)
147 #define IRIS_STAGE_DIRTY_UNCOMPILED_TES (1ull << 8)
148 #define IRIS_STAGE_DIRTY_UNCOMPILED_GS (1ull << 9)
149 #define IRIS_STAGE_DIRTY_UNCOMPILED_FS (1ull << 10)
150 #define IRIS_STAGE_DIRTY_UNCOMPILED_CS (1ull << 11)
151 #define IRIS_STAGE_DIRTY_VS (1ull << 12)
152 #define IRIS_STAGE_DIRTY_TCS (1ull << 13)
153 #define IRIS_STAGE_DIRTY_TES (1ull << 14)
154 #define IRIS_STAGE_DIRTY_GS (1ull << 15)
155 #define IRIS_STAGE_DIRTY_FS (1ull << 16)
156 #define IRIS_STAGE_DIRTY_CS (1ull << 17)
157 #define IRIS_SHIFT_FOR_STAGE_DIRTY_CONSTANTS 18
158 #define IRIS_STAGE_DIRTY_CONSTANTS_VS (1ull << 18)
159 #define IRIS_STAGE_DIRTY_CONSTANTS_TCS (1ull << 19)
160 #define IRIS_STAGE_DIRTY_CONSTANTS_TES (1ull << 20)
161 #define IRIS_STAGE_DIRTY_CONSTANTS_GS (1ull << 21)
162 #define IRIS_STAGE_DIRTY_CONSTANTS_FS (1ull << 22)
163 #define IRIS_STAGE_DIRTY_CONSTANTS_CS (1ull << 23)
164 #define IRIS_SHIFT_FOR_STAGE_DIRTY_BINDINGS 24
165 #define IRIS_STAGE_DIRTY_BINDINGS_VS (1ull << 24)
166 #define IRIS_STAGE_DIRTY_BINDINGS_TCS (1ull << 25)
167 #define IRIS_STAGE_DIRTY_BINDINGS_TES (1ull << 26)
168 #define IRIS_STAGE_DIRTY_BINDINGS_GS (1ull << 27)
169 #define IRIS_STAGE_DIRTY_BINDINGS_FS (1ull << 28)
170 #define IRIS_STAGE_DIRTY_BINDINGS_CS (1ull << 29)
171
172 #define IRIS_ALL_STAGE_DIRTY_FOR_COMPUTE (IRIS_STAGE_DIRTY_CS | \
173 IRIS_STAGE_DIRTY_SAMPLER_STATES_CS | \
174 IRIS_STAGE_DIRTY_UNCOMPILED_CS | \
175 IRIS_STAGE_DIRTY_CONSTANTS_CS | \
176 IRIS_STAGE_DIRTY_BINDINGS_CS)
177
178 #define IRIS_ALL_STAGE_DIRTY_FOR_RENDER (~IRIS_ALL_STAGE_DIRTY_FOR_COMPUTE)
179
180 #define IRIS_ALL_STAGE_DIRTY_BINDINGS_FOR_RENDER (IRIS_STAGE_DIRTY_BINDINGS_VS | \
181 IRIS_STAGE_DIRTY_BINDINGS_TCS | \
182 IRIS_STAGE_DIRTY_BINDINGS_TES | \
183 IRIS_STAGE_DIRTY_BINDINGS_GS | \
184 IRIS_STAGE_DIRTY_BINDINGS_FS)
185
186 #define IRIS_ALL_STAGE_DIRTY_BINDINGS (IRIS_ALL_STAGE_DIRTY_BINDINGS_FOR_RENDER | \
187 IRIS_STAGE_DIRTY_BINDINGS_CS)
188
189 /**
190 * Non-orthogonal state (NOS) dependency flags.
191 *
192 * Shader programs may depend on non-orthogonal state. These flags are
193 * used to indicate that a shader's key depends on the state provided by
194 * a certain Gallium CSO. Changing any CSOs marked as a dependency will
195 * cause the driver to re-compute the shader key, possibly triggering a
196 * shader recompile.
197 */
198 enum iris_nos_dep {
199 IRIS_NOS_FRAMEBUFFER,
200 IRIS_NOS_DEPTH_STENCIL_ALPHA,
201 IRIS_NOS_RASTERIZER,
202 IRIS_NOS_BLEND,
203 IRIS_NOS_LAST_VUE_MAP,
204
205 IRIS_NOS_COUNT,
206 };
207
208 /** @{
209 *
210 * Program cache keys for state based recompiles.
211 */
212
213 /* Provide explicit padding for each member, to ensure that the compiler
214 * initializes every bit in the shader cache keys. The keys will be compared
215 * with memcmp.
216 */
217 PRAGMA_DIAGNOSTIC_PUSH
218 PRAGMA_DIAGNOSTIC_ERROR(-Wpadded)
219
220 /**
221 * Note, we need to take care to have padding explicitly declared
222 * for key since we will directly memcmp the whole struct.
223 */
224 struct iris_base_prog_key {
225 unsigned program_string_id;
226 bool limit_trig_input_range;
227 unsigned padding:24;
228 };
229
230 struct iris_vue_prog_key {
231 struct iris_base_prog_key base;
232
233 unsigned nr_userclip_plane_consts:4;
234 unsigned padding:28;
235 };
236
237 struct iris_vs_prog_key {
238 struct iris_vue_prog_key vue;
239 };
240
241 struct iris_tcs_prog_key {
242 struct iris_vue_prog_key vue;
243
244 enum tess_primitive_mode _tes_primitive_mode;
245
246 uint8_t input_vertices;
247
248 bool quads_workaround;
249 unsigned padding:16;
250
251 /** A bitfield of per-patch outputs written. */
252 uint32_t patch_outputs_written;
253
254 /** A bitfield of per-vertex outputs written. */
255 uint64_t outputs_written;
256 };
257
258 struct iris_tes_prog_key {
259 struct iris_vue_prog_key vue;
260
261 /** A bitfield of per-patch inputs read. */
262 uint32_t patch_inputs_read;
263
264 /** A bitfield of per-vertex inputs read. */
265 uint64_t inputs_read;
266 };
267
268 struct iris_gs_prog_key {
269 struct iris_vue_prog_key vue;
270 };
271
272 struct iris_fs_prog_key {
273 struct iris_base_prog_key base;
274
275 uint64_t input_slots_valid;
276 uint8_t color_outputs_valid;
277
278 unsigned nr_color_regions:5;
279 bool flat_shade:1;
280 bool alpha_test_replicate_alpha:1;
281 bool alpha_to_coverage:1;
282 bool clamp_fragment_color:1;
283 bool persample_interp:1;
284 bool multisample_fbo:1;
285 bool force_dual_color_blend:1;
286 bool coherent_fb_fetch:1;
287 uint64_t padding:43;
288 };
289
290 struct iris_cs_prog_key {
291 struct iris_base_prog_key base;
292 };
293
294 union iris_any_prog_key {
295 struct iris_base_prog_key base;
296 struct iris_vue_prog_key vue;
297 struct iris_vs_prog_key vs;
298 struct iris_tcs_prog_key tcs;
299 struct iris_tes_prog_key tes;
300 struct iris_gs_prog_key gs;
301 struct iris_fs_prog_key fs;
302 struct iris_cs_prog_key cs;
303 };
304
305 /* Restore the pack alignment to default. */
306 PRAGMA_DIAGNOSTIC_POP
307
308 /** @} */
309
310 struct iris_ubo_range
311 {
312 uint16_t block;
313
314 /* In units of 32-byte registers */
315 uint8_t start;
316 uint8_t length;
317 };
318
319 struct iris_fs_data {
320 int urb_setup[VARYING_SLOT_MAX];
321 uint8_t urb_setup_attribs[VARYING_SLOT_MAX];
322 uint8_t urb_setup_attribs_count;
323
324 uint64_t inputs;
325 unsigned num_varying_inputs;
326
327 unsigned msaa_flags_param;
328 uint32_t flat_inputs;
329
330 uint8_t computed_depth_mode;
331 uint8_t max_polygons;
332 uint8_t dispatch_multi;
333
334 bool computed_stencil;
335 bool early_fragment_tests;
336 bool post_depth_coverage;
337 bool inner_coverage;
338 bool dispatch_8;
339 bool dispatch_16;
340 bool dispatch_32;
341 bool dual_src_blend;
342 bool uses_pos_offset;
343 bool uses_omask;
344 bool uses_kill;
345 bool uses_src_depth;
346 bool uses_src_w;
347 bool uses_sample_mask;
348 bool uses_vmask;
349 bool has_side_effects;
350 bool pulls_bary;
351
352 bool uses_sample_offsets;
353 bool uses_npc_bary_coefficients;
354 bool uses_pc_bary_coefficients;
355 bool uses_depth_w_coefficients;
356
357 bool uses_nonperspective_interp_modes;
358
359 bool is_per_sample;
360 };
361
362 struct iris_push_const_block {
363 unsigned dwords; /* Dword count, not reg aligned */
364 unsigned regs;
365 unsigned size; /* Bytes, register aligned */
366 };
367
368 struct iris_cs_data {
369 struct {
370 struct iris_push_const_block cross_thread;
371 struct iris_push_const_block per_thread;
372 } push;
373
374 unsigned local_size[3];
375 unsigned prog_offset[3];
376 unsigned prog_mask;
377
378 uint8_t generate_local_id;
379 enum intel_compute_walk_order walk_order;
380
381 bool uses_barrier;
382 bool first_param_is_builtin_subgroup_id;
383 };
384
385 static inline uint32_t
iris_cs_data_prog_offset(const struct iris_cs_data * prog_data,unsigned dispatch_width)386 iris_cs_data_prog_offset(const struct iris_cs_data *prog_data,
387 unsigned dispatch_width)
388 {
389 assert(dispatch_width == 8 ||
390 dispatch_width == 16 ||
391 dispatch_width == 32);
392 const unsigned index = dispatch_width / 16;
393 assert(prog_data->prog_mask & (1 << index));
394 return prog_data->prog_offset[index];
395 }
396
397 struct iris_vue_data {
398 struct intel_vue_map vue_map;
399 unsigned urb_read_length;
400 uint32_t cull_distance_mask;
401 unsigned urb_entry_size;
402 enum intel_shader_dispatch_mode dispatch_mode;
403 bool include_vue_handles;
404 };
405
406 struct iris_vs_data {
407 struct iris_vue_data base;
408
409 bool uses_vertexid;
410 bool uses_instanceid;
411 bool uses_firstvertex;
412 bool uses_baseinstance;
413 bool uses_drawid;
414 };
415
416 struct iris_tcs_data {
417 struct iris_vue_data base;
418
419 int instances;
420 int patch_count_threshold;
421 bool include_primitive_id;
422 };
423
424 struct iris_tes_data {
425 struct iris_vue_data base;
426
427 enum intel_tess_partitioning partitioning;
428 enum intel_tess_output_topology output_topology;
429 enum intel_tess_domain domain;
430 bool include_primitive_id;
431 };
432
433 struct iris_gs_data {
434 struct iris_vue_data base;
435
436 unsigned vertices_in;
437 unsigned output_vertex_size_hwords;
438 unsigned output_topology;
439 unsigned control_data_header_size_hwords;
440 unsigned control_data_format;
441 int static_vertex_count;
442 int invocations;
443 bool include_primitive_id;
444 };
445
446 struct iris_depth_stencil_alpha_state;
447
448 /**
449 * Cache IDs for the in-memory program cache (ice->shaders.cache).
450 */
451 enum iris_program_cache_id {
452 IRIS_CACHE_VS = MESA_SHADER_VERTEX,
453 IRIS_CACHE_TCS = MESA_SHADER_TESS_CTRL,
454 IRIS_CACHE_TES = MESA_SHADER_TESS_EVAL,
455 IRIS_CACHE_GS = MESA_SHADER_GEOMETRY,
456 IRIS_CACHE_FS = MESA_SHADER_FRAGMENT,
457 IRIS_CACHE_CS = MESA_SHADER_COMPUTE,
458 IRIS_CACHE_BLORP,
459 };
460
461 /** @{
462 *
463 * Defines for PIPE_CONTROL operations, which trigger cache flushes,
464 * synchronization, pipelined memory writes, and so on.
465 *
466 * The bits here are not the actual hardware values. The actual fields
467 * move between various generations, so we just have flags for each
468 * potential operation, and use genxml to encode the actual packet.
469 */
470 enum pipe_control_flags
471 {
472 PIPE_CONTROL_FLUSH_LLC = (1 << 1),
473 PIPE_CONTROL_LRI_POST_SYNC_OP = (1 << 2),
474 PIPE_CONTROL_STORE_DATA_INDEX = (1 << 3),
475 PIPE_CONTROL_CS_STALL = (1 << 4),
476 PIPE_CONTROL_GLOBAL_SNAPSHOT_COUNT_RESET = (1 << 5),
477 PIPE_CONTROL_SYNC_GFDT = (1 << 6),
478 PIPE_CONTROL_TLB_INVALIDATE = (1 << 7),
479 PIPE_CONTROL_MEDIA_STATE_CLEAR = (1 << 8),
480 PIPE_CONTROL_WRITE_IMMEDIATE = (1 << 9),
481 PIPE_CONTROL_WRITE_DEPTH_COUNT = (1 << 10),
482 PIPE_CONTROL_WRITE_TIMESTAMP = (1 << 11),
483 PIPE_CONTROL_DEPTH_STALL = (1 << 12),
484 PIPE_CONTROL_RENDER_TARGET_FLUSH = (1 << 13),
485 PIPE_CONTROL_INSTRUCTION_INVALIDATE = (1 << 14),
486 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE = (1 << 15),
487 PIPE_CONTROL_INDIRECT_STATE_POINTERS_DISABLE = (1 << 16),
488 PIPE_CONTROL_NOTIFY_ENABLE = (1 << 17),
489 PIPE_CONTROL_FLUSH_ENABLE = (1 << 18),
490 PIPE_CONTROL_DATA_CACHE_FLUSH = (1 << 19),
491 PIPE_CONTROL_VF_CACHE_INVALIDATE = (1 << 20),
492 PIPE_CONTROL_CONST_CACHE_INVALIDATE = (1 << 21),
493 PIPE_CONTROL_STATE_CACHE_INVALIDATE = (1 << 22),
494 PIPE_CONTROL_STALL_AT_SCOREBOARD = (1 << 23),
495 PIPE_CONTROL_DEPTH_CACHE_FLUSH = (1 << 24),
496 PIPE_CONTROL_TILE_CACHE_FLUSH = (1 << 25), /* Not available in Gfx20+ */
497 PIPE_CONTROL_FLUSH_HDC = (1 << 26),
498 PIPE_CONTROL_PSS_STALL_SYNC = (1 << 27),
499 PIPE_CONTROL_L3_READ_ONLY_CACHE_INVALIDATE = (1 << 28),
500 PIPE_CONTROL_UNTYPED_DATAPORT_CACHE_FLUSH = (1 << 29),
501 PIPE_CONTROL_CCS_CACHE_FLUSH = (1 << 30),
502 };
503
504 #define PIPE_CONTROL_CACHE_FLUSH_BITS \
505 (PIPE_CONTROL_DEPTH_CACHE_FLUSH | \
506 PIPE_CONTROL_DATA_CACHE_FLUSH | \
507 PIPE_CONTROL_TILE_CACHE_FLUSH | \
508 PIPE_CONTROL_FLUSH_HDC | \
509 PIPE_CONTROL_UNTYPED_DATAPORT_CACHE_FLUSH | \
510 PIPE_CONTROL_RENDER_TARGET_FLUSH)
511
512 #define PIPE_CONTROL_CACHE_INVALIDATE_BITS \
513 (PIPE_CONTROL_STATE_CACHE_INVALIDATE | \
514 PIPE_CONTROL_CONST_CACHE_INVALIDATE | \
515 PIPE_CONTROL_VF_CACHE_INVALIDATE | \
516 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \
517 PIPE_CONTROL_INSTRUCTION_INVALIDATE)
518
519 #define PIPE_CONTROL_L3_RO_INVALIDATE_BITS \
520 (PIPE_CONTROL_L3_READ_ONLY_CACHE_INVALIDATE | \
521 PIPE_CONTROL_CONST_CACHE_INVALIDATE)
522
523 #define PIPE_CONTROL_GRAPHICS_BITS \
524 (PIPE_CONTROL_RENDER_TARGET_FLUSH | \
525 PIPE_CONTROL_DEPTH_CACHE_FLUSH | \
526 PIPE_CONTROL_TILE_CACHE_FLUSH | \
527 PIPE_CONTROL_DEPTH_STALL | \
528 PIPE_CONTROL_STALL_AT_SCOREBOARD | \
529 PIPE_CONTROL_PSS_STALL_SYNC | \
530 PIPE_CONTROL_VF_CACHE_INVALIDATE | \
531 PIPE_CONTROL_GLOBAL_SNAPSHOT_COUNT_RESET | \
532 PIPE_CONTROL_L3_READ_ONLY_CACHE_INVALIDATE |\
533 PIPE_CONTROL_WRITE_DEPTH_COUNT)
534
535 enum iris_predicate_state {
536 /* The first two states are used if we can determine whether to draw
537 * without having to look at the values in the query object buffer. This
538 * will happen if there is no conditional render in progress, if the query
539 * object is already completed or if something else has already added
540 * samples to the preliminary result.
541 */
542 IRIS_PREDICATE_STATE_RENDER,
543 IRIS_PREDICATE_STATE_DONT_RENDER,
544
545 /* In this case whether to draw or not depends on the result of an
546 * MI_PREDICATE command so the predicate enable bit needs to be checked.
547 */
548 IRIS_PREDICATE_STATE_USE_BIT,
549 };
550
551 /** @} */
552
553 /**
554 * An uncompiled, API-facing shader. This is the Gallium CSO for shaders.
555 * It primarily contains the NIR for the shader.
556 *
557 * Each API-facing shader can be compiled into multiple shader variants,
558 * based on non-orthogonal state dependencies, recorded in the shader key.
559 *
560 * See iris_compiled_shader, which represents a compiled shader variant.
561 */
562 struct iris_uncompiled_shader {
563 struct pipe_reference ref;
564
565 /**
566 * NIR for the shader.
567 *
568 * Even for shaders that originate as TGSI, this pointer will be non-NULL.
569 */
570 struct nir_shader *nir;
571
572 struct pipe_stream_output_info stream_output;
573
574 /* A SHA1 of the serialized NIR for the disk cache. */
575 unsigned char nir_sha1[20];
576
577 /* Hash value based on shader source program */
578 unsigned source_hash;
579
580 unsigned program_id;
581
582 /** Bitfield of (1 << IRIS_NOS_*) flags. */
583 unsigned nos;
584
585 /** Have any shader variants been compiled yet? */
586 bool compiled_once;
587
588 /* Whether shader uses atomic operations. */
589 bool uses_atomic_load_store;
590
591 /** Size (in bytes) of the kernel input data */
592 unsigned kernel_input_size;
593
594 /** Size (in bytes) of the local (shared) data passed as kernel inputs */
595 unsigned kernel_shared_size;
596
597 /** List of iris_compiled_shader variants */
598 struct list_head variants;
599
600 /** Lock for the variants list */
601 simple_mtx_t lock;
602
603 /** For parallel shader compiles */
604 struct util_queue_fence ready;
605 };
606
607 enum iris_surface_group {
608 IRIS_SURFACE_GROUP_RENDER_TARGET,
609 IRIS_SURFACE_GROUP_RENDER_TARGET_READ,
610 IRIS_SURFACE_GROUP_CS_WORK_GROUPS,
611 IRIS_SURFACE_GROUP_TEXTURE_LOW64,
612 IRIS_SURFACE_GROUP_TEXTURE_HIGH64,
613 IRIS_SURFACE_GROUP_IMAGE,
614 IRIS_SURFACE_GROUP_UBO,
615 IRIS_SURFACE_GROUP_SSBO,
616
617 IRIS_SURFACE_GROUP_COUNT,
618 };
619
620 enum {
621 /* Invalid value for a binding table index. */
622 IRIS_SURFACE_NOT_USED = 0xa0a0a0a0,
623 };
624
625 struct iris_binding_table {
626 uint32_t size_bytes;
627
628 /** Number of surfaces in each group, before compacting. */
629 uint32_t sizes[IRIS_SURFACE_GROUP_COUNT];
630
631 /** Initial offset of each group. */
632 uint32_t offsets[IRIS_SURFACE_GROUP_COUNT];
633
634 /** Mask of surfaces used in each group. */
635 uint64_t used_mask[IRIS_SURFACE_GROUP_COUNT];
636
637 uint64_t samplers_used_mask;
638 };
639
640 /**
641 * A compiled shader variant, containing a pointer to the GPU assembly,
642 * as well as program data and other packets needed by state upload.
643 *
644 * There can be several iris_compiled_shader variants per API-level shader
645 * (iris_uncompiled_shader), due to state-based recompiles (brw_*_prog_key).
646 */
647 struct iris_compiled_shader {
648 struct pipe_reference ref;
649
650 /** Link in the iris_uncompiled_shader::variants list */
651 struct list_head link;
652
653 /** Key for this variant (but not for BLORP programs) */
654 union iris_any_prog_key key;
655
656 /**
657 * Is the variant fully compiled and ready?
658 *
659 * Variants are added to \c iris_uncompiled_shader::variants before
660 * compilation actually occurs. This signals that compilation has
661 * completed.
662 */
663 struct util_queue_fence ready;
664
665 /** Variant is ready, but compilation failed. */
666 bool compilation_failed;
667
668 /** Reference to the uploaded assembly. */
669 struct iris_state_ref assembly;
670
671 /** Pointer to the assembly in the BO's map. */
672 void *map;
673
674 /** The program data (owned by the program cache hash table) */
675 struct brw_stage_prog_data *brw_prog_data;
676 struct elk_stage_prog_data *elk_prog_data;
677
678 /** A list of system values to be uploaded as uniforms. */
679 uint32_t *system_values;
680 unsigned num_system_values;
681
682 /** Size (in bytes) of the kernel input data */
683 unsigned kernel_input_size;
684
685 /** Number of constbufs expected by the shader. */
686 unsigned num_cbufs;
687
688 /**
689 * Derived 3DSTATE_STREAMOUT and 3DSTATE_SO_DECL_LIST packets
690 * (the VUE-based information for transform feedback outputs).
691 */
692 uint32_t *streamout;
693
694 struct iris_binding_table bt;
695
696 gl_shader_stage stage;
697
698 /**
699 * Data derived from prog_data.
700 */
701 struct iris_ubo_range ubo_ranges[4];
702
703 unsigned nr_params;
704 unsigned total_scratch;
705 unsigned total_shared;
706 unsigned program_size;
707 unsigned const_data_offset;
708 unsigned dispatch_grf_start_reg;
709 bool has_ubo_pull;
710 bool use_alt_mode;
711
712 union {
713 struct iris_fs_data fs;
714 struct iris_cs_data cs;
715 struct iris_vs_data vs;
716 struct iris_tcs_data tcs;
717 struct iris_tes_data tes;
718 struct iris_gs_data gs;
719 };
720
721 /**
722 * Shader packets and other data derived from prog_data. These must be
723 * completely determined from prog_data.
724 */
725 uint8_t derived_data[0];
726 };
727
728 static inline uint64_t
KSP(const struct iris_compiled_shader * shader)729 KSP(const struct iris_compiled_shader *shader)
730 {
731 struct iris_resource *res = (void *) shader->assembly.res;
732 return iris_bo_offset_from_base_address(res->bo) + shader->assembly.offset;
733 }
734
735 #define DEFINE_IRIS_SHADER_DATA(TYPE, STAGE, FIELD) \
736 static inline TYPE * \
737 iris_ ## FIELD ## _data(struct iris_compiled_shader *shader) \
738 { \
739 assert(shader->stage == STAGE); \
740 return &shader->FIELD; \
741 } \
742 static inline const TYPE * \
743 iris_ ## FIELD ## _data_const(const struct iris_compiled_shader *shader) \
744 { \
745 assert(shader->stage == STAGE); \
746 return &shader->FIELD; \
747 }
748
DEFINE_IRIS_SHADER_DATA(struct iris_fs_data,MESA_SHADER_FRAGMENT,fs)749 DEFINE_IRIS_SHADER_DATA(struct iris_fs_data, MESA_SHADER_FRAGMENT, fs)
750 DEFINE_IRIS_SHADER_DATA(struct iris_cs_data, MESA_SHADER_COMPUTE, cs)
751 DEFINE_IRIS_SHADER_DATA(struct iris_vs_data, MESA_SHADER_VERTEX, vs)
752 DEFINE_IRIS_SHADER_DATA(struct iris_tcs_data, MESA_SHADER_TESS_CTRL, tcs)
753 DEFINE_IRIS_SHADER_DATA(struct iris_tes_data, MESA_SHADER_TESS_EVAL, tes)
754 DEFINE_IRIS_SHADER_DATA(struct iris_gs_data, MESA_SHADER_GEOMETRY, gs)
755
756 #undef DEFINE_IRIS_SHADER_DATA
757
758 static inline struct iris_vue_data *
759 iris_vue_data(struct iris_compiled_shader *shader)
760 {
761 switch (shader->stage) {
762 case MESA_SHADER_VERTEX: return &shader->vs.base;
763 case MESA_SHADER_TESS_CTRL: return &shader->tcs.base;
764 case MESA_SHADER_TESS_EVAL: return &shader->tes.base;
765 case MESA_SHADER_GEOMETRY: return &shader->gs.base;
766 default:
767 unreachable("invalid shader stage for vue prog data");
768 return NULL;
769 }
770 }
771
772 /**
773 * API context state that is replicated per shader stage.
774 */
775 struct iris_shader_state {
776 /** Uniform Buffers */
777 struct pipe_shader_buffer constbuf[PIPE_MAX_CONSTANT_BUFFERS];
778 struct iris_state_ref constbuf_surf_state[PIPE_MAX_CONSTANT_BUFFERS];
779
780 bool sysvals_need_upload;
781
782 /** Shader Storage Buffers */
783 struct pipe_shader_buffer ssbo[PIPE_MAX_SHADER_BUFFERS];
784 struct iris_state_ref ssbo_surf_state[PIPE_MAX_SHADER_BUFFERS];
785
786 /** Shader Storage Images (image load store) */
787 struct iris_image_view image[PIPE_MAX_SHADER_IMAGES];
788
789 struct iris_state_ref sampler_table;
790 struct iris_sampler_state *samplers[IRIS_MAX_SAMPLERS];
791 struct iris_sampler_view *textures[IRIS_MAX_TEXTURES];
792
793 /** Bitfield of which constant buffers are bound (non-null). */
794 uint32_t bound_cbufs;
795 uint32_t dirty_cbufs;
796
797 /** Bitfield of which image views are bound (non-null). */
798 uint64_t bound_image_views;
799
800 /** Bitfield of which sampler views are bound (non-null). */
801 BITSET_DECLARE(bound_sampler_views, IRIS_MAX_TEXTURES);
802
803 /** Bitfield of which shader storage buffers are bound (non-null). */
804 uint32_t bound_ssbos;
805
806 /** Bitfield of which shader storage buffers are writable. */
807 uint32_t writable_ssbos;
808
809 /** Array of aux usages used for our shader's images in the current draw */
810 enum isl_aux_usage image_aux_usage[PIPE_MAX_SHADER_IMAGES];
811 };
812
813 /**
814 * Gallium CSO for stream output (transform feedback) targets.
815 */
816 struct iris_stream_output_target {
817 struct pipe_stream_output_target base;
818
819 /** Storage holding the offset where we're writing in the buffer */
820 struct iris_state_ref offset;
821
822 /** Stride (bytes-per-vertex) during this transform feedback operation */
823 uint16_t stride;
824
825 /** Does the next 3DSTATE_SO_BUFFER need to zero the offsets? */
826 bool zero_offset;
827 };
828
829 enum iris_context_priority {
830 IRIS_CONTEXT_MEDIUM_PRIORITY = 0,
831 IRIS_CONTEXT_LOW_PRIORITY,
832 IRIS_CONTEXT_HIGH_PRIORITY
833 };
834
835 /**
836 * The API context (derived from pipe_context).
837 *
838 * Most driver state is tracked here.
839 */
840 struct iris_context {
841 struct pipe_context ctx;
842 struct threaded_context *thrctx;
843
844 /** A debug callback for KHR_debug output. */
845 struct util_debug_callback dbg;
846
847 /** Whether the context protected (through EGL_EXT_protected_content) */
848 bool protected;
849
850 /** Whether a banned context was already signalled */
851 bool context_reset_signaled;
852
853 /** A device reset status callback for notifying that the GPU is hosed. */
854 struct pipe_device_reset_callback reset;
855
856 /** A set of dmabuf resources dirtied beyond their default aux-states. */
857 struct set *dirty_dmabufs;
858
859 /** Slab allocator for iris_transfer_map objects. */
860 struct slab_child_pool transfer_pool;
861
862 /** Slab allocator for threaded_context's iris_transfer_map objects */
863 struct slab_child_pool transfer_pool_unsync;
864
865 struct blorp_context blorp;
866
867 struct iris_batch batches[IRIS_BATCH_COUNT];
868 enum iris_context_priority priority;
869 bool has_engines_context; /* i915 specific */
870
871 struct u_upload_mgr *query_buffer_uploader;
872
873 struct intel_ds_device ds;
874
875 struct {
876 struct {
877 /**
878 * Either the value of BaseVertex for indexed draw calls or the value
879 * of the argument <first> for non-indexed draw calls.
880 */
881 int firstvertex;
882 int baseinstance;
883 } params;
884
885 /**
886 * Are the above values the ones stored in the draw_params buffer?
887 * If so, we can compare them against new values to see if anything
888 * changed. If not, we need to assume they changed.
889 */
890 bool params_valid;
891
892 /**
893 * Resource and offset that stores draw_parameters from the indirect
894 * buffer or to the buffer that stures the previous values for non
895 * indirect draws.
896 */
897 struct iris_state_ref draw_params;
898
899 struct {
900 /**
901 * The value of DrawID. This always comes in from it's own vertex
902 * buffer since it's not part of the indirect draw parameters.
903 */
904 int drawid;
905
906 /**
907 * Stores if an indexed or non-indexed draw (~0/0). Useful to
908 * calculate BaseVertex as an AND of firstvertex and is_indexed_draw.
909 */
910 int is_indexed_draw;
911 } derived_params;
912
913 /**
914 * Resource and offset used for GL_ARB_shader_draw_parameters which
915 * contains parameters that are not present in the indirect buffer as
916 * drawid and is_indexed_draw. They will go in their own vertex element.
917 */
918 struct iris_state_ref derived_draw_params;
919
920 struct {
921 /**
922 * Generation fragment shader
923 */
924 struct iris_compiled_shader *shader;
925
926 /**
927 * Ring buffer where to generate indirect draw commands
928 */
929 struct iris_bo *ring_bo;
930
931 /**
932 * Allocated iris_gen_indirect_params
933 */
934 struct iris_state_ref params;
935
936 /**
937 * Vertices used to dispatch the generated fragment shaders
938 */
939 struct iris_state_ref vertices;
940 } generation;
941 } draw;
942
943 struct {
944 struct iris_uncompiled_shader *uncompiled[MESA_SHADER_STAGES];
945 struct iris_compiled_shader *prog[MESA_SHADER_STAGES];
946 struct iris_compiled_shader *last_vue_shader;
947 struct {
948 struct intel_urb_config cfg;
949 bool constrained;
950 } urb;
951
952 /** Last urb emitted by the driver. */
953 struct intel_urb_config last_urb;
954
955 /** Uploader for shader assembly from the driver thread */
956 struct u_upload_mgr *uploader_driver;
957 /** Uploader for shader assembly from the threaded context */
958 struct u_upload_mgr *uploader_unsync;
959 struct hash_table *cache;
960
961 /** Is a GS or TES outputting points or lines? */
962 bool output_topology_is_points_or_lines;
963
964 /**
965 * Scratch buffers for various sizes and stages.
966 *
967 * Indexed by the "Per-Thread Scratch Space" field's 4-bit encoding,
968 * and shader stage.
969 */
970 struct iris_bo *scratch_bos[1 << 4][MESA_SHADER_STAGES];
971
972 /**
973 * Scratch buffer surface states on Gfx12.5+
974 */
975 struct iris_state_ref scratch_surfs[1 << 4];
976 } shaders;
977
978 struct intel_perf_context *perf_ctx;
979
980 /** Frame number for u_trace */
981 struct {
982 uint32_t begin_frame;
983 uint32_t end_frame;
984 uint64_t last_full_timestamp;
985 void *last_compute_walker;
986 } utrace;
987
988 /** Frame number for debug prints */
989 uint32_t frame;
990
991 /** Track draw call count for adding GPU breakpoint on 3DPRIMITIVE */
992 uint32_t draw_call_count;
993
994 struct {
995 uint64_t dirty;
996 uint64_t stage_dirty;
997 uint64_t stage_dirty_for_nos[IRIS_NOS_COUNT];
998
999 unsigned num_viewports;
1000 unsigned sample_mask;
1001 struct iris_blend_state *cso_blend;
1002 struct iris_rasterizer_state *cso_rast;
1003 struct iris_depth_stencil_alpha_state *cso_zsa;
1004 struct iris_vertex_element_state *cso_vertex_elements;
1005 struct pipe_blend_color blend_color;
1006 struct pipe_poly_stipple poly_stipple;
1007 struct pipe_viewport_state viewports[IRIS_MAX_VIEWPORTS];
1008 struct pipe_scissor_state scissors[IRIS_MAX_VIEWPORTS];
1009 struct pipe_stencil_ref stencil_ref;
1010 struct pipe_framebuffer_state framebuffer;
1011 struct pipe_clip_state clip_planes;
1012
1013 float default_outer_level[4];
1014 float default_inner_level[2];
1015
1016 /** Bitfield of which vertex buffers are bound (non-null). */
1017 uint64_t bound_vertex_buffers;
1018
1019 uint8_t patch_vertices;
1020 bool primitive_restart;
1021 unsigned cut_index;
1022 enum mesa_prim prim_mode:8;
1023 bool prim_is_points_or_lines;
1024 uint8_t vertices_per_patch;
1025
1026 bool window_space_position;
1027
1028 /** The last compute group size */
1029 uint32_t last_block[3];
1030
1031 /** The last compute grid size */
1032 uint32_t last_grid[3];
1033 /** The last compute grid dimensions */
1034 uint32_t last_grid_dim;
1035 /** Reference to the BO containing the compute grid size */
1036 struct iris_state_ref grid_size;
1037 /** Reference to the SURFACE_STATE for the compute grid resource */
1038 struct iris_state_ref grid_surf_state;
1039
1040 /**
1041 * Array of aux usages for drawing, altered to account for any
1042 * self-dependencies from resources bound for sampling and rendering.
1043 */
1044 enum isl_aux_usage draw_aux_usage[IRIS_MAX_DRAW_BUFFERS];
1045
1046 /** Aux usage of the fb's depth buffer (which may or may not exist). */
1047 enum isl_aux_usage hiz_usage;
1048
1049 enum intel_urb_deref_block_size urb_deref_block_size;
1050
1051 /** Are depth writes enabled? (Depth buffer may or may not exist.) */
1052 bool depth_writes_enabled;
1053
1054 /** Are stencil writes enabled? (Stencil buffer may or may not exist.) */
1055 bool stencil_writes_enabled;
1056
1057 /** Current/upcoming ds_write_state for Wa_18019816803. */
1058 bool ds_write_state;
1059
1060 /** State tracking for Wa_14018912822. */
1061 bool color_blend_zero;
1062 bool alpha_blend_zero;
1063
1064 /** State tracking for Wa_18020335297. */
1065 bool viewport_ptr_set;
1066
1067 /** State for Wa_14015055625, Wa_14019166699 */
1068 bool uses_primitive_id;
1069
1070 /** Do we have integer RT in current framebuffer state? */
1071 bool has_integer_rt;
1072
1073 /** GenX-specific current state */
1074 struct iris_genx_state *genx;
1075
1076 struct iris_shader_state shaders[MESA_SHADER_STAGES];
1077
1078 /** Do vertex shader uses shader draw parameters ? */
1079 bool vs_uses_draw_params;
1080 bool vs_uses_derived_draw_params;
1081 bool vs_needs_sgvs_element;
1082
1083 /** Do vertex shader uses edge flag ? */
1084 bool vs_needs_edge_flag;
1085
1086 /** Do any samplers need border color? One bit per shader stage. */
1087 uint8_t need_border_colors;
1088
1089 /** Global resource bindings */
1090 struct pipe_resource *global_bindings[IRIS_MAX_GLOBAL_BINDINGS];
1091
1092 struct pipe_stream_output_target *so_target[PIPE_MAX_SO_BUFFERS];
1093 bool streamout_active;
1094
1095 bool statistics_counters_enabled;
1096
1097 /** Current conditional rendering mode */
1098 enum iris_predicate_state predicate;
1099
1100 /**
1101 * Query BO with a MI_PREDICATE_RESULT snapshot calculated on the
1102 * render context that needs to be uploaded to the compute context.
1103 */
1104 struct iris_bo *compute_predicate;
1105
1106 /** Is a PIPE_QUERY_PRIMITIVES_GENERATED query active? */
1107 bool prims_generated_query_active;
1108
1109 /** Is a PIPE_QUERY_OCCLUSION_COUNTER query active? */
1110 bool occlusion_query_active;
1111
1112 /** 3DSTATE_STREAMOUT and 3DSTATE_SO_DECL_LIST packets */
1113 uint32_t *streamout;
1114
1115 /** The SURFACE_STATE for a 1x1x1 null surface. */
1116 struct iris_state_ref unbound_tex;
1117
1118 /** The SURFACE_STATE for a framebuffer-sized null surface. */
1119 struct iris_state_ref null_fb;
1120
1121 struct u_upload_mgr *surface_uploader;
1122 struct u_upload_mgr *scratch_surface_uploader;
1123 struct u_upload_mgr *dynamic_uploader;
1124
1125 struct iris_binder binder;
1126
1127 /** The high 16-bits of the last VBO/index buffer addresses */
1128 uint16_t last_vbo_high_bits[33];
1129 uint16_t last_index_bo_high_bits;
1130
1131 /**
1132 * Resources containing streamed state which our render context
1133 * currently points to. Used to re-add these to the validation
1134 * list when we start a new batch and haven't resubmitted commands.
1135 */
1136 struct {
1137 struct pipe_resource *cc_vp;
1138 struct pipe_resource *sf_cl_vp;
1139 struct pipe_resource *color_calc;
1140 struct pipe_resource *scissor;
1141 struct pipe_resource *blend;
1142 struct pipe_resource *index_buffer;
1143 struct pipe_resource *cs_thread_ids;
1144 struct pipe_resource *cs_desc;
1145 } last_res;
1146
1147 /** Records the size of variable-length state for INTEL_DEBUG=bat */
1148 struct hash_table_u64 *sizes;
1149
1150 /** Last rendering scale argument provided to genX(emit_hashing_mode). */
1151 unsigned current_hash_scale;
1152
1153 /** Resource holding the pixel pipe hashing tables. */
1154 struct pipe_resource *pixel_hashing_tables;
1155
1156 bool use_tbimr;
1157 } state;
1158 };
1159
1160 /**
1161 * Push constant data handed over to the indirect draw generation shader
1162 */
1163 struct iris_gen_indirect_params {
1164 /**
1165 * Address of iris_context:draw:generation:ring_bo
1166 */
1167 uint64_t generated_cmds_addr;
1168 /**
1169 * Address of indirect data to draw with
1170 */
1171 uint64_t indirect_data_addr;
1172 /**
1173 * Address inside iris_context:draw:generation:ring_bo where to draw ids
1174 */
1175 uint64_t draw_id_addr;
1176 /**
1177 * Address of the indirect count (can be null, in which case max_draw_count
1178 * is used)
1179 */
1180 uint64_t draw_count_addr;
1181 /**
1182 * Address to jump to in order to generate more draws
1183 */
1184 uint64_t gen_addr;
1185 /**
1186 * Address to jump to to end generated draws
1187 */
1188 uint64_t end_addr;
1189 /**
1190 * Stride between the indirect draw data
1191 */
1192 uint32_t indirect_data_stride;
1193 /**
1194 * Base index of the current generated draws in the ring buffer (increments
1195 * by ring_count)
1196 */
1197 uint32_t draw_base;
1198 /**
1199 * Maximum number of generated draw if draw_count_addr is null
1200 */
1201 uint32_t max_draw_count;
1202 /**
1203 * bits 0-7: ANV_GENERATED_FLAG_*
1204 * bits 8-15: vertex buffer mocs
1205 * bits 16-23: stride between generated commands
1206 */
1207 uint32_t flags;
1208 /**
1209 * Number of items to generate in the ring buffer
1210 */
1211 uint32_t ring_count;
1212 };
1213
1214 #define perf_debug(dbg, ...) do { \
1215 if (INTEL_DEBUG(DEBUG_PERF)) \
1216 dbg_printf(__VA_ARGS__); \
1217 if (unlikely(dbg)) \
1218 util_debug_message(dbg, PERF_INFO, __VA_ARGS__); \
1219 } while(0)
1220
1221 struct pipe_context *
1222 iris_create_context(struct pipe_screen *screen, void *priv, unsigned flags);
1223 void iris_destroy_context(struct pipe_context *ctx);
1224
1225 void iris_lost_context_state(struct iris_batch *batch);
1226
1227 void iris_mark_dirty_dmabuf(struct iris_context *ice,
1228 struct pipe_resource *res);
1229 void iris_flush_dirty_dmabufs(struct iris_context *ice);
1230
1231 void iris_init_blit_functions(struct pipe_context *ctx);
1232 void iris_init_clear_functions(struct pipe_context *ctx);
1233 void iris_init_program_functions(struct pipe_context *ctx);
1234 void iris_init_screen_program_functions(struct pipe_screen *pscreen);
1235 void iris_init_resource_functions(struct pipe_context *ctx);
1236 void iris_init_perfquery_functions(struct pipe_context *ctx);
1237 void iris_update_compiled_shaders(struct iris_context *ice);
1238 void iris_update_compiled_compute_shader(struct iris_context *ice);
1239 void iris_fill_cs_push_const_buffer(struct iris_screen *screen,
1240 struct iris_compiled_shader *shader,
1241 unsigned threads,
1242 uint32_t *dst);
1243
1244
1245 /* iris_blit.c */
1246 #define IRIS_BLORP_RELOC_FLAGS_EXEC_OBJECT_WRITE (1 << 2)
1247
1248 void iris_blorp_surf_for_resource(struct iris_batch *batch,
1249 struct blorp_surf *surf,
1250 struct pipe_resource *p_res,
1251 enum isl_aux_usage aux_usage,
1252 unsigned level,
1253 bool is_render_target);
1254 void iris_copy_region(struct blorp_context *blorp,
1255 struct iris_batch *batch,
1256 struct pipe_resource *dst,
1257 unsigned dst_level,
1258 unsigned dstx, unsigned dsty, unsigned dstz,
1259 struct pipe_resource *src,
1260 unsigned src_level,
1261 const struct pipe_box *src_box);
1262
1263 static inline enum blorp_batch_flags
iris_blorp_flags_for_batch(struct iris_batch * batch)1264 iris_blorp_flags_for_batch(struct iris_batch *batch)
1265 {
1266 if (batch->name == IRIS_BATCH_COMPUTE)
1267 return BLORP_BATCH_USE_COMPUTE;
1268
1269 if (batch->name == IRIS_BATCH_BLITTER)
1270 return BLORP_BATCH_USE_BLITTER;
1271
1272 return 0;
1273 }
1274
1275 static inline isl_surf_usage_flags_t
iris_blorp_batch_usage(struct iris_batch * batch,bool is_dest)1276 iris_blorp_batch_usage(struct iris_batch *batch, bool is_dest)
1277 {
1278 switch (batch->name) {
1279 case IRIS_BATCH_RENDER:
1280 return is_dest ? ISL_SURF_USAGE_RENDER_TARGET_BIT : ISL_SURF_USAGE_TEXTURE_BIT;
1281 case IRIS_BATCH_COMPUTE:
1282 return is_dest ? ISL_SURF_USAGE_STORAGE_BIT : ISL_SURF_USAGE_TEXTURE_BIT;
1283 case IRIS_BATCH_BLITTER:
1284 return is_dest ? ISL_SURF_USAGE_BLITTER_DST_BIT : ISL_SURF_USAGE_BLITTER_SRC_BIT;
1285 default:
1286 unreachable("Unhandled batch type");
1287 }
1288 }
1289
1290 /* iris_draw.c */
1291
1292 void iris_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *info,
1293 unsigned drawid_offset,
1294 const struct pipe_draw_indirect_info *indirect,
1295 const struct pipe_draw_start_count_bias *draws,
1296 unsigned num_draws);
1297 void iris_launch_grid(struct pipe_context *, const struct pipe_grid_info *);
1298
1299 /* iris_pipe_control.c */
1300
1301 void iris_emit_pipe_control_flush(struct iris_batch *batch,
1302 const char *reason, uint32_t flags);
1303 void iris_emit_pipe_control_write(struct iris_batch *batch,
1304 const char *reason, uint32_t flags,
1305 struct iris_bo *bo, uint32_t offset,
1306 uint64_t imm);
1307 void iris_emit_end_of_pipe_sync(struct iris_batch *batch,
1308 const char *reason, uint32_t flags);
1309 void iris_emit_buffer_barrier_for(struct iris_batch *batch,
1310 struct iris_bo *bo,
1311 enum iris_domain access);
1312 void iris_flush_all_caches(struct iris_batch *batch);
1313
1314 #define iris_handle_always_flush_cache(batch) \
1315 if (unlikely(batch->screen->driconf.always_flush_cache)) \
1316 iris_flush_all_caches(batch);
1317
1318 void iris_init_flush_functions(struct pipe_context *ctx);
1319
1320 /* iris_program.c */
1321 void iris_compiler_init(struct iris_screen *screen);
1322 void iris_upload_ubo_ssbo_surf_state(struct iris_context *ice,
1323 struct pipe_shader_buffer *buf,
1324 struct iris_state_ref *surf_state,
1325 isl_surf_usage_flags_t usage);
1326 const struct shader_info *iris_get_shader_info(const struct iris_context *ice,
1327 gl_shader_stage stage);
1328 struct iris_bo *iris_get_scratch_space(struct iris_context *ice,
1329 unsigned per_thread_scratch,
1330 gl_shader_stage stage);
1331 const struct iris_state_ref *iris_get_scratch_surf(struct iris_context *ice,
1332 unsigned per_thread_scratch);
1333 uint32_t iris_group_index_to_bti(const struct iris_binding_table *bt,
1334 enum iris_surface_group group,
1335 uint32_t index);
1336 uint32_t iris_bti_to_group_index(const struct iris_binding_table *bt,
1337 enum iris_surface_group group,
1338 uint32_t bti);
1339 void iris_apply_brw_prog_data(struct iris_compiled_shader *shader,
1340 struct brw_stage_prog_data *prog_data);
1341 void iris_apply_elk_prog_data(struct iris_compiled_shader *shader,
1342 struct elk_stage_prog_data *prog_data);
1343 struct intel_cs_dispatch_info
1344 iris_get_cs_dispatch_info(const struct intel_device_info *devinfo,
1345 const struct iris_compiled_shader *shader,
1346 const uint32_t block[3]);
1347 unsigned
1348 iris_cs_push_const_total_size(const struct iris_compiled_shader *shader,
1349 unsigned threads);
1350 uint32_t
1351 iris_fs_barycentric_modes(const struct iris_compiled_shader *shader,
1352 enum intel_msaa_flags pushed_msaa_flags);
1353 bool iris_use_tcs_multi_patch(struct iris_screen *screen);
1354 bool iris_indirect_ubos_use_sampler(struct iris_screen *screen);
1355 const void *iris_get_compiler_options(struct pipe_screen *pscreen,
1356 enum pipe_shader_ir ir,
1357 enum pipe_shader_type pstage);
1358
1359 /* iris_disk_cache.c */
1360
1361 void iris_disk_cache_store(struct disk_cache *cache,
1362 const struct iris_uncompiled_shader *ish,
1363 const struct iris_compiled_shader *shader,
1364 const void *prog_key,
1365 uint32_t prog_key_size);
1366 bool
1367 iris_disk_cache_retrieve(struct iris_screen *screen,
1368 struct u_upload_mgr *uploader,
1369 struct iris_uncompiled_shader *ish,
1370 struct iris_compiled_shader *shader,
1371 const void *prog_key,
1372 uint32_t prog_key_size);
1373
1374 /* iris_program_cache.c */
1375
1376 void iris_init_program_cache(struct iris_context *ice);
1377 void iris_destroy_program_cache(struct iris_context *ice);
1378 struct iris_compiled_shader *iris_find_cached_shader(struct iris_context *ice,
1379 enum iris_program_cache_id,
1380 uint32_t key_size,
1381 const void *key);
1382
1383 struct iris_compiled_shader *iris_create_shader_variant(const struct iris_screen *,
1384 void *mem_ctx,
1385 gl_shader_stage stage,
1386 enum iris_program_cache_id cache_id,
1387 uint32_t key_size,
1388 const void *key);
1389
1390 void iris_finalize_program(struct iris_compiled_shader *shader,
1391 uint32_t *streamout,
1392 uint32_t *system_values,
1393 unsigned num_system_values,
1394 unsigned kernel_input_size,
1395 unsigned num_cbufs,
1396 const struct iris_binding_table *bt);
1397
1398 void iris_upload_shader(struct iris_screen *screen,
1399 struct iris_uncompiled_shader *,
1400 struct iris_compiled_shader *,
1401 struct hash_table *driver_ht,
1402 struct u_upload_mgr *uploader,
1403 enum iris_program_cache_id,
1404 uint32_t key_size,
1405 const void *key,
1406 const void *assembly);
1407 void iris_delete_shader_variant(struct iris_compiled_shader *shader);
1408
1409 void iris_destroy_shader_state(struct pipe_context *ctx, void *state);
1410
1411 static inline void
iris_uncompiled_shader_reference(struct pipe_context * ctx,struct iris_uncompiled_shader ** dst,struct iris_uncompiled_shader * src)1412 iris_uncompiled_shader_reference(struct pipe_context *ctx,
1413 struct iris_uncompiled_shader **dst,
1414 struct iris_uncompiled_shader *src)
1415 {
1416 if (*dst == src)
1417 return;
1418
1419 struct iris_uncompiled_shader *old_dst = *dst;
1420
1421 if (pipe_reference(old_dst != NULL ? &old_dst->ref : NULL,
1422 src != NULL ? &src->ref : NULL)) {
1423 iris_destroy_shader_state(ctx, *dst);
1424 }
1425
1426 *dst = src;
1427 }
1428
1429 static inline void
iris_shader_variant_reference(struct iris_compiled_shader ** dst,struct iris_compiled_shader * src)1430 iris_shader_variant_reference(struct iris_compiled_shader **dst,
1431 struct iris_compiled_shader *src)
1432 {
1433 struct iris_compiled_shader *old_dst = *dst;
1434
1435 if (pipe_reference(old_dst ? &old_dst->ref: NULL, src ? &src->ref : NULL))
1436 iris_delete_shader_variant(old_dst);
1437
1438 *dst = src;
1439 }
1440
1441 bool iris_blorp_lookup_shader(struct blorp_batch *blorp_batch,
1442 const void *key,
1443 uint32_t key_size,
1444 uint32_t *kernel_out,
1445 void *prog_data_out);
1446 bool iris_blorp_upload_shader(struct blorp_batch *blorp_batch, uint32_t stage,
1447 const void *key, uint32_t key_size,
1448 const void *kernel, uint32_t kernel_size,
1449 const void *prog_data,
1450 uint32_t prog_data_size,
1451 uint32_t *kernel_out,
1452 void *prog_data_out);
1453
1454 void iris_ensure_indirect_generation_shader(struct iris_batch *batch);
1455
1456
1457 /* iris_resolve.c */
1458
1459 void iris_predraw_resolve_inputs(struct iris_context *ice,
1460 struct iris_batch *batch,
1461 bool *draw_aux_buffer_disabled,
1462 gl_shader_stage stage,
1463 bool consider_framebuffer);
1464 void iris_predraw_resolve_framebuffer(struct iris_context *ice,
1465 struct iris_batch *batch,
1466 bool *draw_aux_buffer_disabled);
1467 void iris_predraw_flush_buffers(struct iris_context *ice,
1468 struct iris_batch *batch,
1469 gl_shader_stage stage);
1470 void iris_postdraw_update_resolve_tracking(struct iris_context *ice);
1471 void iris_postdraw_update_image_resolve_tracking(struct iris_context *ice,
1472 gl_shader_stage stage);
1473 int iris_get_driver_query_info(struct pipe_screen *pscreen, unsigned index,
1474 struct pipe_driver_query_info *info);
1475 int iris_get_driver_query_group_info(struct pipe_screen *pscreen,
1476 unsigned index,
1477 struct pipe_driver_query_group_info *info);
1478
1479 /* iris_state.c */
1480 void gfx9_toggle_preemption(struct iris_context *ice,
1481 struct iris_batch *batch,
1482 const struct pipe_draw_info *draw);
1483 static const bool
iris_execute_indirect_draw_supported(const struct iris_context * ice,const struct pipe_draw_indirect_info * indirect,const struct pipe_draw_info * draw)1484 iris_execute_indirect_draw_supported(const struct iris_context *ice,
1485 const struct pipe_draw_indirect_info *indirect,
1486 const struct pipe_draw_info *draw)
1487 {
1488 const struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
1489 const struct iris_vs_data *vs_data =
1490 iris_vs_data(ice->shaders.prog[MESA_SHADER_VERTEX]);
1491 const bool is_multiview = draw->view_mask != 0;
1492 const size_t struct_size = draw->index_size ?
1493 sizeof(uint32_t) * 5 :
1494 sizeof(uint32_t) * 4;
1495 const bool aligned_stride =
1496 indirect && (indirect->stride == 0 || indirect->stride == struct_size);
1497
1498 return (screen->devinfo->has_indirect_unroll &&
1499 aligned_stride &&
1500 (indirect &&
1501 !indirect->count_from_stream_output) &&
1502 !is_multiview &&
1503 !(vs_data->uses_firstvertex ||
1504 vs_data->uses_baseinstance ||
1505 vs_data->uses_drawid));
1506 }
1507
1508 #ifdef genX
1509 # include "iris_genx_protos.h"
1510 #else
1511 # define genX(x) gfx8_##x
1512 # include "iris_genx_protos.h"
1513 # undef genX
1514 # define genX(x) gfx9_##x
1515 # include "iris_genx_protos.h"
1516 # undef genX
1517 # define genX(x) gfx11_##x
1518 # include "iris_genx_protos.h"
1519 # undef genX
1520 # define genX(x) gfx12_##x
1521 # include "iris_genx_protos.h"
1522 # undef genX
1523 # define genX(x) gfx125_##x
1524 # include "iris_genx_protos.h"
1525 # undef genX
1526 # define genX(x) gfx20_##x
1527 # include "iris_genx_protos.h"
1528 # undef genX
1529 #endif
1530
1531 #endif
1532