1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 #include "nir/nir_builder.h"
8 #include "radv_meta.h"
9 #include "vk_command_pool.h"
10 #include "vk_common_entrypoints.h"
11
12 static VkResult create_pipeline(struct radv_device *device, VkImageAspectFlagBits aspect, enum glsl_sampler_dim tex_dim,
13 VkFormat format, VkPipeline *pipeline);
14
15 static nir_shader *
build_nir_vertex_shader(struct radv_device * dev)16 build_nir_vertex_shader(struct radv_device *dev)
17 {
18 const struct glsl_type *vec4 = glsl_vec4_type();
19 nir_builder b = radv_meta_init_shader(dev, MESA_SHADER_VERTEX, "meta_blit_vs");
20
21 nir_variable *pos_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "gl_Position");
22 pos_out->data.location = VARYING_SLOT_POS;
23
24 nir_variable *tex_pos_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "v_tex_pos");
25 tex_pos_out->data.location = VARYING_SLOT_VAR0;
26 tex_pos_out->data.interpolation = INTERP_MODE_SMOOTH;
27
28 nir_def *outvec = nir_gen_rect_vertices(&b, NULL, NULL);
29
30 nir_store_var(&b, pos_out, outvec, 0xf);
31
32 nir_def *src_box = nir_load_push_constant(&b, 4, 32, nir_imm_int(&b, 0), .range = 16);
33 nir_def *src0_z = nir_load_push_constant(&b, 1, 32, nir_imm_int(&b, 0), .base = 16, .range = 4);
34
35 nir_def *vertex_id = nir_load_vertex_id_zero_base(&b);
36
37 /* vertex 0 - src0_x, src0_y, src0_z */
38 /* vertex 1 - src0_x, src1_y, src0_z*/
39 /* vertex 2 - src1_x, src0_y, src0_z */
40 /* so channel 0 is vertex_id != 2 ? src_x : src_x + w
41 channel 1 is vertex id != 1 ? src_y : src_y + w */
42
43 nir_def *c0cmp = nir_ine_imm(&b, vertex_id, 2);
44 nir_def *c1cmp = nir_ine_imm(&b, vertex_id, 1);
45
46 nir_def *comp[4];
47 comp[0] = nir_bcsel(&b, c0cmp, nir_channel(&b, src_box, 0), nir_channel(&b, src_box, 2));
48
49 comp[1] = nir_bcsel(&b, c1cmp, nir_channel(&b, src_box, 1), nir_channel(&b, src_box, 3));
50 comp[2] = src0_z;
51 comp[3] = nir_imm_float(&b, 1.0);
52 nir_def *out_tex_vec = nir_vec(&b, comp, 4);
53 nir_store_var(&b, tex_pos_out, out_tex_vec, 0xf);
54 return b.shader;
55 }
56
57 static nir_shader *
build_nir_copy_fragment_shader(struct radv_device * dev,enum glsl_sampler_dim tex_dim)58 build_nir_copy_fragment_shader(struct radv_device *dev, enum glsl_sampler_dim tex_dim)
59 {
60 const struct glsl_type *vec4 = glsl_vec4_type();
61 nir_builder b = radv_meta_init_shader(dev, MESA_SHADER_FRAGMENT, "meta_blit_fs.%d", tex_dim);
62
63 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec4, "v_tex_pos");
64 tex_pos_in->data.location = VARYING_SLOT_VAR0;
65
66 /* Swizzle the array index which comes in as Z coordinate into the right
67 * position.
68 */
69 unsigned swz[] = {0, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 1), 2};
70 nir_def *const tex_pos =
71 nir_swizzle(&b, nir_load_var(&b, tex_pos_in), swz, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 3));
72
73 const struct glsl_type *sampler_type =
74 glsl_sampler_type(tex_dim, false, tex_dim != GLSL_SAMPLER_DIM_3D, glsl_get_base_type(vec4));
75 nir_variable *sampler = nir_variable_create(b.shader, nir_var_uniform, sampler_type, "s_tex");
76 sampler->data.descriptor_set = 0;
77 sampler->data.binding = 0;
78
79 nir_deref_instr *tex_deref = nir_build_deref_var(&b, sampler);
80 nir_def *color = nir_tex_deref(&b, tex_deref, tex_deref, tex_pos);
81
82 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
83 color_out->data.location = FRAG_RESULT_DATA0;
84 nir_store_var(&b, color_out, color, 0xf);
85
86 return b.shader;
87 }
88
89 static nir_shader *
build_nir_copy_fragment_shader_depth(struct radv_device * dev,enum glsl_sampler_dim tex_dim)90 build_nir_copy_fragment_shader_depth(struct radv_device *dev, enum glsl_sampler_dim tex_dim)
91 {
92 const struct glsl_type *vec4 = glsl_vec4_type();
93 nir_builder b = radv_meta_init_shader(dev, MESA_SHADER_FRAGMENT, "meta_blit_depth_fs.%d", tex_dim);
94
95 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec4, "v_tex_pos");
96 tex_pos_in->data.location = VARYING_SLOT_VAR0;
97
98 /* Swizzle the array index which comes in as Z coordinate into the right
99 * position.
100 */
101 unsigned swz[] = {0, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 1), 2};
102 nir_def *const tex_pos =
103 nir_swizzle(&b, nir_load_var(&b, tex_pos_in), swz, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 3));
104
105 const struct glsl_type *sampler_type =
106 glsl_sampler_type(tex_dim, false, tex_dim != GLSL_SAMPLER_DIM_3D, glsl_get_base_type(vec4));
107 nir_variable *sampler = nir_variable_create(b.shader, nir_var_uniform, sampler_type, "s_tex");
108 sampler->data.descriptor_set = 0;
109 sampler->data.binding = 0;
110
111 nir_deref_instr *tex_deref = nir_build_deref_var(&b, sampler);
112 nir_def *color = nir_tex_deref(&b, tex_deref, tex_deref, tex_pos);
113
114 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
115 color_out->data.location = FRAG_RESULT_DEPTH;
116 nir_store_var(&b, color_out, color, 0x1);
117
118 return b.shader;
119 }
120
121 static nir_shader *
build_nir_copy_fragment_shader_stencil(struct radv_device * dev,enum glsl_sampler_dim tex_dim)122 build_nir_copy_fragment_shader_stencil(struct radv_device *dev, enum glsl_sampler_dim tex_dim)
123 {
124 const struct glsl_type *vec4 = glsl_vec4_type();
125 nir_builder b = radv_meta_init_shader(dev, MESA_SHADER_FRAGMENT, "meta_blit_stencil_fs.%d", tex_dim);
126
127 nir_variable *tex_pos_in = nir_variable_create(b.shader, nir_var_shader_in, vec4, "v_tex_pos");
128 tex_pos_in->data.location = VARYING_SLOT_VAR0;
129
130 /* Swizzle the array index which comes in as Z coordinate into the right
131 * position.
132 */
133 unsigned swz[] = {0, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 1), 2};
134 nir_def *const tex_pos =
135 nir_swizzle(&b, nir_load_var(&b, tex_pos_in), swz, (tex_dim == GLSL_SAMPLER_DIM_1D ? 2 : 3));
136
137 const struct glsl_type *sampler_type =
138 glsl_sampler_type(tex_dim, false, tex_dim != GLSL_SAMPLER_DIM_3D, glsl_get_base_type(vec4));
139 nir_variable *sampler = nir_variable_create(b.shader, nir_var_uniform, sampler_type, "s_tex");
140 sampler->data.descriptor_set = 0;
141 sampler->data.binding = 0;
142
143 nir_deref_instr *tex_deref = nir_build_deref_var(&b, sampler);
144 nir_def *color = nir_tex_deref(&b, tex_deref, tex_deref, tex_pos);
145
146 nir_variable *color_out = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color");
147 color_out->data.location = FRAG_RESULT_STENCIL;
148 nir_store_var(&b, color_out, color, 0x1);
149
150 return b.shader;
151 }
152
153 static enum glsl_sampler_dim
translate_sampler_dim(VkImageType type)154 translate_sampler_dim(VkImageType type)
155 {
156 switch (type) {
157 case VK_IMAGE_TYPE_1D:
158 return GLSL_SAMPLER_DIM_1D;
159 case VK_IMAGE_TYPE_2D:
160 return GLSL_SAMPLER_DIM_2D;
161 case VK_IMAGE_TYPE_3D:
162 return GLSL_SAMPLER_DIM_3D;
163 default:
164 unreachable("Unhandled image type");
165 }
166 }
167
168 static VkResult
get_pipeline(struct radv_device * device,const struct radv_image_view * src_iview,const struct radv_image_view * dst_iview,VkPipeline * pipeline_out)169 get_pipeline(struct radv_device *device, const struct radv_image_view *src_iview,
170 const struct radv_image_view *dst_iview, VkPipeline *pipeline_out)
171 {
172 struct radv_meta_state *state = &device->meta_state;
173 const struct radv_image *src_image = src_iview->image;
174 const struct radv_image *dst_image = dst_iview->image;
175 VkFormat format = VK_FORMAT_UNDEFINED;
176 VkPipeline *pipeline;
177 unsigned fs_key = 0;
178 VkResult result = VK_SUCCESS;
179
180 mtx_lock(&state->mtx);
181 switch (src_image->vk.aspects) {
182 case VK_IMAGE_ASPECT_COLOR_BIT: {
183 fs_key = radv_format_meta_fs_key(device, dst_image->vk.format);
184 format = radv_fs_key_format_exemplars[fs_key];
185
186 switch (src_image->vk.image_type) {
187 case VK_IMAGE_TYPE_1D:
188 pipeline = &device->meta_state.blit.pipeline_1d_src[fs_key];
189 break;
190 case VK_IMAGE_TYPE_2D:
191 pipeline = &device->meta_state.blit.pipeline_2d_src[fs_key];
192 break;
193 case VK_IMAGE_TYPE_3D:
194 pipeline = &device->meta_state.blit.pipeline_3d_src[fs_key];
195 break;
196 default:
197 unreachable("bad VkImageType");
198 }
199 break;
200 }
201 case VK_IMAGE_ASPECT_DEPTH_BIT: {
202 format = VK_FORMAT_D32_SFLOAT;
203
204 switch (src_image->vk.image_type) {
205 case VK_IMAGE_TYPE_1D:
206 pipeline = &device->meta_state.blit.depth_only_1d_pipeline;
207 break;
208 case VK_IMAGE_TYPE_2D:
209 pipeline = &device->meta_state.blit.depth_only_2d_pipeline;
210 break;
211 case VK_IMAGE_TYPE_3D:
212 pipeline = &device->meta_state.blit.depth_only_3d_pipeline;
213 break;
214 default:
215 unreachable("bad VkImageType");
216 }
217 break;
218 }
219 case VK_IMAGE_ASPECT_STENCIL_BIT: {
220 format = VK_FORMAT_S8_UINT;
221
222 switch (src_image->vk.image_type) {
223 case VK_IMAGE_TYPE_1D:
224 pipeline = &device->meta_state.blit.stencil_only_1d_pipeline;
225 break;
226 case VK_IMAGE_TYPE_2D:
227 pipeline = &device->meta_state.blit.stencil_only_2d_pipeline;
228 break;
229 case VK_IMAGE_TYPE_3D:
230 pipeline = &device->meta_state.blit.stencil_only_3d_pipeline;
231 break;
232 default:
233 unreachable("bad VkImageType");
234 }
235 break;
236 }
237 default:
238 unreachable("bad VkImageType");
239 }
240
241 if (!*pipeline) {
242 result = create_pipeline(device, src_iview->vk.aspects, translate_sampler_dim(src_image->vk.image_type), format,
243 pipeline);
244 if (result != VK_SUCCESS)
245 goto fail;
246 }
247
248 *pipeline_out = *pipeline;
249
250 fail:
251 mtx_unlock(&state->mtx);
252 return result;
253 }
254
255 static void
meta_emit_blit(struct radv_cmd_buffer * cmd_buffer,struct radv_image_view * src_iview,VkImageLayout src_image_layout,float src_offset_0[3],float src_offset_1[3],struct radv_image_view * dst_iview,VkImageLayout dst_image_layout,VkRect2D dst_box,VkSampler sampler)256 meta_emit_blit(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview, VkImageLayout src_image_layout,
257 float src_offset_0[3], float src_offset_1[3], struct radv_image_view *dst_iview,
258 VkImageLayout dst_image_layout, VkRect2D dst_box, VkSampler sampler)
259 {
260 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
261 const struct radv_image *src_image = src_iview->image;
262 const struct radv_image *dst_image = dst_iview->image;
263 uint32_t src_width = u_minify(src_image->vk.extent.width, src_iview->vk.base_mip_level);
264 uint32_t src_height = u_minify(src_image->vk.extent.height, src_iview->vk.base_mip_level);
265 uint32_t src_depth = u_minify(src_image->vk.extent.depth, src_iview->vk.base_mip_level);
266 uint32_t dst_width = u_minify(dst_image->vk.extent.width, dst_iview->vk.base_mip_level);
267 uint32_t dst_height = u_minify(dst_image->vk.extent.height, dst_iview->vk.base_mip_level);
268 VkPipeline pipeline;
269 VkResult result;
270
271 assert(src_image->vk.samples == dst_image->vk.samples);
272
273 result = get_pipeline(device, src_iview, dst_iview, &pipeline);
274 if (result != VK_SUCCESS) {
275 vk_command_buffer_set_error(&cmd_buffer->vk, result);
276 return;
277 }
278
279 float vertex_push_constants[5] = {
280 src_offset_0[0] / (float)src_width, src_offset_0[1] / (float)src_height, src_offset_1[0] / (float)src_width,
281 src_offset_1[1] / (float)src_height, src_offset_0[2] / (float)src_depth,
282 };
283
284 vk_common_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), device->meta_state.blit.pipeline_layout,
285 VK_SHADER_STAGE_VERTEX_BIT, 0, 20, vertex_push_constants);
286
287 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline);
288
289 radv_meta_push_descriptor_set(cmd_buffer, VK_PIPELINE_BIND_POINT_GRAPHICS, device->meta_state.blit.pipeline_layout,
290 0, 1,
291 (VkWriteDescriptorSet[]){{.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
292 .dstBinding = 0,
293 .dstArrayElement = 0,
294 .descriptorCount = 1,
295 .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
296 .pImageInfo = (VkDescriptorImageInfo[]){
297 {
298 .sampler = sampler,
299 .imageView = radv_image_view_to_handle(src_iview),
300 .imageLayout = VK_IMAGE_LAYOUT_GENERAL,
301 },
302 }}});
303
304 VkRenderingInfo rendering_info = {
305 .sType = VK_STRUCTURE_TYPE_RENDERING_INFO,
306 .flags = VK_RENDERING_INPUT_ATTACHMENT_NO_CONCURRENT_WRITES_BIT_MESA,
307 .renderArea =
308 {
309 .offset = {0, 0},
310 .extent = {dst_width, dst_height},
311 },
312 .layerCount = 1,
313 };
314
315 VkRenderingAttachmentInfo color_att;
316 if (src_image->vk.aspects == VK_IMAGE_ASPECT_COLOR_BIT) {
317 unsigned dst_layout = radv_meta_dst_layout_from_layout(dst_image_layout);
318 VkImageLayout layout = radv_meta_dst_layout_to_layout(dst_layout);
319
320 color_att = (VkRenderingAttachmentInfo){
321 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
322 .imageView = radv_image_view_to_handle(dst_iview),
323 .imageLayout = layout,
324 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
325 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
326 };
327 rendering_info.colorAttachmentCount = 1;
328 rendering_info.pColorAttachments = &color_att;
329 }
330
331 VkRenderingAttachmentInfo depth_att;
332 if (src_image->vk.aspects & VK_IMAGE_ASPECT_DEPTH_BIT) {
333 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst_image_layout);
334 VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
335
336 depth_att = (VkRenderingAttachmentInfo){
337 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
338 .imageView = radv_image_view_to_handle(dst_iview),
339 .imageLayout = layout,
340 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
341 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
342 };
343 rendering_info.pDepthAttachment = &depth_att;
344 }
345
346 VkRenderingAttachmentInfo stencil_att;
347 if (src_image->vk.aspects & VK_IMAGE_ASPECT_STENCIL_BIT) {
348 enum radv_blit_ds_layout ds_layout = radv_meta_blit_ds_to_type(dst_image_layout);
349 VkImageLayout layout = radv_meta_blit_ds_to_layout(ds_layout);
350
351 stencil_att = (VkRenderingAttachmentInfo){
352 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO,
353 .imageView = radv_image_view_to_handle(dst_iview),
354 .imageLayout = layout,
355 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD,
356 .storeOp = VK_ATTACHMENT_STORE_OP_STORE,
357 };
358 rendering_info.pStencilAttachment = &stencil_att;
359 }
360
361 radv_CmdBeginRendering(radv_cmd_buffer_to_handle(cmd_buffer), &rendering_info);
362
363 radv_CmdDraw(radv_cmd_buffer_to_handle(cmd_buffer), 3, 1, 0, 0);
364
365 radv_CmdEndRendering(radv_cmd_buffer_to_handle(cmd_buffer));
366 }
367
368 static bool
flip_coords(unsigned * src0,unsigned * src1,unsigned * dst0,unsigned * dst1)369 flip_coords(unsigned *src0, unsigned *src1, unsigned *dst0, unsigned *dst1)
370 {
371 bool flip = false;
372 if (*src0 > *src1) {
373 unsigned tmp = *src0;
374 *src0 = *src1;
375 *src1 = tmp;
376 flip = !flip;
377 }
378
379 if (*dst0 > *dst1) {
380 unsigned tmp = *dst0;
381 *dst0 = *dst1;
382 *dst1 = tmp;
383 flip = !flip;
384 }
385 return flip;
386 }
387
388 static void
blit_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,struct radv_image * dst_image,VkImageLayout dst_image_layout,const VkImageBlit2 * region,VkFilter filter)389 blit_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkImageLayout src_image_layout,
390 struct radv_image *dst_image, VkImageLayout dst_image_layout, const VkImageBlit2 *region, VkFilter filter)
391 {
392 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
393 const VkImageSubresourceLayers *src_res = ®ion->srcSubresource;
394 const VkImageSubresourceLayers *dst_res = ®ion->dstSubresource;
395 struct radv_meta_saved_state saved_state;
396 VkSampler sampler;
397
398 /* From the Vulkan 1.0 spec:
399 *
400 * vkCmdBlitImage must not be used for multisampled source or
401 * destination images. Use vkCmdResolveImage for this purpose.
402 */
403 assert(src_image->vk.samples == 1);
404 assert(dst_image->vk.samples == 1);
405
406 radv_CreateSampler(radv_device_to_handle(device),
407 &(VkSamplerCreateInfo){
408 .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
409 .magFilter = filter,
410 .minFilter = filter,
411 .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
412 .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
413 .addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
414 },
415 &cmd_buffer->vk.pool->alloc, &sampler);
416
417 /* VK_EXT_conditional_rendering says that blit commands should not be
418 * affected by conditional rendering.
419 */
420 radv_meta_save(&saved_state, cmd_buffer,
421 RADV_META_SAVE_GRAPHICS_PIPELINE | RADV_META_SAVE_CONSTANTS | RADV_META_SAVE_DESCRIPTORS |
422 RADV_META_SUSPEND_PREDICATING);
423
424 unsigned dst_start, dst_end;
425 if (dst_image->vk.image_type == VK_IMAGE_TYPE_3D) {
426 assert(dst_res->baseArrayLayer == 0);
427 dst_start = region->dstOffsets[0].z;
428 dst_end = region->dstOffsets[1].z;
429 } else {
430 dst_start = dst_res->baseArrayLayer;
431 dst_end = dst_start + vk_image_subresource_layer_count(&dst_image->vk, dst_res);
432 }
433
434 unsigned src_start, src_end;
435 if (src_image->vk.image_type == VK_IMAGE_TYPE_3D) {
436 assert(src_res->baseArrayLayer == 0);
437 src_start = region->srcOffsets[0].z;
438 src_end = region->srcOffsets[1].z;
439 } else {
440 src_start = src_res->baseArrayLayer;
441 src_end = src_start + vk_image_subresource_layer_count(&src_image->vk, src_res);
442 }
443
444 bool flip_z = flip_coords(&src_start, &src_end, &dst_start, &dst_end);
445 float src_z_step = (float)(src_end - src_start) / (float)(dst_end - dst_start);
446
447 /* There is no interpolation to the pixel center during
448 * rendering, so add the 0.5 offset ourselves here. */
449 float depth_center_offset = 0;
450 if (src_image->vk.image_type == VK_IMAGE_TYPE_3D)
451 depth_center_offset = 0.5 / (dst_end - dst_start) * (src_end - src_start);
452
453 if (flip_z) {
454 src_start = src_end;
455 src_z_step *= -1;
456 depth_center_offset *= -1;
457 }
458
459 unsigned src_x0 = region->srcOffsets[0].x;
460 unsigned src_x1 = region->srcOffsets[1].x;
461 unsigned dst_x0 = region->dstOffsets[0].x;
462 unsigned dst_x1 = region->dstOffsets[1].x;
463
464 unsigned src_y0 = region->srcOffsets[0].y;
465 unsigned src_y1 = region->srcOffsets[1].y;
466 unsigned dst_y0 = region->dstOffsets[0].y;
467 unsigned dst_y1 = region->dstOffsets[1].y;
468
469 VkRect2D dst_box;
470 dst_box.offset.x = MIN2(dst_x0, dst_x1);
471 dst_box.offset.y = MIN2(dst_y0, dst_y1);
472 dst_box.extent.width = dst_x1 - dst_x0;
473 dst_box.extent.height = dst_y1 - dst_y0;
474
475 const VkOffset2D dst_offset_0 = {
476 .x = dst_x0,
477 .y = dst_y0,
478 };
479 const VkOffset2D dst_offset_1 = {
480 .x = dst_x1,
481 .y = dst_y1,
482 };
483
484 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
485 &(VkViewport){.x = dst_offset_0.x,
486 .y = dst_offset_0.y,
487 .width = dst_offset_1.x - dst_offset_0.x,
488 .height = dst_offset_1.y - dst_offset_0.y,
489 .minDepth = 0.0f,
490 .maxDepth = 1.0f});
491
492 radv_CmdSetScissor(
493 radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
494 &(VkRect2D){
495 .offset = (VkOffset2D){MIN2(dst_offset_0.x, dst_offset_1.x), MIN2(dst_offset_0.y, dst_offset_1.y)},
496 .extent = (VkExtent2D){abs(dst_offset_1.x - dst_offset_0.x), abs(dst_offset_1.y - dst_offset_0.y)},
497 });
498
499 const unsigned num_layers = dst_end - dst_start;
500 for (unsigned i = 0; i < num_layers; i++) {
501 struct radv_image_view dst_iview, src_iview;
502
503 float src_offset_0[3] = {
504 src_x0,
505 src_y0,
506 src_start + i * src_z_step + depth_center_offset,
507 };
508 float src_offset_1[3] = {
509 src_x1,
510 src_y1,
511 src_start + i * src_z_step + depth_center_offset,
512 };
513 const uint32_t dst_array_slice = dst_start + i;
514
515 /* 3D images have just 1 layer */
516 const uint32_t src_array_slice = src_image->vk.image_type == VK_IMAGE_TYPE_3D ? 0 : src_start + i;
517
518 radv_image_view_init(&dst_iview, device,
519 &(VkImageViewCreateInfo){
520 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
521 .image = radv_image_to_handle(dst_image),
522 .viewType = radv_meta_get_view_type(dst_image),
523 .format = dst_image->vk.format,
524 .subresourceRange = {.aspectMask = dst_res->aspectMask,
525 .baseMipLevel = dst_res->mipLevel,
526 .levelCount = 1,
527 .baseArrayLayer = dst_array_slice,
528 .layerCount = 1},
529 },
530 0, NULL);
531 radv_image_view_init(&src_iview, device,
532 &(VkImageViewCreateInfo){
533 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
534 .image = radv_image_to_handle(src_image),
535 .viewType = radv_meta_get_view_type(src_image),
536 .format = src_image->vk.format,
537 .subresourceRange = {.aspectMask = src_res->aspectMask,
538 .baseMipLevel = src_res->mipLevel,
539 .levelCount = 1,
540 .baseArrayLayer = src_array_slice,
541 .layerCount = 1},
542 },
543 0, NULL);
544 meta_emit_blit(cmd_buffer, &src_iview, src_image_layout, src_offset_0, src_offset_1, &dst_iview, dst_image_layout,
545 dst_box, sampler);
546
547 radv_image_view_finish(&dst_iview);
548 radv_image_view_finish(&src_iview);
549 }
550
551 radv_meta_restore(&saved_state, cmd_buffer);
552
553 radv_DestroySampler(radv_device_to_handle(device), sampler, &cmd_buffer->vk.pool->alloc);
554 }
555
556 VKAPI_ATTR void VKAPI_CALL
radv_CmdBlitImage2(VkCommandBuffer commandBuffer,const VkBlitImageInfo2 * pBlitImageInfo)557 radv_CmdBlitImage2(VkCommandBuffer commandBuffer, const VkBlitImageInfo2 *pBlitImageInfo)
558 {
559 VK_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
560 VK_FROM_HANDLE(radv_image, src_image, pBlitImageInfo->srcImage);
561 VK_FROM_HANDLE(radv_image, dst_image, pBlitImageInfo->dstImage);
562
563 for (unsigned r = 0; r < pBlitImageInfo->regionCount; r++) {
564 blit_image(cmd_buffer, src_image, pBlitImageInfo->srcImageLayout, dst_image, pBlitImageInfo->dstImageLayout,
565 &pBlitImageInfo->pRegions[r], pBlitImageInfo->filter);
566 }
567 }
568
569 void
radv_device_finish_meta_blit_state(struct radv_device * device)570 radv_device_finish_meta_blit_state(struct radv_device *device)
571 {
572 struct radv_meta_state *state = &device->meta_state;
573
574 for (unsigned i = 0; i < NUM_META_FS_KEYS; ++i) {
575 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.pipeline_1d_src[i], &state->alloc);
576 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.pipeline_2d_src[i], &state->alloc);
577 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.pipeline_3d_src[i], &state->alloc);
578 }
579
580 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.depth_only_1d_pipeline, &state->alloc);
581 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.depth_only_2d_pipeline, &state->alloc);
582 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.depth_only_3d_pipeline, &state->alloc);
583
584 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.stencil_only_1d_pipeline, &state->alloc);
585 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.stencil_only_2d_pipeline, &state->alloc);
586 radv_DestroyPipeline(radv_device_to_handle(device), state->blit.stencil_only_3d_pipeline, &state->alloc);
587
588 radv_DestroyPipelineLayout(radv_device_to_handle(device), state->blit.pipeline_layout, &state->alloc);
589 device->vk.dispatch_table.DestroyDescriptorSetLayout(radv_device_to_handle(device), state->blit.ds_layout,
590 &state->alloc);
591 }
592
593 static VkResult
create_pipeline(struct radv_device * device,VkImageAspectFlagBits aspect,enum glsl_sampler_dim tex_dim,VkFormat format,VkPipeline * pipeline)594 create_pipeline(struct radv_device *device, VkImageAspectFlagBits aspect, enum glsl_sampler_dim tex_dim,
595 VkFormat format, VkPipeline *pipeline)
596 {
597 VkResult result = VK_SUCCESS;
598
599 if (!device->meta_state.blit.ds_layout) {
600 const VkDescriptorSetLayoutBinding binding = {
601 .binding = 0,
602 .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
603 .descriptorCount = 1,
604 .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT,
605 };
606
607 result = radv_meta_create_descriptor_set_layout(device, 1, &binding, &device->meta_state.blit.ds_layout);
608 if (result != VK_SUCCESS)
609 return result;
610 }
611
612 if (!device->meta_state.blit.pipeline_layout) {
613 const VkPushConstantRange push_constant_range = {VK_SHADER_STAGE_VERTEX_BIT, 0, 20};
614
615 result = radv_meta_create_pipeline_layout(device, &device->meta_state.blit.ds_layout, 1, &push_constant_range,
616 &device->meta_state.blit.pipeline_layout);
617 if (result != VK_SUCCESS)
618 return result;
619 }
620
621 nir_shader *fs;
622 nir_shader *vs = build_nir_vertex_shader(device);
623
624 VkPipelineRenderingCreateInfo rendering_create_info = {
625 .sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO,
626 };
627
628 switch (aspect) {
629 case VK_IMAGE_ASPECT_COLOR_BIT:
630 fs = build_nir_copy_fragment_shader(device, tex_dim);
631 rendering_create_info.colorAttachmentCount = 1;
632 rendering_create_info.pColorAttachmentFormats = &format;
633 break;
634 case VK_IMAGE_ASPECT_DEPTH_BIT:
635 fs = build_nir_copy_fragment_shader_depth(device, tex_dim);
636 rendering_create_info.depthAttachmentFormat = format;
637 break;
638 case VK_IMAGE_ASPECT_STENCIL_BIT:
639 fs = build_nir_copy_fragment_shader_stencil(device, tex_dim);
640 rendering_create_info.stencilAttachmentFormat = format;
641 break;
642 default:
643 unreachable("Unhandled aspect");
644 }
645 VkPipelineVertexInputStateCreateInfo vi_create_info = {
646 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
647 .vertexBindingDescriptionCount = 0,
648 .vertexAttributeDescriptionCount = 0,
649 };
650
651 VkPipelineShaderStageCreateInfo pipeline_shader_stages[] = {
652 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
653 .stage = VK_SHADER_STAGE_VERTEX_BIT,
654 .module = vk_shader_module_handle_from_nir(vs),
655 .pName = "main",
656 .pSpecializationInfo = NULL},
657 {.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
658 .stage = VK_SHADER_STAGE_FRAGMENT_BIT,
659 .module = vk_shader_module_handle_from_nir(fs),
660 .pName = "main",
661 .pSpecializationInfo = NULL},
662 };
663
664 VkGraphicsPipelineCreateInfo vk_pipeline_info = {
665 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
666 .pNext = &rendering_create_info,
667 .stageCount = ARRAY_SIZE(pipeline_shader_stages),
668 .pStages = pipeline_shader_stages,
669 .pVertexInputState = &vi_create_info,
670 .pInputAssemblyState =
671 &(VkPipelineInputAssemblyStateCreateInfo){
672 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
673 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP,
674 .primitiveRestartEnable = false,
675 },
676 .pViewportState =
677 &(VkPipelineViewportStateCreateInfo){
678 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
679 .viewportCount = 1,
680 .scissorCount = 1,
681 },
682 .pRasterizationState =
683 &(VkPipelineRasterizationStateCreateInfo){.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
684 .rasterizerDiscardEnable = false,
685 .polygonMode = VK_POLYGON_MODE_FILL,
686 .cullMode = VK_CULL_MODE_NONE,
687 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE,
688 .lineWidth = 1.0f},
689 .pMultisampleState =
690 &(VkPipelineMultisampleStateCreateInfo){
691 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
692 .rasterizationSamples = 1,
693 .sampleShadingEnable = false,
694 .pSampleMask = (VkSampleMask[]){UINT32_MAX},
695 },
696 .pDynamicState =
697 &(VkPipelineDynamicStateCreateInfo){
698 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO,
699 .dynamicStateCount = 2,
700 .pDynamicStates =
701 (VkDynamicState[]){
702 VK_DYNAMIC_STATE_VIEWPORT,
703 VK_DYNAMIC_STATE_SCISSOR,
704 },
705 },
706 .flags = 0,
707 .layout = device->meta_state.blit.pipeline_layout,
708 .renderPass = VK_NULL_HANDLE,
709 .subpass = 0,
710 };
711
712 VkPipelineColorBlendStateCreateInfo color_blend_info = {
713 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
714 .attachmentCount = 1,
715 .pAttachments =
716 (VkPipelineColorBlendAttachmentState[]){
717 {.colorWriteMask = VK_COLOR_COMPONENT_A_BIT | VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT |
718 VK_COLOR_COMPONENT_B_BIT},
719 },
720 .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}};
721
722 VkPipelineDepthStencilStateCreateInfo depth_info = {
723 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
724 .depthTestEnable = true,
725 .depthWriteEnable = true,
726 .depthCompareOp = VK_COMPARE_OP_ALWAYS,
727 };
728
729 VkPipelineDepthStencilStateCreateInfo stencil_info = {
730 .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO,
731 .depthTestEnable = false,
732 .depthWriteEnable = false,
733 .stencilTestEnable = true,
734 .front = {.failOp = VK_STENCIL_OP_REPLACE,
735 .passOp = VK_STENCIL_OP_REPLACE,
736 .depthFailOp = VK_STENCIL_OP_REPLACE,
737 .compareOp = VK_COMPARE_OP_ALWAYS,
738 .compareMask = 0xff,
739 .writeMask = 0xff,
740 .reference = 0},
741 .back = {.failOp = VK_STENCIL_OP_REPLACE,
742 .passOp = VK_STENCIL_OP_REPLACE,
743 .depthFailOp = VK_STENCIL_OP_REPLACE,
744 .compareOp = VK_COMPARE_OP_ALWAYS,
745 .compareMask = 0xff,
746 .writeMask = 0xff,
747 .reference = 0},
748 .depthCompareOp = VK_COMPARE_OP_ALWAYS,
749 };
750
751 switch (aspect) {
752 case VK_IMAGE_ASPECT_COLOR_BIT:
753 vk_pipeline_info.pColorBlendState = &color_blend_info;
754 break;
755 case VK_IMAGE_ASPECT_DEPTH_BIT:
756 vk_pipeline_info.pDepthStencilState = &depth_info;
757 break;
758 case VK_IMAGE_ASPECT_STENCIL_BIT:
759 vk_pipeline_info.pDepthStencilState = &stencil_info;
760 break;
761 default:
762 unreachable("Unhandled aspect");
763 }
764
765 const struct radv_graphics_pipeline_create_info radv_pipeline_info = {.use_rectlist = true};
766
767 result = radv_graphics_pipeline_create(radv_device_to_handle(device), device->meta_state.cache, &vk_pipeline_info,
768 &radv_pipeline_info, &device->meta_state.alloc, pipeline);
769 ralloc_free(vs);
770 ralloc_free(fs);
771 return result;
772 }
773
774 static VkResult
radv_device_init_meta_blit_color(struct radv_device * device)775 radv_device_init_meta_blit_color(struct radv_device *device)
776 {
777 VkResult result;
778
779 for (unsigned i = 0; i < NUM_META_FS_KEYS; ++i) {
780 VkFormat format = radv_fs_key_format_exemplars[i];
781 unsigned key = radv_format_meta_fs_key(device, format);
782
783 result = create_pipeline(device, VK_IMAGE_ASPECT_COLOR_BIT, GLSL_SAMPLER_DIM_1D, format,
784 &device->meta_state.blit.pipeline_1d_src[key]);
785 if (result != VK_SUCCESS)
786 return result;
787
788 result = create_pipeline(device, VK_IMAGE_ASPECT_COLOR_BIT, GLSL_SAMPLER_DIM_2D, format,
789 &device->meta_state.blit.pipeline_2d_src[key]);
790 if (result != VK_SUCCESS)
791 return result;
792
793 result = create_pipeline(device, VK_IMAGE_ASPECT_COLOR_BIT, GLSL_SAMPLER_DIM_3D, format,
794 &device->meta_state.blit.pipeline_3d_src[key]);
795 if (result != VK_SUCCESS)
796 return result;
797 }
798
799 return VK_SUCCESS;
800 }
801
802 static VkResult
radv_device_init_meta_blit_depth(struct radv_device * device)803 radv_device_init_meta_blit_depth(struct radv_device *device)
804 {
805 VkResult result;
806
807 result = create_pipeline(device, VK_IMAGE_ASPECT_DEPTH_BIT, GLSL_SAMPLER_DIM_1D, VK_FORMAT_D32_SFLOAT,
808 &device->meta_state.blit.depth_only_1d_pipeline);
809 if (result != VK_SUCCESS)
810 return result;
811
812 result = create_pipeline(device, VK_IMAGE_ASPECT_DEPTH_BIT, GLSL_SAMPLER_DIM_2D, VK_FORMAT_D32_SFLOAT,
813 &device->meta_state.blit.depth_only_2d_pipeline);
814 if (result != VK_SUCCESS)
815 return result;
816
817 return create_pipeline(device, VK_IMAGE_ASPECT_DEPTH_BIT, GLSL_SAMPLER_DIM_3D, VK_FORMAT_D32_SFLOAT,
818 &device->meta_state.blit.depth_only_3d_pipeline);
819 }
820
821 static VkResult
radv_device_init_meta_blit_stencil(struct radv_device * device)822 radv_device_init_meta_blit_stencil(struct radv_device *device)
823 {
824 VkResult result;
825
826 result = create_pipeline(device, VK_IMAGE_ASPECT_STENCIL_BIT, GLSL_SAMPLER_DIM_1D, VK_FORMAT_S8_UINT,
827 &device->meta_state.blit.stencil_only_1d_pipeline);
828 if (result != VK_SUCCESS)
829 return result;
830
831 result = create_pipeline(device, VK_IMAGE_ASPECT_STENCIL_BIT, GLSL_SAMPLER_DIM_2D, VK_FORMAT_S8_UINT,
832 &device->meta_state.blit.stencil_only_2d_pipeline);
833 if (result != VK_SUCCESS)
834 return result;
835
836 return create_pipeline(device, VK_IMAGE_ASPECT_STENCIL_BIT, GLSL_SAMPLER_DIM_3D, VK_FORMAT_S8_UINT,
837 &device->meta_state.blit.stencil_only_3d_pipeline);
838 }
839
840 VkResult
radv_device_init_meta_blit_state(struct radv_device * device,bool on_demand)841 radv_device_init_meta_blit_state(struct radv_device *device, bool on_demand)
842 {
843 VkResult result;
844
845 if (on_demand)
846 return VK_SUCCESS;
847
848 result = radv_device_init_meta_blit_color(device);
849 if (result != VK_SUCCESS)
850 return result;
851
852 result = radv_device_init_meta_blit_depth(device);
853 if (result != VK_SUCCESS)
854 return result;
855
856 return radv_device_init_meta_blit_stencil(device);
857 }
858