1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * SPDX-License-Identifier: MIT
5 */
6
7 #include "radv_formats.h"
8 #include "radv_meta.h"
9 #include "radv_sdma.h"
10 #include "vk_format.h"
11
12 static VkFormat
vk_format_for_size(int bs)13 vk_format_for_size(int bs)
14 {
15 switch (bs) {
16 case 1:
17 return VK_FORMAT_R8_UINT;
18 case 2:
19 return VK_FORMAT_R8G8_UINT;
20 case 4:
21 return VK_FORMAT_R8G8B8A8_UINT;
22 case 8:
23 return VK_FORMAT_R16G16B16A16_UINT;
24 case 12:
25 return VK_FORMAT_R32G32B32_UINT;
26 case 16:
27 return VK_FORMAT_R32G32B32A32_UINT;
28 default:
29 unreachable("Invalid format block size");
30 }
31 }
32
33 static struct radv_meta_blit2d_surf
blit_surf_for_image_level_layer(struct radv_image * image,VkImageLayout layout,const VkImageSubresourceLayers * subres,VkImageAspectFlags aspect_mask)34 blit_surf_for_image_level_layer(struct radv_image *image, VkImageLayout layout, const VkImageSubresourceLayers *subres,
35 VkImageAspectFlags aspect_mask)
36 {
37 VkFormat format = radv_get_aspect_format(image, aspect_mask);
38
39 if (!radv_dcc_enabled(image, subres->mipLevel) && !(radv_image_is_tc_compat_htile(image)))
40 format = vk_format_for_size(vk_format_get_blocksize(format));
41
42 format = vk_format_no_srgb(format);
43
44 return (struct radv_meta_blit2d_surf){
45 .format = format,
46 .bs = vk_format_get_blocksize(format),
47 .level = subres->mipLevel,
48 .layer = subres->baseArrayLayer,
49 .image = image,
50 .aspect_mask = aspect_mask,
51 .current_layout = layout,
52 };
53 }
54
55 static bool
alloc_transfer_temp_bo(struct radv_cmd_buffer * cmd_buffer)56 alloc_transfer_temp_bo(struct radv_cmd_buffer *cmd_buffer)
57 {
58 if (cmd_buffer->transfer.copy_temp)
59 return true;
60
61 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
62 const VkResult r =
63 radv_bo_create(device, &cmd_buffer->vk.base, RADV_SDMA_TRANSFER_TEMP_BYTES, 4096, RADEON_DOMAIN_VRAM,
64 RADEON_FLAG_NO_CPU_ACCESS | RADEON_FLAG_NO_INTERPROCESS_SHARING, RADV_BO_PRIORITY_SCRATCH, 0, true,
65 &cmd_buffer->transfer.copy_temp);
66
67 if (r != VK_SUCCESS) {
68 vk_command_buffer_set_error(&cmd_buffer->vk, r);
69 return false;
70 }
71
72 radv_cs_add_buffer(device->ws, cmd_buffer->cs, cmd_buffer->transfer.copy_temp);
73 return true;
74 }
75
76 static void
transfer_copy_buffer_image(struct radv_cmd_buffer * cmd_buffer,struct radv_buffer * buffer,struct radv_image * image,const VkBufferImageCopy2 * region,bool to_image)77 transfer_copy_buffer_image(struct radv_cmd_buffer *cmd_buffer, struct radv_buffer *buffer, struct radv_image *image,
78 const VkBufferImageCopy2 *region, bool to_image)
79 {
80 const struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
81 struct radeon_cmdbuf *cs = cmd_buffer->cs;
82 const VkImageAspectFlags aspect_mask = region->imageSubresource.aspectMask;
83 const unsigned binding_idx = image->disjoint ? radv_plane_from_aspect(aspect_mask) : 0;
84
85 radv_cs_add_buffer(device->ws, cs, image->bindings[binding_idx].bo);
86 radv_cs_add_buffer(device->ws, cs, buffer->bo);
87
88 struct radv_sdma_surf buf = radv_sdma_get_buf_surf(buffer, image, region, aspect_mask);
89 const struct radv_sdma_surf img =
90 radv_sdma_get_surf(device, image, region->imageSubresource, region->imageOffset, aspect_mask);
91 const VkExtent3D extent = radv_sdma_get_copy_extent(image, region->imageSubresource, region->imageExtent);
92
93 if (radv_sdma_use_unaligned_buffer_image_copy(device, &buf, &img, extent)) {
94 if (!alloc_transfer_temp_bo(cmd_buffer))
95 return;
96
97 radv_sdma_copy_buffer_image_unaligned(device, cs, &buf, &img, extent, cmd_buffer->transfer.copy_temp, to_image);
98 return;
99 }
100
101 radv_sdma_copy_buffer_image(device, cs, &buf, &img, extent, to_image);
102 }
103
104 static void
copy_buffer_to_image(struct radv_cmd_buffer * cmd_buffer,struct radv_buffer * buffer,struct radv_image * image,VkImageLayout layout,const VkBufferImageCopy2 * region)105 copy_buffer_to_image(struct radv_cmd_buffer *cmd_buffer, struct radv_buffer *buffer, struct radv_image *image,
106 VkImageLayout layout, const VkBufferImageCopy2 *region)
107 {
108 if (cmd_buffer->qf == RADV_QUEUE_TRANSFER) {
109 transfer_copy_buffer_image(cmd_buffer, buffer, image, region, true);
110 return;
111 }
112
113 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
114 struct radv_meta_saved_state saved_state;
115 bool cs;
116
117 /* The Vulkan 1.0 spec says "dstImage must have a sample count equal to
118 * VK_SAMPLE_COUNT_1_BIT."
119 */
120 assert(image->vk.samples == 1);
121
122 cs = cmd_buffer->qf == RADV_QUEUE_COMPUTE || !radv_image_is_renderable(device, image);
123
124 /* VK_EXT_conditional_rendering says that copy commands should not be
125 * affected by conditional rendering.
126 */
127 radv_meta_save(&saved_state, cmd_buffer,
128 (cs ? RADV_META_SAVE_COMPUTE_PIPELINE : RADV_META_SAVE_GRAPHICS_PIPELINE) | RADV_META_SAVE_CONSTANTS |
129 RADV_META_SAVE_DESCRIPTORS | RADV_META_SUSPEND_PREDICATING);
130
131 /**
132 * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
133 * extent is the size in texels of the source image to copy in width,
134 * height and depth. 1D images use only x and width. 2D images use x, y,
135 * width and height. 3D images use x, y, z, width, height and depth.
136 *
137 *
138 * Also, convert the offsets and extent from units of texels to units of
139 * blocks - which is the highest resolution accessible in this command.
140 */
141 const VkOffset3D img_offset_el = vk_image_offset_to_elements(&image->vk, region->imageOffset);
142
143 /* Start creating blit rect */
144 const VkExtent3D img_extent_el = vk_image_extent_to_elements(&image->vk, region->imageExtent);
145 struct radv_meta_blit2d_rect rect = {
146 .width = img_extent_el.width,
147 .height = img_extent_el.height,
148 };
149
150 /* Create blit surfaces */
151 struct radv_meta_blit2d_surf img_bsurf =
152 blit_surf_for_image_level_layer(image, layout, ®ion->imageSubresource, region->imageSubresource.aspectMask);
153
154 if (!radv_is_buffer_format_supported(img_bsurf.format, NULL)) {
155 uint32_t queue_mask = radv_image_queue_family_mask(image, cmd_buffer->qf, cmd_buffer->qf);
156 bool compressed =
157 radv_layout_dcc_compressed(device, image, region->imageSubresource.mipLevel, layout, queue_mask);
158 if (compressed) {
159 radv_describe_barrier_start(cmd_buffer, RGP_BARRIER_UNKNOWN_REASON);
160
161 radv_decompress_dcc(cmd_buffer, image,
162 &(VkImageSubresourceRange){
163 .aspectMask = region->imageSubresource.aspectMask,
164 .baseMipLevel = region->imageSubresource.mipLevel,
165 .levelCount = 1,
166 .baseArrayLayer = region->imageSubresource.baseArrayLayer,
167 .layerCount = vk_image_subresource_layer_count(&image->vk, ®ion->imageSubresource),
168 });
169 img_bsurf.disable_compression = true;
170
171 radv_describe_barrier_end(cmd_buffer);
172 }
173 img_bsurf.format = vk_format_for_size(vk_format_get_blocksize(img_bsurf.format));
174 }
175
176 const struct vk_image_buffer_layout buf_layout = vk_image_buffer_copy_layout(&image->vk, region);
177 struct radv_meta_blit2d_buffer buf_bsurf = {
178 .bs = img_bsurf.bs,
179 .format = img_bsurf.format,
180 .buffer = buffer,
181 .offset = region->bufferOffset,
182 .pitch = buf_layout.row_stride_B / buf_layout.element_size_B,
183 };
184
185 if (image->vk.image_type == VK_IMAGE_TYPE_3D)
186 img_bsurf.layer = img_offset_el.z;
187 /* Loop through each 3D or array slice */
188 unsigned num_slices_3d = img_extent_el.depth;
189 unsigned num_slices_array = vk_image_subresource_layer_count(&image->vk, ®ion->imageSubresource);
190 unsigned slice_3d = 0;
191 unsigned slice_array = 0;
192 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
193
194 rect.dst_x = img_offset_el.x;
195 rect.dst_y = img_offset_el.y;
196
197 /* Perform Blit */
198 if (cs) {
199 radv_meta_buffer_to_image_cs(cmd_buffer, &buf_bsurf, &img_bsurf, &rect);
200 } else {
201 radv_meta_blit2d(cmd_buffer, NULL, &buf_bsurf, &img_bsurf, &rect);
202 }
203
204 /* Once we've done the blit, all of the actual information about
205 * the image is embedded in the command buffer so we can just
206 * increment the offset directly in the image effectively
207 * re-binding it to different backing memory.
208 */
209 buf_bsurf.offset += buf_layout.image_stride_B;
210 img_bsurf.layer++;
211 if (image->vk.image_type == VK_IMAGE_TYPE_3D)
212 slice_3d++;
213 else
214 slice_array++;
215 }
216
217 radv_meta_restore(&saved_state, cmd_buffer);
218 }
219
220 VKAPI_ATTR void VKAPI_CALL
radv_CmdCopyBufferToImage2(VkCommandBuffer commandBuffer,const VkCopyBufferToImageInfo2 * pCopyBufferToImageInfo)221 radv_CmdCopyBufferToImage2(VkCommandBuffer commandBuffer, const VkCopyBufferToImageInfo2 *pCopyBufferToImageInfo)
222 {
223 VK_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
224 VK_FROM_HANDLE(radv_buffer, src_buffer, pCopyBufferToImageInfo->srcBuffer);
225 VK_FROM_HANDLE(radv_image, dst_image, pCopyBufferToImageInfo->dstImage);
226 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
227 const struct radv_physical_device *pdev = radv_device_physical(device);
228
229 for (unsigned r = 0; r < pCopyBufferToImageInfo->regionCount; r++) {
230 copy_buffer_to_image(cmd_buffer, src_buffer, dst_image, pCopyBufferToImageInfo->dstImageLayout,
231 &pCopyBufferToImageInfo->pRegions[r]);
232 }
233
234 if (radv_is_format_emulated(pdev, dst_image->vk.format)) {
235 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_PS_PARTIAL_FLUSH |
236 radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
237 VK_ACCESS_TRANSFER_WRITE_BIT, dst_image) |
238 radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
239 VK_ACCESS_TRANSFER_READ_BIT, dst_image);
240
241 const enum util_format_layout format_layout = vk_format_description(dst_image->vk.format)->layout;
242 for (unsigned r = 0; r < pCopyBufferToImageInfo->regionCount; r++) {
243 if (format_layout == UTIL_FORMAT_LAYOUT_ASTC) {
244 radv_meta_decode_astc(cmd_buffer, dst_image, pCopyBufferToImageInfo->dstImageLayout,
245 &pCopyBufferToImageInfo->pRegions[r].imageSubresource,
246 pCopyBufferToImageInfo->pRegions[r].imageOffset,
247 pCopyBufferToImageInfo->pRegions[r].imageExtent);
248 } else {
249 radv_meta_decode_etc(cmd_buffer, dst_image, pCopyBufferToImageInfo->dstImageLayout,
250 &pCopyBufferToImageInfo->pRegions[r].imageSubresource,
251 pCopyBufferToImageInfo->pRegions[r].imageOffset,
252 pCopyBufferToImageInfo->pRegions[r].imageExtent);
253 }
254 }
255 }
256 }
257
258 static void
copy_image_to_buffer(struct radv_cmd_buffer * cmd_buffer,struct radv_buffer * buffer,struct radv_image * image,VkImageLayout layout,const VkBufferImageCopy2 * region)259 copy_image_to_buffer(struct radv_cmd_buffer *cmd_buffer, struct radv_buffer *buffer, struct radv_image *image,
260 VkImageLayout layout, const VkBufferImageCopy2 *region)
261 {
262 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
263 if (cmd_buffer->qf == RADV_QUEUE_TRANSFER) {
264 transfer_copy_buffer_image(cmd_buffer, buffer, image, region, false);
265 return;
266 }
267
268 struct radv_meta_saved_state saved_state;
269
270 /* VK_EXT_conditional_rendering says that copy commands should not be
271 * affected by conditional rendering.
272 */
273 radv_meta_save(&saved_state, cmd_buffer,
274 RADV_META_SAVE_COMPUTE_PIPELINE | RADV_META_SAVE_CONSTANTS | RADV_META_SAVE_DESCRIPTORS |
275 RADV_META_SUSPEND_PREDICATING);
276
277 /**
278 * From the Vulkan 1.0.6 spec: 18.3 Copying Data Between Images
279 * extent is the size in texels of the source image to copy in width,
280 * height and depth. 1D images use only x and width. 2D images use x, y,
281 * width and height. 3D images use x, y, z, width, height and depth.
282 *
283 *
284 * Also, convert the offsets and extent from units of texels to units of
285 * blocks - which is the highest resolution accessible in this command.
286 */
287 const VkOffset3D img_offset_el = vk_image_offset_to_elements(&image->vk, region->imageOffset);
288 const VkExtent3D bufferExtent = {
289 .width = region->bufferRowLength ? region->bufferRowLength : region->imageExtent.width,
290 .height = region->bufferImageHeight ? region->bufferImageHeight : region->imageExtent.height,
291 };
292 const VkExtent3D buf_extent_el = vk_image_extent_to_elements(&image->vk, bufferExtent);
293
294 /* Start creating blit rect */
295 const VkExtent3D img_extent_el = vk_image_extent_to_elements(&image->vk, region->imageExtent);
296 struct radv_meta_blit2d_rect rect = {
297 .width = img_extent_el.width,
298 .height = img_extent_el.height,
299 };
300
301 /* Create blit surfaces */
302 struct radv_meta_blit2d_surf img_info =
303 blit_surf_for_image_level_layer(image, layout, ®ion->imageSubresource, region->imageSubresource.aspectMask);
304
305 if (!radv_is_buffer_format_supported(img_info.format, NULL)) {
306 uint32_t queue_mask = radv_image_queue_family_mask(image, cmd_buffer->qf, cmd_buffer->qf);
307 bool compressed =
308 radv_layout_dcc_compressed(device, image, region->imageSubresource.mipLevel, layout, queue_mask);
309 if (compressed) {
310 radv_describe_barrier_start(cmd_buffer, RGP_BARRIER_UNKNOWN_REASON);
311
312 radv_decompress_dcc(cmd_buffer, image,
313 &(VkImageSubresourceRange){
314 .aspectMask = region->imageSubresource.aspectMask,
315 .baseMipLevel = region->imageSubresource.mipLevel,
316 .levelCount = 1,
317 .baseArrayLayer = region->imageSubresource.baseArrayLayer,
318 .layerCount = vk_image_subresource_layer_count(&image->vk, ®ion->imageSubresource),
319 });
320 img_info.disable_compression = true;
321
322 radv_describe_barrier_end(cmd_buffer);
323 }
324 img_info.format = vk_format_for_size(vk_format_get_blocksize(img_info.format));
325 }
326
327 struct radv_meta_blit2d_buffer buf_info = {
328 .bs = img_info.bs,
329 .format = img_info.format,
330 .buffer = buffer,
331 .offset = region->bufferOffset,
332 .pitch = buf_extent_el.width,
333 };
334
335 if (image->vk.image_type == VK_IMAGE_TYPE_3D)
336 img_info.layer = img_offset_el.z;
337 /* Loop through each 3D or array slice */
338 unsigned num_slices_3d = img_extent_el.depth;
339 unsigned num_slices_array = vk_image_subresource_layer_count(&image->vk, ®ion->imageSubresource);
340 unsigned slice_3d = 0;
341 unsigned slice_array = 0;
342 while (slice_3d < num_slices_3d && slice_array < num_slices_array) {
343
344 rect.src_x = img_offset_el.x;
345 rect.src_y = img_offset_el.y;
346
347 /* Perform Blit */
348 radv_meta_image_to_buffer(cmd_buffer, &img_info, &buf_info, &rect);
349
350 buf_info.offset += buf_extent_el.width * buf_extent_el.height * buf_info.bs;
351 img_info.layer++;
352 if (image->vk.image_type == VK_IMAGE_TYPE_3D)
353 slice_3d++;
354 else
355 slice_array++;
356 }
357
358 radv_meta_restore(&saved_state, cmd_buffer);
359 }
360
361 VKAPI_ATTR void VKAPI_CALL
radv_CmdCopyImageToBuffer2(VkCommandBuffer commandBuffer,const VkCopyImageToBufferInfo2 * pCopyImageToBufferInfo)362 radv_CmdCopyImageToBuffer2(VkCommandBuffer commandBuffer, const VkCopyImageToBufferInfo2 *pCopyImageToBufferInfo)
363 {
364 VK_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
365 VK_FROM_HANDLE(radv_image, src_image, pCopyImageToBufferInfo->srcImage);
366 VK_FROM_HANDLE(radv_buffer, dst_buffer, pCopyImageToBufferInfo->dstBuffer);
367
368 for (unsigned r = 0; r < pCopyImageToBufferInfo->regionCount; r++) {
369 copy_image_to_buffer(cmd_buffer, dst_buffer, src_image, pCopyImageToBufferInfo->srcImageLayout,
370 &pCopyImageToBufferInfo->pRegions[r]);
371 }
372 }
373
374 static void
transfer_copy_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,struct radv_image * dst_image,VkImageLayout dst_image_layout,const VkImageCopy2 * region)375 transfer_copy_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkImageLayout src_image_layout,
376 struct radv_image *dst_image, VkImageLayout dst_image_layout, const VkImageCopy2 *region)
377 {
378 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
379 struct radeon_cmdbuf *cs = cmd_buffer->cs;
380 unsigned int dst_aspect_mask_remaining = region->dstSubresource.aspectMask;
381
382 u_foreach_bit (b, region->srcSubresource.aspectMask) {
383 const VkImageAspectFlags src_aspect_mask = BITFIELD_BIT(b);
384 const VkImageAspectFlags dst_aspect_mask = BITFIELD_BIT(u_bit_scan(&dst_aspect_mask_remaining));
385 const unsigned src_binding_idx = src_image->disjoint ? radv_plane_from_aspect(src_aspect_mask) : 0;
386 const unsigned dst_binding_idx = dst_image->disjoint ? radv_plane_from_aspect(dst_aspect_mask) : 0;
387
388 radv_cs_add_buffer(device->ws, cs, src_image->bindings[src_binding_idx].bo);
389 radv_cs_add_buffer(device->ws, cs, dst_image->bindings[dst_binding_idx].bo);
390
391 const struct radv_sdma_surf src =
392 radv_sdma_get_surf(device, src_image, region->srcSubresource, region->srcOffset, src_aspect_mask);
393 const struct radv_sdma_surf dst =
394 radv_sdma_get_surf(device, dst_image, region->dstSubresource, region->dstOffset, dst_aspect_mask);
395 const VkExtent3D extent = radv_sdma_get_copy_extent(src_image, region->srcSubresource, region->extent);
396
397 if (radv_sdma_use_t2t_scanline_copy(device, &src, &dst, extent)) {
398 if (!alloc_transfer_temp_bo(cmd_buffer))
399 return;
400
401 radv_sdma_copy_image_t2t_scanline(device, cs, &src, &dst, extent, cmd_buffer->transfer.copy_temp);
402 } else {
403 radv_sdma_copy_image(device, cs, &src, &dst, extent);
404 }
405 }
406 }
407
408 static void
copy_image(struct radv_cmd_buffer * cmd_buffer,struct radv_image * src_image,VkImageLayout src_image_layout,struct radv_image * dst_image,VkImageLayout dst_image_layout,const VkImageCopy2 * region)409 copy_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkImageLayout src_image_layout,
410 struct radv_image *dst_image, VkImageLayout dst_image_layout, const VkImageCopy2 *region)
411 {
412 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
413 const struct radv_physical_device *pdev = radv_device_physical(device);
414
415 if (cmd_buffer->qf == RADV_QUEUE_TRANSFER) {
416 transfer_copy_image(cmd_buffer, src_image, src_image_layout, dst_image, dst_image_layout, region);
417 return;
418 }
419
420 struct radv_meta_saved_state saved_state;
421 bool cs;
422
423 /* From the Vulkan 1.0 spec:
424 *
425 * vkCmdCopyImage can be used to copy image data between multisample images, but both images must have the same
426 * number of samples.
427 */
428 assert(src_image->vk.samples == dst_image->vk.samples);
429 /* From the Vulkan 1.3 spec:
430 *
431 * Multi-planar images can only be copied on a per-plane basis, and the subresources used in each region when
432 * copying to or from such images must specify only one plane, though different regions can specify different
433 * planes.
434 */
435 assert(src_image->plane_count == 1 || util_is_power_of_two_nonzero(region->srcSubresource.aspectMask));
436 assert(dst_image->plane_count == 1 || util_is_power_of_two_nonzero(region->dstSubresource.aspectMask));
437
438 cs = cmd_buffer->qf == RADV_QUEUE_COMPUTE || !radv_image_is_renderable(device, dst_image);
439
440 /* VK_EXT_conditional_rendering says that copy commands should not be
441 * affected by conditional rendering.
442 */
443 radv_meta_save(&saved_state, cmd_buffer,
444 (cs ? RADV_META_SAVE_COMPUTE_PIPELINE : RADV_META_SAVE_GRAPHICS_PIPELINE) | RADV_META_SAVE_CONSTANTS |
445 RADV_META_SAVE_DESCRIPTORS | RADV_META_SUSPEND_PREDICATING);
446
447 if (cs) {
448 /* For partial copies, HTILE should be decompressed before copying because the metadata is
449 * re-initialized to the uncompressed state after.
450 */
451 uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, cmd_buffer->qf);
452
453 if (radv_layout_is_htile_compressed(device, dst_image, dst_image_layout, queue_mask) &&
454 (region->dstOffset.x || region->dstOffset.y || region->dstOffset.z ||
455 region->extent.width != dst_image->vk.extent.width || region->extent.height != dst_image->vk.extent.height ||
456 region->extent.depth != dst_image->vk.extent.depth)) {
457 radv_describe_barrier_start(cmd_buffer, RGP_BARRIER_UNKNOWN_REASON);
458
459 u_foreach_bit (i, region->dstSubresource.aspectMask) {
460 unsigned aspect_mask = 1u << i;
461 radv_expand_depth_stencil(
462 cmd_buffer, dst_image,
463 &(VkImageSubresourceRange){
464 .aspectMask = aspect_mask,
465 .baseMipLevel = region->dstSubresource.mipLevel,
466 .levelCount = 1,
467 .baseArrayLayer = region->dstSubresource.baseArrayLayer,
468 .layerCount = vk_image_subresource_layer_count(&dst_image->vk, ®ion->dstSubresource),
469 },
470 NULL);
471 }
472
473 radv_describe_barrier_end(cmd_buffer);
474 }
475 }
476
477 /* Create blit surfaces */
478 struct radv_meta_blit2d_surf b_src = blit_surf_for_image_level_layer(
479 src_image, src_image_layout, ®ion->srcSubresource, region->srcSubresource.aspectMask);
480
481 struct radv_meta_blit2d_surf b_dst = blit_surf_for_image_level_layer(
482 dst_image, dst_image_layout, ®ion->dstSubresource, region->dstSubresource.aspectMask);
483
484 uint32_t dst_queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, cmd_buffer->qf);
485 bool dst_compressed =
486 radv_layout_dcc_compressed(device, dst_image, region->dstSubresource.mipLevel, dst_image_layout, dst_queue_mask);
487 uint32_t src_queue_mask = radv_image_queue_family_mask(src_image, cmd_buffer->qf, cmd_buffer->qf);
488 bool src_compressed =
489 radv_layout_dcc_compressed(device, src_image, region->srcSubresource.mipLevel, src_image_layout, src_queue_mask);
490 bool need_dcc_sign_reinterpret = false;
491
492 if (!src_compressed ||
493 (radv_dcc_formats_compatible(pdev->info.gfx_level, b_src.format, b_dst.format, &need_dcc_sign_reinterpret) &&
494 !need_dcc_sign_reinterpret)) {
495 b_src.format = b_dst.format;
496 } else if (!dst_compressed) {
497 b_dst.format = b_src.format;
498 } else {
499 radv_describe_barrier_start(cmd_buffer, RGP_BARRIER_UNKNOWN_REASON);
500
501 radv_decompress_dcc(cmd_buffer, dst_image,
502 &(VkImageSubresourceRange){
503 .aspectMask = region->dstSubresource.aspectMask,
504 .baseMipLevel = region->dstSubresource.mipLevel,
505 .levelCount = 1,
506 .baseArrayLayer = region->dstSubresource.baseArrayLayer,
507 .layerCount = vk_image_subresource_layer_count(&dst_image->vk, ®ion->dstSubresource),
508 });
509 b_dst.format = b_src.format;
510 b_dst.disable_compression = true;
511
512 radv_describe_barrier_end(cmd_buffer);
513 }
514
515 /**
516 * From the Vulkan 1.0.6 spec: 18.4 Copying Data Between Buffers and Images
517 * imageExtent is the size in texels of the image to copy in width, height
518 * and depth. 1D images use only x and width. 2D images use x, y, width
519 * and height. 3D images use x, y, z, width, height and depth.
520 *
521 * Also, convert the offsets and extent from units of texels to units of
522 * blocks - which is the highest resolution accessible in this command.
523 */
524 const VkOffset3D dst_offset_el = vk_image_offset_to_elements(&dst_image->vk, region->dstOffset);
525 const VkOffset3D src_offset_el = vk_image_offset_to_elements(&src_image->vk, region->srcOffset);
526
527 /*
528 * From Vulkan 1.0.68, "Copying Data Between Images":
529 * "When copying between compressed and uncompressed formats
530 * the extent members represent the texel dimensions of the
531 * source image and not the destination."
532 * However, we must use the destination image type to avoid
533 * clamping depth when copying multiple layers of a 2D image to
534 * a 3D image.
535 */
536 const VkExtent3D img_extent_el = vk_image_extent_to_elements(&src_image->vk, region->extent);
537
538 /* Start creating blit rect */
539 struct radv_meta_blit2d_rect rect = {
540 .width = img_extent_el.width,
541 .height = img_extent_el.height,
542 };
543
544 unsigned num_slices = vk_image_subresource_layer_count(&src_image->vk, ®ion->srcSubresource);
545
546 if (src_image->vk.image_type == VK_IMAGE_TYPE_3D) {
547 b_src.layer = src_offset_el.z;
548 num_slices = img_extent_el.depth;
549 }
550
551 if (dst_image->vk.image_type == VK_IMAGE_TYPE_3D)
552 b_dst.layer = dst_offset_el.z;
553
554 for (unsigned slice = 0; slice < num_slices; slice++) {
555 /* Finish creating blit rect */
556 rect.dst_x = dst_offset_el.x;
557 rect.dst_y = dst_offset_el.y;
558 rect.src_x = src_offset_el.x;
559 rect.src_y = src_offset_el.y;
560
561 /* Perform Blit */
562 if (cs) {
563 radv_meta_image_to_image_cs(cmd_buffer, &b_src, &b_dst, &rect);
564 } else {
565 if (radv_can_use_fmask_copy(cmd_buffer, b_src.image, b_dst.image, &rect)) {
566 radv_fmask_copy(cmd_buffer, &b_src, &b_dst);
567 } else {
568 radv_meta_blit2d(cmd_buffer, &b_src, NULL, &b_dst, &rect);
569 }
570 }
571
572 b_src.layer++;
573 b_dst.layer++;
574 }
575
576 if (cs) {
577 /* Fixup HTILE after a copy on compute. */
578 uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, cmd_buffer->qf);
579
580 if (radv_layout_is_htile_compressed(device, dst_image, dst_image_layout, queue_mask)) {
581 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_INV_VCACHE;
582
583 VkImageSubresourceRange range = {
584 .aspectMask = region->dstSubresource.aspectMask,
585 .baseMipLevel = region->dstSubresource.mipLevel,
586 .levelCount = 1,
587 .baseArrayLayer = region->dstSubresource.baseArrayLayer,
588 .layerCount = vk_image_subresource_layer_count(&dst_image->vk, ®ion->dstSubresource),
589 };
590
591 uint32_t htile_value = radv_get_htile_initial_value(device, dst_image);
592
593 cmd_buffer->state.flush_bits |= radv_clear_htile(cmd_buffer, dst_image, &range, htile_value);
594 }
595 }
596
597 radv_meta_restore(&saved_state, cmd_buffer);
598 }
599
600 VKAPI_ATTR void VKAPI_CALL
radv_CmdCopyImage2(VkCommandBuffer commandBuffer,const VkCopyImageInfo2 * pCopyImageInfo)601 radv_CmdCopyImage2(VkCommandBuffer commandBuffer, const VkCopyImageInfo2 *pCopyImageInfo)
602 {
603 VK_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
604 VK_FROM_HANDLE(radv_image, src_image, pCopyImageInfo->srcImage);
605 VK_FROM_HANDLE(radv_image, dst_image, pCopyImageInfo->dstImage);
606 struct radv_device *device = radv_cmd_buffer_device(cmd_buffer);
607 const struct radv_physical_device *pdev = radv_device_physical(device);
608
609 for (unsigned r = 0; r < pCopyImageInfo->regionCount; r++) {
610 copy_image(cmd_buffer, src_image, pCopyImageInfo->srcImageLayout, dst_image, pCopyImageInfo->dstImageLayout,
611 &pCopyImageInfo->pRegions[r]);
612 }
613
614 if (radv_is_format_emulated(pdev, dst_image->vk.format)) {
615 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_CS_PARTIAL_FLUSH | RADV_CMD_FLAG_PS_PARTIAL_FLUSH |
616 radv_src_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
617 VK_ACCESS_TRANSFER_WRITE_BIT, dst_image) |
618 radv_dst_access_flush(cmd_buffer, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
619 VK_ACCESS_TRANSFER_READ_BIT, dst_image);
620
621 const enum util_format_layout format_layout = vk_format_description(dst_image->vk.format)->layout;
622 for (unsigned r = 0; r < pCopyImageInfo->regionCount; r++) {
623 VkExtent3D dst_extent = pCopyImageInfo->pRegions[r].extent;
624 if (src_image->vk.format != dst_image->vk.format) {
625 dst_extent.width = dst_extent.width / vk_format_get_blockwidth(src_image->vk.format) *
626 vk_format_get_blockwidth(dst_image->vk.format);
627 dst_extent.height = dst_extent.height / vk_format_get_blockheight(src_image->vk.format) *
628 vk_format_get_blockheight(dst_image->vk.format);
629 }
630 if (format_layout == UTIL_FORMAT_LAYOUT_ASTC) {
631 radv_meta_decode_astc(cmd_buffer, dst_image, pCopyImageInfo->dstImageLayout,
632 &pCopyImageInfo->pRegions[r].dstSubresource, pCopyImageInfo->pRegions[r].dstOffset,
633 dst_extent);
634 } else {
635 radv_meta_decode_etc(cmd_buffer, dst_image, pCopyImageInfo->dstImageLayout,
636 &pCopyImageInfo->pRegions[r].dstSubresource, pCopyImageInfo->pRegions[r].dstOffset,
637 dst_extent);
638 }
639 }
640 }
641 }
642