1 /*
2 * Copyright © 2022 Raspberry Pi Ltd
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "v3dv_private.h"
25 #include "compiler/nir/nir_builder.h"
26
27 #include "vk_common_entrypoints.h"
28
29 static nir_shader *
get_set_event_cs(const nir_shader_compiler_options * options)30 get_set_event_cs(const nir_shader_compiler_options *options)
31 {
32 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE, options,
33 "set event cs");
34
35 nir_def *buf =
36 nir_vulkan_resource_index(&b, 2, 32, nir_imm_int(&b, 0),
37 .desc_set = 0,
38 .binding = 0,
39 .desc_type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER);
40
41 nir_def *offset =
42 nir_load_push_constant(&b, 1, 32, nir_imm_int(&b, 0), .base = 0, .range = 4);
43
44 nir_def *value =
45 nir_load_push_constant(&b, 1, 8, nir_imm_int(&b, 0), .base = 4, .range = 4);
46
47 nir_store_ssbo(&b, value, buf, offset,
48 .access = 0, .write_mask = 0x1, .align_mul = 4);
49
50 return b.shader;
51 }
52
53 static nir_shader *
get_wait_event_cs(const nir_shader_compiler_options * options)54 get_wait_event_cs(const nir_shader_compiler_options *options)
55 {
56 nir_builder b = nir_builder_init_simple_shader(MESA_SHADER_COMPUTE, options,
57 "wait event cs");
58
59 nir_def *buf =
60 nir_vulkan_resource_index(&b, 2, 32, nir_imm_int(&b, 0),
61 .desc_set = 0,
62 .binding = 0,
63 .desc_type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER);
64
65 nir_def *offset =
66 nir_load_push_constant(&b, 1, 32, nir_imm_int(&b, 0), .base = 0, .range = 4);
67
68 nir_loop *loop = nir_push_loop(&b);
69 nir_def *load =
70 nir_load_ssbo(&b, 1, 8, buf, offset, .access = 0, .align_mul = 4);
71 nir_def *value = nir_i2i32(&b, load);
72
73 nir_if *if_stmt = nir_push_if(&b, nir_ieq_imm(&b, value, 1));
74 nir_jump(&b, nir_jump_break);
75 nir_pop_if(&b, if_stmt);
76 nir_pop_loop(&b, loop);
77
78 return b.shader;
79 }
80
81 static bool
create_event_pipelines(struct v3dv_device * device)82 create_event_pipelines(struct v3dv_device *device)
83 {
84 VkResult result;
85
86 if (!device->events.descriptor_set_layout) {
87 /* Pipeline layout:
88 * - 1 storage buffer for the BO with the events state.
89 * - 2 push constants:
90 * 0B: offset of the event in the buffer (4 bytes).
91 * 4B: value for the event (1 byte), only used with the set_event_pipeline.
92 */
93 VkDescriptorSetLayoutBinding descriptor_set_layout_binding = {
94 .binding = 0,
95 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
96 .descriptorCount = 1,
97 .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT,
98 };
99
100 VkDescriptorSetLayoutCreateInfo descriptor_set_layout_info = {
101 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
102 .bindingCount = 1,
103 .pBindings = &descriptor_set_layout_binding,
104 };
105
106 result =
107 v3dv_CreateDescriptorSetLayout(v3dv_device_to_handle(device),
108 &descriptor_set_layout_info,
109 &device->vk.alloc,
110 &device->events.descriptor_set_layout);
111
112 if (result != VK_SUCCESS)
113 return false;
114 }
115
116 if (!device->events.pipeline_layout) {
117 VkPipelineLayoutCreateInfo pipeline_layout_info = {
118 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
119 .setLayoutCount = 1,
120 .pSetLayouts = &device->events.descriptor_set_layout,
121 .pushConstantRangeCount = 1,
122 .pPushConstantRanges =
123 &(VkPushConstantRange) { VK_SHADER_STAGE_COMPUTE_BIT, 0, 5 },
124 };
125
126 result =
127 v3dv_CreatePipelineLayout(v3dv_device_to_handle(device),
128 &pipeline_layout_info,
129 &device->vk.alloc,
130 &device->events.pipeline_layout);
131
132 if (result != VK_SUCCESS)
133 return false;
134 }
135
136 VkPipeline pipeline;
137
138 const nir_shader_compiler_options *options =
139 v3dv_pipeline_get_nir_options(&device->devinfo);
140
141 if (!device->events.set_event_pipeline) {
142 nir_shader *set_event_cs_nir = get_set_event_cs(options);
143 result = v3dv_create_compute_pipeline_from_nir(device,
144 set_event_cs_nir,
145 device->events.pipeline_layout,
146 &pipeline);
147 ralloc_free(set_event_cs_nir);
148 if (result != VK_SUCCESS)
149 return false;
150
151 device->events.set_event_pipeline = pipeline;
152 }
153
154 if (!device->events.wait_event_pipeline) {
155 nir_shader *wait_event_cs_nir = get_wait_event_cs(options);
156 result = v3dv_create_compute_pipeline_from_nir(device,
157 wait_event_cs_nir,
158 device->events.pipeline_layout,
159 &pipeline);
160 ralloc_free(wait_event_cs_nir);
161 if (result != VK_SUCCESS)
162 return false;
163
164 device->events.wait_event_pipeline = pipeline;
165 }
166
167 return true;
168 }
169
170 static void
destroy_event_pipelines(struct v3dv_device * device)171 destroy_event_pipelines(struct v3dv_device *device)
172 {
173 VkDevice _device = v3dv_device_to_handle(device);
174
175 v3dv_DestroyPipeline(_device, device->events.set_event_pipeline,
176 &device->vk.alloc);
177 device->events.set_event_pipeline = VK_NULL_HANDLE;
178
179 v3dv_DestroyPipeline(_device, device->events.wait_event_pipeline,
180 &device->vk.alloc);
181 device->events.wait_event_pipeline = VK_NULL_HANDLE;
182
183 v3dv_DestroyPipelineLayout(_device, device->events.pipeline_layout,
184 &device->vk.alloc);
185 device->events.pipeline_layout = VK_NULL_HANDLE;
186
187 v3dv_DestroyDescriptorSetLayout(_device,
188 device->events.descriptor_set_layout,
189 &device->vk.alloc);
190 device->events.descriptor_set_layout = VK_NULL_HANDLE;
191 }
192
193 static void
init_event(struct v3dv_device * device,struct v3dv_event * event,uint32_t index)194 init_event(struct v3dv_device *device, struct v3dv_event *event, uint32_t index)
195 {
196 vk_object_base_init(&device->vk, &event->base, VK_OBJECT_TYPE_EVENT);
197 event->index = index;
198 list_addtail(&event->link, &device->events.free_list);
199 }
200
201 VkResult
v3dv_event_allocate_resources(struct v3dv_device * device)202 v3dv_event_allocate_resources(struct v3dv_device *device)
203 {
204 VkResult result = VK_SUCCESS;
205 VkDevice _device = v3dv_device_to_handle(device);
206
207 /* BO with event states. Make sure we always align to a page size (4096)
208 * to ensure we use all the memory the kernel will allocate for the BO.
209 *
210 * CTS has tests that require over 8192 active events (yes, really) so
211 * let's make sure we allow for that.
212 */
213 const uint32_t bo_size = 3 * 4096;
214 struct v3dv_bo *bo = v3dv_bo_alloc(device, bo_size, "events", true);
215 if (!bo) {
216 result = vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
217 goto fail;
218 }
219
220 device->events.bo = bo;
221
222 if (!v3dv_bo_map(device, bo, bo_size)) {
223 result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
224 goto fail;
225 }
226
227 /* Pre-allocate our events, each event requires 1 byte of BO storage */
228 device->events.event_count = bo_size;
229 device->events.events =
230 vk_zalloc2(&device->vk.alloc, NULL,
231 device->events.event_count * sizeof(struct v3dv_event), 8,
232 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
233 if (!device->events.events) {
234 result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
235 goto fail;
236 }
237
238 list_inithead(&device->events.free_list);
239 for (int i = 0; i < device->events.event_count; i++)
240 init_event(device, &device->events.events[i], i);
241
242 /* Vulkan buffer for the event state BO */
243 VkBufferCreateInfo buf_info = {
244 .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
245 .size = bo->size,
246 .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT,
247 };
248 result = v3dv_CreateBuffer(_device, &buf_info, NULL,
249 &device->events.buffer);
250 if (result != VK_SUCCESS)
251 goto fail;
252
253 struct v3dv_device_memory *mem =
254 vk_object_zalloc(&device->vk, NULL, sizeof(*mem),
255 VK_OBJECT_TYPE_DEVICE_MEMORY);
256 if (!mem) {
257 result = VK_ERROR_OUT_OF_HOST_MEMORY;
258 goto fail;
259 }
260
261 mem->bo = bo;
262 mem->type = &device->pdevice->memory.memoryTypes[0];
263
264 device->events.mem = v3dv_device_memory_to_handle(mem);
265 VkBindBufferMemoryInfo bind_info = {
266 .sType = VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO,
267 .buffer = device->events.buffer,
268 .memory = device->events.mem,
269 .memoryOffset = 0,
270 };
271 v3dv_BindBufferMemory2(_device, 1, &bind_info);
272
273 /* Pipelines */
274 if (!create_event_pipelines(device)) {
275 result = VK_ERROR_OUT_OF_HOST_MEMORY;
276 goto fail;
277 }
278
279 /* Descriptor pool & set to access the buffer */
280 VkDescriptorPoolSize pool_size = {
281 .type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
282 .descriptorCount = 1,
283 };
284 VkDescriptorPoolCreateInfo pool_info = {
285 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO,
286 .flags = VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT,
287 .maxSets = 1,
288 .poolSizeCount = 1,
289 .pPoolSizes = &pool_size,
290 };
291 result =
292 v3dv_CreateDescriptorPool(_device, &pool_info, NULL,
293 &device->events.descriptor_pool);
294
295 if (result != VK_SUCCESS)
296 goto fail;
297
298 VkDescriptorSetAllocateInfo alloc_info = {
299 .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO,
300 .descriptorPool = device->events.descriptor_pool,
301 .descriptorSetCount = 1,
302 .pSetLayouts = &device->events.descriptor_set_layout,
303 };
304 result = v3dv_AllocateDescriptorSets(_device, &alloc_info,
305 &device->events.descriptor_set);
306 if (result != VK_SUCCESS)
307 goto fail;
308
309 VkDescriptorBufferInfo desc_buf_info = {
310 .buffer = device->events.buffer,
311 .offset = 0,
312 .range = VK_WHOLE_SIZE,
313 };
314
315 VkWriteDescriptorSet write = {
316 .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
317 .dstSet = device->events.descriptor_set,
318 .dstBinding = 0,
319 .dstArrayElement = 0,
320 .descriptorCount = 1,
321 .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER,
322 .pBufferInfo = &desc_buf_info,
323 };
324 v3dv_UpdateDescriptorSets(_device, 1, &write, 0, NULL);
325
326 return VK_SUCCESS;
327
328 fail:
329 v3dv_event_free_resources(device);
330 return result;
331 }
332
333 void
v3dv_event_free_resources(struct v3dv_device * device)334 v3dv_event_free_resources(struct v3dv_device *device)
335 {
336 if (device->events.bo) {
337 v3dv_bo_free(device, device->events.bo);
338 device->events.bo = NULL;
339 }
340
341 if (device->events.events) {
342 vk_free2(&device->vk.alloc, NULL, device->events.events);
343 device->events.events = NULL;
344 }
345
346 if (device->events.mem) {
347 vk_object_free(&device->vk, NULL,
348 v3dv_device_memory_from_handle(device->events.mem));
349 device->events.mem = VK_NULL_HANDLE;
350 }
351
352 v3dv_DestroyBuffer(v3dv_device_to_handle(device),
353 device->events.buffer, NULL);
354 device->events.buffer = VK_NULL_HANDLE;
355
356 v3dv_FreeDescriptorSets(v3dv_device_to_handle(device),
357 device->events.descriptor_pool,
358 1, &device->events.descriptor_set);
359 device->events.descriptor_set = VK_NULL_HANDLE;
360
361 v3dv_DestroyDescriptorPool(v3dv_device_to_handle(device),
362 device->events.descriptor_pool,
363 NULL);
364 device->events.descriptor_pool = VK_NULL_HANDLE;
365
366 destroy_event_pipelines(device);
367 }
368
369 static struct v3dv_event *
allocate_event(struct v3dv_device * device)370 allocate_event(struct v3dv_device *device)
371 {
372 mtx_lock(&device->events.lock);
373 if (list_is_empty(&device->events.free_list)) {
374 mtx_unlock(&device->events.lock);
375 return NULL;
376 }
377
378 struct v3dv_event *event =
379 list_first_entry(&device->events.free_list, struct v3dv_event, link);
380 list_del(&event->link);
381 mtx_unlock(&device->events.lock);
382
383 return event;
384 }
385
386 static void
free_event(struct v3dv_device * device,uint32_t index)387 free_event(struct v3dv_device *device, uint32_t index)
388 {
389 assert(index < device->events.event_count);
390 mtx_lock(&device->events.lock);
391 list_addtail(&device->events.events[index].link, &device->events.free_list);
392 mtx_unlock(&device->events.lock);
393 }
394
395 static void
event_set_value(struct v3dv_device * device,struct v3dv_event * event,uint8_t value)396 event_set_value(struct v3dv_device *device,
397 struct v3dv_event *event,
398 uint8_t value)
399 {
400 assert(value == 0 || value == 1);
401 uint8_t *data = (uint8_t *) device->events.bo->map;
402 data[event->index] = value;
403 }
404
405 static uint8_t
event_get_value(struct v3dv_device * device,struct v3dv_event * event)406 event_get_value(struct v3dv_device *device, struct v3dv_event *event)
407 {
408 uint8_t *data = (uint8_t *) device->events.bo->map;
409 return data[event->index];
410 }
411
412 VKAPI_ATTR VkResult VKAPI_CALL
v3dv_CreateEvent(VkDevice _device,const VkEventCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkEvent * pEvent)413 v3dv_CreateEvent(VkDevice _device,
414 const VkEventCreateInfo *pCreateInfo,
415 const VkAllocationCallbacks *pAllocator,
416 VkEvent *pEvent)
417 {
418 V3DV_FROM_HANDLE(v3dv_device, device, _device);
419 VkResult result = VK_SUCCESS;
420
421 struct v3dv_event *event = allocate_event(device);
422 if (!event) {
423 result = vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
424 goto fail;
425 }
426
427 event_set_value(device, event, 0);
428 *pEvent = v3dv_event_to_handle(event);
429 return VK_SUCCESS;
430
431 fail:
432 return result;
433 }
434
435 VKAPI_ATTR void VKAPI_CALL
v3dv_DestroyEvent(VkDevice _device,VkEvent _event,const VkAllocationCallbacks * pAllocator)436 v3dv_DestroyEvent(VkDevice _device,
437 VkEvent _event,
438 const VkAllocationCallbacks *pAllocator)
439 {
440 V3DV_FROM_HANDLE(v3dv_device, device, _device);
441 V3DV_FROM_HANDLE(v3dv_event, event, _event);
442
443 if (!event)
444 return;
445
446 free_event(device, event->index);
447 }
448
449 VKAPI_ATTR VkResult VKAPI_CALL
v3dv_GetEventStatus(VkDevice _device,VkEvent _event)450 v3dv_GetEventStatus(VkDevice _device, VkEvent _event)
451 {
452 V3DV_FROM_HANDLE(v3dv_device, device, _device);
453 V3DV_FROM_HANDLE(v3dv_event, event, _event);
454 if (vk_device_is_lost(&device->vk))
455 return VK_ERROR_DEVICE_LOST;
456 return event_get_value(device, event) ? VK_EVENT_SET : VK_EVENT_RESET;
457 }
458
459 VKAPI_ATTR VkResult VKAPI_CALL
v3dv_SetEvent(VkDevice _device,VkEvent _event)460 v3dv_SetEvent(VkDevice _device, VkEvent _event)
461 {
462 V3DV_FROM_HANDLE(v3dv_device, device, _device);
463 V3DV_FROM_HANDLE(v3dv_event, event, _event);
464 event_set_value(device, event, 1);
465 return VK_SUCCESS;
466 }
467
468 VKAPI_ATTR VkResult VKAPI_CALL
v3dv_ResetEvent(VkDevice _device,VkEvent _event)469 v3dv_ResetEvent(VkDevice _device, VkEvent _event)
470 {
471 V3DV_FROM_HANDLE(v3dv_device, device, _device);
472 V3DV_FROM_HANDLE(v3dv_event, event, _event);
473 event_set_value(device, event, 0);
474 return VK_SUCCESS;
475 }
476
477 static void
cmd_buffer_emit_set_event(struct v3dv_cmd_buffer * cmd_buffer,struct v3dv_event * event,uint8_t value)478 cmd_buffer_emit_set_event(struct v3dv_cmd_buffer *cmd_buffer,
479 struct v3dv_event *event,
480 uint8_t value)
481 {
482 assert(value == 0 || value == 1);
483
484 struct v3dv_device *device = cmd_buffer->device;
485 VkCommandBuffer commandBuffer = v3dv_cmd_buffer_to_handle(cmd_buffer);
486
487 v3dv_cmd_buffer_meta_state_push(cmd_buffer, true);
488
489 v3dv_CmdBindPipeline(commandBuffer,
490 VK_PIPELINE_BIND_POINT_COMPUTE,
491 device->events.set_event_pipeline);
492
493 v3dv_CmdBindDescriptorSets(commandBuffer,
494 VK_PIPELINE_BIND_POINT_COMPUTE,
495 device->events.pipeline_layout,
496 0, 1, &device->events.descriptor_set, 0, NULL);
497
498 assert(event->index < device->events.event_count);
499 uint32_t offset = event->index;
500 v3dv_CmdPushConstants(commandBuffer,
501 device->events.pipeline_layout,
502 VK_SHADER_STAGE_COMPUTE_BIT,
503 0, 4, &offset);
504
505 v3dv_CmdPushConstants(commandBuffer,
506 device->events.pipeline_layout,
507 VK_SHADER_STAGE_COMPUTE_BIT,
508 4, 1, &value);
509
510 vk_common_CmdDispatch(commandBuffer, 1, 1, 1);
511
512 v3dv_cmd_buffer_meta_state_pop(cmd_buffer, false);
513 }
514
515 static void
cmd_buffer_emit_wait_event(struct v3dv_cmd_buffer * cmd_buffer,struct v3dv_event * event)516 cmd_buffer_emit_wait_event(struct v3dv_cmd_buffer *cmd_buffer,
517 struct v3dv_event *event)
518 {
519 struct v3dv_device *device = cmd_buffer->device;
520 VkCommandBuffer commandBuffer = v3dv_cmd_buffer_to_handle(cmd_buffer);
521
522 v3dv_cmd_buffer_meta_state_push(cmd_buffer, true);
523
524 v3dv_CmdBindPipeline(commandBuffer,
525 VK_PIPELINE_BIND_POINT_COMPUTE,
526 device->events.wait_event_pipeline);
527
528 v3dv_CmdBindDescriptorSets(commandBuffer,
529 VK_PIPELINE_BIND_POINT_COMPUTE,
530 device->events.pipeline_layout,
531 0, 1, &device->events.descriptor_set, 0, NULL);
532
533 assert(event->index < device->events.event_count);
534 uint32_t offset = event->index;
535 v3dv_CmdPushConstants(commandBuffer,
536 device->events.pipeline_layout,
537 VK_SHADER_STAGE_COMPUTE_BIT,
538 0, 4, &offset);
539
540 vk_common_CmdDispatch(commandBuffer, 1, 1, 1);
541
542 v3dv_cmd_buffer_meta_state_pop(cmd_buffer, false);
543 }
544
545 VKAPI_ATTR void VKAPI_CALL
v3dv_CmdSetEvent2(VkCommandBuffer commandBuffer,VkEvent _event,const VkDependencyInfo * pDependencyInfo)546 v3dv_CmdSetEvent2(VkCommandBuffer commandBuffer,
547 VkEvent _event,
548 const VkDependencyInfo *pDependencyInfo)
549 {
550 V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer);
551 V3DV_FROM_HANDLE(v3dv_event, event, _event);
552
553 /* Event (re)sets can only happen outside a render pass instance so we
554 * should not be in the middle of job recording.
555 */
556 assert(cmd_buffer->state.pass == NULL);
557 assert(cmd_buffer->state.job == NULL);
558
559 /* We need to add the compute stage to the dstStageMask of all dependencies,
560 * so let's go ahead and patch the dependency info we receive.
561 */
562 struct v3dv_device *device = cmd_buffer->device;
563
564 uint32_t memory_barrier_count = pDependencyInfo->memoryBarrierCount;
565 VkMemoryBarrier2 *memory_barriers = memory_barrier_count ?
566 vk_alloc2(&device->vk.alloc, NULL,
567 memory_barrier_count * sizeof(memory_barriers[0]), 8,
568 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND) : NULL;
569 for (int i = 0; i < memory_barrier_count; i++) {
570 memory_barriers[i] = pDependencyInfo->pMemoryBarriers[i];
571 memory_barriers[i].dstStageMask |= VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT;
572 }
573
574 uint32_t buffer_barrier_count = pDependencyInfo->bufferMemoryBarrierCount;
575 VkBufferMemoryBarrier2 *buffer_barriers = buffer_barrier_count ?
576 vk_alloc2(&device->vk.alloc, NULL,
577 buffer_barrier_count * sizeof(buffer_barriers[0]), 8,
578 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND) : NULL;
579 for (int i = 0; i < buffer_barrier_count; i++) {
580 buffer_barriers[i] = pDependencyInfo->pBufferMemoryBarriers[i];
581 buffer_barriers[i].dstStageMask |= VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT;
582 }
583
584 uint32_t image_barrier_count = pDependencyInfo->imageMemoryBarrierCount;
585 VkImageMemoryBarrier2 *image_barriers = image_barrier_count ?
586 vk_alloc2(&device->vk.alloc, NULL,
587 image_barrier_count * sizeof(image_barriers[0]), 8,
588 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND) : NULL;
589 for (int i = 0; i < image_barrier_count; i++) {
590 image_barriers[i] = pDependencyInfo->pImageMemoryBarriers[i];
591 image_barriers[i].dstStageMask |= VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT;
592 }
593
594 VkDependencyInfo info = {
595 .sType = pDependencyInfo->sType,
596 .dependencyFlags = pDependencyInfo->dependencyFlags,
597 .memoryBarrierCount = memory_barrier_count,
598 .pMemoryBarriers = memory_barriers,
599 .bufferMemoryBarrierCount = buffer_barrier_count,
600 .pBufferMemoryBarriers = buffer_barriers,
601 .imageMemoryBarrierCount = image_barrier_count,
602 .pImageMemoryBarriers = image_barriers,
603 };
604
605 v3dv_cmd_buffer_emit_pipeline_barrier(cmd_buffer, &info);
606
607 cmd_buffer_emit_set_event(cmd_buffer, event, 1);
608
609 if (memory_barriers)
610 vk_free2(&device->vk.alloc, NULL, memory_barriers);
611 if (buffer_barriers)
612 vk_free2(&device->vk.alloc, NULL, buffer_barriers);
613 if (image_barriers)
614 vk_free2(&device->vk.alloc, NULL, image_barriers);
615 }
616
617 VKAPI_ATTR void VKAPI_CALL
v3dv_CmdResetEvent2(VkCommandBuffer commandBuffer,VkEvent _event,VkPipelineStageFlags2 stageMask)618 v3dv_CmdResetEvent2(VkCommandBuffer commandBuffer,
619 VkEvent _event,
620 VkPipelineStageFlags2 stageMask)
621 {
622 V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer);
623 V3DV_FROM_HANDLE(v3dv_event, event, _event);
624
625 /* Event (re)sets can only happen outside a render pass instance so we
626 * should not be in the middle of job recording.
627 */
628 assert(cmd_buffer->state.pass == NULL);
629 assert(cmd_buffer->state.job == NULL);
630
631 VkMemoryBarrier2 barrier = {
632 .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2,
633 .srcStageMask = stageMask,
634 .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT,
635 };
636 VkDependencyInfo info = {
637 .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
638 .memoryBarrierCount = 1,
639 .pMemoryBarriers = &barrier,
640 };
641 v3dv_cmd_buffer_emit_pipeline_barrier(cmd_buffer, &info);
642
643 cmd_buffer_emit_set_event(cmd_buffer, event, 0);
644 }
645
646 VKAPI_ATTR void VKAPI_CALL
v3dv_CmdWaitEvents2(VkCommandBuffer commandBuffer,uint32_t eventCount,const VkEvent * pEvents,const VkDependencyInfo * pDependencyInfo)647 v3dv_CmdWaitEvents2(VkCommandBuffer commandBuffer,
648 uint32_t eventCount,
649 const VkEvent *pEvents,
650 const VkDependencyInfo *pDependencyInfo)
651 {
652 V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer);
653 for (uint32_t i = 0; i < eventCount; i++) {
654 struct v3dv_event *event = v3dv_event_from_handle(pEvents[i]);;
655 cmd_buffer_emit_wait_event(cmd_buffer, event);
656 }
657
658 /* We need to add the compute stage to the srcStageMask of all dependencies,
659 * so let's go ahead and patch the dependency info we receive.
660 */
661 struct v3dv_device *device = cmd_buffer->device;
662 for (int e = 0; e < eventCount; e++) {
663 const VkDependencyInfo *info = &pDependencyInfo[e];
664
665 uint32_t memory_barrier_count = info->memoryBarrierCount;
666 VkMemoryBarrier2 *memory_barriers = memory_barrier_count ?
667 vk_alloc2(&device->vk.alloc, NULL,
668 memory_barrier_count * sizeof(memory_barriers[0]), 8,
669 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND) : NULL;
670 for (int i = 0; i < memory_barrier_count; i++) {
671 memory_barriers[i] = info->pMemoryBarriers[i];
672 memory_barriers[i].srcStageMask |= VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT;
673 }
674
675 uint32_t buffer_barrier_count = info->bufferMemoryBarrierCount;
676 VkBufferMemoryBarrier2 *buffer_barriers = buffer_barrier_count ?
677 vk_alloc2(&device->vk.alloc, NULL,
678 buffer_barrier_count * sizeof(buffer_barriers[0]), 8,
679 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND) : NULL;
680 for (int i = 0; i < buffer_barrier_count; i++) {
681 buffer_barriers[i] = info->pBufferMemoryBarriers[i];
682 buffer_barriers[i].srcStageMask |= VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT;
683 }
684
685 uint32_t image_barrier_count = info->imageMemoryBarrierCount;
686 VkImageMemoryBarrier2 *image_barriers = image_barrier_count ?
687 vk_alloc2(&device->vk.alloc, NULL,
688 image_barrier_count * sizeof(image_barriers[0]), 8,
689 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND) : NULL;
690 for (int i = 0; i < image_barrier_count; i++) {
691 image_barriers[i] = info->pImageMemoryBarriers[i];
692 image_barriers[i].srcStageMask |= VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT;
693 }
694
695 VkDependencyInfo new_info = {
696 .sType = info->sType,
697 .dependencyFlags = info->dependencyFlags,
698 .memoryBarrierCount = memory_barrier_count,
699 .pMemoryBarriers = memory_barriers,
700 .bufferMemoryBarrierCount = buffer_barrier_count,
701 .pBufferMemoryBarriers = buffer_barriers,
702 .imageMemoryBarrierCount = image_barrier_count,
703 .pImageMemoryBarriers = image_barriers,
704 };
705
706 v3dv_cmd_buffer_emit_pipeline_barrier(cmd_buffer, &new_info);
707
708 if (memory_barriers)
709 vk_free2(&device->vk.alloc, NULL, memory_barriers);
710 if (buffer_barriers)
711 vk_free2(&device->vk.alloc, NULL, buffer_barriers);
712 if (image_barriers)
713 vk_free2(&device->vk.alloc, NULL, image_barriers);
714 }
715 }
716