1 /*
2 * Copyright © 2022 Collabora Ltd. and Red Hat Inc.
3 * SPDX-License-Identifier: MIT
4 */
5 #include "nvk_event.h"
6
7 #include "nvk_cmd_buffer.h"
8 #include "nvk_device.h"
9 #include "nvk_entrypoints.h"
10 #include "nvk_mme.h"
11
12 #include "nv_push_cl906f.h"
13 #include "nv_push_cl9097.h"
14
15 #define NVK_EVENT_MEM_SIZE sizeof(VkResult)
16
17 VKAPI_ATTR VkResult VKAPI_CALL
nvk_CreateEvent(VkDevice device,const VkEventCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkEvent * pEvent)18 nvk_CreateEvent(VkDevice device,
19 const VkEventCreateInfo *pCreateInfo,
20 const VkAllocationCallbacks *pAllocator,
21 VkEvent *pEvent)
22 {
23 VK_FROM_HANDLE(nvk_device, dev, device);
24 struct nvk_event *event;
25 VkResult result;
26
27 event = vk_object_zalloc(&dev->vk, pAllocator, sizeof(*event),
28 VK_OBJECT_TYPE_EVENT);
29 if (!event)
30 return vk_error(dev, VK_ERROR_OUT_OF_HOST_MEMORY);
31
32 result = nvk_heap_alloc(dev, &dev->event_heap,
33 NVK_EVENT_MEM_SIZE, NVK_EVENT_MEM_SIZE,
34 &event->addr, (void **)&event->status);
35 if (result != VK_SUCCESS) {
36 vk_object_free(&dev->vk, pAllocator, event);
37 return result;
38 }
39
40 *event->status = VK_EVENT_RESET;
41
42 *pEvent = nvk_event_to_handle(event);
43
44 return VK_SUCCESS;
45 }
46
47 VKAPI_ATTR void VKAPI_CALL
nvk_DestroyEvent(VkDevice device,VkEvent _event,const VkAllocationCallbacks * pAllocator)48 nvk_DestroyEvent(VkDevice device,
49 VkEvent _event,
50 const VkAllocationCallbacks *pAllocator)
51 {
52 VK_FROM_HANDLE(nvk_device, dev, device);
53 VK_FROM_HANDLE(nvk_event, event, _event);
54
55 if (!event)
56 return;
57
58 nvk_heap_free(dev, &dev->event_heap, event->addr, NVK_EVENT_MEM_SIZE);
59
60 vk_object_free(&dev->vk, pAllocator, event);
61 }
62
63 VKAPI_ATTR VkResult VKAPI_CALL
nvk_GetEventStatus(VkDevice device,VkEvent _event)64 nvk_GetEventStatus(VkDevice device,
65 VkEvent _event)
66 {
67 VK_FROM_HANDLE(nvk_event, event, _event);
68
69 return *event->status;
70 }
71
72 VKAPI_ATTR VkResult VKAPI_CALL
nvk_SetEvent(VkDevice device,VkEvent _event)73 nvk_SetEvent(VkDevice device,
74 VkEvent _event)
75 {
76 VK_FROM_HANDLE(nvk_event, event, _event);
77
78 *event->status = VK_EVENT_SET;
79
80 return VK_SUCCESS;
81 }
82
83 VKAPI_ATTR VkResult VKAPI_CALL
nvk_ResetEvent(VkDevice device,VkEvent _event)84 nvk_ResetEvent(VkDevice device,
85 VkEvent _event)
86 {
87 VK_FROM_HANDLE(nvk_event, event, _event);
88
89 *event->status = VK_EVENT_RESET;
90
91 return VK_SUCCESS;
92 }
93
94 static bool
clear_bits64(uint64_t * bitfield,uint64_t bits)95 clear_bits64(uint64_t *bitfield, uint64_t bits)
96 {
97 bool has_bits = (*bitfield & bits) != 0;
98 *bitfield &= ~bits;
99 return has_bits;
100 }
101
102 uint32_t
vk_stage_flags_to_nv9097_pipeline_location(VkPipelineStageFlags2 flags)103 vk_stage_flags_to_nv9097_pipeline_location(VkPipelineStageFlags2 flags)
104 {
105 if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT |
106 VK_PIPELINE_STAGE_2_ALL_TRANSFER_BIT |
107 VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT |
108 VK_PIPELINE_STAGE_2_ALL_GRAPHICS_BIT |
109 VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT|
110 VK_PIPELINE_STAGE_2_COPY_BIT|
111 VK_PIPELINE_STAGE_2_RESOLVE_BIT|
112 VK_PIPELINE_STAGE_2_BLIT_BIT|
113 VK_PIPELINE_STAGE_2_CLEAR_BIT))
114 return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_ALL;
115
116 if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT))
117 return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_DEPTH_TEST;
118
119 if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT))
120 return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_PIXEL_SHADER;
121
122 if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT))
123 return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_ZCULL;
124
125 if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_TRANSFORM_FEEDBACK_BIT_EXT))
126 return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_STREAMING_OUTPUT;
127
128 if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_GEOMETRY_SHADER_BIT |
129 VK_PIPELINE_STAGE_2_PRE_RASTERIZATION_SHADERS_BIT))
130 return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_GEOMETRY_SHADER;
131
132 if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_TESSELLATION_EVALUATION_SHADER_BIT))
133 return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_TESSELATION_SHADER;
134
135 if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_TESSELLATION_CONTROL_SHADER_BIT))
136 return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_TESSELATION_INIT_SHADER;
137
138 if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_VERTEX_SHADER_BIT))
139 return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_VERTEX_SHADER;
140
141 if (clear_bits64(&flags, VK_PIPELINE_STAGE_2_VERTEX_INPUT_BIT |
142 VK_PIPELINE_STAGE_2_INDEX_INPUT_BIT |
143 VK_PIPELINE_STAGE_2_VERTEX_ATTRIBUTE_INPUT_BIT))
144 return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_DATA_ASSEMBLER;
145
146 clear_bits64(&flags, VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT |
147 VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT |
148 VK_PIPELINE_STAGE_2_HOST_BIT |
149 VK_PIPELINE_STAGE_2_CONDITIONAL_RENDERING_BIT_EXT);
150
151 /* TODO: Doing this on 3D will likely cause a WFI which is probably ok but,
152 * if we tracked which subchannel we've used most recently, we can probably
153 * do better than that.
154 */
155 clear_bits64(&flags, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
156
157 assert(flags == 0);
158
159 return NV9097_SET_REPORT_SEMAPHORE_D_PIPELINE_LOCATION_NONE;
160 }
161
162 VKAPI_ATTR void VKAPI_CALL
nvk_CmdSetEvent2(VkCommandBuffer commandBuffer,VkEvent _event,const VkDependencyInfo * pDependencyInfo)163 nvk_CmdSetEvent2(VkCommandBuffer commandBuffer,
164 VkEvent _event,
165 const VkDependencyInfo *pDependencyInfo)
166 {
167 VK_FROM_HANDLE(nvk_cmd_buffer, cmd, commandBuffer);
168 VK_FROM_HANDLE(nvk_event, event, _event);
169
170 nvk_cmd_flush_wait_dep(cmd, pDependencyInfo, false);
171
172 VkPipelineStageFlags2 stages = 0;
173 for (uint32_t i = 0; i < pDependencyInfo->memoryBarrierCount; i++)
174 stages |= pDependencyInfo->pMemoryBarriers[i].srcStageMask;
175 for (uint32_t i = 0; i < pDependencyInfo->bufferMemoryBarrierCount; i++)
176 stages |= pDependencyInfo->pBufferMemoryBarriers[i].srcStageMask;
177 for (uint32_t i = 0; i < pDependencyInfo->imageMemoryBarrierCount; i++)
178 stages |= pDependencyInfo->pImageMemoryBarriers[i].srcStageMask;
179
180 struct nv_push *p = nvk_cmd_buffer_push(cmd, 5);
181 P_MTHD(p, NV9097, SET_REPORT_SEMAPHORE_A);
182 P_NV9097_SET_REPORT_SEMAPHORE_A(p, event->addr >> 32);
183 P_NV9097_SET_REPORT_SEMAPHORE_B(p, event->addr);
184 P_NV9097_SET_REPORT_SEMAPHORE_C(p, VK_EVENT_SET);
185 P_NV9097_SET_REPORT_SEMAPHORE_D(p, {
186 .operation = OPERATION_RELEASE,
187 .release = RELEASE_AFTER_ALL_PRECEEDING_WRITES_COMPLETE,
188 .pipeline_location = vk_stage_flags_to_nv9097_pipeline_location(stages),
189 .structure_size = STRUCTURE_SIZE_ONE_WORD,
190 });
191 }
192
193 VKAPI_ATTR void VKAPI_CALL
nvk_CmdResetEvent2(VkCommandBuffer commandBuffer,VkEvent _event,VkPipelineStageFlags2 stageMask)194 nvk_CmdResetEvent2(VkCommandBuffer commandBuffer,
195 VkEvent _event,
196 VkPipelineStageFlags2 stageMask)
197 {
198 VK_FROM_HANDLE(nvk_cmd_buffer, cmd, commandBuffer);
199 VK_FROM_HANDLE(nvk_event, event, _event);
200
201 struct nv_push *p = nvk_cmd_buffer_push(cmd, 5);
202 P_MTHD(p, NV9097, SET_REPORT_SEMAPHORE_A);
203 P_NV9097_SET_REPORT_SEMAPHORE_A(p, event->addr >> 32);
204 P_NV9097_SET_REPORT_SEMAPHORE_B(p, event->addr);
205 P_NV9097_SET_REPORT_SEMAPHORE_C(p, VK_EVENT_RESET);
206 P_NV9097_SET_REPORT_SEMAPHORE_D(p, {
207 .operation = OPERATION_RELEASE,
208 .release = RELEASE_AFTER_ALL_PRECEEDING_WRITES_COMPLETE,
209 .pipeline_location =
210 vk_stage_flags_to_nv9097_pipeline_location(stageMask),
211 .structure_size = STRUCTURE_SIZE_ONE_WORD,
212 });
213 }
214
215 VKAPI_ATTR void VKAPI_CALL
nvk_CmdWaitEvents2(VkCommandBuffer commandBuffer,uint32_t eventCount,const VkEvent * pEvents,const VkDependencyInfo * pDependencyInfos)216 nvk_CmdWaitEvents2(VkCommandBuffer commandBuffer,
217 uint32_t eventCount,
218 const VkEvent *pEvents,
219 const VkDependencyInfo *pDependencyInfos)
220 {
221 VK_FROM_HANDLE(nvk_cmd_buffer, cmd, commandBuffer);
222
223 for (uint32_t i = 0; i < eventCount; i++) {
224 VK_FROM_HANDLE(nvk_event, event, pEvents[i]);
225
226 struct nv_push *p = nvk_cmd_buffer_push(cmd, 5);
227 __push_mthd(p, SUBC_NV9097, NV906F_SEMAPHOREA);
228 P_NV906F_SEMAPHOREA(p, event->addr >> 32);
229 P_NV906F_SEMAPHOREB(p, (event->addr & UINT32_MAX) >> 2);
230 P_NV906F_SEMAPHOREC(p, VK_EVENT_SET);
231 P_NV906F_SEMAPHORED(p, {
232 .operation = OPERATION_ACQUIRE,
233 .acquire_switch = ACQUIRE_SWITCH_ENABLED,
234 .release_size = RELEASE_SIZE_4BYTE,
235 });
236 }
237
238 nvk_cmd_invalidate_deps(cmd, eventCount, pDependencyInfos);
239 }
240