1*61046927SAndroid Build Coastguard Worker /*
2*61046927SAndroid Build Coastguard Worker * Copyright © 2022 Collabora Ltd. and Red Hat Inc.
3*61046927SAndroid Build Coastguard Worker * SPDX-License-Identifier: MIT
4*61046927SAndroid Build Coastguard Worker */
5*61046927SAndroid Build Coastguard Worker #include "nvk_queue.h"
6*61046927SAndroid Build Coastguard Worker
7*61046927SAndroid Build Coastguard Worker #include "nvk_buffer.h"
8*61046927SAndroid Build Coastguard Worker #include "nvk_cmd_buffer.h"
9*61046927SAndroid Build Coastguard Worker #include "nvk_device.h"
10*61046927SAndroid Build Coastguard Worker #include "nvk_image.h"
11*61046927SAndroid Build Coastguard Worker #include "nvk_physical_device.h"
12*61046927SAndroid Build Coastguard Worker #include "nv_push.h"
13*61046927SAndroid Build Coastguard Worker
14*61046927SAndroid Build Coastguard Worker #include "nv_push_cl9039.h"
15*61046927SAndroid Build Coastguard Worker #include "nv_push_cl9097.h"
16*61046927SAndroid Build Coastguard Worker #include "nv_push_cl90b5.h"
17*61046927SAndroid Build Coastguard Worker #include "nv_push_cla0c0.h"
18*61046927SAndroid Build Coastguard Worker #include "cla1c0.h"
19*61046927SAndroid Build Coastguard Worker #include "nv_push_clc3c0.h"
20*61046927SAndroid Build Coastguard Worker #include "nv_push_clc397.h"
21*61046927SAndroid Build Coastguard Worker
22*61046927SAndroid Build Coastguard Worker static void
nvk_queue_state_init(struct nvk_queue_state * qs)23*61046927SAndroid Build Coastguard Worker nvk_queue_state_init(struct nvk_queue_state *qs)
24*61046927SAndroid Build Coastguard Worker {
25*61046927SAndroid Build Coastguard Worker memset(qs, 0, sizeof(*qs));
26*61046927SAndroid Build Coastguard Worker }
27*61046927SAndroid Build Coastguard Worker
28*61046927SAndroid Build Coastguard Worker static void
nvk_queue_state_finish(struct nvk_device * dev,struct nvk_queue_state * qs)29*61046927SAndroid Build Coastguard Worker nvk_queue_state_finish(struct nvk_device *dev,
30*61046927SAndroid Build Coastguard Worker struct nvk_queue_state *qs)
31*61046927SAndroid Build Coastguard Worker {
32*61046927SAndroid Build Coastguard Worker if (qs->images.mem)
33*61046927SAndroid Build Coastguard Worker nvkmd_mem_unref(qs->images.mem);
34*61046927SAndroid Build Coastguard Worker if (qs->samplers.mem)
35*61046927SAndroid Build Coastguard Worker nvkmd_mem_unref(qs->samplers.mem);
36*61046927SAndroid Build Coastguard Worker if (qs->slm.mem)
37*61046927SAndroid Build Coastguard Worker nvkmd_mem_unref(qs->slm.mem);
38*61046927SAndroid Build Coastguard Worker if (qs->push.mem)
39*61046927SAndroid Build Coastguard Worker nvkmd_mem_unref(qs->push.mem);
40*61046927SAndroid Build Coastguard Worker }
41*61046927SAndroid Build Coastguard Worker
42*61046927SAndroid Build Coastguard Worker static void
nvk_queue_state_dump_push(struct nvk_device * dev,struct nvk_queue_state * qs,FILE * fp)43*61046927SAndroid Build Coastguard Worker nvk_queue_state_dump_push(struct nvk_device *dev,
44*61046927SAndroid Build Coastguard Worker struct nvk_queue_state *qs, FILE *fp)
45*61046927SAndroid Build Coastguard Worker {
46*61046927SAndroid Build Coastguard Worker struct nvk_physical_device *pdev = nvk_device_physical(dev);
47*61046927SAndroid Build Coastguard Worker
48*61046927SAndroid Build Coastguard Worker struct nv_push push = {
49*61046927SAndroid Build Coastguard Worker .start = (uint32_t *)qs->push.mem->map,
50*61046927SAndroid Build Coastguard Worker .end = (uint32_t *)qs->push.mem->map + qs->push.dw_count,
51*61046927SAndroid Build Coastguard Worker };
52*61046927SAndroid Build Coastguard Worker vk_push_print(fp, &push, &pdev->info);
53*61046927SAndroid Build Coastguard Worker }
54*61046927SAndroid Build Coastguard Worker
55*61046927SAndroid Build Coastguard Worker VkResult
nvk_queue_state_update(struct nvk_device * dev,struct nvk_queue_state * qs)56*61046927SAndroid Build Coastguard Worker nvk_queue_state_update(struct nvk_device *dev,
57*61046927SAndroid Build Coastguard Worker struct nvk_queue_state *qs)
58*61046927SAndroid Build Coastguard Worker {
59*61046927SAndroid Build Coastguard Worker struct nvk_physical_device *pdev = nvk_device_physical(dev);
60*61046927SAndroid Build Coastguard Worker struct nvkmd_mem *mem;
61*61046927SAndroid Build Coastguard Worker uint32_t alloc_count, bytes_per_warp, bytes_per_tpc;
62*61046927SAndroid Build Coastguard Worker bool dirty = false;
63*61046927SAndroid Build Coastguard Worker
64*61046927SAndroid Build Coastguard Worker mem = nvk_descriptor_table_get_mem_ref(&dev->images, &alloc_count);
65*61046927SAndroid Build Coastguard Worker if (qs->images.mem != mem || qs->images.alloc_count != alloc_count) {
66*61046927SAndroid Build Coastguard Worker if (qs->images.mem)
67*61046927SAndroid Build Coastguard Worker nvkmd_mem_unref(qs->images.mem);
68*61046927SAndroid Build Coastguard Worker qs->images.mem = mem;
69*61046927SAndroid Build Coastguard Worker qs->images.alloc_count = alloc_count;
70*61046927SAndroid Build Coastguard Worker dirty = true;
71*61046927SAndroid Build Coastguard Worker } else {
72*61046927SAndroid Build Coastguard Worker /* No change */
73*61046927SAndroid Build Coastguard Worker if (mem)
74*61046927SAndroid Build Coastguard Worker nvkmd_mem_unref(mem);
75*61046927SAndroid Build Coastguard Worker }
76*61046927SAndroid Build Coastguard Worker
77*61046927SAndroid Build Coastguard Worker mem = nvk_descriptor_table_get_mem_ref(&dev->samplers, &alloc_count);
78*61046927SAndroid Build Coastguard Worker if (qs->samplers.mem != mem || qs->samplers.alloc_count != alloc_count) {
79*61046927SAndroid Build Coastguard Worker if (qs->samplers.mem)
80*61046927SAndroid Build Coastguard Worker nvkmd_mem_unref(qs->samplers.mem);
81*61046927SAndroid Build Coastguard Worker qs->samplers.mem = mem;
82*61046927SAndroid Build Coastguard Worker qs->samplers.alloc_count = alloc_count;
83*61046927SAndroid Build Coastguard Worker dirty = true;
84*61046927SAndroid Build Coastguard Worker } else {
85*61046927SAndroid Build Coastguard Worker /* No change */
86*61046927SAndroid Build Coastguard Worker if (mem)
87*61046927SAndroid Build Coastguard Worker nvkmd_mem_unref(mem);
88*61046927SAndroid Build Coastguard Worker }
89*61046927SAndroid Build Coastguard Worker
90*61046927SAndroid Build Coastguard Worker mem = nvk_slm_area_get_mem_ref(&dev->slm, &bytes_per_warp, &bytes_per_tpc);
91*61046927SAndroid Build Coastguard Worker if (qs->slm.mem != mem || qs->slm.bytes_per_warp != bytes_per_warp ||
92*61046927SAndroid Build Coastguard Worker qs->slm.bytes_per_tpc != bytes_per_tpc) {
93*61046927SAndroid Build Coastguard Worker if (qs->slm.mem)
94*61046927SAndroid Build Coastguard Worker nvkmd_mem_unref(qs->slm.mem);
95*61046927SAndroid Build Coastguard Worker qs->slm.mem = mem;
96*61046927SAndroid Build Coastguard Worker qs->slm.bytes_per_warp = bytes_per_warp;
97*61046927SAndroid Build Coastguard Worker qs->slm.bytes_per_tpc = bytes_per_tpc;
98*61046927SAndroid Build Coastguard Worker dirty = true;
99*61046927SAndroid Build Coastguard Worker } else {
100*61046927SAndroid Build Coastguard Worker /* No change */
101*61046927SAndroid Build Coastguard Worker if (mem)
102*61046927SAndroid Build Coastguard Worker nvkmd_mem_unref(mem);
103*61046927SAndroid Build Coastguard Worker }
104*61046927SAndroid Build Coastguard Worker
105*61046927SAndroid Build Coastguard Worker /* TODO: We're currently depending on kernel reference counting to protect
106*61046927SAndroid Build Coastguard Worker * us here. If we ever stop reference counting in the kernel, we will
107*61046927SAndroid Build Coastguard Worker * either need to delay destruction or hold on to our extra BO references
108*61046927SAndroid Build Coastguard Worker * and insert a GPU stall here if anything has changed before dropping our
109*61046927SAndroid Build Coastguard Worker * old references.
110*61046927SAndroid Build Coastguard Worker */
111*61046927SAndroid Build Coastguard Worker
112*61046927SAndroid Build Coastguard Worker if (!dirty)
113*61046927SAndroid Build Coastguard Worker return VK_SUCCESS;
114*61046927SAndroid Build Coastguard Worker
115*61046927SAndroid Build Coastguard Worker struct nvkmd_mem *push_mem;
116*61046927SAndroid Build Coastguard Worker VkResult result = nvkmd_dev_alloc_mapped_mem(dev->nvkmd, &dev->vk.base,
117*61046927SAndroid Build Coastguard Worker 256 * 4, 0, NVKMD_MEM_LOCAL,
118*61046927SAndroid Build Coastguard Worker NVKMD_MEM_MAP_WR, &push_mem);
119*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
120*61046927SAndroid Build Coastguard Worker return result;
121*61046927SAndroid Build Coastguard Worker
122*61046927SAndroid Build Coastguard Worker struct nv_push push;
123*61046927SAndroid Build Coastguard Worker nv_push_init(&push, push_mem->map, 256);
124*61046927SAndroid Build Coastguard Worker struct nv_push *p = &push;
125*61046927SAndroid Build Coastguard Worker
126*61046927SAndroid Build Coastguard Worker if (qs->images.mem) {
127*61046927SAndroid Build Coastguard Worker /* Compute */
128*61046927SAndroid Build Coastguard Worker P_MTHD(p, NVA0C0, SET_TEX_HEADER_POOL_A);
129*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_TEX_HEADER_POOL_A(p, qs->images.mem->va->addr >> 32);
130*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_TEX_HEADER_POOL_B(p, qs->images.mem->va->addr);
131*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_TEX_HEADER_POOL_C(p, qs->images.alloc_count - 1);
132*61046927SAndroid Build Coastguard Worker P_IMMD(p, NVA0C0, INVALIDATE_TEXTURE_HEADER_CACHE_NO_WFI, {
133*61046927SAndroid Build Coastguard Worker .lines = LINES_ALL
134*61046927SAndroid Build Coastguard Worker });
135*61046927SAndroid Build Coastguard Worker
136*61046927SAndroid Build Coastguard Worker /* 3D */
137*61046927SAndroid Build Coastguard Worker P_MTHD(p, NV9097, SET_TEX_HEADER_POOL_A);
138*61046927SAndroid Build Coastguard Worker P_NV9097_SET_TEX_HEADER_POOL_A(p, qs->images.mem->va->addr >> 32);
139*61046927SAndroid Build Coastguard Worker P_NV9097_SET_TEX_HEADER_POOL_B(p, qs->images.mem->va->addr);
140*61046927SAndroid Build Coastguard Worker P_NV9097_SET_TEX_HEADER_POOL_C(p, qs->images.alloc_count - 1);
141*61046927SAndroid Build Coastguard Worker P_IMMD(p, NV9097, INVALIDATE_TEXTURE_HEADER_CACHE_NO_WFI, {
142*61046927SAndroid Build Coastguard Worker .lines = LINES_ALL
143*61046927SAndroid Build Coastguard Worker });
144*61046927SAndroid Build Coastguard Worker }
145*61046927SAndroid Build Coastguard Worker
146*61046927SAndroid Build Coastguard Worker if (qs->samplers.mem) {
147*61046927SAndroid Build Coastguard Worker /* Compute */
148*61046927SAndroid Build Coastguard Worker P_MTHD(p, NVA0C0, SET_TEX_SAMPLER_POOL_A);
149*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_TEX_SAMPLER_POOL_A(p, qs->samplers.mem->va->addr >> 32);
150*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_TEX_SAMPLER_POOL_B(p, qs->samplers.mem->va->addr);
151*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_TEX_SAMPLER_POOL_C(p, qs->samplers.alloc_count - 1);
152*61046927SAndroid Build Coastguard Worker P_IMMD(p, NVA0C0, INVALIDATE_SAMPLER_CACHE_NO_WFI, {
153*61046927SAndroid Build Coastguard Worker .lines = LINES_ALL
154*61046927SAndroid Build Coastguard Worker });
155*61046927SAndroid Build Coastguard Worker
156*61046927SAndroid Build Coastguard Worker /* 3D */
157*61046927SAndroid Build Coastguard Worker P_MTHD(p, NV9097, SET_TEX_SAMPLER_POOL_A);
158*61046927SAndroid Build Coastguard Worker P_NV9097_SET_TEX_SAMPLER_POOL_A(p, qs->samplers.mem->va->addr >> 32);
159*61046927SAndroid Build Coastguard Worker P_NV9097_SET_TEX_SAMPLER_POOL_B(p, qs->samplers.mem->va->addr);
160*61046927SAndroid Build Coastguard Worker P_NV9097_SET_TEX_SAMPLER_POOL_C(p, qs->samplers.alloc_count - 1);
161*61046927SAndroid Build Coastguard Worker P_IMMD(p, NV9097, INVALIDATE_SAMPLER_CACHE_NO_WFI, {
162*61046927SAndroid Build Coastguard Worker .lines = LINES_ALL
163*61046927SAndroid Build Coastguard Worker });
164*61046927SAndroid Build Coastguard Worker }
165*61046927SAndroid Build Coastguard Worker
166*61046927SAndroid Build Coastguard Worker if (qs->slm.mem) {
167*61046927SAndroid Build Coastguard Worker const uint64_t slm_addr = qs->slm.mem->va->addr;
168*61046927SAndroid Build Coastguard Worker const uint64_t slm_size = qs->slm.mem->size_B;
169*61046927SAndroid Build Coastguard Worker const uint64_t slm_per_warp = qs->slm.bytes_per_warp;
170*61046927SAndroid Build Coastguard Worker const uint64_t slm_per_tpc = qs->slm.bytes_per_tpc;
171*61046927SAndroid Build Coastguard Worker assert(!(slm_per_tpc & 0x7fff));
172*61046927SAndroid Build Coastguard Worker
173*61046927SAndroid Build Coastguard Worker /* Compute */
174*61046927SAndroid Build Coastguard Worker P_MTHD(p, NVA0C0, SET_SHADER_LOCAL_MEMORY_A);
175*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_SHADER_LOCAL_MEMORY_A(p, slm_addr >> 32);
176*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_SHADER_LOCAL_MEMORY_B(p, slm_addr);
177*61046927SAndroid Build Coastguard Worker
178*61046927SAndroid Build Coastguard Worker P_MTHD(p, NVA0C0, SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_A);
179*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_A(p, slm_per_tpc >> 32);
180*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_B(p, slm_per_tpc);
181*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_SHADER_LOCAL_MEMORY_NON_THROTTLED_C(p, 0xff);
182*61046927SAndroid Build Coastguard Worker
183*61046927SAndroid Build Coastguard Worker if (pdev->info.cls_compute < VOLTA_COMPUTE_A) {
184*61046927SAndroid Build Coastguard Worker P_MTHD(p, NVA0C0, SET_SHADER_LOCAL_MEMORY_THROTTLED_A);
185*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_SHADER_LOCAL_MEMORY_THROTTLED_A(p, slm_per_tpc >> 32);
186*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_SHADER_LOCAL_MEMORY_THROTTLED_B(p, slm_per_tpc);
187*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_SHADER_LOCAL_MEMORY_THROTTLED_C(p, 0xff);
188*61046927SAndroid Build Coastguard Worker }
189*61046927SAndroid Build Coastguard Worker
190*61046927SAndroid Build Coastguard Worker /* 3D */
191*61046927SAndroid Build Coastguard Worker P_MTHD(p, NV9097, SET_SHADER_LOCAL_MEMORY_A);
192*61046927SAndroid Build Coastguard Worker P_NV9097_SET_SHADER_LOCAL_MEMORY_A(p, slm_addr >> 32);
193*61046927SAndroid Build Coastguard Worker P_NV9097_SET_SHADER_LOCAL_MEMORY_B(p, slm_addr);
194*61046927SAndroid Build Coastguard Worker P_NV9097_SET_SHADER_LOCAL_MEMORY_C(p, slm_size >> 32);
195*61046927SAndroid Build Coastguard Worker P_NV9097_SET_SHADER_LOCAL_MEMORY_D(p, slm_size);
196*61046927SAndroid Build Coastguard Worker P_NV9097_SET_SHADER_LOCAL_MEMORY_E(p, slm_per_warp);
197*61046927SAndroid Build Coastguard Worker }
198*61046927SAndroid Build Coastguard Worker
199*61046927SAndroid Build Coastguard Worker /* We set memory windows unconditionally. Otherwise, the memory window
200*61046927SAndroid Build Coastguard Worker * might be in a random place and cause us to fault off into nowhere.
201*61046927SAndroid Build Coastguard Worker */
202*61046927SAndroid Build Coastguard Worker if (pdev->info.cls_compute >= VOLTA_COMPUTE_A) {
203*61046927SAndroid Build Coastguard Worker uint64_t temp = 0xfeULL << 24;
204*61046927SAndroid Build Coastguard Worker P_MTHD(p, NVC3C0, SET_SHADER_SHARED_MEMORY_WINDOW_A);
205*61046927SAndroid Build Coastguard Worker P_NVC3C0_SET_SHADER_SHARED_MEMORY_WINDOW_A(p, temp >> 32);
206*61046927SAndroid Build Coastguard Worker P_NVC3C0_SET_SHADER_SHARED_MEMORY_WINDOW_B(p, temp & 0xffffffff);
207*61046927SAndroid Build Coastguard Worker
208*61046927SAndroid Build Coastguard Worker temp = 0xffULL << 24;
209*61046927SAndroid Build Coastguard Worker P_MTHD(p, NVC3C0, SET_SHADER_LOCAL_MEMORY_WINDOW_A);
210*61046927SAndroid Build Coastguard Worker P_NVC3C0_SET_SHADER_LOCAL_MEMORY_WINDOW_A(p, temp >> 32);
211*61046927SAndroid Build Coastguard Worker P_NVC3C0_SET_SHADER_LOCAL_MEMORY_WINDOW_B(p, temp & 0xffffffff);
212*61046927SAndroid Build Coastguard Worker } else {
213*61046927SAndroid Build Coastguard Worker P_MTHD(p, NVA0C0, SET_SHADER_LOCAL_MEMORY_WINDOW);
214*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_SHADER_LOCAL_MEMORY_WINDOW(p, 0xff << 24);
215*61046927SAndroid Build Coastguard Worker
216*61046927SAndroid Build Coastguard Worker P_MTHD(p, NVA0C0, SET_SHADER_SHARED_MEMORY_WINDOW);
217*61046927SAndroid Build Coastguard Worker P_NVA0C0_SET_SHADER_SHARED_MEMORY_WINDOW(p, 0xfe << 24);
218*61046927SAndroid Build Coastguard Worker }
219*61046927SAndroid Build Coastguard Worker
220*61046927SAndroid Build Coastguard Worker /* From nvc0_screen.c:
221*61046927SAndroid Build Coastguard Worker *
222*61046927SAndroid Build Coastguard Worker * "Reduce likelihood of collision with real buffers by placing the
223*61046927SAndroid Build Coastguard Worker * hole at the top of the 4G area. This will have to be dealt with
224*61046927SAndroid Build Coastguard Worker * for real eventually by blocking off that area from the VM."
225*61046927SAndroid Build Coastguard Worker *
226*61046927SAndroid Build Coastguard Worker * Really?!? TODO: Fix this for realz. Annoyingly, we only have a
227*61046927SAndroid Build Coastguard Worker * 32-bit pointer for this in 3D rather than a full 48 like we have for
228*61046927SAndroid Build Coastguard Worker * compute.
229*61046927SAndroid Build Coastguard Worker */
230*61046927SAndroid Build Coastguard Worker P_IMMD(p, NV9097, SET_SHADER_LOCAL_MEMORY_WINDOW, 0xff << 24);
231*61046927SAndroid Build Coastguard Worker
232*61046927SAndroid Build Coastguard Worker if (qs->push.mem)
233*61046927SAndroid Build Coastguard Worker nvkmd_mem_unref(qs->push.mem);
234*61046927SAndroid Build Coastguard Worker
235*61046927SAndroid Build Coastguard Worker qs->push.mem = push_mem;
236*61046927SAndroid Build Coastguard Worker qs->push.dw_count = nv_push_dw_count(&push);
237*61046927SAndroid Build Coastguard Worker
238*61046927SAndroid Build Coastguard Worker return VK_SUCCESS;
239*61046927SAndroid Build Coastguard Worker }
240*61046927SAndroid Build Coastguard Worker
241*61046927SAndroid Build Coastguard Worker static VkResult
nvk_queue_submit_bind(struct nvk_queue * queue,struct vk_queue_submit * submit)242*61046927SAndroid Build Coastguard Worker nvk_queue_submit_bind(struct nvk_queue *queue,
243*61046927SAndroid Build Coastguard Worker struct vk_queue_submit *submit)
244*61046927SAndroid Build Coastguard Worker {
245*61046927SAndroid Build Coastguard Worker VkResult result;
246*61046927SAndroid Build Coastguard Worker
247*61046927SAndroid Build Coastguard Worker result = nvkmd_ctx_wait(queue->bind_ctx, &queue->vk.base,
248*61046927SAndroid Build Coastguard Worker submit->wait_count, submit->waits);
249*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
250*61046927SAndroid Build Coastguard Worker return result;
251*61046927SAndroid Build Coastguard Worker
252*61046927SAndroid Build Coastguard Worker for (uint32_t i = 0; i < submit->buffer_bind_count; i++) {
253*61046927SAndroid Build Coastguard Worker result = nvk_queue_buffer_bind(queue, &submit->buffer_binds[i]);
254*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
255*61046927SAndroid Build Coastguard Worker return result;
256*61046927SAndroid Build Coastguard Worker }
257*61046927SAndroid Build Coastguard Worker
258*61046927SAndroid Build Coastguard Worker for (uint32_t i = 0; i < submit->image_bind_count; i++) {
259*61046927SAndroid Build Coastguard Worker result = nvk_queue_image_bind(queue, &submit->image_binds[i]);
260*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
261*61046927SAndroid Build Coastguard Worker return result;
262*61046927SAndroid Build Coastguard Worker }
263*61046927SAndroid Build Coastguard Worker
264*61046927SAndroid Build Coastguard Worker for (uint32_t i = 0; i < submit->image_opaque_bind_count; i++) {
265*61046927SAndroid Build Coastguard Worker result = nvk_queue_image_opaque_bind(queue, &submit->image_opaque_binds[i]);
266*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
267*61046927SAndroid Build Coastguard Worker return result;
268*61046927SAndroid Build Coastguard Worker }
269*61046927SAndroid Build Coastguard Worker
270*61046927SAndroid Build Coastguard Worker result = nvkmd_ctx_signal(queue->bind_ctx, &queue->vk.base,
271*61046927SAndroid Build Coastguard Worker submit->signal_count, submit->signals);
272*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
273*61046927SAndroid Build Coastguard Worker return result;
274*61046927SAndroid Build Coastguard Worker
275*61046927SAndroid Build Coastguard Worker return VK_SUCCESS;
276*61046927SAndroid Build Coastguard Worker }
277*61046927SAndroid Build Coastguard Worker
278*61046927SAndroid Build Coastguard Worker static VkResult
nvk_queue_submit_exec(struct nvk_queue * queue,struct vk_queue_submit * submit)279*61046927SAndroid Build Coastguard Worker nvk_queue_submit_exec(struct nvk_queue *queue,
280*61046927SAndroid Build Coastguard Worker struct vk_queue_submit *submit)
281*61046927SAndroid Build Coastguard Worker {
282*61046927SAndroid Build Coastguard Worker struct nvk_device *dev = nvk_queue_device(queue);
283*61046927SAndroid Build Coastguard Worker struct nvk_physical_device *pdev = nvk_device_physical(dev);
284*61046927SAndroid Build Coastguard Worker VkResult result;
285*61046927SAndroid Build Coastguard Worker
286*61046927SAndroid Build Coastguard Worker const bool sync = pdev->debug_flags & NVK_DEBUG_PUSH_SYNC;
287*61046927SAndroid Build Coastguard Worker
288*61046927SAndroid Build Coastguard Worker uint64_t upload_time_point;
289*61046927SAndroid Build Coastguard Worker result = nvk_upload_queue_flush(dev, &dev->upload, &upload_time_point);
290*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
291*61046927SAndroid Build Coastguard Worker return result;
292*61046927SAndroid Build Coastguard Worker
293*61046927SAndroid Build Coastguard Worker if (upload_time_point > 0) {
294*61046927SAndroid Build Coastguard Worker struct vk_sync_wait wait = {
295*61046927SAndroid Build Coastguard Worker .sync = dev->upload.sync,
296*61046927SAndroid Build Coastguard Worker .stage_mask = ~0,
297*61046927SAndroid Build Coastguard Worker .wait_value = upload_time_point,
298*61046927SAndroid Build Coastguard Worker };
299*61046927SAndroid Build Coastguard Worker result = nvkmd_ctx_wait(queue->exec_ctx, &queue->vk.base, 1, &wait);
300*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
301*61046927SAndroid Build Coastguard Worker goto fail;
302*61046927SAndroid Build Coastguard Worker }
303*61046927SAndroid Build Coastguard Worker
304*61046927SAndroid Build Coastguard Worker result = nvkmd_ctx_wait(queue->exec_ctx, &queue->vk.base,
305*61046927SAndroid Build Coastguard Worker submit->wait_count, submit->waits);
306*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
307*61046927SAndroid Build Coastguard Worker goto fail;
308*61046927SAndroid Build Coastguard Worker
309*61046927SAndroid Build Coastguard Worker if (queue->state.push.mem != NULL) {
310*61046927SAndroid Build Coastguard Worker struct nvkmd_ctx_exec exec = {
311*61046927SAndroid Build Coastguard Worker .addr = queue->state.push.mem->va->addr,
312*61046927SAndroid Build Coastguard Worker .size_B = queue->state.push.dw_count * 4,
313*61046927SAndroid Build Coastguard Worker };
314*61046927SAndroid Build Coastguard Worker result = nvkmd_ctx_exec(queue->exec_ctx, &queue->vk.base, 1, &exec);
315*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
316*61046927SAndroid Build Coastguard Worker goto fail;
317*61046927SAndroid Build Coastguard Worker }
318*61046927SAndroid Build Coastguard Worker
319*61046927SAndroid Build Coastguard Worker for (unsigned i = 0; i < submit->command_buffer_count; i++) {
320*61046927SAndroid Build Coastguard Worker struct nvk_cmd_buffer *cmd =
321*61046927SAndroid Build Coastguard Worker container_of(submit->command_buffers[i], struct nvk_cmd_buffer, vk);
322*61046927SAndroid Build Coastguard Worker
323*61046927SAndroid Build Coastguard Worker const uint32_t max_execs =
324*61046927SAndroid Build Coastguard Worker util_dynarray_num_elements(&cmd->pushes, struct nvk_cmd_push);
325*61046927SAndroid Build Coastguard Worker STACK_ARRAY(struct nvkmd_ctx_exec, execs, max_execs);
326*61046927SAndroid Build Coastguard Worker uint32_t exec_count = 0;
327*61046927SAndroid Build Coastguard Worker
328*61046927SAndroid Build Coastguard Worker util_dynarray_foreach(&cmd->pushes, struct nvk_cmd_push, push) {
329*61046927SAndroid Build Coastguard Worker if (push->range == 0)
330*61046927SAndroid Build Coastguard Worker continue;
331*61046927SAndroid Build Coastguard Worker
332*61046927SAndroid Build Coastguard Worker execs[exec_count++] = (struct nvkmd_ctx_exec) {
333*61046927SAndroid Build Coastguard Worker .addr = push->addr,
334*61046927SAndroid Build Coastguard Worker .size_B = push->range,
335*61046927SAndroid Build Coastguard Worker .no_prefetch = push->no_prefetch,
336*61046927SAndroid Build Coastguard Worker };
337*61046927SAndroid Build Coastguard Worker }
338*61046927SAndroid Build Coastguard Worker
339*61046927SAndroid Build Coastguard Worker result = nvkmd_ctx_exec(queue->exec_ctx, &queue->vk.base,
340*61046927SAndroid Build Coastguard Worker exec_count, execs);
341*61046927SAndroid Build Coastguard Worker
342*61046927SAndroid Build Coastguard Worker STACK_ARRAY_FINISH(execs);
343*61046927SAndroid Build Coastguard Worker
344*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
345*61046927SAndroid Build Coastguard Worker goto fail;
346*61046927SAndroid Build Coastguard Worker }
347*61046927SAndroid Build Coastguard Worker
348*61046927SAndroid Build Coastguard Worker result = nvkmd_ctx_signal(queue->exec_ctx, &queue->vk.base,
349*61046927SAndroid Build Coastguard Worker submit->signal_count, submit->signals);
350*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
351*61046927SAndroid Build Coastguard Worker goto fail;
352*61046927SAndroid Build Coastguard Worker
353*61046927SAndroid Build Coastguard Worker if (sync) {
354*61046927SAndroid Build Coastguard Worker result = nvkmd_ctx_sync(queue->exec_ctx, &queue->vk.base);
355*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
356*61046927SAndroid Build Coastguard Worker goto fail;
357*61046927SAndroid Build Coastguard Worker }
358*61046927SAndroid Build Coastguard Worker
359*61046927SAndroid Build Coastguard Worker fail:
360*61046927SAndroid Build Coastguard Worker if ((sync && result != VK_SUCCESS) ||
361*61046927SAndroid Build Coastguard Worker (pdev->debug_flags & NVK_DEBUG_PUSH_DUMP)) {
362*61046927SAndroid Build Coastguard Worker nvk_queue_state_dump_push(dev, &queue->state, stderr);
363*61046927SAndroid Build Coastguard Worker
364*61046927SAndroid Build Coastguard Worker for (unsigned i = 0; i < submit->command_buffer_count; i++) {
365*61046927SAndroid Build Coastguard Worker struct nvk_cmd_buffer *cmd =
366*61046927SAndroid Build Coastguard Worker container_of(submit->command_buffers[i], struct nvk_cmd_buffer, vk);
367*61046927SAndroid Build Coastguard Worker
368*61046927SAndroid Build Coastguard Worker nvk_cmd_buffer_dump(cmd, stderr);
369*61046927SAndroid Build Coastguard Worker }
370*61046927SAndroid Build Coastguard Worker }
371*61046927SAndroid Build Coastguard Worker
372*61046927SAndroid Build Coastguard Worker return result;
373*61046927SAndroid Build Coastguard Worker }
374*61046927SAndroid Build Coastguard Worker
375*61046927SAndroid Build Coastguard Worker static VkResult
nvk_queue_submit(struct vk_queue * vk_queue,struct vk_queue_submit * submit)376*61046927SAndroid Build Coastguard Worker nvk_queue_submit(struct vk_queue *vk_queue,
377*61046927SAndroid Build Coastguard Worker struct vk_queue_submit *submit)
378*61046927SAndroid Build Coastguard Worker {
379*61046927SAndroid Build Coastguard Worker struct nvk_queue *queue = container_of(vk_queue, struct nvk_queue, vk);
380*61046927SAndroid Build Coastguard Worker struct nvk_device *dev = nvk_queue_device(queue);
381*61046927SAndroid Build Coastguard Worker VkResult result;
382*61046927SAndroid Build Coastguard Worker
383*61046927SAndroid Build Coastguard Worker if (vk_queue_is_lost(&queue->vk))
384*61046927SAndroid Build Coastguard Worker return VK_ERROR_DEVICE_LOST;
385*61046927SAndroid Build Coastguard Worker
386*61046927SAndroid Build Coastguard Worker if (submit->buffer_bind_count > 0 ||
387*61046927SAndroid Build Coastguard Worker submit->image_bind_count > 0 ||
388*61046927SAndroid Build Coastguard Worker submit->image_opaque_bind_count > 0) {
389*61046927SAndroid Build Coastguard Worker assert(submit->command_buffer_count == 0);
390*61046927SAndroid Build Coastguard Worker result = nvk_queue_submit_bind(queue, submit);
391*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
392*61046927SAndroid Build Coastguard Worker return vk_queue_set_lost(&queue->vk, "Bind operation failed");
393*61046927SAndroid Build Coastguard Worker } else {
394*61046927SAndroid Build Coastguard Worker result = nvk_queue_state_update(dev, &queue->state);
395*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS) {
396*61046927SAndroid Build Coastguard Worker return vk_queue_set_lost(&queue->vk, "Failed to update queue base "
397*61046927SAndroid Build Coastguard Worker "pointers pushbuf");
398*61046927SAndroid Build Coastguard Worker }
399*61046927SAndroid Build Coastguard Worker
400*61046927SAndroid Build Coastguard Worker result = nvk_queue_submit_exec(queue, submit);
401*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
402*61046927SAndroid Build Coastguard Worker return vk_queue_set_lost(&queue->vk, "Submit failed");
403*61046927SAndroid Build Coastguard Worker }
404*61046927SAndroid Build Coastguard Worker
405*61046927SAndroid Build Coastguard Worker return VK_SUCCESS;
406*61046927SAndroid Build Coastguard Worker }
407*61046927SAndroid Build Coastguard Worker
408*61046927SAndroid Build Coastguard Worker static VkResult
nvk_queue_submit_simple(struct nvk_queue * queue,uint32_t dw_count,const uint32_t * dw)409*61046927SAndroid Build Coastguard Worker nvk_queue_submit_simple(struct nvk_queue *queue,
410*61046927SAndroid Build Coastguard Worker uint32_t dw_count, const uint32_t *dw)
411*61046927SAndroid Build Coastguard Worker {
412*61046927SAndroid Build Coastguard Worker struct nvk_device *dev = nvk_queue_device(queue);
413*61046927SAndroid Build Coastguard Worker struct nvk_physical_device *pdev = nvk_device_physical(dev);
414*61046927SAndroid Build Coastguard Worker VkResult result;
415*61046927SAndroid Build Coastguard Worker
416*61046927SAndroid Build Coastguard Worker if (vk_queue_is_lost(&queue->vk))
417*61046927SAndroid Build Coastguard Worker return VK_ERROR_DEVICE_LOST;
418*61046927SAndroid Build Coastguard Worker
419*61046927SAndroid Build Coastguard Worker struct nvkmd_mem *push_mem;
420*61046927SAndroid Build Coastguard Worker result = nvkmd_dev_alloc_mapped_mem(dev->nvkmd, &dev->vk.base,
421*61046927SAndroid Build Coastguard Worker dw_count * 4, 0,
422*61046927SAndroid Build Coastguard Worker NVKMD_MEM_GART,
423*61046927SAndroid Build Coastguard Worker NVKMD_MEM_MAP_WR, &push_mem);
424*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
425*61046927SAndroid Build Coastguard Worker return result;
426*61046927SAndroid Build Coastguard Worker
427*61046927SAndroid Build Coastguard Worker memcpy(push_mem->map, dw, dw_count * 4);
428*61046927SAndroid Build Coastguard Worker
429*61046927SAndroid Build Coastguard Worker const struct nvkmd_ctx_exec exec = {
430*61046927SAndroid Build Coastguard Worker .addr = push_mem->va->addr,
431*61046927SAndroid Build Coastguard Worker .size_B = dw_count * 4,
432*61046927SAndroid Build Coastguard Worker };
433*61046927SAndroid Build Coastguard Worker result = nvkmd_ctx_exec(queue->exec_ctx, &queue->vk.base, 1, &exec);
434*61046927SAndroid Build Coastguard Worker if (result == VK_SUCCESS)
435*61046927SAndroid Build Coastguard Worker result = nvkmd_ctx_sync(queue->exec_ctx, &queue->vk.base);
436*61046927SAndroid Build Coastguard Worker
437*61046927SAndroid Build Coastguard Worker nvkmd_mem_unref(push_mem);
438*61046927SAndroid Build Coastguard Worker
439*61046927SAndroid Build Coastguard Worker const bool debug_sync = pdev->debug_flags & NVK_DEBUG_PUSH_SYNC;
440*61046927SAndroid Build Coastguard Worker if ((debug_sync && result != VK_SUCCESS) ||
441*61046927SAndroid Build Coastguard Worker (pdev->debug_flags & NVK_DEBUG_PUSH_DUMP)) {
442*61046927SAndroid Build Coastguard Worker struct nv_push push = {
443*61046927SAndroid Build Coastguard Worker .start = (uint32_t *)dw,
444*61046927SAndroid Build Coastguard Worker .end = (uint32_t *)dw + dw_count,
445*61046927SAndroid Build Coastguard Worker };
446*61046927SAndroid Build Coastguard Worker vk_push_print(stderr, &push, &pdev->info);
447*61046927SAndroid Build Coastguard Worker }
448*61046927SAndroid Build Coastguard Worker
449*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
450*61046927SAndroid Build Coastguard Worker return vk_queue_set_lost(&queue->vk, "Submit failed");
451*61046927SAndroid Build Coastguard Worker
452*61046927SAndroid Build Coastguard Worker return VK_SUCCESS;
453*61046927SAndroid Build Coastguard Worker }
454*61046927SAndroid Build Coastguard Worker
455*61046927SAndroid Build Coastguard Worker static VkResult
nvk_queue_init_context_state(struct nvk_queue * queue,enum nvkmd_engines engines)456*61046927SAndroid Build Coastguard Worker nvk_queue_init_context_state(struct nvk_queue *queue,
457*61046927SAndroid Build Coastguard Worker enum nvkmd_engines engines)
458*61046927SAndroid Build Coastguard Worker {
459*61046927SAndroid Build Coastguard Worker struct nvk_device *dev = nvk_queue_device(queue);
460*61046927SAndroid Build Coastguard Worker struct nvk_physical_device *pdev = nvk_device_physical(dev);
461*61046927SAndroid Build Coastguard Worker VkResult result;
462*61046927SAndroid Build Coastguard Worker
463*61046927SAndroid Build Coastguard Worker uint32_t push_data[4096];
464*61046927SAndroid Build Coastguard Worker struct nv_push push;
465*61046927SAndroid Build Coastguard Worker nv_push_init(&push, push_data, ARRAY_SIZE(push_data));
466*61046927SAndroid Build Coastguard Worker struct nv_push *p = &push;
467*61046927SAndroid Build Coastguard Worker
468*61046927SAndroid Build Coastguard Worker /* M2MF state */
469*61046927SAndroid Build Coastguard Worker if (pdev->info.cls_m2mf <= FERMI_MEMORY_TO_MEMORY_FORMAT_A) {
470*61046927SAndroid Build Coastguard Worker /* we absolutely do not support Fermi, but if somebody wants to toy
471*61046927SAndroid Build Coastguard Worker * around with it, this is a must
472*61046927SAndroid Build Coastguard Worker */
473*61046927SAndroid Build Coastguard Worker P_MTHD(p, NV9039, SET_OBJECT);
474*61046927SAndroid Build Coastguard Worker P_NV9039_SET_OBJECT(p, {
475*61046927SAndroid Build Coastguard Worker .class_id = pdev->info.cls_m2mf,
476*61046927SAndroid Build Coastguard Worker .engine_id = 0,
477*61046927SAndroid Build Coastguard Worker });
478*61046927SAndroid Build Coastguard Worker }
479*61046927SAndroid Build Coastguard Worker
480*61046927SAndroid Build Coastguard Worker if (engines & NVKMD_ENGINE_3D) {
481*61046927SAndroid Build Coastguard Worker result = nvk_push_draw_state_init(queue, p);
482*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
483*61046927SAndroid Build Coastguard Worker return result;
484*61046927SAndroid Build Coastguard Worker }
485*61046927SAndroid Build Coastguard Worker
486*61046927SAndroid Build Coastguard Worker if (engines & NVKMD_ENGINE_COMPUTE) {
487*61046927SAndroid Build Coastguard Worker result = nvk_push_dispatch_state_init(queue, p);
488*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
489*61046927SAndroid Build Coastguard Worker return result;
490*61046927SAndroid Build Coastguard Worker }
491*61046927SAndroid Build Coastguard Worker
492*61046927SAndroid Build Coastguard Worker return nvk_queue_submit_simple(queue, nv_push_dw_count(&push), push_data);
493*61046927SAndroid Build Coastguard Worker }
494*61046927SAndroid Build Coastguard Worker
495*61046927SAndroid Build Coastguard Worker VkResult
nvk_queue_init(struct nvk_device * dev,struct nvk_queue * queue,const VkDeviceQueueCreateInfo * pCreateInfo,uint32_t index_in_family)496*61046927SAndroid Build Coastguard Worker nvk_queue_init(struct nvk_device *dev, struct nvk_queue *queue,
497*61046927SAndroid Build Coastguard Worker const VkDeviceQueueCreateInfo *pCreateInfo,
498*61046927SAndroid Build Coastguard Worker uint32_t index_in_family)
499*61046927SAndroid Build Coastguard Worker {
500*61046927SAndroid Build Coastguard Worker struct nvk_physical_device *pdev = nvk_device_physical(dev);
501*61046927SAndroid Build Coastguard Worker VkResult result;
502*61046927SAndroid Build Coastguard Worker
503*61046927SAndroid Build Coastguard Worker assert(pCreateInfo->queueFamilyIndex < pdev->queue_family_count);
504*61046927SAndroid Build Coastguard Worker const struct nvk_queue_family *queue_family =
505*61046927SAndroid Build Coastguard Worker &pdev->queue_families[pCreateInfo->queueFamilyIndex];
506*61046927SAndroid Build Coastguard Worker
507*61046927SAndroid Build Coastguard Worker result = vk_queue_init(&queue->vk, &dev->vk, pCreateInfo, index_in_family);
508*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
509*61046927SAndroid Build Coastguard Worker return result;
510*61046927SAndroid Build Coastguard Worker
511*61046927SAndroid Build Coastguard Worker nvk_queue_state_init(&queue->state);
512*61046927SAndroid Build Coastguard Worker
513*61046927SAndroid Build Coastguard Worker enum nvkmd_engines engines = 0;
514*61046927SAndroid Build Coastguard Worker if (queue_family->queue_flags & VK_QUEUE_GRAPHICS_BIT) {
515*61046927SAndroid Build Coastguard Worker engines |= NVKMD_ENGINE_3D;
516*61046927SAndroid Build Coastguard Worker /* We rely on compute shaders for queries */
517*61046927SAndroid Build Coastguard Worker engines |= NVKMD_ENGINE_COMPUTE;
518*61046927SAndroid Build Coastguard Worker }
519*61046927SAndroid Build Coastguard Worker if (queue_family->queue_flags & VK_QUEUE_COMPUTE_BIT) {
520*61046927SAndroid Build Coastguard Worker engines |= NVKMD_ENGINE_COMPUTE;
521*61046927SAndroid Build Coastguard Worker /* We currently rely on 3D engine MMEs for indirect dispatch */
522*61046927SAndroid Build Coastguard Worker engines |= NVKMD_ENGINE_3D;
523*61046927SAndroid Build Coastguard Worker }
524*61046927SAndroid Build Coastguard Worker if (queue_family->queue_flags & VK_QUEUE_TRANSFER_BIT)
525*61046927SAndroid Build Coastguard Worker engines |= NVKMD_ENGINE_COPY;
526*61046927SAndroid Build Coastguard Worker
527*61046927SAndroid Build Coastguard Worker if (engines) {
528*61046927SAndroid Build Coastguard Worker result = nvkmd_dev_create_ctx(dev->nvkmd, &dev->vk.base,
529*61046927SAndroid Build Coastguard Worker engines, &queue->exec_ctx);
530*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
531*61046927SAndroid Build Coastguard Worker goto fail_init;
532*61046927SAndroid Build Coastguard Worker
533*61046927SAndroid Build Coastguard Worker result = nvkmd_dev_alloc_mem(dev->nvkmd, &dev->vk.base,
534*61046927SAndroid Build Coastguard Worker 4096, 0, NVKMD_MEM_LOCAL,
535*61046927SAndroid Build Coastguard Worker &queue->draw_cb0);
536*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
537*61046927SAndroid Build Coastguard Worker goto fail_exec_ctx;
538*61046927SAndroid Build Coastguard Worker
539*61046927SAndroid Build Coastguard Worker result = nvk_upload_queue_fill(dev, &dev->upload,
540*61046927SAndroid Build Coastguard Worker queue->draw_cb0->va->addr, 0,
541*61046927SAndroid Build Coastguard Worker queue->draw_cb0->size_B);
542*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
543*61046927SAndroid Build Coastguard Worker goto fail_draw_cb0;
544*61046927SAndroid Build Coastguard Worker }
545*61046927SAndroid Build Coastguard Worker
546*61046927SAndroid Build Coastguard Worker if (queue_family->queue_flags & VK_QUEUE_SPARSE_BINDING_BIT) {
547*61046927SAndroid Build Coastguard Worker result = nvkmd_dev_create_ctx(dev->nvkmd, &dev->vk.base,
548*61046927SAndroid Build Coastguard Worker NVKMD_ENGINE_BIND, &queue->bind_ctx);
549*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
550*61046927SAndroid Build Coastguard Worker goto fail_draw_cb0;
551*61046927SAndroid Build Coastguard Worker }
552*61046927SAndroid Build Coastguard Worker
553*61046927SAndroid Build Coastguard Worker result = nvk_queue_init_context_state(queue, engines);
554*61046927SAndroid Build Coastguard Worker if (result != VK_SUCCESS)
555*61046927SAndroid Build Coastguard Worker goto fail_bind_ctx;
556*61046927SAndroid Build Coastguard Worker
557*61046927SAndroid Build Coastguard Worker queue->vk.driver_submit = nvk_queue_submit;
558*61046927SAndroid Build Coastguard Worker
559*61046927SAndroid Build Coastguard Worker return VK_SUCCESS;
560*61046927SAndroid Build Coastguard Worker
561*61046927SAndroid Build Coastguard Worker fail_bind_ctx:
562*61046927SAndroid Build Coastguard Worker if (queue->bind_ctx != NULL)
563*61046927SAndroid Build Coastguard Worker nvkmd_ctx_destroy(queue->bind_ctx);
564*61046927SAndroid Build Coastguard Worker fail_draw_cb0:
565*61046927SAndroid Build Coastguard Worker if (queue->draw_cb0 != NULL)
566*61046927SAndroid Build Coastguard Worker nvkmd_mem_unref(queue->draw_cb0);
567*61046927SAndroid Build Coastguard Worker fail_exec_ctx:
568*61046927SAndroid Build Coastguard Worker if (queue->exec_ctx != NULL)
569*61046927SAndroid Build Coastguard Worker nvkmd_ctx_destroy(queue->exec_ctx);
570*61046927SAndroid Build Coastguard Worker fail_init:
571*61046927SAndroid Build Coastguard Worker nvk_queue_state_finish(dev, &queue->state);
572*61046927SAndroid Build Coastguard Worker vk_queue_finish(&queue->vk);
573*61046927SAndroid Build Coastguard Worker
574*61046927SAndroid Build Coastguard Worker return result;
575*61046927SAndroid Build Coastguard Worker }
576*61046927SAndroid Build Coastguard Worker
577*61046927SAndroid Build Coastguard Worker void
nvk_queue_finish(struct nvk_device * dev,struct nvk_queue * queue)578*61046927SAndroid Build Coastguard Worker nvk_queue_finish(struct nvk_device *dev, struct nvk_queue *queue)
579*61046927SAndroid Build Coastguard Worker {
580*61046927SAndroid Build Coastguard Worker if (queue->draw_cb0 != NULL) {
581*61046927SAndroid Build Coastguard Worker nvk_upload_queue_sync(dev, &dev->upload);
582*61046927SAndroid Build Coastguard Worker nvkmd_mem_unref(queue->draw_cb0);
583*61046927SAndroid Build Coastguard Worker }
584*61046927SAndroid Build Coastguard Worker nvk_queue_state_finish(dev, &queue->state);
585*61046927SAndroid Build Coastguard Worker if (queue->bind_ctx != NULL)
586*61046927SAndroid Build Coastguard Worker nvkmd_ctx_destroy(queue->bind_ctx);
587*61046927SAndroid Build Coastguard Worker if (queue->exec_ctx != NULL)
588*61046927SAndroid Build Coastguard Worker nvkmd_ctx_destroy(queue->exec_ctx);
589*61046927SAndroid Build Coastguard Worker vk_queue_finish(&queue->vk);
590*61046927SAndroid Build Coastguard Worker }
591