1 /*
2 * Copyright 2019 Google LLC
3 * SPDX-License-Identifier: MIT
4 *
5 * based in part on anv and radv which are:
6 * Copyright © 2015 Intel Corporation
7 * Copyright © 2016 Red Hat.
8 * Copyright © 2016 Bas Nieuwenhuizen
9 */
10
11 #include "vn_common.h"
12
13 #include <stdarg.h>
14
15 #include "util/log.h"
16 #include "util/os_misc.h"
17 #include "util/u_debug.h"
18 #include "venus-protocol/vn_protocol_driver_info.h"
19 #include "vk_enum_to_str.h"
20
21 #include "vn_instance.h"
22 #include "vn_ring.h"
23
24 static const struct debug_control vn_debug_options[] = {
25 /* clang-format off */
26 { "init", VN_DEBUG_INIT },
27 { "result", VN_DEBUG_RESULT },
28 { "vtest", VN_DEBUG_VTEST },
29 { "wsi", VN_DEBUG_WSI },
30 { "no_abort", VN_DEBUG_NO_ABORT },
31 { "log_ctx_info", VN_DEBUG_LOG_CTX_INFO },
32 { "cache", VN_DEBUG_CACHE },
33 { "no_sparse", VN_DEBUG_NO_SPARSE },
34 { "no_gpl", VN_DEBUG_NO_GPL },
35 { NULL, 0 },
36 /* clang-format on */
37 };
38
39 static const struct debug_control vn_perf_options[] = {
40 /* clang-format off */
41 { "no_async_set_alloc", VN_PERF_NO_ASYNC_SET_ALLOC },
42 { "no_async_buffer_create", VN_PERF_NO_ASYNC_BUFFER_CREATE },
43 { "no_async_queue_submit", VN_PERF_NO_ASYNC_QUEUE_SUBMIT },
44 { "no_event_feedback", VN_PERF_NO_EVENT_FEEDBACK },
45 { "no_fence_feedback", VN_PERF_NO_FENCE_FEEDBACK },
46 { "no_cmd_batching", VN_PERF_NO_CMD_BATCHING },
47 { "no_semaphore_feedback", VN_PERF_NO_SEMAPHORE_FEEDBACK },
48 { "no_query_feedback", VN_PERF_NO_QUERY_FEEDBACK },
49 { "no_async_mem_alloc", VN_PERF_NO_ASYNC_MEM_ALLOC },
50 { "no_tiled_wsi_image", VN_PERF_NO_TILED_WSI_IMAGE },
51 { "no_multi_ring", VN_PERF_NO_MULTI_RING },
52 { "no_async_image_create", VN_PERF_NO_ASYNC_IMAGE_CREATE },
53 { "no_async_image_format", VN_PERF_NO_ASYNC_IMAGE_FORMAT },
54 { NULL, 0 },
55 /* clang-format on */
56 };
57
58 uint64_t vn_next_obj_id = 1;
59 struct vn_env vn_env;
60
61 static void
vn_env_init_once(void)62 vn_env_init_once(void)
63 {
64 vn_env.debug =
65 parse_debug_string(os_get_option("VN_DEBUG"), vn_debug_options);
66 vn_env.perf =
67 parse_debug_string(os_get_option("VN_PERF"), vn_perf_options);
68 }
69
70 void
vn_env_init(void)71 vn_env_init(void)
72 {
73 static once_flag once = ONCE_FLAG_INIT;
74 call_once(&once, vn_env_init_once);
75
76 /* log per VkInstance creation */
77 if (VN_DEBUG(INIT)) {
78 vn_log(NULL,
79 "vn_env is as below:"
80 "\n\tdebug = 0x%" PRIx64 ""
81 "\n\tperf = 0x%" PRIx64 "",
82 vn_env.debug, vn_env.perf);
83 }
84 }
85
86 void
vn_trace_init(void)87 vn_trace_init(void)
88 {
89 #if DETECT_OS_ANDROID
90 atrace_init();
91 #else
92 util_cpu_trace_init();
93 #endif
94 }
95
96 void
vn_log(struct vn_instance * instance,const char * format,...)97 vn_log(struct vn_instance *instance, const char *format, ...)
98 {
99 va_list ap;
100
101 va_start(ap, format);
102 mesa_log_v(MESA_LOG_DEBUG, "MESA-VIRTIO", format, ap);
103 va_end(ap);
104
105 /* instance may be NULL or partially initialized */
106 }
107
108 VkResult
vn_log_result(struct vn_instance * instance,VkResult result,const char * where)109 vn_log_result(struct vn_instance *instance,
110 VkResult result,
111 const char *where)
112 {
113 vn_log(instance, "%s: %s", where, vk_Result_to_str(result));
114 return result;
115 }
116
117 uint32_t
vn_extension_get_spec_version(const char * name)118 vn_extension_get_spec_version(const char *name)
119 {
120 const int32_t index = vn_info_extension_index(name);
121 return index >= 0 ? vn_info_extension_get(index)->spec_version : 0;
122 }
123
124 static inline bool
vn_watchdog_timeout(const struct vn_watchdog * watchdog)125 vn_watchdog_timeout(const struct vn_watchdog *watchdog)
126 {
127 return !watchdog->alive;
128 }
129
130 static inline void
vn_watchdog_release(struct vn_watchdog * watchdog)131 vn_watchdog_release(struct vn_watchdog *watchdog)
132 {
133 if (vn_gettid() == watchdog->tid) {
134 watchdog->tid = 0;
135 mtx_unlock(&watchdog->mutex);
136 }
137 }
138
139 static bool
vn_watchdog_acquire(struct vn_watchdog * watchdog,bool alive)140 vn_watchdog_acquire(struct vn_watchdog *watchdog, bool alive)
141 {
142 pid_t tid = vn_gettid();
143 if (!watchdog->tid && tid != watchdog->tid &&
144 mtx_trylock(&watchdog->mutex) == thrd_success) {
145 /* register as the only waiting thread that monitors the ring. */
146 watchdog->tid = tid;
147 }
148
149 if (tid != watchdog->tid)
150 return false;
151
152 watchdog->alive = alive;
153 return true;
154 }
155
156 void
vn_relax_fini(struct vn_relax_state * state)157 vn_relax_fini(struct vn_relax_state *state)
158 {
159 vn_watchdog_release(&state->instance->ring.watchdog);
160 }
161
162 static inline const char *
vn_relax_reason_string(enum vn_relax_reason reason)163 vn_relax_reason_string(enum vn_relax_reason reason)
164 {
165 /* deliberately avoid default case for -Wswitch to catch upon compile */
166 switch (reason) {
167 case VN_RELAX_REASON_RING_SEQNO:
168 return "ring seqno";
169 case VN_RELAX_REASON_TLS_RING_SEQNO:
170 return "tls ring seqno";
171 case VN_RELAX_REASON_RING_SPACE:
172 return "ring space";
173 case VN_RELAX_REASON_FENCE:
174 return "fence";
175 case VN_RELAX_REASON_SEMAPHORE:
176 return "semaphore";
177 case VN_RELAX_REASON_QUERY:
178 return "query";
179 }
180 return "";
181 }
182
183 static struct vn_relax_profile
vn_relax_get_profile(enum vn_relax_reason reason)184 vn_relax_get_profile(enum vn_relax_reason reason)
185 {
186 /* This is the helper to map a vn_relax_reason to a profile. For new
187 * profiles added, we MUST also update the pre-calculated "first_warn_time"
188 * in vn_watchdog_init() if the "first warn" comes sooner.
189 */
190
191 /* deliberately avoid default case for -Wswitch to catch upon compile */
192 switch (reason) {
193 case VN_RELAX_REASON_RING_SEQNO:
194 /* warn every 4096 iters after having already slept ~3.5s:
195 * (yielded 255 times)
196 * stuck in wait with iter at 4096 (3.5s slept already)
197 * stuck in wait with iter at 8192 (14s slept already)
198 * stuck in wait with iter at 12288 (35s slept already)
199 * ...
200 * aborting after 895s
201 */
202 return (struct vn_relax_profile){
203 .base_sleep_us = 160,
204 .busy_wait_order = 8,
205 .warn_order = 12,
206 .abort_order = 16,
207 };
208 case VN_RELAX_REASON_TLS_RING_SEQNO:
209 case VN_RELAX_REASON_RING_SPACE:
210 case VN_RELAX_REASON_FENCE:
211 case VN_RELAX_REASON_SEMAPHORE:
212 case VN_RELAX_REASON_QUERY:
213 /* warn every 1024 iters after having already slept ~3.5s:
214 * (yielded 15 times)
215 * stuck in wait with iter at 1024 (3.5s slept already)
216 * stuck in wait with iter at 2048 (14s slept already)
217 * stuck in wait with iter at 3072 (35s slept already)
218 * ...
219 * aborting after 895s
220 */
221 return (struct vn_relax_profile){
222 .base_sleep_us = 160,
223 .busy_wait_order = 4,
224 .warn_order = 10,
225 .abort_order = 14,
226 };
227 }
228
229 unreachable("unhandled vn_relax_reason");
230 }
231
232 struct vn_relax_state
vn_relax_init(struct vn_instance * instance,enum vn_relax_reason reason)233 vn_relax_init(struct vn_instance *instance, enum vn_relax_reason reason)
234 {
235 struct vn_ring *ring = instance->ring.ring;
236 struct vn_watchdog *watchdog = &instance->ring.watchdog;
237 if (vn_watchdog_acquire(watchdog, true))
238 vn_ring_unset_status_bits(ring, VK_RING_STATUS_ALIVE_BIT_MESA);
239
240 return (struct vn_relax_state){
241 .instance = instance,
242 .iter = 0,
243 .profile = vn_relax_get_profile(reason),
244 .reason_str = vn_relax_reason_string(reason),
245 };
246 }
247
248 void
vn_relax(struct vn_relax_state * state)249 vn_relax(struct vn_relax_state *state)
250 {
251 const uint32_t base_sleep_us = state->profile.base_sleep_us;
252 const uint32_t busy_wait_order = state->profile.busy_wait_order;
253 const uint32_t warn_order = state->profile.warn_order;
254 const uint32_t abort_order = state->profile.abort_order;
255
256 uint32_t *iter = &state->iter;
257 (*iter)++;
258 if (*iter < (1 << busy_wait_order)) {
259 thrd_yield();
260 return;
261 }
262
263 if (unlikely(*iter % (1 << warn_order) == 0)) {
264 struct vn_instance *instance = state->instance;
265 vn_log(instance, "stuck in %s wait with iter at %d", state->reason_str,
266 *iter);
267
268 struct vn_ring *ring = instance->ring.ring;
269 const uint32_t status = vn_ring_load_status(ring);
270 if (status & VK_RING_STATUS_FATAL_BIT_MESA) {
271 vn_log(instance, "aborting on ring fatal error at iter %d", *iter);
272 abort();
273 }
274
275 struct vn_watchdog *watchdog = &instance->ring.watchdog;
276 const bool alive = status & VK_RING_STATUS_ALIVE_BIT_MESA;
277 if (vn_watchdog_acquire(watchdog, alive))
278 vn_ring_unset_status_bits(ring, VK_RING_STATUS_ALIVE_BIT_MESA);
279
280 if (vn_watchdog_timeout(watchdog) && !VN_DEBUG(NO_ABORT)) {
281 vn_log(instance, "aborting on expired ring alive status at iter %d",
282 *iter);
283 abort();
284 }
285
286 if (*iter >= (1 << abort_order) && !VN_DEBUG(NO_ABORT)) {
287 vn_log(instance, "aborting");
288 abort();
289 }
290 }
291
292 const uint32_t shift = util_last_bit(*iter) - busy_wait_order - 1;
293 os_time_sleep(base_sleep_us << shift);
294 }
295
296 struct vn_ring *
vn_tls_get_ring(struct vn_instance * instance)297 vn_tls_get_ring(struct vn_instance *instance)
298 {
299 if (VN_PERF(NO_MULTI_RING))
300 return instance->ring.ring;
301
302 struct vn_tls *tls = vn_tls_get();
303 if (unlikely(!tls)) {
304 /* only allow to fallback on missing tls */
305 return instance->ring.ring;
306 }
307
308 /* look up tls_ring owned by instance */
309 list_for_each_entry(struct vn_tls_ring, tls_ring, &tls->tls_rings,
310 tls_head) {
311 mtx_lock(&tls_ring->mutex);
312 if (tls_ring->instance == instance) {
313 mtx_unlock(&tls_ring->mutex);
314 assert(tls_ring->ring);
315 return tls_ring->ring;
316 }
317 mtx_unlock(&tls_ring->mutex);
318 }
319
320 struct vn_tls_ring *tls_ring = calloc(1, sizeof(*tls_ring));
321 if (!tls_ring)
322 return NULL;
323
324 /* keep the extra for potential roundtrip sync on tls ring */
325 static const size_t extra_size = sizeof(uint32_t);
326
327 /* only need a small ring for synchronous cmds on tls ring */
328 static const size_t buf_size = 16 * 1024;
329
330 /* single cmd can use the entire ring shmem on tls ring */
331 static const uint8_t direct_order = 0;
332
333 struct vn_ring_layout layout;
334 vn_ring_get_layout(buf_size, extra_size, &layout);
335
336 tls_ring->ring =
337 vn_ring_create(instance, &layout, direct_order, true /* is_tls_ring */);
338 if (!tls_ring->ring) {
339 free(tls_ring);
340 return NULL;
341 }
342
343 mtx_init(&tls_ring->mutex, mtx_plain);
344 tls_ring->instance = instance;
345 list_add(&tls_ring->tls_head, &tls->tls_rings);
346 list_add(&tls_ring->vk_head, &instance->ring.tls_rings);
347
348 return tls_ring->ring;
349 }
350
351 void
vn_tls_destroy_ring(struct vn_tls_ring * tls_ring)352 vn_tls_destroy_ring(struct vn_tls_ring *tls_ring)
353 {
354 mtx_lock(&tls_ring->mutex);
355 if (tls_ring->ring) {
356 vn_ring_destroy(tls_ring->ring);
357 tls_ring->ring = NULL;
358 tls_ring->instance = NULL;
359 mtx_unlock(&tls_ring->mutex);
360 } else {
361 mtx_unlock(&tls_ring->mutex);
362 mtx_destroy(&tls_ring->mutex);
363 free(tls_ring);
364 }
365 }
366
367 static void
vn_tls_free(void * tls)368 vn_tls_free(void *tls)
369 {
370 if (tls) {
371 list_for_each_entry_safe(struct vn_tls_ring, tls_ring,
372 &((struct vn_tls *)tls)->tls_rings, tls_head)
373 vn_tls_destroy_ring(tls_ring);
374 }
375 free(tls);
376 }
377
378 static tss_t vn_tls_key;
379 static bool vn_tls_key_valid;
380
381 static void
vn_tls_key_create_once(void)382 vn_tls_key_create_once(void)
383 {
384 vn_tls_key_valid = tss_create(&vn_tls_key, vn_tls_free) == thrd_success;
385 if (!vn_tls_key_valid && VN_DEBUG(INIT))
386 vn_log(NULL, "WARNING: failed to create vn_tls_key");
387 }
388
389 struct vn_tls *
vn_tls_get(void)390 vn_tls_get(void)
391 {
392 static once_flag once = ONCE_FLAG_INIT;
393 call_once(&once, vn_tls_key_create_once);
394 if (unlikely(!vn_tls_key_valid))
395 return NULL;
396
397 struct vn_tls *tls = tss_get(vn_tls_key);
398 if (likely(tls))
399 return tls;
400
401 tls = calloc(1, sizeof(*tls));
402 if (!tls)
403 return NULL;
404
405 /* initialize tls */
406 tls->async_pipeline_create = false;
407 list_inithead(&tls->tls_rings);
408
409 if (tss_set(vn_tls_key, tls) != thrd_success) {
410 free(tls);
411 return NULL;
412 }
413
414 return tls;
415 }
416