1 /*
2 * Copyright (c) 2008-2024 Broadcom. All Rights Reserved.
3 * The term “Broadcom” refers to Broadcom Inc.
4 * and/or its subsidiaries.
5 * SPDX-License-Identifier: MIT
6 */
7
8 #include "util/u_math.h"
9 #include "util/u_memory.h"
10 #include "util/crc32.h"
11
12 #include "svga_debug.h"
13 #include "svga_format.h"
14 #include "svga_winsys.h"
15 #include "svga_screen.h"
16 #include "svga_screen_cache.h"
17 #include "svga_context.h"
18 #include "svga_cmd.h"
19
20 #define SVGA_SURFACE_CACHE_ENABLED 1
21
22
23 /**
24 * Return the size of the surface described by the key (in bytes).
25 */
26 unsigned
svga_surface_size(const struct svga_host_surface_cache_key * key)27 svga_surface_size(const struct svga_host_surface_cache_key *key)
28 {
29 unsigned bw, bh, bpb, total_size, i;
30
31 assert(key->numMipLevels > 0);
32 assert(key->numFaces > 0);
33 assert(key->arraySize > 0);
34
35 if (key->format == SVGA3D_BUFFER) {
36 /* Special case: we don't want to count vertex/index buffers
37 * against the cache size limit, so view them as zero-sized.
38 */
39 return 0;
40 }
41
42 svga_format_size(key->format, &bw, &bh, &bpb);
43
44 total_size = 0;
45
46 for (i = 0; i < key->numMipLevels; i++) {
47 unsigned w = u_minify(key->size.width, i);
48 unsigned h = u_minify(key->size.height, i);
49 unsigned d = u_minify(key->size.depth, i);
50 unsigned img_size = ((w + bw - 1) / bw) * ((h + bh - 1) / bh) * d * bpb;
51 total_size += img_size;
52 }
53
54 total_size *= key->numFaces * key->arraySize * MAX2(1, key->sampleCount);
55
56 return total_size;
57 }
58
59
60 /**
61 * Compute the bucket for this key.
62 */
63 static inline unsigned
svga_screen_cache_bucket(const struct svga_host_surface_cache_key * key)64 svga_screen_cache_bucket(const struct svga_host_surface_cache_key *key)
65 {
66 return util_hash_crc32(key, sizeof *key) % SVGA_HOST_SURFACE_CACHE_BUCKETS;
67 }
68
69
70 /**
71 * Search the cache for a surface that matches the key. If a match is
72 * found, remove it from the cache and return the surface pointer.
73 * Return NULL otherwise.
74 */
75 static struct svga_winsys_surface *
svga_screen_cache_lookup(struct svga_screen * svgascreen,const struct svga_host_surface_cache_key * key)76 svga_screen_cache_lookup(struct svga_screen *svgascreen,
77 const struct svga_host_surface_cache_key *key)
78 {
79 struct svga_host_surface_cache *cache = &svgascreen->cache;
80 struct svga_winsys_screen *sws = svgascreen->sws;
81 struct svga_host_surface_cache_entry *entry;
82 struct svga_winsys_surface *handle = NULL;
83 struct list_head *curr, *next;
84 unsigned bucket;
85 unsigned tries = 0;
86
87 assert(key->cachable);
88
89 bucket = svga_screen_cache_bucket(key);
90
91 mtx_lock(&cache->mutex);
92
93 curr = cache->bucket[bucket].next;
94 next = curr->next;
95 while (curr != &cache->bucket[bucket]) {
96 ++tries;
97
98 entry = list_entry(curr, struct svga_host_surface_cache_entry, bucket_head);
99
100 assert(entry->handle);
101
102 /* If the key matches and the fence is signalled (the surface is no
103 * longer needed) the lookup was successful. We found a surface that
104 * can be reused.
105 * We unlink the surface from the cache entry and we add the entry to
106 * the 'empty' list.
107 */
108 if (memcmp(&entry->key, key, sizeof *key) == 0 &&
109 sws->fence_signalled(sws, entry->fence, 0) == 0) {
110 unsigned surf_size;
111
112 assert(sws->surface_is_flushed(sws, entry->handle));
113
114 handle = entry->handle; /* Reference is transfered here. */
115 entry->handle = NULL;
116
117 /* Remove from hash table */
118 list_del(&entry->bucket_head);
119
120 /* remove from LRU list */
121 list_del(&entry->head);
122
123 /* Add the cache entry (but not the surface!) to the empty list */
124 list_add(&entry->head, &cache->empty);
125
126 /* update the cache size */
127 surf_size = svga_surface_size(&entry->key);
128 assert(surf_size <= cache->total_size);
129 if (surf_size > cache->total_size)
130 cache->total_size = 0; /* should never happen, but be safe */
131 else
132 cache->total_size -= surf_size;
133
134 break;
135 }
136
137 curr = next;
138 next = curr->next;
139 }
140
141 mtx_unlock(&cache->mutex);
142
143 if (SVGA_DEBUG & DEBUG_DMA)
144 debug_printf("%s: cache %s after %u tries (bucket %d)\n", __func__,
145 handle ? "hit" : "miss", tries, bucket);
146
147 return handle;
148 }
149
150
151 /**
152 * Free the least recently used entries in the surface cache until the
153 * cache size is <= the target size OR there are no unused entries left
154 * to discard. We don't do any flushing to try to free up additional
155 * surfaces.
156 */
157 static void
svga_screen_cache_shrink(struct svga_screen * svgascreen,unsigned target_size)158 svga_screen_cache_shrink(struct svga_screen *svgascreen,
159 unsigned target_size)
160 {
161 struct svga_host_surface_cache *cache = &svgascreen->cache;
162 struct svga_winsys_screen *sws = svgascreen->sws;
163 struct svga_host_surface_cache_entry *entry = NULL, *next_entry;
164
165 /* Walk over the list of unused buffers in reverse order: from oldest
166 * to newest.
167 */
168 LIST_FOR_EACH_ENTRY_SAFE_REV(entry, next_entry, &cache->unused, head) {
169 if (entry->key.format != SVGA3D_BUFFER) {
170 /* we don't want to discard vertex/index buffers */
171
172 cache->total_size -= svga_surface_size(&entry->key);
173
174 assert(entry->handle);
175 sws->surface_reference(sws, &entry->handle, NULL);
176
177 list_del(&entry->bucket_head);
178 list_del(&entry->head);
179 list_add(&entry->head, &cache->empty);
180
181 if (cache->total_size <= target_size) {
182 /* all done */
183 break;
184 }
185 }
186 }
187 }
188
189
190 /**
191 * Add a surface to the cache. This is done when the driver deletes
192 * the surface. Note: transfers a handle reference.
193 */
194 static void
svga_screen_cache_add(struct svga_screen * svgascreen,const struct svga_host_surface_cache_key * key,bool to_invalidate,struct svga_winsys_surface ** p_handle)195 svga_screen_cache_add(struct svga_screen *svgascreen,
196 const struct svga_host_surface_cache_key *key,
197 bool to_invalidate,
198 struct svga_winsys_surface **p_handle)
199 {
200 struct svga_host_surface_cache *cache = &svgascreen->cache;
201 struct svga_winsys_screen *sws = svgascreen->sws;
202 struct svga_host_surface_cache_entry *entry = NULL;
203 struct svga_winsys_surface *handle = *p_handle;
204 unsigned surf_size;
205
206 assert(key->cachable);
207
208 if (!handle)
209 return;
210
211 surf_size = svga_surface_size(key);
212
213 *p_handle = NULL;
214 mtx_lock(&cache->mutex);
215
216 if (surf_size >= SVGA_HOST_SURFACE_CACHE_BYTES) {
217 /* this surface is too large to cache, just free it */
218 sws->surface_reference(sws, &handle, NULL);
219 mtx_unlock(&cache->mutex);
220 return;
221 }
222
223 if (cache->total_size + surf_size > SVGA_HOST_SURFACE_CACHE_BYTES) {
224 /* Adding this surface would exceed the cache size.
225 * Try to discard least recently used entries until we hit the
226 * new target cache size.
227 */
228 unsigned target_size = SVGA_HOST_SURFACE_CACHE_BYTES - surf_size;
229
230 svga_screen_cache_shrink(svgascreen, target_size);
231
232 if (cache->total_size > target_size) {
233 /* we weren't able to shrink the cache as much as we wanted so
234 * just discard this surface.
235 */
236 sws->surface_reference(sws, &handle, NULL);
237 mtx_unlock(&cache->mutex);
238 return;
239 }
240 }
241
242 if (!list_is_empty(&cache->empty)) {
243 /* An empty entry has no surface associated with it.
244 * Use the first empty entry.
245 */
246 entry = list_entry(cache->empty.next,
247 struct svga_host_surface_cache_entry,
248 head);
249
250 /* Remove from LRU list */
251 list_del(&entry->head);
252 }
253 else if (!list_is_empty(&cache->unused)) {
254 /* free the last used buffer and reuse its entry */
255 entry = list_entry(cache->unused.prev,
256 struct svga_host_surface_cache_entry,
257 head);
258 SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
259 "unref sid %p (make space)\n", entry->handle);
260
261 cache->total_size -= svga_surface_size(&entry->key);
262
263 sws->surface_reference(sws, &entry->handle, NULL);
264
265 /* Remove from hash table */
266 list_del(&entry->bucket_head);
267
268 /* Remove from LRU list */
269 list_del(&entry->head);
270 }
271
272 if (entry) {
273 assert(entry->handle == NULL);
274 entry->handle = handle;
275 memcpy(&entry->key, key, sizeof entry->key);
276
277 SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
278 "cache sid %p\n", entry->handle);
279
280 /* If we don't have gb objects, we don't need to invalidate. */
281 if (sws->have_gb_objects) {
282 if (to_invalidate)
283 list_add(&entry->head, &cache->validated);
284 else
285 list_add(&entry->head, &cache->invalidated);
286 }
287 else
288 list_add(&entry->head, &cache->invalidated);
289
290 cache->total_size += surf_size;
291 }
292 else {
293 /* Couldn't cache the buffer -- this really shouldn't happen */
294 SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
295 "unref sid %p (couldn't find space)\n", handle);
296 sws->surface_reference(sws, &handle, NULL);
297 }
298
299 mtx_unlock(&cache->mutex);
300 }
301
302
303 /* Maximum number of invalidate surface commands in a command buffer */
304 # define SVGA_MAX_SURFACE_TO_INVALIDATE 1000
305
306 /**
307 * Called during the screen flush to move all buffers not in a validate list
308 * into the unused list.
309 */
310 void
svga_screen_cache_flush(struct svga_screen * svgascreen,struct svga_context * svga,struct pipe_fence_handle * fence)311 svga_screen_cache_flush(struct svga_screen *svgascreen,
312 struct svga_context *svga,
313 struct pipe_fence_handle *fence)
314 {
315 struct svga_host_surface_cache *cache = &svgascreen->cache;
316 struct svga_winsys_screen *sws = svgascreen->sws;
317 struct svga_host_surface_cache_entry *entry;
318 struct list_head *curr, *next;
319 unsigned bucket;
320
321 mtx_lock(&cache->mutex);
322
323 /* Loop over entries in the invalidated list */
324 curr = cache->invalidated.next;
325 next = curr->next;
326 while (curr != &cache->invalidated) {
327 entry = list_entry(curr, struct svga_host_surface_cache_entry, head);
328
329 assert(entry->handle);
330
331 if (sws->surface_is_flushed(sws, entry->handle)) {
332 /* remove entry from the invalidated list */
333 list_del(&entry->head);
334
335 sws->fence_reference(sws, &entry->fence, fence);
336
337 /* Add entry to the unused list */
338 list_add(&entry->head, &cache->unused);
339
340 /* Add entry to the hash table bucket */
341 bucket = svga_screen_cache_bucket(&entry->key);
342 list_add(&entry->bucket_head, &cache->bucket[bucket]);
343 }
344
345 curr = next;
346 next = curr->next;
347 }
348
349 unsigned nsurf = 0;
350 curr = cache->validated.next;
351 next = curr->next;
352 while (curr != &cache->validated) {
353 entry = list_entry(curr, struct svga_host_surface_cache_entry, head);
354
355 assert(entry->handle);
356 assert(svga_have_gb_objects(svga));
357
358 if (sws->surface_is_flushed(sws, entry->handle)) {
359 /* remove entry from the validated list */
360 list_del(&entry->head);
361
362 /* It is now safe to invalidate the surface content.
363 * It will be done using the current context.
364 */
365 if (SVGA_TRY(SVGA3D_InvalidateGBSurface(svga->swc, entry->handle))
366 != PIPE_OK) {
367 ASSERTED enum pipe_error ret;
368
369 /* Even though surface invalidation here is done after the command
370 * buffer is flushed, it is still possible that it will
371 * fail because there might be just enough of this command that is
372 * filling up the command buffer, so in this case we will call
373 * the winsys flush directly to flush the buffer.
374 * Note, we don't want to call svga_context_flush() here because
375 * this function itself is called inside svga_context_flush().
376 */
377 svga_retry_enter(svga);
378 svga->swc->flush(svga->swc, NULL);
379 nsurf = 0;
380 ret = SVGA3D_InvalidateGBSurface(svga->swc, entry->handle);
381 svga_retry_exit(svga);
382 assert(ret == PIPE_OK);
383 }
384
385 /* add the entry to the invalidated list */
386
387 list_add(&entry->head, &cache->invalidated);
388 nsurf++;
389 }
390
391 curr = next;
392 next = curr->next;
393 }
394
395 mtx_unlock(&cache->mutex);
396
397 /**
398 * In some rare cases (when running ARK survival), we hit the max number
399 * of surface relocations with invalidated surfaces during context flush.
400 * So if the number of invalidated surface exceeds a certain limit (1000),
401 * we'll do another winsys flush.
402 */
403 if (nsurf > SVGA_MAX_SURFACE_TO_INVALIDATE) {
404 svga->swc->flush(svga->swc, NULL);
405 }
406 }
407
408
409 /**
410 * Free all the surfaces in the cache.
411 * Called when destroying the svga screen object.
412 */
413 void
svga_screen_cache_cleanup(struct svga_screen * svgascreen)414 svga_screen_cache_cleanup(struct svga_screen *svgascreen)
415 {
416 struct svga_host_surface_cache *cache = &svgascreen->cache;
417 struct svga_winsys_screen *sws = svgascreen->sws;
418 unsigned i;
419
420 for (i = 0; i < SVGA_HOST_SURFACE_CACHE_SIZE; ++i) {
421 if (cache->entries[i].handle) {
422 SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
423 "unref sid %p (shutdown)\n", cache->entries[i].handle);
424 sws->surface_reference(sws, &cache->entries[i].handle, NULL);
425
426 cache->total_size -= svga_surface_size(&cache->entries[i].key);
427 }
428
429 if (cache->entries[i].fence)
430 sws->fence_reference(sws, &cache->entries[i].fence, NULL);
431 }
432
433 mtx_destroy(&cache->mutex);
434 }
435
436
437 enum pipe_error
svga_screen_cache_init(struct svga_screen * svgascreen)438 svga_screen_cache_init(struct svga_screen *svgascreen)
439 {
440 struct svga_host_surface_cache *cache = &svgascreen->cache;
441 unsigned i;
442
443 assert(cache->total_size == 0);
444
445 (void) mtx_init(&cache->mutex, mtx_plain);
446
447 for (i = 0; i < SVGA_HOST_SURFACE_CACHE_BUCKETS; ++i)
448 list_inithead(&cache->bucket[i]);
449
450 list_inithead(&cache->unused);
451
452 list_inithead(&cache->validated);
453
454 list_inithead(&cache->invalidated);
455
456 list_inithead(&cache->empty);
457 for (i = 0; i < SVGA_HOST_SURFACE_CACHE_SIZE; ++i)
458 list_addtail(&cache->entries[i].head, &cache->empty);
459
460 return PIPE_OK;
461 }
462
463
464 /**
465 * Allocate a new host-side surface. If the surface is marked as cachable,
466 * first try re-using a surface in the cache of freed surfaces. Otherwise,
467 * allocate a new surface.
468 * \param bind_flags bitmask of PIPE_BIND_x flags
469 * \param usage one of PIPE_USAGE_x values
470 * \param validated return True if the surface is a reused surface
471 */
472 struct svga_winsys_surface *
svga_screen_surface_create(struct svga_screen * svgascreen,unsigned bind_flags,enum pipe_resource_usage usage,bool * validated,struct svga_host_surface_cache_key * key)473 svga_screen_surface_create(struct svga_screen *svgascreen,
474 unsigned bind_flags, enum pipe_resource_usage usage,
475 bool *validated,
476 struct svga_host_surface_cache_key *key)
477 {
478 struct svga_winsys_screen *sws = svgascreen->sws;
479 struct svga_winsys_surface *handle = NULL;
480 bool cachable = SVGA_SURFACE_CACHE_ENABLED && key->cachable;
481
482 SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
483 "%s sz %dx%dx%d mips %d faces %d arraySize %d cachable %d\n",
484 __func__,
485 key->size.width,
486 key->size.height,
487 key->size.depth,
488 key->numMipLevels,
489 key->numFaces,
490 key->arraySize,
491 key->cachable);
492
493 if (cachable) {
494 /* Try to re-cycle a previously freed, cached surface */
495 if (key->format == SVGA3D_BUFFER) {
496 SVGA3dSurfaceAllFlags hint_flag;
497
498 /* For buffers, round the buffer size up to the nearest power
499 * of two to increase the probability of cache hits. Keep
500 * texture surface dimensions unchanged.
501 */
502 uint32_t size = 1;
503 while (size < key->size.width)
504 size <<= 1;
505 key->size.width = size;
506
507 /* Determine whether the buffer is static or dynamic.
508 * This is a bit of a heuristic which can be tuned as needed.
509 */
510 if (usage == PIPE_USAGE_DEFAULT ||
511 usage == PIPE_USAGE_IMMUTABLE) {
512 hint_flag = SVGA3D_SURFACE_HINT_STATIC;
513 }
514 else if (bind_flags & PIPE_BIND_INDEX_BUFFER) {
515 /* Index buffers don't change too often. Mark them as static.
516 */
517 hint_flag = SVGA3D_SURFACE_HINT_STATIC;
518 }
519 else {
520 /* Since we're reusing buffers we're effectively transforming all
521 * of them into dynamic buffers.
522 *
523 * It would be nice to not cache long lived static buffers. But there
524 * is no way to detect the long lived from short lived ones yet. A
525 * good heuristic would be buffer size.
526 */
527 hint_flag = SVGA3D_SURFACE_HINT_DYNAMIC;
528 }
529
530 key->flags &= ~(SVGA3D_SURFACE_HINT_STATIC |
531 SVGA3D_SURFACE_HINT_DYNAMIC);
532 key->flags |= hint_flag;
533 }
534
535 handle = svga_screen_cache_lookup(svgascreen, key);
536 if (handle) {
537 if (key->format == SVGA3D_BUFFER)
538 SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
539 "reuse sid %p sz %d (buffer)\n", handle,
540 key->size.width);
541 else
542 SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
543 "reuse sid %p sz %dx%dx%d mips %d faces %d arraySize %d\n", handle,
544 key->size.width,
545 key->size.height,
546 key->size.depth,
547 key->numMipLevels,
548 key->numFaces,
549 key->arraySize);
550 *validated = true;
551 }
552 }
553
554 if (!handle) {
555 /* Unable to recycle surface, allocate a new one */
556 unsigned usage = 0;
557
558 /* mark the surface as shareable if the surface is not
559 * cachable or the RENDER_TARGET bind flag is set.
560 */
561 if (!key->cachable ||
562 ((bind_flags & PIPE_BIND_RENDER_TARGET) != 0))
563 usage |= SVGA_SURFACE_USAGE_SHARED;
564 if (key->scanout)
565 usage |= SVGA_SURFACE_USAGE_SCANOUT;
566 if (key->coherent)
567 usage |= SVGA_SURFACE_USAGE_COHERENT;
568
569 handle = sws->surface_create(sws,
570 key->flags,
571 key->format,
572 usage,
573 key->size,
574 key->numFaces * key->arraySize,
575 key->numMipLevels,
576 key->sampleCount);
577 if (handle)
578 SVGA_DBG(DEBUG_CACHE|DEBUG_DMA,
579 " CREATE sid %p sz %dx%dx%d\n",
580 handle,
581 key->size.width,
582 key->size.height,
583 key->size.depth);
584
585 *validated = false;
586 }
587
588 return handle;
589 }
590
591
592 /**
593 * Release a surface. We don't actually free the surface- we put
594 * it into the cache of freed surfaces (if it's cachable).
595 */
596 void
svga_screen_surface_destroy(struct svga_screen * svgascreen,const struct svga_host_surface_cache_key * key,bool to_invalidate,struct svga_winsys_surface ** p_handle)597 svga_screen_surface_destroy(struct svga_screen *svgascreen,
598 const struct svga_host_surface_cache_key *key,
599 bool to_invalidate,
600 struct svga_winsys_surface **p_handle)
601 {
602 struct svga_winsys_screen *sws = svgascreen->sws;
603
604 /* We only set the cachable flag for surfaces of which we are the
605 * exclusive owner. So just hold onto our existing reference in
606 * that case.
607 */
608 if (SVGA_SURFACE_CACHE_ENABLED && key->cachable) {
609 svga_screen_cache_add(svgascreen, key, to_invalidate, p_handle);
610 }
611 else {
612 SVGA_DBG(DEBUG_DMA,
613 "unref sid %p (uncachable)\n", *p_handle);
614 sws->surface_reference(sws, p_handle, NULL);
615 }
616 }
617
618
619 /**
620 * Print/dump the contents of the screen cache. For debugging.
621 */
622 void
svga_screen_cache_dump(const struct svga_screen * svgascreen)623 svga_screen_cache_dump(const struct svga_screen *svgascreen)
624 {
625 const struct svga_host_surface_cache *cache = &svgascreen->cache;
626 unsigned bucket;
627 unsigned count = 0;
628
629 debug_printf("svga3d surface cache:\n");
630 for (bucket = 0; bucket < SVGA_HOST_SURFACE_CACHE_BUCKETS; bucket++) {
631 struct list_head *curr;
632 curr = cache->bucket[bucket].next;
633 while (curr && curr != &cache->bucket[bucket]) {
634 struct svga_host_surface_cache_entry *entry =
635 list_entry(curr, struct svga_host_surface_cache_entry,bucket_head);
636 if (entry->key.format == SVGA3D_BUFFER) {
637 debug_printf(" %p: buffer %u bytes\n",
638 entry->handle,
639 entry->key.size.width);
640 }
641 else {
642 debug_printf(" %p: %u x %u x %u format %u\n",
643 entry->handle,
644 entry->key.size.width,
645 entry->key.size.height,
646 entry->key.size.depth,
647 entry->key.format);
648 }
649 curr = curr->next;
650 count++;
651 }
652 }
653
654 debug_printf("%u surfaces, %u bytes\n", count, cache->total_size);
655 }
656