1 /*
2 * Copyright 2021 Alyssa Rosenzweig
3 * Copyright 2019 Collabora, Ltd.
4 * SPDX-License-Identifier: MIT
5 */
6
7 #include "agx_bo.h"
8 #include <inttypes.h>
9 #include "agx_device.h"
10 #include "decode.h"
11
12 /* Helper to calculate the bucket index of a BO */
13 static unsigned
agx_bucket_index(unsigned size)14 agx_bucket_index(unsigned size)
15 {
16 /* Round down to POT to compute a bucket index */
17 unsigned bucket_index = util_logbase2(size);
18
19 /* Clamp to supported buckets. Huge allocations use the largest bucket */
20 bucket_index = CLAMP(bucket_index, MIN_BO_CACHE_BUCKET, MAX_BO_CACHE_BUCKET);
21
22 /* Reindex from 0 */
23 return (bucket_index - MIN_BO_CACHE_BUCKET);
24 }
25
26 static struct list_head *
agx_bucket(struct agx_device * dev,unsigned size)27 agx_bucket(struct agx_device *dev, unsigned size)
28 {
29 return &dev->bo_cache.buckets[agx_bucket_index(size)];
30 }
31
32 static void
agx_bo_cache_remove_locked(struct agx_device * dev,struct agx_bo * bo)33 agx_bo_cache_remove_locked(struct agx_device *dev, struct agx_bo *bo)
34 {
35 simple_mtx_assert_locked(&dev->bo_cache.lock);
36 list_del(&bo->bucket_link);
37 list_del(&bo->lru_link);
38 dev->bo_cache.size -= bo->size;
39 }
40
41 /* Tries to fetch a BO of sufficient size with the appropriate flags from the
42 * BO cache. If it succeeds, it returns that BO and removes the BO from the
43 * cache. If it fails, it returns NULL signaling the caller to allocate a new
44 * BO. */
45
46 struct agx_bo *
agx_bo_cache_fetch(struct agx_device * dev,size_t size,size_t align,uint32_t flags,const bool dontwait)47 agx_bo_cache_fetch(struct agx_device *dev, size_t size, size_t align,
48 uint32_t flags, const bool dontwait)
49 {
50 simple_mtx_lock(&dev->bo_cache.lock);
51 struct list_head *bucket = agx_bucket(dev, size);
52 struct agx_bo *bo = NULL;
53
54 /* Iterate the bucket looking for something suitable */
55 list_for_each_entry_safe(struct agx_bo, entry, bucket, bucket_link) {
56 if (entry->size < size || entry->flags != flags)
57 continue;
58
59 /* Do not return more than 2x oversized BOs. */
60 if (entry->size > 2 * size)
61 continue;
62
63 if (align > entry->align)
64 continue;
65
66 /* This one works, use it */
67 agx_bo_cache_remove_locked(dev, entry);
68 bo = entry;
69 break;
70 }
71 simple_mtx_unlock(&dev->bo_cache.lock);
72
73 return bo;
74 }
75
76 static void
agx_bo_cache_evict_stale_bos(struct agx_device * dev,unsigned tv_sec)77 agx_bo_cache_evict_stale_bos(struct agx_device *dev, unsigned tv_sec)
78 {
79 struct timespec time;
80
81 clock_gettime(CLOCK_MONOTONIC, &time);
82 list_for_each_entry_safe(struct agx_bo, entry, &dev->bo_cache.lru,
83 lru_link) {
84 /* We want all entries that have been used more than 1 sec ago to be
85 * dropped, others can be kept. Note the <= 2 check and not <= 1. It's
86 * here to account for the fact that we're only testing ->tv_sec, not
87 * ->tv_nsec. That means we might keep entries that are between 1 and 2
88 * seconds old, but we don't really care, as long as unused BOs are
89 * dropped at some point.
90 */
91 if (time.tv_sec - entry->last_used <= 2)
92 break;
93
94 agx_bo_cache_remove_locked(dev, entry);
95 agx_bo_free(dev, entry);
96 }
97 }
98
99 static void
agx_bo_cache_put_locked(struct agx_device * dev,struct agx_bo * bo)100 agx_bo_cache_put_locked(struct agx_device *dev, struct agx_bo *bo)
101 {
102 struct list_head *bucket = agx_bucket(dev, bo->size);
103 struct timespec time;
104
105 /* Add us to the bucket */
106 list_addtail(&bo->bucket_link, bucket);
107
108 /* Add us to the LRU list and update the last_used field. */
109 list_addtail(&bo->lru_link, &dev->bo_cache.lru);
110 clock_gettime(CLOCK_MONOTONIC, &time);
111 bo->last_used = time.tv_sec;
112
113 /* Update statistics */
114 dev->bo_cache.size += bo->size;
115
116 if (0) {
117 printf("BO cache: %zu KiB (+%zu KiB from %s, hit/miss %" PRIu64
118 "/%" PRIu64 ")\n",
119 DIV_ROUND_UP(dev->bo_cache.size, 1024),
120 DIV_ROUND_UP(bo->size, 1024), bo->label,
121 p_atomic_read(&dev->bo_cache.hits),
122 p_atomic_read(&dev->bo_cache.misses));
123 }
124
125 /* Update label for debug */
126 bo->label = "Unused (BO cache)";
127
128 /* Let's do some cleanup in the BO cache while we hold the lock. */
129 agx_bo_cache_evict_stale_bos(dev, time.tv_sec);
130 }
131
132 /* Tries to add a BO to the cache. Returns if it was successful */
133 static bool
agx_bo_cache_put(struct agx_device * dev,struct agx_bo * bo)134 agx_bo_cache_put(struct agx_device *dev, struct agx_bo *bo)
135 {
136 if (bo->flags & AGX_BO_SHARED) {
137 return false;
138 } else {
139 simple_mtx_lock(&dev->bo_cache.lock);
140 agx_bo_cache_put_locked(dev, bo);
141 simple_mtx_unlock(&dev->bo_cache.lock);
142
143 return true;
144 }
145 }
146
147 void
agx_bo_cache_evict_all(struct agx_device * dev)148 agx_bo_cache_evict_all(struct agx_device *dev)
149 {
150 simple_mtx_lock(&dev->bo_cache.lock);
151 for (unsigned i = 0; i < ARRAY_SIZE(dev->bo_cache.buckets); ++i) {
152 struct list_head *bucket = &dev->bo_cache.buckets[i];
153
154 list_for_each_entry_safe(struct agx_bo, entry, bucket, bucket_link) {
155 agx_bo_cache_remove_locked(dev, entry);
156 agx_bo_free(dev, entry);
157 }
158 }
159 simple_mtx_unlock(&dev->bo_cache.lock);
160 }
161
162 void
agx_bo_reference(struct agx_bo * bo)163 agx_bo_reference(struct agx_bo *bo)
164 {
165 if (bo) {
166 ASSERTED int count = p_atomic_inc_return(&bo->refcnt);
167 assert(count != 1);
168 }
169 }
170
171 void
agx_bo_unreference(struct agx_device * dev,struct agx_bo * bo)172 agx_bo_unreference(struct agx_device *dev, struct agx_bo *bo)
173 {
174 if (!bo)
175 return;
176
177 /* Don't return to cache if there are still references */
178 if (p_atomic_dec_return(&bo->refcnt))
179 return;
180
181 pthread_mutex_lock(&dev->bo_map_lock);
182
183 /* Someone might have imported this BO while we were waiting for the
184 * lock, let's make sure it's still not referenced before freeing it.
185 */
186 if (p_atomic_read(&bo->refcnt) == 0) {
187 assert(!p_atomic_read_relaxed(&bo->writer));
188
189 if (dev->debug & AGX_DBG_TRACE)
190 agxdecode_track_free(dev->agxdecode, bo);
191
192 if (!agx_bo_cache_put(dev, bo))
193 agx_bo_free(dev, bo);
194 }
195
196 pthread_mutex_unlock(&dev->bo_map_lock);
197 }
198
199 struct agx_bo *
agx_bo_create(struct agx_device * dev,unsigned size,unsigned align,enum agx_bo_flags flags,const char * label)200 agx_bo_create(struct agx_device *dev, unsigned size, unsigned align,
201 enum agx_bo_flags flags, const char *label)
202 {
203 struct agx_bo *bo;
204 assert(size > 0);
205
206 /* To maximize BO cache usage, don't allocate tiny BOs */
207 size = ALIGN_POT(size, 16384);
208
209 /* See if we have a BO already in the cache */
210 bo = agx_bo_cache_fetch(dev, size, align, flags, true);
211
212 /* Update stats based on the first attempt to fetch */
213 if (bo != NULL)
214 p_atomic_inc(&dev->bo_cache.hits);
215 else
216 p_atomic_inc(&dev->bo_cache.misses);
217
218 /* Otherwise, allocate a fresh BO. If allocation fails, we can try waiting
219 * for something in the cache. But if there's no nothing suitable, we should
220 * flush the cache to make space for the new allocation.
221 */
222 if (!bo)
223 bo = dev->ops.bo_alloc(dev, size, align, flags);
224 if (!bo)
225 bo = agx_bo_cache_fetch(dev, size, align, flags, false);
226 if (!bo) {
227 agx_bo_cache_evict_all(dev);
228 bo = dev->ops.bo_alloc(dev, size, align, flags);
229 }
230
231 if (!bo) {
232 fprintf(stderr, "BO creation failed\n");
233 return NULL;
234 }
235
236 bo->label = label;
237 p_atomic_set(&bo->refcnt, 1);
238
239 if (dev->debug & AGX_DBG_TRACE)
240 agxdecode_track_alloc(dev->agxdecode, bo);
241
242 return bo;
243 }
244