1 /*
2 * Copyright © 2020, VideoLAN and dav1d authors
3 * Copyright © 2020, Two Orioles, LLC
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice, this
10 * list of conditions and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice,
13 * this list of conditions and the following disclaimer in the documentation
14 * and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 #include "config.h"
29
30 #include <stdint.h>
31
32 #include "src/internal.h"
33
34 #if TRACK_HEAP_ALLOCATIONS
35 #include <stdio.h>
36
37 #include "src/log.h"
38
39 #define DEFAULT_ALIGN 16
40
41 typedef struct {
42 size_t sz;
43 unsigned align;
44 enum AllocationType type;
45 } Dav1dAllocationData;
46
47 typedef struct {
48 size_t curr_sz;
49 size_t peak_sz;
50 unsigned num_allocs;
51 unsigned num_reuses;
52 } AllocStats;
53
54 static AllocStats tracked_allocs[N_ALLOC_TYPES];
55 static size_t curr_total_sz;
56 static size_t peak_total_sz;
57 static pthread_mutex_t track_alloc_mutex = PTHREAD_MUTEX_INITIALIZER;
58
track_alloc(const enum AllocationType type,char * ptr,const size_t sz,const size_t align)59 static void *track_alloc(const enum AllocationType type, char *ptr,
60 const size_t sz, const size_t align)
61 {
62 assert(align >= sizeof(Dav1dAllocationData));
63 if (ptr) {
64 ptr += align;
65 Dav1dAllocationData *const d = &((Dav1dAllocationData*)ptr)[-1];
66 AllocStats *const s = &tracked_allocs[type];
67
68 d->sz = sz;
69 d->align = (unsigned)align;
70 d->type = type;
71
72 pthread_mutex_lock(&track_alloc_mutex);
73 s->num_allocs++;
74 s->curr_sz += sz;
75 if (s->curr_sz > s->peak_sz)
76 s->peak_sz = s->curr_sz;
77
78 curr_total_sz += sz;
79 if (curr_total_sz > peak_total_sz)
80 peak_total_sz = curr_total_sz;
81 pthread_mutex_unlock(&track_alloc_mutex);
82 }
83 return ptr;
84 }
85
track_free(char * const ptr)86 static void *track_free(char *const ptr) {
87 const Dav1dAllocationData *const d = &((Dav1dAllocationData*)ptr)[-1];
88 const size_t sz = d->sz;
89
90 pthread_mutex_lock(&track_alloc_mutex);
91 tracked_allocs[d->type].curr_sz -= sz;
92 curr_total_sz -= sz;
93 pthread_mutex_unlock(&track_alloc_mutex);
94
95 return ptr - d->align;
96 }
97
dav1d_track_reuse(const enum AllocationType type)98 static void dav1d_track_reuse(const enum AllocationType type) {
99 pthread_mutex_lock(&track_alloc_mutex);
100 tracked_allocs[type].num_reuses++;
101 pthread_mutex_unlock(&track_alloc_mutex);
102 }
103
dav1d_malloc(const enum AllocationType type,const size_t sz)104 void *dav1d_malloc(const enum AllocationType type, const size_t sz) {
105 void *const ptr = malloc(sz + DEFAULT_ALIGN);
106 return track_alloc(type, ptr, sz, DEFAULT_ALIGN);
107 }
108
dav1d_alloc_aligned(const enum AllocationType type,const size_t sz,const size_t align)109 void *dav1d_alloc_aligned(const enum AllocationType type,
110 const size_t sz, const size_t align)
111 {
112 void *const ptr = dav1d_alloc_aligned_internal(align, sz + align);
113 return track_alloc(type, ptr, sz, align);
114 }
115
dav1d_realloc(const enum AllocationType type,void * ptr,const size_t sz)116 void *dav1d_realloc(const enum AllocationType type,
117 void *ptr, const size_t sz)
118 {
119 if (!ptr)
120 return dav1d_malloc(type, sz);
121 ptr = realloc((char*)ptr - DEFAULT_ALIGN, sz + DEFAULT_ALIGN);
122 if (ptr)
123 ptr = track_free((char*)ptr + DEFAULT_ALIGN);
124 return track_alloc(type, ptr, sz, DEFAULT_ALIGN);
125 }
126
dav1d_free(void * ptr)127 void dav1d_free(void *ptr) {
128 if (ptr)
129 free(track_free(ptr));
130 }
131
dav1d_free_aligned(void * ptr)132 void dav1d_free_aligned(void *ptr) {
133 if (ptr) {
134 dav1d_free_aligned_internal(track_free(ptr));
135 }
136 }
137
cmp_stats(const void * const a,const void * const b)138 static COLD int cmp_stats(const void *const a, const void *const b) {
139 const size_t a_sz = ((const AllocStats*)a)->peak_sz;
140 const size_t b_sz = ((const AllocStats*)b)->peak_sz;
141 return a_sz < b_sz ? -1 : a_sz > b_sz;
142 }
143
144 /* Insert spaces as thousands separators for better readability */
format_tsep(char * const s,const size_t n,const size_t value)145 static COLD int format_tsep(char *const s, const size_t n, const size_t value) {
146 if (value < 1000)
147 return snprintf(s, n, "%u", (unsigned)value);
148
149 const int len = format_tsep(s, n, value / 1000);
150 assert((size_t)len < n);
151 return len + snprintf(s + len, n - len, " %03u", (unsigned)(value % 1000));
152 }
153
dav1d_log_alloc_stats(Dav1dContext * const c)154 COLD void dav1d_log_alloc_stats(Dav1dContext *const c) {
155 static const char *const type_names[N_ALLOC_TYPES] = {
156 [ALLOC_BLOCK ] = "Block data",
157 [ALLOC_CDEF ] = "CDEF line buffers",
158 [ALLOC_CDF ] = "CDF contexts",
159 [ALLOC_COEF ] = "Coefficient data",
160 [ALLOC_COMMON_CTX] = "Common context data",
161 [ALLOC_DAV1DDATA ] = "Dav1dData",
162 [ALLOC_IPRED ] = "Intra pred edges",
163 [ALLOC_LF ] = "Loopfilter data",
164 [ALLOC_LR ] = "Looprestoration data",
165 [ALLOC_OBU_HDR ] = "OBU headers",
166 [ALLOC_OBU_META ] = "OBU metadata",
167 [ALLOC_PAL ] = "Palette data",
168 [ALLOC_PIC ] = "Picture buffers",
169 [ALLOC_PIC_CTX ] = "Picture context data",
170 [ALLOC_REFMVS ] = "Reference mv data",
171 [ALLOC_SEGMAP ] = "Segmentation maps",
172 [ALLOC_THREAD_CTX] = "Thread context data",
173 [ALLOC_TILE ] = "Tile data",
174 };
175
176 struct {
177 AllocStats stats;
178 enum AllocationType type;
179 } data[N_ALLOC_TYPES];
180 unsigned total_allocs = 0;
181 unsigned total_reuses = 0;
182
183 pthread_mutex_lock(&track_alloc_mutex);
184 for (int i = 0; i < N_ALLOC_TYPES; i++) {
185 AllocStats *const s = &data[i].stats;
186 *s = tracked_allocs[i];
187 data[i].type = i;
188 total_allocs += s->num_allocs;
189 total_reuses += s->num_reuses;
190 }
191 size_t total_sz = peak_total_sz;
192 pthread_mutex_unlock(&track_alloc_mutex);
193
194 /* Sort types by memory usage */
195 qsort(&data, N_ALLOC_TYPES, sizeof(*data), cmp_stats);
196
197 const double inv_total_share = 100.0 / total_sz;
198 char total_sz_buf[32];
199 const int sz_len = 4 + format_tsep(total_sz_buf, sizeof(total_sz_buf), total_sz);
200
201 dav1d_log(c, "\n Type Allocs Reuses Share Peak size\n"
202 "---------------------------------------------------------------------\n");
203 for (int i = N_ALLOC_TYPES - 1; i >= 0; i--) {
204 const AllocStats *const s = &data[i].stats;
205 if (s->num_allocs) {
206 const double share = s->peak_sz * inv_total_share;
207 char sz_buf[32];
208 format_tsep(sz_buf, sizeof(sz_buf), s->peak_sz);
209 dav1d_log(c, " %-20s%10u%10u%8.1f%%%*s\n", type_names[data[i].type],
210 s->num_allocs, s->num_reuses, share, sz_len, sz_buf);
211 }
212 }
213 dav1d_log(c, "---------------------------------------------------------------------\n"
214 "%31u%10u %s\n",
215 total_allocs, total_reuses, total_sz_buf);
216 }
217 #endif /* TRACK_HEAP_ALLOCATIONS */
218
mem_pool_destroy(Dav1dMemPool * const pool)219 static COLD void mem_pool_destroy(Dav1dMemPool *const pool) {
220 pthread_mutex_destroy(&pool->lock);
221 dav1d_free(pool);
222 }
223
dav1d_mem_pool_push(Dav1dMemPool * const pool,Dav1dMemPoolBuffer * const buf)224 void dav1d_mem_pool_push(Dav1dMemPool *const pool, Dav1dMemPoolBuffer *const buf) {
225 pthread_mutex_lock(&pool->lock);
226 const int ref_cnt = --pool->ref_cnt;
227 if (!pool->end) {
228 buf->next = pool->buf;
229 pool->buf = buf;
230 pthread_mutex_unlock(&pool->lock);
231 assert(ref_cnt > 0);
232 } else {
233 pthread_mutex_unlock(&pool->lock);
234 dav1d_free_aligned(buf->data);
235 if (!ref_cnt) mem_pool_destroy(pool);
236 }
237 }
238
dav1d_mem_pool_pop(Dav1dMemPool * const pool,const size_t size)239 Dav1dMemPoolBuffer *dav1d_mem_pool_pop(Dav1dMemPool *const pool, const size_t size) {
240 assert(!(size & (sizeof(void*) - 1)));
241 pthread_mutex_lock(&pool->lock);
242 Dav1dMemPoolBuffer *buf = pool->buf;
243 pool->ref_cnt++;
244 uint8_t *data;
245 if (buf) {
246 pool->buf = buf->next;
247 pthread_mutex_unlock(&pool->lock);
248 data = buf->data;
249 if ((uintptr_t)buf - (uintptr_t)data != size) {
250 /* Reallocate if the size has changed */
251 dav1d_free_aligned(data);
252 goto alloc;
253 }
254 #if TRACK_HEAP_ALLOCATIONS
255 dav1d_track_reuse(pool->type);
256 #endif
257 } else {
258 pthread_mutex_unlock(&pool->lock);
259 alloc:
260 data = dav1d_alloc_aligned(pool->type,
261 size + sizeof(Dav1dMemPoolBuffer), 64);
262 if (!data) {
263 pthread_mutex_lock(&pool->lock);
264 const int ref_cnt = --pool->ref_cnt;
265 pthread_mutex_unlock(&pool->lock);
266 if (!ref_cnt) mem_pool_destroy(pool);
267 return NULL;
268 }
269 buf = (Dav1dMemPoolBuffer*)(data + size);
270 buf->data = data;
271 }
272
273 return buf;
274 }
275
dav1d_mem_pool_init(const enum AllocationType type,Dav1dMemPool ** const ppool)276 COLD int dav1d_mem_pool_init(const enum AllocationType type,
277 Dav1dMemPool **const ppool)
278 {
279 Dav1dMemPool *const pool = dav1d_malloc(ALLOC_COMMON_CTX,
280 sizeof(Dav1dMemPool));
281 if (pool) {
282 if (!pthread_mutex_init(&pool->lock, NULL)) {
283 pool->buf = NULL;
284 pool->ref_cnt = 1;
285 pool->end = 0;
286 #if TRACK_HEAP_ALLOCATIONS
287 pool->type = type;
288 #endif
289 *ppool = pool;
290 return 0;
291 }
292 dav1d_free(pool);
293 }
294 *ppool = NULL;
295 return DAV1D_ERR(ENOMEM);
296 }
297
dav1d_mem_pool_end(Dav1dMemPool * const pool)298 COLD void dav1d_mem_pool_end(Dav1dMemPool *const pool) {
299 if (pool) {
300 pthread_mutex_lock(&pool->lock);
301 Dav1dMemPoolBuffer *buf = pool->buf;
302 const int ref_cnt = --pool->ref_cnt;
303 pool->buf = NULL;
304 pool->end = 1;
305 pthread_mutex_unlock(&pool->lock);
306
307 while (buf) {
308 void *const data = buf->data;
309 buf = buf->next;
310 dav1d_free_aligned(data);
311 }
312 if (!ref_cnt) mem_pool_destroy(pool);
313 }
314 }
315