1 /*
2 * Copyright 2023 Rose Hudson
3 * Copyright 2022 Amazon.com, Inc. or its affiliates.
4 * Copyright 2018 Intel Corporation
5 * SPDX-License-Identifier: MIT
6 */
7
8 #include <assert.h>
9 #include <stdio.h>
10
11 #include "asahi/compiler/agx_debug.h"
12 #include "compiler/shader_enums.h"
13 #include "util/blob.h"
14 #include "util/build_id.h"
15 #include "util/disk_cache.h"
16 #include "util/mesa-sha1.h"
17 #include "agx_bo.h"
18 #include "agx_device.h"
19 #include "agx_disk_cache.h"
20 #include "agx_state.h"
21
22 /* Flags that are allowed and do not disable the disk cache */
23 #define ALLOWED_FLAGS (AGX_DBG_NO16)
24
25 /**
26 * Compute a disk cache key for the given uncompiled shader and shader key.
27 */
28 static void
agx_disk_cache_compute_key(struct disk_cache * cache,const struct agx_uncompiled_shader * uncompiled,const union asahi_shader_key * shader_key,cache_key cache_key)29 agx_disk_cache_compute_key(struct disk_cache *cache,
30 const struct agx_uncompiled_shader *uncompiled,
31 const union asahi_shader_key *shader_key,
32 cache_key cache_key)
33 {
34 uint8_t data[sizeof(uncompiled->nir_sha1) + sizeof(*shader_key)];
35 int hash_size = sizeof(uncompiled->nir_sha1);
36 int key_size;
37 if (uncompiled->type == PIPE_SHADER_VERTEX ||
38 uncompiled->type == PIPE_SHADER_TESS_EVAL)
39 key_size = sizeof(shader_key->vs);
40 else if (uncompiled->type == PIPE_SHADER_GEOMETRY)
41 key_size = sizeof(shader_key->gs);
42 else if (uncompiled->type == PIPE_SHADER_FRAGMENT)
43 key_size = sizeof(shader_key->fs);
44 else if (uncompiled->type == PIPE_SHADER_COMPUTE ||
45 uncompiled->type == PIPE_SHADER_TESS_CTRL)
46 key_size = 0;
47 else
48 unreachable("Unsupported shader stage");
49
50 memcpy(data, uncompiled->nir_sha1, hash_size);
51
52 if (key_size)
53 memcpy(data + hash_size, shader_key, key_size);
54
55 disk_cache_compute_key(cache, data, hash_size + key_size, cache_key);
56 }
57
58 static void
write_shader(struct blob * blob,const struct agx_compiled_shader * binary,bool is_root_gs)59 write_shader(struct blob *blob, const struct agx_compiled_shader *binary,
60 bool is_root_gs)
61 {
62 blob_write_uint32(blob, binary->b.binary_size);
63
64 if (binary->b.binary_size) {
65 blob_write_bytes(blob, binary->b.binary, binary->b.binary_size);
66 }
67
68 blob_write_bytes(blob, &binary->b.info, sizeof(binary->b.info));
69 blob_write_bytes(blob, &binary->uvs, sizeof(binary->uvs));
70 blob_write_bytes(blob, &binary->attrib_components_read,
71 sizeof(binary->attrib_components_read));
72 blob_write_bytes(blob, &binary->epilog_key, sizeof(binary->epilog_key));
73 blob_write_uint32(blob, binary->push_range_count);
74 blob_write_bytes(blob, binary->push,
75 sizeof(binary->push[0]) * binary->push_range_count);
76
77 if (is_root_gs) {
78 blob_write_uint32(blob, binary->gs_count_words);
79 blob_write_uint32(blob, binary->gs_output_mode);
80 write_shader(blob, binary->pre_gs, false);
81
82 blob_write_uint8(blob, binary->gs_copy != NULL);
83 if (binary->gs_copy)
84 write_shader(blob, binary->gs_copy, false);
85
86 blob_write_uint8(blob, binary->gs_count != NULL);
87 if (binary->gs_count)
88 write_shader(blob, binary->gs_count, false);
89 }
90 }
91
92 static struct agx_compiled_shader *
read_shader(struct agx_screen * screen,struct blob_reader * blob,const struct agx_uncompiled_shader * uncompiled,bool is_root)93 read_shader(struct agx_screen *screen, struct blob_reader *blob,
94 const struct agx_uncompiled_shader *uncompiled, bool is_root)
95 {
96 struct agx_compiled_shader *binary = CALLOC_STRUCT(agx_compiled_shader);
97 binary->stage = uncompiled->type;
98 binary->so = uncompiled;
99
100 size_t size = blob_read_uint32(blob);
101
102 if (uncompiled->type == PIPE_SHADER_VERTEX ||
103 uncompiled->type == PIPE_SHADER_TESS_EVAL ||
104 uncompiled->type == PIPE_SHADER_FRAGMENT) {
105 binary->b.binary_size = size;
106 binary->b.binary = malloc(binary->b.binary_size);
107 blob_copy_bytes(blob, binary->b.binary, binary->b.binary_size);
108
109 if (size) {
110 binary->bo = agx_bo_create(&screen->dev, size, 0,
111 AGX_BO_EXEC | AGX_BO_LOW_VA, "Executable");
112 memcpy(binary->bo->map, binary->b.binary, size);
113 }
114 } else if (size) {
115 binary->bo = agx_bo_create(&screen->dev, size, 0,
116 AGX_BO_EXEC | AGX_BO_LOW_VA, "Executable");
117 blob_copy_bytes(blob, binary->bo->map, size);
118 }
119
120 blob_copy_bytes(blob, &binary->b.info, sizeof(binary->b.info));
121 blob_copy_bytes(blob, &binary->uvs, sizeof(binary->uvs));
122 blob_copy_bytes(blob, &binary->attrib_components_read,
123 sizeof(binary->attrib_components_read));
124 blob_copy_bytes(blob, &binary->epilog_key, sizeof(binary->epilog_key));
125 binary->push_range_count = blob_read_uint32(blob);
126 blob_copy_bytes(blob, binary->push,
127 sizeof(binary->push[0]) * binary->push_range_count);
128
129 if (is_root && uncompiled->type == PIPE_SHADER_GEOMETRY) {
130 binary->gs_count_words = blob_read_uint32(blob);
131 binary->gs_output_mode = blob_read_uint32(blob);
132 binary->pre_gs = read_shader(screen, blob, uncompiled, false);
133
134 if (blob_read_uint8(blob))
135 binary->gs_copy = read_shader(screen, blob, uncompiled, false);
136
137 if (blob_read_uint8(blob))
138 binary->gs_count = read_shader(screen, blob, uncompiled, false);
139 }
140
141 return binary;
142 }
143
144 /**
145 * Store the given compiled shader in the disk cache.
146 *
147 * This should only be called on newly compiled shaders. No checking is
148 * done to prevent repeated stores of the same shader.
149 */
150 void
agx_disk_cache_store(struct disk_cache * cache,const struct agx_uncompiled_shader * uncompiled,const union asahi_shader_key * key,const struct agx_compiled_shader * binary)151 agx_disk_cache_store(struct disk_cache *cache,
152 const struct agx_uncompiled_shader *uncompiled,
153 const union asahi_shader_key *key,
154 const struct agx_compiled_shader *binary)
155 {
156 #ifdef ENABLE_SHADER_CACHE
157 if (!cache)
158 return;
159
160 cache_key cache_key;
161 agx_disk_cache_compute_key(cache, uncompiled, key, cache_key);
162
163 struct blob blob;
164 blob_init(&blob);
165
166 write_shader(&blob, binary, uncompiled->type == PIPE_SHADER_GEOMETRY);
167
168 disk_cache_put(cache, cache_key, blob.data, blob.size, NULL);
169 blob_finish(&blob);
170 #endif
171 }
172
173 /**
174 * Search for a compiled shader in the disk cache.
175 */
176 struct agx_compiled_shader *
agx_disk_cache_retrieve(struct agx_screen * screen,const struct agx_uncompiled_shader * uncompiled,const union asahi_shader_key * key)177 agx_disk_cache_retrieve(struct agx_screen *screen,
178 const struct agx_uncompiled_shader *uncompiled,
179 const union asahi_shader_key *key)
180 {
181 #ifdef ENABLE_SHADER_CACHE
182 struct disk_cache *cache = screen->disk_cache;
183 if (!cache)
184 return NULL;
185
186 cache_key cache_key;
187 agx_disk_cache_compute_key(cache, uncompiled, key, cache_key);
188
189 size_t size;
190 void *buffer = disk_cache_get(cache, cache_key, &size);
191 if (!buffer)
192 return NULL;
193
194 struct blob_reader blob;
195 blob_reader_init(&blob, buffer, size);
196
197 struct agx_compiled_shader *binary =
198 read_shader(screen, &blob, uncompiled, true);
199
200 free(buffer);
201 return binary;
202 #else
203 return NULL;
204 #endif
205 }
206
207 /**
208 * Initialise the on-disk shader cache.
209 */
210 void
agx_disk_cache_init(struct agx_screen * screen)211 agx_disk_cache_init(struct agx_screen *screen)
212 {
213 #ifdef ENABLE_SHADER_CACHE
214 if (agx_get_compiler_debug() || (screen->dev.debug & ~ALLOWED_FLAGS))
215 return;
216
217 const char *renderer = screen->pscreen.get_name(&screen->pscreen);
218
219 const struct build_id_note *note =
220 build_id_find_nhdr_for_addr(agx_disk_cache_init);
221 assert(note && build_id_length(note) == 20);
222
223 const uint8_t *id_sha1 = build_id_data(note);
224 assert(id_sha1);
225
226 char timestamp[41];
227 _mesa_sha1_format(timestamp, id_sha1);
228
229 uint64_t driver_flags = screen->dev.debug;
230 screen->disk_cache = disk_cache_create(renderer, timestamp, driver_flags);
231 #endif
232 }
233