1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "util/blob.h"
25 #include "util/hash_table.h"
26 #include "util/u_debug.h"
27 #include "util/disk_cache.h"
28 #include "util/mesa-sha1.h"
29 #include "nir/nir_serialize.h"
30 #include "anv_private.h"
31 #include "nir/nir_xfb_info.h"
32 #include "vk_util.h"
33
34 static bool
35 anv_shader_bin_serialize(struct vk_pipeline_cache_object *object,
36 struct blob *blob);
37
38 struct vk_pipeline_cache_object *
39 anv_shader_bin_deserialize(struct vk_pipeline_cache *cache,
40 const void *key_data, size_t key_size,
41 struct blob_reader *blob);
42
43 static void
anv_shader_bin_destroy(struct vk_device * _device,struct vk_pipeline_cache_object * object)44 anv_shader_bin_destroy(struct vk_device *_device,
45 struct vk_pipeline_cache_object *object)
46 {
47 struct anv_device *device =
48 container_of(_device, struct anv_device, vk);
49 struct anv_shader_bin *shader =
50 container_of(object, struct anv_shader_bin, base);
51
52 anv_state_pool_free(&device->instruction_state_pool, shader->kernel);
53 vk_pipeline_cache_object_finish(&shader->base);
54 vk_free(&device->vk.alloc, shader);
55 }
56
57 static const struct vk_pipeline_cache_object_ops anv_shader_bin_ops = {
58 .serialize = anv_shader_bin_serialize,
59 .deserialize = anv_shader_bin_deserialize,
60 .destroy = anv_shader_bin_destroy,
61 };
62
63 const struct vk_pipeline_cache_object_ops *const anv_cache_import_ops[2] = {
64 &anv_shader_bin_ops,
65 NULL
66 };
67
68 struct anv_shader_bin *
anv_shader_bin_create(struct anv_device * device,gl_shader_stage stage,const void * key_data,uint32_t key_size,const void * kernel_data,uint32_t kernel_size,const struct elk_stage_prog_data * prog_data_in,uint32_t prog_data_size,const struct elk_compile_stats * stats,uint32_t num_stats,const nir_xfb_info * xfb_info_in,const struct anv_pipeline_bind_map * bind_map)69 anv_shader_bin_create(struct anv_device *device,
70 gl_shader_stage stage,
71 const void *key_data, uint32_t key_size,
72 const void *kernel_data, uint32_t kernel_size,
73 const struct elk_stage_prog_data *prog_data_in,
74 uint32_t prog_data_size,
75 const struct elk_compile_stats *stats, uint32_t num_stats,
76 const nir_xfb_info *xfb_info_in,
77 const struct anv_pipeline_bind_map *bind_map)
78 {
79 VK_MULTIALLOC(ma);
80 VK_MULTIALLOC_DECL(&ma, struct anv_shader_bin, shader, 1);
81 VK_MULTIALLOC_DECL_SIZE(&ma, void, obj_key_data, key_size);
82 VK_MULTIALLOC_DECL_SIZE(&ma, struct elk_stage_prog_data, prog_data,
83 prog_data_size);
84 VK_MULTIALLOC_DECL(&ma, struct elk_shader_reloc, prog_data_relocs,
85 prog_data_in->num_relocs);
86 VK_MULTIALLOC_DECL(&ma, uint32_t, prog_data_param, prog_data_in->nr_params);
87
88 VK_MULTIALLOC_DECL_SIZE(&ma, nir_xfb_info, xfb_info,
89 xfb_info_in == NULL ? 0 :
90 nir_xfb_info_size(xfb_info_in->output_count));
91
92 VK_MULTIALLOC_DECL(&ma, struct anv_pipeline_binding, surface_to_descriptor,
93 bind_map->surface_count);
94 VK_MULTIALLOC_DECL(&ma, struct anv_pipeline_binding, sampler_to_descriptor,
95 bind_map->sampler_count);
96
97 if (!vk_multialloc_alloc(&ma, &device->vk.alloc,
98 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE))
99 return NULL;
100
101 memcpy(obj_key_data, key_data, key_size);
102 vk_pipeline_cache_object_init(&device->vk, &shader->base,
103 &anv_shader_bin_ops, obj_key_data, key_size);
104
105 shader->stage = stage;
106
107 shader->kernel =
108 anv_state_pool_alloc(&device->instruction_state_pool, kernel_size, 64);
109 memcpy(shader->kernel.map, kernel_data, kernel_size);
110 shader->kernel_size = kernel_size;
111
112 uint64_t shader_data_addr = INSTRUCTION_STATE_POOL_MIN_ADDRESS +
113 shader->kernel.offset +
114 prog_data_in->const_data_offset;
115
116 int rv_count = 0;
117 struct elk_shader_reloc_value reloc_values[5];
118 reloc_values[rv_count++] = (struct elk_shader_reloc_value) {
119 .id = ELK_SHADER_RELOC_CONST_DATA_ADDR_LOW,
120 .value = shader_data_addr,
121 };
122 reloc_values[rv_count++] = (struct elk_shader_reloc_value) {
123 .id = ELK_SHADER_RELOC_CONST_DATA_ADDR_HIGH,
124 .value = shader_data_addr >> 32,
125 };
126 reloc_values[rv_count++] = (struct elk_shader_reloc_value) {
127 .id = ELK_SHADER_RELOC_SHADER_START_OFFSET,
128 .value = shader->kernel.offset,
129 };
130 elk_write_shader_relocs(&device->physical->compiler->isa,
131 shader->kernel.map, prog_data_in,
132 reloc_values, rv_count);
133
134 memcpy(prog_data, prog_data_in, prog_data_size);
135 typed_memcpy(prog_data_relocs, prog_data_in->relocs,
136 prog_data_in->num_relocs);
137 prog_data->relocs = prog_data_relocs;
138 memset(prog_data_param, 0,
139 prog_data->nr_params * sizeof(*prog_data_param));
140 prog_data->param = prog_data_param;
141 shader->prog_data = prog_data;
142 shader->prog_data_size = prog_data_size;
143
144 assert(num_stats <= ARRAY_SIZE(shader->stats));
145 typed_memcpy(shader->stats, stats, num_stats);
146 shader->num_stats = num_stats;
147
148 if (xfb_info_in) {
149 *xfb_info = *xfb_info_in;
150 typed_memcpy(xfb_info->outputs, xfb_info_in->outputs,
151 xfb_info_in->output_count);
152 shader->xfb_info = xfb_info;
153 } else {
154 shader->xfb_info = NULL;
155 }
156
157 shader->bind_map = *bind_map;
158 typed_memcpy(surface_to_descriptor, bind_map->surface_to_descriptor,
159 bind_map->surface_count);
160 shader->bind_map.surface_to_descriptor = surface_to_descriptor;
161 typed_memcpy(sampler_to_descriptor, bind_map->sampler_to_descriptor,
162 bind_map->sampler_count);
163 shader->bind_map.sampler_to_descriptor = sampler_to_descriptor;
164
165 return shader;
166 }
167
168 static bool
anv_shader_bin_serialize(struct vk_pipeline_cache_object * object,struct blob * blob)169 anv_shader_bin_serialize(struct vk_pipeline_cache_object *object,
170 struct blob *blob)
171 {
172 struct anv_shader_bin *shader =
173 container_of(object, struct anv_shader_bin, base);
174
175 blob_write_uint32(blob, shader->stage);
176
177 blob_write_uint32(blob, shader->kernel_size);
178 blob_write_bytes(blob, shader->kernel.map, shader->kernel_size);
179
180 blob_write_uint32(blob, shader->prog_data_size);
181 blob_write_bytes(blob, shader->prog_data, shader->prog_data_size);
182 blob_write_bytes(blob, shader->prog_data->relocs,
183 shader->prog_data->num_relocs *
184 sizeof(shader->prog_data->relocs[0]));
185
186 blob_write_uint32(blob, shader->num_stats);
187 blob_write_bytes(blob, shader->stats,
188 shader->num_stats * sizeof(shader->stats[0]));
189
190 if (shader->xfb_info) {
191 uint32_t xfb_info_size =
192 nir_xfb_info_size(shader->xfb_info->output_count);
193 blob_write_uint32(blob, xfb_info_size);
194 blob_write_bytes(blob, shader->xfb_info, xfb_info_size);
195 } else {
196 blob_write_uint32(blob, 0);
197 }
198
199 blob_write_bytes(blob, shader->bind_map.surface_sha1,
200 sizeof(shader->bind_map.surface_sha1));
201 blob_write_bytes(blob, shader->bind_map.sampler_sha1,
202 sizeof(shader->bind_map.sampler_sha1));
203 blob_write_bytes(blob, shader->bind_map.push_sha1,
204 sizeof(shader->bind_map.push_sha1));
205 blob_write_uint32(blob, shader->bind_map.surface_count);
206 blob_write_uint32(blob, shader->bind_map.sampler_count);
207 blob_write_bytes(blob, shader->bind_map.surface_to_descriptor,
208 shader->bind_map.surface_count *
209 sizeof(*shader->bind_map.surface_to_descriptor));
210 blob_write_bytes(blob, shader->bind_map.sampler_to_descriptor,
211 shader->bind_map.sampler_count *
212 sizeof(*shader->bind_map.sampler_to_descriptor));
213 blob_write_bytes(blob, shader->bind_map.push_ranges,
214 sizeof(shader->bind_map.push_ranges));
215
216 return !blob->out_of_memory;
217 }
218
219 struct vk_pipeline_cache_object *
anv_shader_bin_deserialize(struct vk_pipeline_cache * cache,const void * key_data,size_t key_size,struct blob_reader * blob)220 anv_shader_bin_deserialize(struct vk_pipeline_cache *cache,
221 const void *key_data, size_t key_size,
222 struct blob_reader *blob)
223 {
224 struct anv_device *device =
225 container_of(cache->base.device, struct anv_device, vk);
226
227 gl_shader_stage stage = blob_read_uint32(blob);
228
229 uint32_t kernel_size = blob_read_uint32(blob);
230 const void *kernel_data = blob_read_bytes(blob, kernel_size);
231
232 uint32_t prog_data_size = blob_read_uint32(blob);
233 const void *prog_data_bytes = blob_read_bytes(blob, prog_data_size);
234 if (blob->overrun)
235 return NULL;
236
237 union elk_any_prog_data prog_data;
238 memcpy(&prog_data, prog_data_bytes,
239 MIN2(sizeof(prog_data), prog_data_size));
240 prog_data.base.relocs =
241 blob_read_bytes(blob, prog_data.base.num_relocs *
242 sizeof(prog_data.base.relocs[0]));
243
244 uint32_t num_stats = blob_read_uint32(blob);
245 const struct elk_compile_stats *stats =
246 blob_read_bytes(blob, num_stats * sizeof(stats[0]));
247
248 const nir_xfb_info *xfb_info = NULL;
249 uint32_t xfb_size = blob_read_uint32(blob);
250 if (xfb_size)
251 xfb_info = blob_read_bytes(blob, xfb_size);
252
253 struct anv_pipeline_bind_map bind_map;
254 blob_copy_bytes(blob, bind_map.surface_sha1, sizeof(bind_map.surface_sha1));
255 blob_copy_bytes(blob, bind_map.sampler_sha1, sizeof(bind_map.sampler_sha1));
256 blob_copy_bytes(blob, bind_map.push_sha1, sizeof(bind_map.push_sha1));
257 bind_map.surface_count = blob_read_uint32(blob);
258 bind_map.sampler_count = blob_read_uint32(blob);
259 bind_map.surface_to_descriptor = (void *)
260 blob_read_bytes(blob, bind_map.surface_count *
261 sizeof(*bind_map.surface_to_descriptor));
262 bind_map.sampler_to_descriptor = (void *)
263 blob_read_bytes(blob, bind_map.sampler_count *
264 sizeof(*bind_map.sampler_to_descriptor));
265 blob_copy_bytes(blob, bind_map.push_ranges, sizeof(bind_map.push_ranges));
266
267 if (blob->overrun)
268 return NULL;
269
270 struct anv_shader_bin *shader =
271 anv_shader_bin_create(device, stage,
272 key_data, key_size,
273 kernel_data, kernel_size,
274 &prog_data.base, prog_data_size,
275 stats, num_stats, xfb_info, &bind_map);
276 if (shader == NULL)
277 return NULL;
278
279 return &shader->base;
280 }
281
282 struct anv_shader_bin *
anv_device_search_for_kernel(struct anv_device * device,struct vk_pipeline_cache * cache,const void * key_data,uint32_t key_size,bool * user_cache_hit)283 anv_device_search_for_kernel(struct anv_device *device,
284 struct vk_pipeline_cache *cache,
285 const void *key_data, uint32_t key_size,
286 bool *user_cache_hit)
287 {
288 /* Use the default pipeline cache if none is specified */
289 if (cache == NULL)
290 cache = device->default_pipeline_cache;
291
292 bool cache_hit = false;
293 struct vk_pipeline_cache_object *object =
294 vk_pipeline_cache_lookup_object(cache, key_data, key_size,
295 &anv_shader_bin_ops, &cache_hit);
296 if (user_cache_hit != NULL) {
297 *user_cache_hit = object != NULL && cache_hit &&
298 cache != device->default_pipeline_cache;
299 }
300 if (object == NULL)
301 return NULL;
302
303 return container_of(object, struct anv_shader_bin, base);
304 }
305
306 struct anv_shader_bin *
anv_device_upload_kernel(struct anv_device * device,struct vk_pipeline_cache * cache,gl_shader_stage stage,const void * key_data,uint32_t key_size,const void * kernel_data,uint32_t kernel_size,const struct elk_stage_prog_data * prog_data,uint32_t prog_data_size,const struct elk_compile_stats * stats,uint32_t num_stats,const nir_xfb_info * xfb_info,const struct anv_pipeline_bind_map * bind_map)307 anv_device_upload_kernel(struct anv_device *device,
308 struct vk_pipeline_cache *cache,
309 gl_shader_stage stage,
310 const void *key_data, uint32_t key_size,
311 const void *kernel_data, uint32_t kernel_size,
312 const struct elk_stage_prog_data *prog_data,
313 uint32_t prog_data_size,
314 const struct elk_compile_stats *stats,
315 uint32_t num_stats,
316 const nir_xfb_info *xfb_info,
317 const struct anv_pipeline_bind_map *bind_map)
318 {
319 /* Use the default pipeline cache if none is specified */
320 if (cache == NULL)
321 cache = device->default_pipeline_cache;
322
323 struct anv_shader_bin *shader =
324 anv_shader_bin_create(device, stage,
325 key_data, key_size,
326 kernel_data, kernel_size,
327 prog_data, prog_data_size,
328 stats, num_stats,
329 xfb_info, bind_map);
330 if (shader == NULL)
331 return NULL;
332
333 struct vk_pipeline_cache_object *cached =
334 vk_pipeline_cache_add_object(cache, &shader->base);
335
336 return container_of(cached, struct anv_shader_bin, base);
337 }
338
339 #define SHA1_KEY_SIZE 20
340
341 struct nir_shader *
anv_device_search_for_nir(struct anv_device * device,struct vk_pipeline_cache * cache,const nir_shader_compiler_options * nir_options,unsigned char sha1_key[SHA1_KEY_SIZE],void * mem_ctx)342 anv_device_search_for_nir(struct anv_device *device,
343 struct vk_pipeline_cache *cache,
344 const nir_shader_compiler_options *nir_options,
345 unsigned char sha1_key[SHA1_KEY_SIZE],
346 void *mem_ctx)
347 {
348 if (cache == NULL)
349 cache = device->default_pipeline_cache;
350
351 return vk_pipeline_cache_lookup_nir(cache, sha1_key, SHA1_KEY_SIZE,
352 nir_options, NULL, mem_ctx);
353 }
354
355 void
anv_device_upload_nir(struct anv_device * device,struct vk_pipeline_cache * cache,const struct nir_shader * nir,unsigned char sha1_key[SHA1_KEY_SIZE])356 anv_device_upload_nir(struct anv_device *device,
357 struct vk_pipeline_cache *cache,
358 const struct nir_shader *nir,
359 unsigned char sha1_key[SHA1_KEY_SIZE])
360 {
361 if (cache == NULL)
362 cache = device->default_pipeline_cache;
363
364 vk_pipeline_cache_add_nir(cache, sha1_key, SHA1_KEY_SIZE, nir);
365 }
366