1 /*
2 * Copyright 2022 Red Hat.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "util/compiler.h"
25 #include "gallivm/lp_bld.h"
26 #include "gallivm/lp_bld_init.h"
27 #include "gallivm/lp_bld_struct.h"
28 #include "gallivm/lp_bld_sample.h"
29 #include "gallivm/lp_bld_const.h"
30 #include "gallivm/lp_bld_debug.h"
31 #include "gallivm/lp_bld_ir_common.h"
32 #include "draw/draw_vertex_header.h"
33 #include "lp_bld_jit_types.h"
34
35
36 static LLVMTypeRef
lp_build_create_jit_buffer_type(struct gallivm_state * gallivm)37 lp_build_create_jit_buffer_type(struct gallivm_state *gallivm)
38 {
39 LLVMContextRef lc = gallivm->context;
40 LLVMTypeRef buffer_type;
41 LLVMTypeRef elem_types[LP_JIT_BUFFER_NUM_FIELDS];
42
43 elem_types[LP_JIT_BUFFER_BASE] = LLVMPointerType(LLVMInt32TypeInContext(lc), 0);
44 elem_types[LP_JIT_BUFFER_NUM_ELEMENTS] = LLVMInt32TypeInContext(lc);
45
46 buffer_type = LLVMStructTypeInContext(lc, elem_types,
47 ARRAY_SIZE(elem_types), 0);
48
49 LP_CHECK_MEMBER_OFFSET(struct lp_jit_buffer, f,
50 gallivm->target, buffer_type,
51 LP_JIT_BUFFER_BASE);
52
53 LP_CHECK_MEMBER_OFFSET(struct lp_jit_buffer, num_elements,
54 gallivm->target, buffer_type,
55 LP_JIT_BUFFER_NUM_ELEMENTS);
56 return buffer_type;
57 }
58
59 LLVMValueRef
lp_llvm_descriptor_base(struct gallivm_state * gallivm,LLVMValueRef buffers_ptr,LLVMValueRef index,unsigned buffers_limit)60 lp_llvm_descriptor_base(struct gallivm_state *gallivm,
61 LLVMValueRef buffers_ptr,
62 LLVMValueRef index, unsigned buffers_limit)
63 {
64 LLVMBuilderRef builder = gallivm->builder;
65
66 LLVMValueRef desc_set_index = LLVMBuildExtractValue(builder, index, 0, "");
67 if (LLVMGetTypeKind(LLVMTypeOf(desc_set_index)) == LLVMVectorTypeKind)
68 desc_set_index = LLVMBuildExtractElement(builder, desc_set_index, lp_build_const_int32(gallivm, 0), "");
69 LLVMValueRef desc_set_base = lp_llvm_buffer_base(gallivm, buffers_ptr, desc_set_index, buffers_limit);
70
71 LLVMValueRef binding_index = LLVMBuildExtractValue(builder, index, 1, "");
72 if (LLVMGetTypeKind(LLVMTypeOf(binding_index)) == LLVMVectorTypeKind)
73 binding_index = LLVMBuildExtractElement(builder, binding_index, lp_build_const_int32(gallivm, 0), "");
74
75 LLVMValueRef binding_offset = LLVMBuildMul(builder, binding_index, lp_build_const_int32(gallivm, sizeof(struct lp_descriptor)), "");
76 LLVMTypeRef int64_type = LLVMInt64TypeInContext(gallivm->context);
77 binding_offset = LLVMBuildIntCast2(builder, binding_offset, int64_type, false, "");
78
79 LLVMValueRef desc_ptr = LLVMBuildPtrToInt(builder, desc_set_base, int64_type, "");
80 return LLVMBuildAdd(builder, desc_ptr, binding_offset, "");
81 }
82
83 static LLVMValueRef
lp_llvm_buffer_member(struct gallivm_state * gallivm,LLVMValueRef buffers_ptr,LLVMValueRef buffers_offset,unsigned buffers_limit,unsigned member_index,const char * member_name)84 lp_llvm_buffer_member(struct gallivm_state *gallivm,
85 LLVMValueRef buffers_ptr,
86 LLVMValueRef buffers_offset,
87 unsigned buffers_limit,
88 unsigned member_index,
89 const char *member_name)
90 {
91 LLVMBuilderRef builder = gallivm->builder;
92
93 LLVMTypeRef buffer_type = lp_build_create_jit_buffer_type(gallivm);
94
95 LLVMValueRef ptr;
96 if (LLVMGetTypeKind(LLVMTypeOf(buffers_offset)) == LLVMArrayTypeKind) {
97 LLVMValueRef desc_ptr = lp_llvm_descriptor_base(gallivm, buffers_ptr, buffers_offset, buffers_limit);
98
99 LLVMTypeRef buffer_ptr_type = LLVMPointerType(buffer_type, 0);
100 desc_ptr = LLVMBuildIntToPtr(builder, desc_ptr, buffer_ptr_type, "");
101
102 LLVMValueRef indices[2] = {
103 lp_build_const_int32(gallivm, 0),
104 lp_build_const_int32(gallivm, member_index),
105 };
106 ptr = LLVMBuildGEP2(builder, buffer_type, desc_ptr, indices, ARRAY_SIZE(indices), "");
107 } else {
108 LLVMValueRef indices[3];
109
110 indices[0] = lp_build_const_int32(gallivm, 0);
111 LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntULT, buffers_offset, lp_build_const_int32(gallivm, buffers_limit), "");
112 indices[1] = LLVMBuildSelect(gallivm->builder, cond, buffers_offset, lp_build_const_int32(gallivm, 0), "");
113 indices[2] = lp_build_const_int32(gallivm, member_index);
114
115 LLVMTypeRef buffers_type = LLVMArrayType(buffer_type, buffers_limit);
116 ptr = LLVMBuildGEP2(builder, buffers_type, buffers_ptr, indices, ARRAY_SIZE(indices), "");
117 }
118
119 LLVMTypeRef res_type = LLVMStructGetTypeAtIndex(buffer_type, member_index);
120 LLVMValueRef res = LLVMBuildLoad2(builder, res_type, ptr, "");
121
122 lp_build_name(res, "buffer.%s", member_name);
123
124 return res;
125 }
126
127 LLVMValueRef
lp_llvm_buffer_base(struct gallivm_state * gallivm,LLVMValueRef buffers_ptr,LLVMValueRef buffers_offset,unsigned buffers_limit)128 lp_llvm_buffer_base(struct gallivm_state *gallivm,
129 LLVMValueRef buffers_ptr, LLVMValueRef buffers_offset, unsigned buffers_limit)
130 {
131 return lp_llvm_buffer_member(gallivm, buffers_ptr, buffers_offset, buffers_limit, LP_JIT_BUFFER_BASE, "base");
132 }
133
134 LLVMValueRef
lp_llvm_buffer_num_elements(struct gallivm_state * gallivm,LLVMValueRef buffers_ptr,LLVMValueRef buffers_offset,unsigned buffers_limit)135 lp_llvm_buffer_num_elements(struct gallivm_state *gallivm,
136 LLVMValueRef buffers_ptr, LLVMValueRef buffers_offset, unsigned buffers_limit)
137 {
138 return lp_llvm_buffer_member(gallivm, buffers_ptr, buffers_offset, buffers_limit, LP_JIT_BUFFER_NUM_ELEMENTS, "num_elements");
139 }
140
141 static LLVMTypeRef
lp_build_create_jit_texture_type(struct gallivm_state * gallivm)142 lp_build_create_jit_texture_type(struct gallivm_state *gallivm)
143 {
144 LLVMContextRef lc = gallivm->context;
145 LLVMTypeRef texture_type;
146 LLVMTypeRef elem_types[LP_JIT_TEXTURE_NUM_FIELDS];
147
148 /* struct lp_jit_texture */
149 elem_types[LP_JIT_SAMPLER_INDEX_DUMMY] =
150 elem_types[LP_JIT_TEXTURE_WIDTH] = LLVMInt32TypeInContext(lc);
151 elem_types[LP_JIT_TEXTURE_HEIGHT] =
152 elem_types[LP_JIT_TEXTURE_DEPTH] = LLVMInt16TypeInContext(lc);
153 elem_types[LP_JIT_TEXTURE_FIRST_LEVEL] =
154 elem_types[LP_JIT_TEXTURE_LAST_LEVEL] = LLVMInt8TypeInContext(lc);
155 elem_types[LP_JIT_TEXTURE_BASE] = LLVMPointerType(LLVMInt8TypeInContext(lc), 0);
156 elem_types[LP_JIT_TEXTURE_ROW_STRIDE] =
157 elem_types[LP_JIT_TEXTURE_IMG_STRIDE] =
158 elem_types[LP_JIT_TEXTURE_MIP_OFFSETS] =
159 LLVMArrayType(LLVMInt32TypeInContext(lc), PIPE_MAX_TEXTURE_LEVELS);
160
161 texture_type = LLVMStructTypeInContext(lc, elem_types,
162 ARRAY_SIZE(elem_types), 0);
163
164 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, width,
165 gallivm->target, texture_type,
166 LP_JIT_TEXTURE_WIDTH);
167 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, height,
168 gallivm->target, texture_type,
169 LP_JIT_TEXTURE_HEIGHT);
170 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, depth,
171 gallivm->target, texture_type,
172 LP_JIT_TEXTURE_DEPTH);
173 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, base,
174 gallivm->target, texture_type,
175 LP_JIT_TEXTURE_BASE);
176 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, row_stride,
177 gallivm->target, texture_type,
178 LP_JIT_TEXTURE_ROW_STRIDE);
179 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, img_stride,
180 gallivm->target, texture_type,
181 LP_JIT_TEXTURE_IMG_STRIDE);
182 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, first_level,
183 gallivm->target, texture_type,
184 LP_JIT_TEXTURE_FIRST_LEVEL);
185 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, last_level,
186 gallivm->target, texture_type,
187 LP_JIT_TEXTURE_LAST_LEVEL);
188 LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, mip_offsets,
189 gallivm->target, texture_type,
190 LP_JIT_TEXTURE_MIP_OFFSETS);
191 LP_CHECK_STRUCT_SIZE(struct lp_jit_texture,
192 gallivm->target, texture_type);
193 return texture_type;
194 }
195
196 static LLVMTypeRef
lp_build_create_jit_sampler_type(struct gallivm_state * gallivm)197 lp_build_create_jit_sampler_type(struct gallivm_state *gallivm)
198 {
199 LLVMContextRef lc = gallivm->context;
200 LLVMTypeRef sampler_type;
201 LLVMTypeRef elem_types[LP_JIT_SAMPLER_NUM_FIELDS];
202 elem_types[LP_JIT_SAMPLER_MIN_LOD] =
203 elem_types[LP_JIT_SAMPLER_MAX_LOD] =
204 elem_types[LP_JIT_SAMPLER_LOD_BIAS] =
205 elem_types[LP_JIT_SAMPLER_MAX_ANISO] = LLVMFloatTypeInContext(lc);
206 elem_types[LP_JIT_SAMPLER_BORDER_COLOR] =
207 LLVMArrayType(LLVMFloatTypeInContext(lc), 4);
208
209 sampler_type = LLVMStructTypeInContext(lc, elem_types,
210 ARRAY_SIZE(elem_types), 0);
211
212 LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, min_lod,
213 gallivm->target, sampler_type,
214 LP_JIT_SAMPLER_MIN_LOD);
215 LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, max_lod,
216 gallivm->target, sampler_type,
217 LP_JIT_SAMPLER_MAX_LOD);
218 LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, lod_bias,
219 gallivm->target, sampler_type,
220 LP_JIT_SAMPLER_LOD_BIAS);
221 LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, border_color,
222 gallivm->target, sampler_type,
223 LP_JIT_SAMPLER_BORDER_COLOR);
224 LP_CHECK_MEMBER_OFFSET(struct lp_jit_sampler, max_aniso,
225 gallivm->target, sampler_type,
226 LP_JIT_SAMPLER_MAX_ANISO);
227 LP_CHECK_STRUCT_SIZE(struct lp_jit_sampler,
228 gallivm->target, sampler_type);
229 return sampler_type;
230 }
231
232 static LLVMTypeRef
lp_build_create_jit_image_type(struct gallivm_state * gallivm)233 lp_build_create_jit_image_type(struct gallivm_state *gallivm)
234 {
235 LLVMContextRef lc = gallivm->context;
236 LLVMTypeRef image_type;
237 LLVMTypeRef elem_types[LP_JIT_IMAGE_NUM_FIELDS];
238 elem_types[LP_JIT_IMAGE_WIDTH] = LLVMInt32TypeInContext(lc);
239 elem_types[LP_JIT_IMAGE_HEIGHT] =
240 elem_types[LP_JIT_IMAGE_DEPTH] = LLVMInt16TypeInContext(lc);
241 elem_types[LP_JIT_IMAGE_NUM_SAMPLES] = LLVMInt8TypeInContext(lc);
242 elem_types[LP_JIT_IMAGE_BASE] =
243 elem_types[LP_JIT_IMAGE_RESIDENCY] = LLVMPointerType(LLVMInt8TypeInContext(lc), 0);
244 elem_types[LP_JIT_IMAGE_ROW_STRIDE] =
245 elem_types[LP_JIT_IMAGE_IMG_STRIDE] =
246 elem_types[LP_JIT_IMAGE_SAMPLE_STRIDE] =
247 elem_types[LP_JIT_IMAGE_BASE_OFFSET] = LLVMInt32TypeInContext(lc);
248
249 image_type = LLVMStructTypeInContext(lc, elem_types,
250 ARRAY_SIZE(elem_types), 0);
251 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, width,
252 gallivm->target, image_type,
253 LP_JIT_IMAGE_WIDTH);
254 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, height,
255 gallivm->target, image_type,
256 LP_JIT_IMAGE_HEIGHT);
257 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, depth,
258 gallivm->target, image_type,
259 LP_JIT_IMAGE_DEPTH);
260 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, base,
261 gallivm->target, image_type,
262 LP_JIT_IMAGE_BASE);
263 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, row_stride,
264 gallivm->target, image_type,
265 LP_JIT_IMAGE_ROW_STRIDE);
266 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, img_stride,
267 gallivm->target, image_type,
268 LP_JIT_IMAGE_IMG_STRIDE);
269 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, num_samples,
270 gallivm->target, image_type,
271 LP_JIT_IMAGE_NUM_SAMPLES);
272 LP_CHECK_MEMBER_OFFSET(struct lp_jit_image, sample_stride,
273 gallivm->target, image_type,
274 LP_JIT_IMAGE_SAMPLE_STRIDE);
275 return image_type;
276 }
277
278 LLVMTypeRef
lp_build_jit_resources_type(struct gallivm_state * gallivm)279 lp_build_jit_resources_type(struct gallivm_state *gallivm)
280 {
281 LLVMTypeRef elem_types[LP_JIT_RES_COUNT];
282 LLVMTypeRef resources_type;
283 LLVMTypeRef texture_type, sampler_type, image_type, buffer_type;
284
285 buffer_type = lp_build_create_jit_buffer_type(gallivm);
286 texture_type = lp_build_create_jit_texture_type(gallivm);
287 sampler_type = lp_build_create_jit_sampler_type(gallivm);
288 image_type = lp_build_create_jit_image_type(gallivm);
289 elem_types[LP_JIT_RES_CONSTANTS] = LLVMArrayType(buffer_type,
290 LP_MAX_TGSI_CONST_BUFFERS);
291 elem_types[LP_JIT_RES_SSBOS] =
292 LLVMArrayType(buffer_type, LP_MAX_TGSI_SHADER_BUFFERS);
293 elem_types[LP_JIT_RES_TEXTURES] = LLVMArrayType(texture_type,
294 PIPE_MAX_SHADER_SAMPLER_VIEWS);
295 elem_types[LP_JIT_RES_SAMPLERS] = LLVMArrayType(sampler_type,
296 PIPE_MAX_SAMPLERS);
297 elem_types[LP_JIT_RES_IMAGES] = LLVMArrayType(image_type,
298 PIPE_MAX_SHADER_IMAGES);
299 elem_types[LP_JIT_RES_ANISO_FILTER_TABLE] = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
300
301 resources_type = LLVMStructTypeInContext(gallivm->context, elem_types,
302 ARRAY_SIZE(elem_types), 0);
303
304 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, constants,
305 gallivm->target, resources_type,
306 LP_JIT_RES_CONSTANTS);
307 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, ssbos,
308 gallivm->target, resources_type,
309 LP_JIT_RES_SSBOS);
310 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, textures,
311 gallivm->target, resources_type,
312 LP_JIT_RES_TEXTURES);
313 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, samplers,
314 gallivm->target, resources_type,
315 LP_JIT_RES_SAMPLERS);
316 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, images,
317 gallivm->target, resources_type,
318 LP_JIT_RES_IMAGES);
319 LP_CHECK_MEMBER_OFFSET(struct lp_jit_resources, aniso_filter_table,
320 gallivm->target, resources_type,
321 LP_JIT_RES_ANISO_FILTER_TABLE);
322
323 return resources_type;
324 }
325
326
327 /**
328 * Fetch the specified member of the lp_jit_texture structure.
329 * \param emit_load if TRUE, emit the LLVM load instruction to actually
330 * fetch the field's value. Otherwise, just emit the
331 * GEP code to address the field.
332 *
333 * @sa http://llvm.org/docs/GetElementPtr.html
334 */
335 static LLVMValueRef
lp_build_llvm_texture_member(struct gallivm_state * gallivm,LLVMTypeRef resources_type,LLVMValueRef resources_ptr,unsigned texture_unit,LLVMValueRef texture_unit_offset,unsigned member_index,const char * member_name,bool emit_load,LLVMTypeRef * out_type)336 lp_build_llvm_texture_member(struct gallivm_state *gallivm,
337 LLVMTypeRef resources_type,
338 LLVMValueRef resources_ptr,
339 unsigned texture_unit,
340 LLVMValueRef texture_unit_offset,
341 unsigned member_index,
342 const char *member_name,
343 bool emit_load,
344 LLVMTypeRef *out_type)
345 {
346 LLVMBuilderRef builder = gallivm->builder;
347
348 LLVMValueRef ptr;
349 if (gallivm->texture_descriptor) {
350 static_assert(offsetof(struct lp_descriptor, texture) == 0, "Invalid texture offset!");
351 LLVMValueRef texture_ptr = gallivm->texture_descriptor;
352
353 LLVMTypeRef texture_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_TEXTURES);
354 LLVMTypeRef texture_type = LLVMGetElementType(texture_ptr_type);
355 texture_ptr_type = LLVMPointerType(texture_type, 0);
356
357 texture_ptr = LLVMBuildIntToPtr(builder, texture_ptr, texture_ptr_type, "");
358
359 LLVMValueRef indices[2] = {
360 lp_build_const_int32(gallivm, 0),
361 lp_build_const_int32(gallivm, member_index),
362 };
363 ptr = LLVMBuildGEP2(builder, texture_type, texture_ptr, indices, ARRAY_SIZE(indices), "");
364 } else {
365 LLVMValueRef indices[4];
366
367 assert(texture_unit < PIPE_MAX_SHADER_SAMPLER_VIEWS);
368
369 /* resources[0] */
370 indices[0] = lp_build_const_int32(gallivm, 0);
371 /* resources[0].textures */
372 indices[1] = lp_build_const_int32(gallivm, LP_JIT_RES_TEXTURES);
373 /* resources[0].textures[unit] */
374 indices[2] = lp_build_const_int32(gallivm, texture_unit);
375 if (texture_unit_offset) {
376 indices[2] = LLVMBuildAdd(gallivm->builder, indices[2],
377 texture_unit_offset, "");
378 LLVMValueRef cond =
379 LLVMBuildICmp(gallivm->builder, LLVMIntULT,
380 indices[2],
381 lp_build_const_int32(gallivm,
382 PIPE_MAX_SHADER_SAMPLER_VIEWS), "");
383 indices[2] = LLVMBuildSelect(gallivm->builder, cond, indices[2],
384 lp_build_const_int32(gallivm,
385 texture_unit), "");
386 }
387 /* resources[0].textures[unit].member */
388 indices[3] = lp_build_const_int32(gallivm, member_index);
389
390 ptr = LLVMBuildGEP2(builder, resources_type, resources_ptr, indices, ARRAY_SIZE(indices), "");
391 }
392
393 LLVMValueRef res;
394 if (emit_load) {
395 LLVMTypeRef tex_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_TEXTURES);
396 LLVMTypeRef res_type = LLVMStructGetTypeAtIndex(LLVMGetElementType(tex_type), member_index);
397 res = LLVMBuildLoad2(builder, res_type, ptr, "");
398 } else
399 res = ptr;
400
401 if (out_type) {
402 LLVMTypeRef tex_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_TEXTURES);
403 LLVMTypeRef res_type = LLVMStructGetTypeAtIndex(LLVMGetElementType(tex_type), member_index);
404 *out_type = res_type;
405 }
406
407 lp_build_name(res, "resources.texture%u.%s", texture_unit, member_name);
408
409 return res;
410 }
411
412 static LLVMValueRef
lp_build_llvm_texture_residency(struct gallivm_state * gallivm,LLVMTypeRef resources_type,LLVMValueRef resources_ptr,unsigned texture_unit,LLVMValueRef texture_unit_offset)413 lp_build_llvm_texture_residency(struct gallivm_state *gallivm,
414 LLVMTypeRef resources_type,
415 LLVMValueRef resources_ptr,
416 unsigned texture_unit,
417 LLVMValueRef texture_unit_offset)
418 {
419 LLVMBuilderRef builder = gallivm->builder;
420
421 static_assert(offsetof(struct lp_descriptor, texture) == 0, "Invalid texture offset");
422 LLVMValueRef texture_ptr = gallivm->texture_descriptor;
423
424 LLVMTypeRef texture_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_TEXTURES);
425 LLVMTypeRef texture_type = LLVMGetElementType(texture_ptr_type);
426 texture_ptr_type = LLVMPointerType(texture_type, 0);
427
428 texture_ptr = LLVMBuildIntToPtr(builder, texture_ptr, texture_ptr_type, "");
429
430 static_assert(offsetof(struct lp_jit_texture, row_stride) == offsetof(struct lp_jit_texture, residency),
431 "Invalid texture descriptor layout");
432 LLVMValueRef indices[2] = {
433 lp_build_const_int32(gallivm, 0),
434 lp_build_const_int32(gallivm, LP_JIT_TEXTURE_ROW_STRIDE),
435 };
436 LLVMValueRef ptr = LLVMBuildGEP2(builder, texture_type, texture_ptr, indices, ARRAY_SIZE(indices), "");
437
438 LLVMTypeRef residency_type = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
439 ptr = LLVMBuildBitCast(builder, ptr, LLVMPointerType(residency_type, 0), "");
440
441 return LLVMBuildLoad2(builder, residency_type, ptr, "");
442 }
443
444
445 /**
446 * Helper macro to instantiate the functions that generate the code to
447 * fetch the members of lp_jit_texture to fulfill the sampler code
448 * generator requests.
449 *
450 * This complexity is the price we have to pay to keep the texture
451 * sampler code generator a reusable module without dependencies to
452 * llvmpipe internals.
453 */
454 #define LP_BUILD_LLVM_TEXTURE_MEMBER(_name, _index, _emit_load) \
455 static LLVMValueRef \
456 lp_build_llvm_texture_##_name(struct gallivm_state *gallivm, \
457 LLVMTypeRef resources_type, \
458 LLVMValueRef resources_ptr, \
459 unsigned texture_unit, \
460 LLVMValueRef texture_unit_offset) \
461 { \
462 return lp_build_llvm_texture_member(gallivm, resources_type, resources_ptr, \
463 texture_unit, texture_unit_offset, \
464 _index, #_name, _emit_load, NULL ); \
465 }
466
467 #define LP_BUILD_LLVM_TEXTURE_MEMBER_OUTTYPE(_name, _index, _emit_load) \
468 static LLVMValueRef \
469 lp_build_llvm_texture_##_name(struct gallivm_state *gallivm, \
470 LLVMTypeRef resources_type, \
471 LLVMValueRef resources_ptr, \
472 unsigned texture_unit, \
473 LLVMValueRef texture_unit_offset, \
474 LLVMTypeRef *out_type) \
475 { \
476 return lp_build_llvm_texture_member(gallivm, resources_type, resources_ptr, \
477 texture_unit, texture_unit_offset, \
478 _index, #_name, _emit_load, out_type ); \
479 }
480
LP_BUILD_LLVM_TEXTURE_MEMBER(width,LP_JIT_TEXTURE_WIDTH,true)481 LP_BUILD_LLVM_TEXTURE_MEMBER(width, LP_JIT_TEXTURE_WIDTH, true)
482 LP_BUILD_LLVM_TEXTURE_MEMBER(height, LP_JIT_TEXTURE_HEIGHT, true)
483 LP_BUILD_LLVM_TEXTURE_MEMBER(depth, LP_JIT_TEXTURE_DEPTH, true)
484 LP_BUILD_LLVM_TEXTURE_MEMBER(first_level, LP_JIT_TEXTURE_FIRST_LEVEL, true)
485 LP_BUILD_LLVM_TEXTURE_MEMBER(last_level, LP_JIT_TEXTURE_LAST_LEVEL, true)
486 LP_BUILD_LLVM_TEXTURE_MEMBER(base_ptr, LP_JIT_TEXTURE_BASE, true)
487 LP_BUILD_LLVM_TEXTURE_MEMBER_OUTTYPE(row_stride, LP_JIT_TEXTURE_ROW_STRIDE, false)
488 LP_BUILD_LLVM_TEXTURE_MEMBER_OUTTYPE(img_stride, LP_JIT_TEXTURE_IMG_STRIDE, false)
489 LP_BUILD_LLVM_TEXTURE_MEMBER_OUTTYPE(mip_offsets, LP_JIT_TEXTURE_MIP_OFFSETS, false)
490
491 /**
492 * Fetch the specified member of the lp_jit_sampler structure.
493 * \param emit_load if TRUE, emit the LLVM load instruction to actually
494 * fetch the field's value. Otherwise, just emit the
495 * GEP code to address the field.
496 *
497 * @sa http://llvm.org/docs/GetElementPtr.html
498 */
499 static LLVMValueRef
500 lp_build_llvm_sampler_member(struct gallivm_state *gallivm,
501 LLVMTypeRef resources_type,
502 LLVMValueRef resources_ptr,
503 unsigned sampler_unit,
504 unsigned member_index,
505 const char *member_name,
506 bool emit_load)
507 {
508 LLVMBuilderRef builder = gallivm->builder;
509
510 LLVMValueRef ptr;
511 if (gallivm->sampler_descriptor) {
512 LLVMValueRef sampler_offset = lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, sampler));
513 LLVMValueRef sampler_ptr = LLVMBuildAdd(builder, gallivm->sampler_descriptor, sampler_offset, "");
514
515 LLVMTypeRef sampler_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_SAMPLERS);
516 LLVMTypeRef sampler_type = LLVMGetElementType(sampler_ptr_type);
517 sampler_ptr_type = LLVMPointerType(sampler_type, 0);
518
519 sampler_ptr = LLVMBuildIntToPtr(builder, sampler_ptr, sampler_ptr_type, "");
520
521 LLVMValueRef indices[2] = {
522 lp_build_const_int32(gallivm, 0),
523 lp_build_const_int32(gallivm, member_index),
524 };
525 ptr = LLVMBuildGEP2(builder, sampler_type, sampler_ptr, indices, ARRAY_SIZE(indices), "");
526 } else {
527 LLVMValueRef indices[4];
528
529 assert(sampler_unit < PIPE_MAX_SAMPLERS);
530
531 /* resources[0] */
532 indices[0] = lp_build_const_int32(gallivm, 0);
533 /* resources[0].samplers */
534 indices[1] = lp_build_const_int32(gallivm, LP_JIT_RES_SAMPLERS);
535 /* resources[0].samplers[unit] */
536 indices[2] = lp_build_const_int32(gallivm, sampler_unit);
537 /* resources[0].samplers[unit].member */
538 indices[3] = lp_build_const_int32(gallivm, member_index);
539
540 ptr = LLVMBuildGEP2(builder, resources_type, resources_ptr, indices, ARRAY_SIZE(indices), "");
541 }
542
543 LLVMValueRef res;
544 if (emit_load) {
545 LLVMTypeRef samp_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_SAMPLERS);
546 LLVMTypeRef res_type = LLVMStructGetTypeAtIndex(LLVMGetElementType(samp_type), member_index);
547 res = LLVMBuildLoad2(builder, res_type, ptr, "");
548 } else
549 res = ptr;
550
551 lp_build_name(res, "resources.sampler%u.%s", sampler_unit, member_name);
552
553 return res;
554 }
555
556
557 #define LP_BUILD_LLVM_SAMPLER_MEMBER(_name, _index, _emit_load) \
558 static LLVMValueRef \
559 lp_build_llvm_sampler_##_name( struct gallivm_state *gallivm, \
560 LLVMTypeRef resources_type, \
561 LLVMValueRef resources_ptr, \
562 unsigned sampler_unit) \
563 { \
564 return lp_build_llvm_sampler_member(gallivm, resources_type, resources_ptr, \
565 sampler_unit, _index, #_name, _emit_load ); \
566 }
567
568
LP_BUILD_LLVM_SAMPLER_MEMBER(min_lod,LP_JIT_SAMPLER_MIN_LOD,true)569 LP_BUILD_LLVM_SAMPLER_MEMBER(min_lod, LP_JIT_SAMPLER_MIN_LOD, true)
570 LP_BUILD_LLVM_SAMPLER_MEMBER(max_lod, LP_JIT_SAMPLER_MAX_LOD, true)
571 LP_BUILD_LLVM_SAMPLER_MEMBER(lod_bias, LP_JIT_SAMPLER_LOD_BIAS, true)
572 LP_BUILD_LLVM_SAMPLER_MEMBER(border_color, LP_JIT_SAMPLER_BORDER_COLOR, false)
573 LP_BUILD_LLVM_SAMPLER_MEMBER(max_aniso, LP_JIT_SAMPLER_MAX_ANISO, true)
574
575 /**
576 * Fetch the specified member of the lp_jit_image structure.
577 * \param emit_load if TRUE, emit the LLVM load instruction to actually
578 * fetch the field's value. Otherwise, just emit the
579 * GEP code to address the field.
580 *
581 * @sa http://llvm.org/docs/GetElementPtr.html
582 */
583 static LLVMValueRef
584 lp_build_llvm_image_member(struct gallivm_state *gallivm,
585 LLVMTypeRef resources_type,
586 LLVMValueRef resources_ptr,
587 unsigned image_unit,
588 LLVMValueRef image_unit_offset,
589 unsigned member_index,
590 const char *member_name,
591 bool emit_load)
592 {
593 LLVMBuilderRef builder = gallivm->builder;
594
595 LLVMValueRef ptr;
596 if (gallivm->texture_descriptor) {
597 LLVMValueRef image_offset = lp_build_const_int64(gallivm, offsetof(struct lp_descriptor, image));
598 LLVMValueRef image_ptr = LLVMBuildAdd(builder, gallivm->texture_descriptor, image_offset, "");
599
600 LLVMTypeRef image_ptr_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_IMAGES);
601 LLVMTypeRef image_type = LLVMGetElementType(image_ptr_type);
602 image_ptr_type = LLVMPointerType(image_type, 0);
603
604 image_ptr = LLVMBuildIntToPtr(builder, image_ptr, image_ptr_type, "");
605
606 LLVMValueRef indices[2] = {
607 lp_build_const_int32(gallivm, 0),
608 lp_build_const_int32(gallivm, member_index),
609 };
610 ptr = LLVMBuildGEP2(builder, image_type, image_ptr, indices, ARRAY_SIZE(indices), "");
611 } else {
612 LLVMValueRef indices[4];
613
614 assert(image_unit < PIPE_MAX_SHADER_IMAGES);
615
616 /* resources[0] */
617 indices[0] = lp_build_const_int32(gallivm, 0);
618 /* resources[0].images */
619 indices[1] = lp_build_const_int32(gallivm, LP_JIT_RES_IMAGES);
620 /* resources[0].images[unit] */
621 indices[2] = lp_build_const_int32(gallivm, image_unit);
622 if (image_unit_offset) {
623 indices[2] = LLVMBuildAdd(gallivm->builder, indices[2], image_unit_offset, "");
624 LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntULT, indices[2], lp_build_const_int32(gallivm, PIPE_MAX_SHADER_IMAGES), "");
625 indices[2] = LLVMBuildSelect(gallivm->builder, cond, indices[2], lp_build_const_int32(gallivm, image_unit), "");
626 }
627 /* resources[0].images[unit].member */
628 indices[3] = lp_build_const_int32(gallivm, member_index);
629
630 ptr = LLVMBuildGEP2(builder, resources_type, resources_ptr, indices, ARRAY_SIZE(indices), "");
631 }
632
633 LLVMValueRef res;
634 if (emit_load) {
635 LLVMTypeRef img_type = LLVMStructGetTypeAtIndex(resources_type, LP_JIT_RES_IMAGES);
636 LLVMTypeRef res_type = LLVMStructGetTypeAtIndex(LLVMGetElementType(img_type), member_index);
637 res = LLVMBuildLoad2(builder, res_type, ptr, "");
638 } else
639 res = ptr;
640
641 lp_build_name(res, "resources.image%u.%s", image_unit, member_name);
642
643 return res;
644 }
645
646 /**
647 * Helper macro to instantiate the functions that generate the code to
648 * fetch the members of lp_jit_image to fulfill the sampler code
649 * generator requests.
650 *
651 * This complexity is the price we have to pay to keep the image
652 * sampler code generator a reusable module without dependencies to
653 * llvmpipe internals.
654 */
655 #define LP_BUILD_LLVM_IMAGE_MEMBER(_name, _index, _emit_load) \
656 static LLVMValueRef \
657 lp_build_llvm_image_##_name( struct gallivm_state *gallivm, \
658 LLVMTypeRef resources_type, \
659 LLVMValueRef resources_ptr, \
660 unsigned image_unit, LLVMValueRef image_unit_offset) \
661 { \
662 return lp_build_llvm_image_member(gallivm, resources_type, resources_ptr, \
663 image_unit, image_unit_offset, \
664 _index, #_name, _emit_load ); \
665 }
666
667 #define LP_BUILD_LLVM_IMAGE_MEMBER_OUTTYPE(_name, _index, _emit_load) \
668 static LLVMValueRef \
669 lp_build_llvm_image_##_name( struct gallivm_state *gallivm, \
670 LLVMTypeRef resources_type, \
671 LLVMValueRef resources_ptr, \
672 unsigned image_unit, LLVMValueRef image_unit_offset, \
673 LLVMTypeRef *out_type) \
674 { \
675 assert(!out_type); \
676 return lp_build_llvm_image_member(gallivm, resources_type, resources_ptr, \
677 image_unit, image_unit_offset, \
678 _index, #_name, _emit_load ); \
679 }
680
LP_BUILD_LLVM_IMAGE_MEMBER(width,LP_JIT_IMAGE_WIDTH,true)681 LP_BUILD_LLVM_IMAGE_MEMBER(width, LP_JIT_IMAGE_WIDTH, true)
682 LP_BUILD_LLVM_IMAGE_MEMBER(height, LP_JIT_IMAGE_HEIGHT, true)
683 LP_BUILD_LLVM_IMAGE_MEMBER(depth, LP_JIT_IMAGE_DEPTH, true)
684 LP_BUILD_LLVM_IMAGE_MEMBER(base_ptr, LP_JIT_IMAGE_BASE, true)
685 LP_BUILD_LLVM_IMAGE_MEMBER_OUTTYPE(row_stride, LP_JIT_IMAGE_ROW_STRIDE, true)
686 LP_BUILD_LLVM_IMAGE_MEMBER_OUTTYPE(img_stride, LP_JIT_IMAGE_IMG_STRIDE, true)
687 LP_BUILD_LLVM_IMAGE_MEMBER(num_samples, LP_JIT_IMAGE_NUM_SAMPLES, true)
688 LP_BUILD_LLVM_IMAGE_MEMBER(sample_stride, LP_JIT_IMAGE_SAMPLE_STRIDE, true)
689 LP_BUILD_LLVM_IMAGE_MEMBER(residency, LP_JIT_IMAGE_RESIDENCY, true)
690 LP_BUILD_LLVM_IMAGE_MEMBER(base_offset, LP_JIT_IMAGE_BASE_OFFSET, true)
691
692 void
693 lp_build_jit_fill_sampler_dynamic_state(struct lp_sampler_dynamic_state *state)
694 {
695 state->width = lp_build_llvm_texture_width;
696 state->height = lp_build_llvm_texture_height;
697 state->depth = lp_build_llvm_texture_depth;
698 state->first_level = lp_build_llvm_texture_first_level;
699 state->last_level = lp_build_llvm_texture_last_level;
700 state->base_ptr = lp_build_llvm_texture_base_ptr;
701 state->row_stride = lp_build_llvm_texture_row_stride;
702 state->img_stride = lp_build_llvm_texture_img_stride;
703 state->mip_offsets = lp_build_llvm_texture_mip_offsets;
704 state->residency = lp_build_llvm_texture_residency;
705
706 state->min_lod = lp_build_llvm_sampler_min_lod;
707 state->max_lod = lp_build_llvm_sampler_max_lod;
708 state->lod_bias = lp_build_llvm_sampler_lod_bias;
709 state->border_color = lp_build_llvm_sampler_border_color;
710 state->max_aniso = lp_build_llvm_sampler_max_aniso;
711 }
712
713 void
lp_build_jit_fill_image_dynamic_state(struct lp_sampler_dynamic_state * state)714 lp_build_jit_fill_image_dynamic_state(struct lp_sampler_dynamic_state *state)
715 {
716 state->width = lp_build_llvm_image_width;
717 state->height = lp_build_llvm_image_height;
718
719 state->depth = lp_build_llvm_image_depth;
720 state->base_ptr = lp_build_llvm_image_base_ptr;
721 state->row_stride = lp_build_llvm_image_row_stride;
722 state->img_stride = lp_build_llvm_image_img_stride;
723 state->last_level = lp_build_llvm_image_num_samples;
724 state->sample_stride = lp_build_llvm_image_sample_stride;
725 state->residency = lp_build_llvm_image_residency;
726 state->base_offset = lp_build_llvm_image_base_offset;
727 }
728
729 /**
730 * Create LLVM type for struct vertex_header;
731 */
732 LLVMTypeRef
lp_build_create_jit_vertex_header_type(struct gallivm_state * gallivm,int data_elems)733 lp_build_create_jit_vertex_header_type(struct gallivm_state *gallivm, int data_elems)
734 {
735 LLVMTargetDataRef target = gallivm->target;
736 LLVMTypeRef elem_types[3];
737 LLVMTypeRef vertex_header;
738 char struct_name[24];
739
740 snprintf(struct_name, 23, "vertex_header%d", data_elems);
741
742 elem_types[LP_JIT_VERTEX_HEADER_VERTEX_ID] = LLVMIntTypeInContext(gallivm->context, 32);
743 elem_types[LP_JIT_VERTEX_HEADER_CLIP_POS] = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
744 elem_types[LP_JIT_VERTEX_HEADER_DATA] = LLVMArrayType(elem_types[1], data_elems);
745
746 vertex_header = LLVMStructTypeInContext(gallivm->context, elem_types,
747 ARRAY_SIZE(elem_types), 0);
748
749 /* these are bit-fields and we can't take address of them
750 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clipmask,
751 target, vertex_header,
752 LP_JIT_VERTEX_HEADER_CLIPMASK);
753 LP_CHECK_MEMBER_OFFSET(struct vertex_header, edgeflag,
754 target, vertex_header,
755 LP_JIT_VERTEX_HEADER_EDGEFLAG);
756 LP_CHECK_MEMBER_OFFSET(struct vertex_header, pad,
757 target, vertex_header,
758 LP_JIT_VERTEX_HEADER_PAD);
759 LP_CHECK_MEMBER_OFFSET(struct vertex_header, vertex_id,
760 target, vertex_header,
761 LP_JIT_VERTEX_HEADER_VERTEX_ID);
762 */
763 (void) target; /* silence unused var warning for non-debug build */
764 LP_CHECK_MEMBER_OFFSET(struct vertex_header, clip_pos,
765 target, vertex_header,
766 LP_JIT_VERTEX_HEADER_CLIP_POS);
767 LP_CHECK_MEMBER_OFFSET(struct vertex_header, data,
768 target, vertex_header,
769 LP_JIT_VERTEX_HEADER_DATA);
770
771 assert(LLVMABISizeOfType(target, vertex_header) ==
772 offsetof(struct vertex_header, data[data_elems]));
773
774 return vertex_header;
775 }
776
777 LLVMTypeRef
lp_build_sample_function_type(struct gallivm_state * gallivm,uint32_t sample_key)778 lp_build_sample_function_type(struct gallivm_state *gallivm, uint32_t sample_key)
779 {
780 struct lp_type type;
781 memset(&type, 0, sizeof type);
782 type.floating = true; /* floating point values */
783 type.sign = true; /* values are signed */
784 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
785 type.width = 32; /* 32-bit float */
786 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
787
788 enum lp_sampler_op_type op_type = (sample_key & LP_SAMPLER_OP_TYPE_MASK) >> LP_SAMPLER_OP_TYPE_SHIFT;
789 enum lp_sampler_lod_control lod_control = (sample_key & LP_SAMPLER_LOD_CONTROL_MASK) >> LP_SAMPLER_LOD_CONTROL_SHIFT;
790
791 LLVMTypeRef arg_types[LP_MAX_TEX_FUNC_ARGS];
792 LLVMTypeRef ret_type;
793 LLVMTypeRef val_type[5];
794 uint32_t num_params = 0;
795
796 LLVMTypeRef coord_type;
797 if (op_type == LP_SAMPLER_OP_FETCH)
798 coord_type = lp_build_vec_type(gallivm, lp_int_type(type));
799 else
800 coord_type = lp_build_vec_type(gallivm, type);
801
802 arg_types[num_params++] = LLVMInt64TypeInContext(gallivm->context);
803 arg_types[num_params++] = LLVMInt64TypeInContext(gallivm->context);
804
805 arg_types[num_params++] = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
806
807 for (unsigned i = 0; i < 4; i++)
808 arg_types[num_params++] = coord_type;
809
810 if (sample_key & LP_SAMPLER_SHADOW)
811 arg_types[num_params++] = lp_build_vec_type(gallivm, type);
812
813 if (sample_key & LP_SAMPLER_FETCH_MS)
814 arg_types[num_params++] = lp_build_vec_type(gallivm, lp_uint_type(type));
815
816 if (sample_key & LP_SAMPLER_OFFSETS)
817 for (uint32_t i = 0; i < 3; i++)
818 arg_types[num_params++] = lp_build_int_vec_type(gallivm, type);
819
820 if (lod_control == LP_SAMPLER_LOD_BIAS || lod_control == LP_SAMPLER_LOD_EXPLICIT)
821 arg_types[num_params++] = coord_type;
822
823 val_type[0] = val_type[1] = val_type[2] = val_type[3] = lp_build_vec_type(gallivm, type);
824 val_type[4] = lp_build_int_vec_type(gallivm, type);
825 ret_type = LLVMStructTypeInContext(gallivm->context, val_type, 5, 0);
826 return LLVMFunctionType(ret_type, arg_types, num_params, false);
827 }
828
829 LLVMTypeRef
lp_build_size_function_type(struct gallivm_state * gallivm,const struct lp_sampler_size_query_params * params)830 lp_build_size_function_type(struct gallivm_state *gallivm,
831 const struct lp_sampler_size_query_params *params)
832 {
833 struct lp_type type;
834 memset(&type, 0, sizeof type);
835 type.floating = true; /* floating point values */
836 type.sign = true; /* values are signed */
837 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
838 type.width = 32; /* 32-bit float */
839 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
840
841 LLVMTypeRef arg_types[LP_MAX_TEX_FUNC_ARGS];
842 LLVMTypeRef ret_type;
843 LLVMTypeRef val_type[4];
844 uint32_t num_params = 0;
845
846 arg_types[num_params++] = LLVMInt64TypeInContext(gallivm->context);
847
848 if (!params->samples_only)
849 arg_types[num_params++] = lp_build_int_vec_type(gallivm, type);
850
851 val_type[0] = val_type[1] = val_type[2] = val_type[3] = lp_build_int_vec_type(gallivm, type);
852 ret_type = LLVMStructTypeInContext(gallivm->context, val_type, 4, 0);
853 return LLVMFunctionType(ret_type, arg_types, num_params, false);
854 }
855
856 LLVMTypeRef
lp_build_image_function_type(struct gallivm_state * gallivm,const struct lp_img_params * params,bool ms)857 lp_build_image_function_type(struct gallivm_state *gallivm,
858 const struct lp_img_params *params, bool ms)
859 {
860 struct lp_type type;
861 memset(&type, 0, sizeof type);
862 type.floating = true; /* floating point values */
863 type.sign = true; /* values are signed */
864 type.norm = false; /* values are not limited to [0,1] or [-1,1] */
865 type.width = 32; /* 32-bit float */
866 type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
867
868 LLVMTypeRef arg_types[LP_MAX_TEX_FUNC_ARGS];
869 LLVMTypeRef ret_type;
870 uint32_t num_params = 0;
871
872 arg_types[num_params++] = LLVMInt64TypeInContext(gallivm->context);
873
874 if (params->img_op != LP_IMG_LOAD && params->img_op != LP_IMG_LOAD_SPARSE)
875 arg_types[num_params++] = lp_build_int_vec_type(gallivm, type);
876
877 for (uint32_t i = 0; i < 3; i++)
878 arg_types[num_params++] = lp_build_vec_type(gallivm, lp_uint_type(type));
879
880 if (ms)
881 arg_types[num_params++] = lp_build_vec_type(gallivm, lp_uint_type(type));
882
883 uint32_t num_inputs = params->img_op != LP_IMG_LOAD && params->img_op != LP_IMG_LOAD_SPARSE ? 4 : 0;
884 if (params->img_op == LP_IMG_ATOMIC_CAS)
885 num_inputs = 8;
886
887 const struct util_format_description *desc = util_format_description(params->format);
888 LLVMTypeRef component_type = lp_build_vec_type(gallivm, lp_build_texel_type(type, desc));
889
890 for (uint32_t i = 0; i < num_inputs; i++)
891 arg_types[num_params++] = component_type;
892
893 if (params->img_op == LP_IMG_LOAD_SPARSE) {
894 LLVMTypeRef val_type[5];
895 val_type[0] = val_type[1] = val_type[2] = val_type[3] = component_type;
896 val_type[4] = lp_build_int_vec_type(gallivm, type);
897 ret_type = LLVMStructTypeInContext(gallivm->context, val_type, 5, 0);
898 } else if (params->img_op != LP_IMG_STORE) {
899 LLVMTypeRef val_type[4];
900 val_type[0] = val_type[1] = val_type[2] = val_type[3] = component_type;
901 ret_type = LLVMStructTypeInContext(gallivm->context, val_type, 4, 0);
902 } else {
903 ret_type = LLVMVoidTypeInContext(gallivm->context);
904 }
905
906 return LLVMFunctionType(ret_type, arg_types, num_params, false);
907 }
908