1 /* Copyright © 2024 Intel Corporation
2 * SPDX-License-Identifier: MIT
3 */
4
5 #include "anv_private.h"
6
7 static void
anv_bind_buffer_memory(struct anv_device * device,const VkBindBufferMemoryInfo * pBindInfo)8 anv_bind_buffer_memory(struct anv_device *device,
9 const VkBindBufferMemoryInfo *pBindInfo)
10 {
11 ANV_FROM_HANDLE(anv_device_memory, mem, pBindInfo->memory);
12 ANV_FROM_HANDLE(anv_buffer, buffer, pBindInfo->buffer);
13
14 assert(pBindInfo->sType == VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO);
15 assert(!anv_buffer_is_sparse(buffer));
16
17 const VkBindMemoryStatusKHR *bind_status =
18 vk_find_struct_const(pBindInfo->pNext, BIND_MEMORY_STATUS_KHR);
19
20 if (mem) {
21 assert(pBindInfo->memoryOffset < mem->vk.size);
22 assert(mem->vk.size - pBindInfo->memoryOffset >= buffer->vk.size);
23 buffer->address = (struct anv_address) {
24 .bo = mem->bo,
25 .offset = pBindInfo->memoryOffset,
26 };
27 } else {
28 buffer->address = ANV_NULL_ADDRESS;
29 }
30
31 ANV_RMV(buffer_bind, device, buffer);
32
33 if (bind_status)
34 *bind_status->pResult = VK_SUCCESS;
35 }
36
anv_BindBufferMemory2(VkDevice _device,uint32_t bindInfoCount,const VkBindBufferMemoryInfo * pBindInfos)37 VkResult anv_BindBufferMemory2(
38 VkDevice _device,
39 uint32_t bindInfoCount,
40 const VkBindBufferMemoryInfo* pBindInfos)
41 {
42 ANV_FROM_HANDLE(anv_device, device, _device);
43
44 for (uint32_t i = 0; i < bindInfoCount; i++)
45 anv_bind_buffer_memory(device, &pBindInfos[i]);
46
47 return VK_SUCCESS;
48 }
49
50 // Buffer functions
51
52 static void
anv_get_buffer_memory_requirements(struct anv_device * device,VkBufferCreateFlags flags,VkDeviceSize size,VkBufferUsageFlags2KHR usage,bool is_sparse,VkMemoryRequirements2 * pMemoryRequirements)53 anv_get_buffer_memory_requirements(struct anv_device *device,
54 VkBufferCreateFlags flags,
55 VkDeviceSize size,
56 VkBufferUsageFlags2KHR usage,
57 bool is_sparse,
58 VkMemoryRequirements2* pMemoryRequirements)
59 {
60 /* The Vulkan spec (git aaed022) says:
61 *
62 * memoryTypeBits is a bitfield and contains one bit set for every
63 * supported memory type for the resource. The bit `1<<i` is set if and
64 * only if the memory type `i` in the VkPhysicalDeviceMemoryProperties
65 * structure for the physical device is supported.
66 *
67 * We have special memory types for descriptor buffers.
68 */
69 uint32_t memory_types;
70 if (flags & VK_BUFFER_CREATE_PROTECTED_BIT)
71 memory_types = device->physical->memory.protected_mem_types;
72 else if (usage & (VK_BUFFER_USAGE_2_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT |
73 VK_BUFFER_USAGE_2_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT))
74 memory_types = device->physical->memory.dynamic_visible_mem_types;
75 else
76 memory_types = device->physical->memory.default_buffer_mem_types;
77
78 /* The GPU appears to write back to main memory in cachelines. Writes to a
79 * buffers should not clobber with writes to another buffers so make sure
80 * those are in different cachelines.
81 */
82 uint32_t alignment = 64;
83
84 /* From the spec, section "Sparse Buffer and Fully-Resident Image Block
85 * Size":
86 * "The sparse block size in bytes for sparse buffers and fully-resident
87 * images is reported as VkMemoryRequirements::alignment. alignment
88 * represents both the memory alignment requirement and the binding
89 * granularity (in bytes) for sparse resources."
90 */
91 if (is_sparse) {
92 alignment = ANV_SPARSE_BLOCK_SIZE;
93 size = align64(size, alignment);
94 }
95
96 pMemoryRequirements->memoryRequirements.size = size;
97 pMemoryRequirements->memoryRequirements.alignment = alignment;
98
99 /* Storage and Uniform buffers should have their size aligned to
100 * 32-bits to avoid boundary checks when last DWord is not complete.
101 * This would ensure that not internal padding would be needed for
102 * 16-bit types.
103 */
104 if (device->robust_buffer_access &&
105 (usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT ||
106 usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT))
107 pMemoryRequirements->memoryRequirements.size = align64(size, 4);
108
109 pMemoryRequirements->memoryRequirements.memoryTypeBits = memory_types;
110
111 vk_foreach_struct(ext, pMemoryRequirements->pNext) {
112 switch (ext->sType) {
113 case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
114 VkMemoryDedicatedRequirements *requirements = (void *)ext;
115 requirements->prefersDedicatedAllocation = false;
116 requirements->requiresDedicatedAllocation = false;
117 break;
118 }
119
120 default:
121 vk_debug_ignored_stype(ext->sType);
122 break;
123 }
124 }
125 }
126
127 static VkBufferUsageFlags2KHR
get_buffer_usages(const VkBufferCreateInfo * create_info)128 get_buffer_usages(const VkBufferCreateInfo *create_info)
129 {
130 const VkBufferUsageFlags2CreateInfoKHR *usage2_info =
131 vk_find_struct_const(create_info->pNext,
132 BUFFER_USAGE_FLAGS_2_CREATE_INFO_KHR);
133 return usage2_info != NULL ? usage2_info->usage : create_info->usage;
134 }
135
anv_GetDeviceBufferMemoryRequirements(VkDevice _device,const VkDeviceBufferMemoryRequirements * pInfo,VkMemoryRequirements2 * pMemoryRequirements)136 void anv_GetDeviceBufferMemoryRequirements(
137 VkDevice _device,
138 const VkDeviceBufferMemoryRequirements* pInfo,
139 VkMemoryRequirements2* pMemoryRequirements)
140 {
141 ANV_FROM_HANDLE(anv_device, device, _device);
142 const bool is_sparse =
143 pInfo->pCreateInfo->flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT;
144 VkBufferUsageFlags2KHR usages = get_buffer_usages(pInfo->pCreateInfo);
145
146 if ((device->physical->sparse_type == ANV_SPARSE_TYPE_NOT_SUPPORTED) &&
147 INTEL_DEBUG(DEBUG_SPARSE) &&
148 pInfo->pCreateInfo->flags & (VK_BUFFER_CREATE_SPARSE_BINDING_BIT |
149 VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT |
150 VK_BUFFER_CREATE_SPARSE_ALIASED_BIT))
151 fprintf(stderr, "=== %s %s:%d flags:0x%08x\n", __func__, __FILE__,
152 __LINE__, pInfo->pCreateInfo->flags);
153
154 anv_get_buffer_memory_requirements(device,
155 pInfo->pCreateInfo->flags,
156 pInfo->pCreateInfo->size,
157 usages,
158 is_sparse,
159 pMemoryRequirements);
160 }
161
anv_CreateBuffer(VkDevice _device,const VkBufferCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkBuffer * pBuffer)162 VkResult anv_CreateBuffer(
163 VkDevice _device,
164 const VkBufferCreateInfo* pCreateInfo,
165 const VkAllocationCallbacks* pAllocator,
166 VkBuffer* pBuffer)
167 {
168 ANV_FROM_HANDLE(anv_device, device, _device);
169 struct anv_buffer *buffer;
170
171 if ((device->physical->sparse_type == ANV_SPARSE_TYPE_NOT_SUPPORTED) &&
172 INTEL_DEBUG(DEBUG_SPARSE) &&
173 pCreateInfo->flags & (VK_BUFFER_CREATE_SPARSE_BINDING_BIT |
174 VK_BUFFER_CREATE_SPARSE_RESIDENCY_BIT |
175 VK_BUFFER_CREATE_SPARSE_ALIASED_BIT))
176 fprintf(stderr, "=== %s %s:%d flags:0x%08x\n", __func__, __FILE__,
177 __LINE__, pCreateInfo->flags);
178
179 /* Don't allow creating buffers bigger than our address space. The real
180 * issue here is that we may align up the buffer size and we don't want
181 * doing so to cause roll-over. However, no one has any business
182 * allocating a buffer larger than our GTT size.
183 */
184 if (pCreateInfo->size > device->physical->gtt_size)
185 return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
186
187 buffer = vk_buffer_create(&device->vk, pCreateInfo,
188 pAllocator, sizeof(*buffer));
189 if (buffer == NULL)
190 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
191
192 buffer->address = ANV_NULL_ADDRESS;
193 if (anv_buffer_is_sparse(buffer)) {
194 enum anv_bo_alloc_flags alloc_flags = 0;
195 uint64_t client_address = 0;
196
197 if (buffer->vk.create_flags & VK_BUFFER_CREATE_DEVICE_ADDRESS_CAPTURE_REPLAY_BIT) {
198 alloc_flags = ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS;
199 const VkBufferOpaqueCaptureAddressCreateInfo *opaque_addr_info =
200 vk_find_struct_const(pCreateInfo->pNext,
201 BUFFER_OPAQUE_CAPTURE_ADDRESS_CREATE_INFO);
202 if (opaque_addr_info)
203 client_address = opaque_addr_info->opaqueCaptureAddress;
204 }
205
206 if (buffer->vk.create_flags & VK_BUFFER_CREATE_DESCRIPTOR_BUFFER_CAPTURE_REPLAY_BIT_EXT) {
207 alloc_flags = ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS;
208
209 const VkOpaqueCaptureDescriptorDataCreateInfoEXT *opaque_info =
210 vk_find_struct_const(pCreateInfo->pNext,
211 OPAQUE_CAPTURE_DESCRIPTOR_DATA_CREATE_INFO_EXT);
212 if (opaque_info)
213 client_address = *((const uint64_t *)opaque_info->opaqueCaptureDescriptorData);
214 }
215
216 VkResult result = anv_init_sparse_bindings(device, buffer->vk.size,
217 &buffer->sparse_data,
218 alloc_flags, client_address,
219 &buffer->address);
220 if (result != VK_SUCCESS) {
221 vk_buffer_destroy(&device->vk, pAllocator, &buffer->vk);
222 return result;
223 }
224 }
225
226 ANV_RMV(buffer_create, device, false, buffer);
227
228 *pBuffer = anv_buffer_to_handle(buffer);
229
230 return VK_SUCCESS;
231 }
232
anv_DestroyBuffer(VkDevice _device,VkBuffer _buffer,const VkAllocationCallbacks * pAllocator)233 void anv_DestroyBuffer(
234 VkDevice _device,
235 VkBuffer _buffer,
236 const VkAllocationCallbacks* pAllocator)
237 {
238 ANV_FROM_HANDLE(anv_device, device, _device);
239 ANV_FROM_HANDLE(anv_buffer, buffer, _buffer);
240
241 if (!buffer)
242 return;
243
244 ANV_RMV(buffer_destroy, device, buffer);
245
246 if (anv_buffer_is_sparse(buffer)) {
247 assert(buffer->address.offset == buffer->sparse_data.address);
248 anv_free_sparse_bindings(device, &buffer->sparse_data);
249 }
250
251 vk_buffer_destroy(&device->vk, pAllocator, &buffer->vk);
252 }
253
anv_GetBufferDeviceAddress(VkDevice device,const VkBufferDeviceAddressInfo * pInfo)254 VkDeviceAddress anv_GetBufferDeviceAddress(
255 VkDevice device,
256 const VkBufferDeviceAddressInfo* pInfo)
257 {
258 ANV_FROM_HANDLE(anv_buffer, buffer, pInfo->buffer);
259
260 assert(!anv_address_is_null(buffer->address));
261
262 return anv_address_physical(buffer->address);
263 }
264
anv_GetBufferOpaqueCaptureAddress(VkDevice device,const VkBufferDeviceAddressInfo * pInfo)265 uint64_t anv_GetBufferOpaqueCaptureAddress(
266 VkDevice device,
267 const VkBufferDeviceAddressInfo* pInfo)
268 {
269 ANV_FROM_HANDLE(anv_buffer, buffer, pInfo->buffer);
270
271 return anv_address_physical(buffer->address);
272 }
273
anv_GetBufferOpaqueCaptureDescriptorDataEXT(VkDevice device,const VkBufferCaptureDescriptorDataInfoEXT * pInfo,void * pData)274 VkResult anv_GetBufferOpaqueCaptureDescriptorDataEXT(
275 VkDevice device,
276 const VkBufferCaptureDescriptorDataInfoEXT* pInfo,
277 void* pData)
278 {
279 ANV_FROM_HANDLE(anv_buffer, buffer, pInfo->buffer);
280
281 *((uint64_t *)pData) = anv_address_physical(buffer->address);
282
283 return VK_SUCCESS;
284 }
285
anv_GetDeviceMemoryOpaqueCaptureAddress(VkDevice device,const VkDeviceMemoryOpaqueCaptureAddressInfo * pInfo)286 uint64_t anv_GetDeviceMemoryOpaqueCaptureAddress(
287 VkDevice device,
288 const VkDeviceMemoryOpaqueCaptureAddressInfo* pInfo)
289 {
290 ANV_FROM_HANDLE(anv_device_memory, memory, pInfo->memory);
291
292 assert(memory->bo->alloc_flags & ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS);
293
294 return intel_48b_address(memory->bo->offset);
295 }
296
297 void
anv_fill_buffer_surface_state(struct anv_device * device,void * surface_state_ptr,enum isl_format format,struct isl_swizzle swizzle,isl_surf_usage_flags_t usage,struct anv_address address,uint32_t range,uint32_t stride)298 anv_fill_buffer_surface_state(struct anv_device *device,
299 void *surface_state_ptr,
300 enum isl_format format,
301 struct isl_swizzle swizzle,
302 isl_surf_usage_flags_t usage,
303 struct anv_address address,
304 uint32_t range, uint32_t stride)
305 {
306 if (address.bo && address.bo->alloc_flags & ANV_BO_ALLOC_PROTECTED)
307 usage |= ISL_SURF_USAGE_PROTECTED_BIT;
308 isl_buffer_fill_state(&device->isl_dev, surface_state_ptr,
309 .address = anv_address_physical(address),
310 .mocs = isl_mocs(&device->isl_dev, usage,
311 address.bo && anv_bo_is_external(address.bo)),
312 .size_B = range,
313 .format = format,
314 .swizzle = swizzle,
315 .stride_B = stride);
316 }
317