1 /*
2 * Copyright © 2021 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "anv_private.h"
25
26 #include "util/os_time.h"
27 #include "util/perf/cpu_trace.h"
28
29 static struct anv_bo_sync *
to_anv_bo_sync(struct vk_sync * sync)30 to_anv_bo_sync(struct vk_sync *sync)
31 {
32 assert(sync->type == &anv_bo_sync_type);
33 return container_of(sync, struct anv_bo_sync, sync);
34 }
35
36 static VkResult
anv_bo_sync_init(struct vk_device * vk_device,struct vk_sync * vk_sync,uint64_t initial_value)37 anv_bo_sync_init(struct vk_device *vk_device,
38 struct vk_sync *vk_sync,
39 uint64_t initial_value)
40 {
41 struct anv_device *device = container_of(vk_device, struct anv_device, vk);
42 struct anv_bo_sync *sync = to_anv_bo_sync(vk_sync);
43
44 sync->state = initial_value ? ANV_BO_SYNC_STATE_SIGNALED :
45 ANV_BO_SYNC_STATE_RESET;
46
47 return anv_device_alloc_bo(device, "bo-sync", 4096,
48 ANV_BO_ALLOC_EXTERNAL |
49 ANV_BO_ALLOC_IMPLICIT_SYNC |
50 ANV_BO_ALLOC_INTERNAL,
51 0 /* explicit_address */,
52 &sync->bo);
53 }
54
55 static void
anv_bo_sync_finish(struct vk_device * vk_device,struct vk_sync * vk_sync)56 anv_bo_sync_finish(struct vk_device *vk_device,
57 struct vk_sync *vk_sync)
58 {
59 struct anv_device *device = container_of(vk_device, struct anv_device, vk);
60 struct anv_bo_sync *sync = to_anv_bo_sync(vk_sync);
61
62 anv_device_release_bo(device, sync->bo);
63 }
64
65 static VkResult
anv_bo_sync_reset(struct vk_device * vk_device,struct vk_sync * vk_sync)66 anv_bo_sync_reset(struct vk_device *vk_device,
67 struct vk_sync *vk_sync)
68 {
69 struct anv_bo_sync *sync = to_anv_bo_sync(vk_sync);
70
71 sync->state = ANV_BO_SYNC_STATE_RESET;
72
73 return VK_SUCCESS;
74 }
75
76 static int64_t
anv_get_relative_timeout(uint64_t abs_timeout)77 anv_get_relative_timeout(uint64_t abs_timeout)
78 {
79 uint64_t now = os_time_get_nano();
80
81 /* We don't want negative timeouts.
82 *
83 * DRM_IOCTL_I915_GEM_WAIT uses a signed 64 bit timeout and is
84 * supposed to block indefinitely timeouts < 0. Unfortunately,
85 * this was broken for a couple of kernel releases. Since there's
86 * no way to know whether or not the kernel we're using is one of
87 * the broken ones, the best we can do is to clamp the timeout to
88 * INT64_MAX. This limits the maximum timeout from 584 years to
89 * 292 years - likely not a big deal.
90 */
91 if (abs_timeout < now)
92 return 0;
93
94 uint64_t rel_timeout = abs_timeout - now;
95 if (rel_timeout > (uint64_t) INT64_MAX)
96 rel_timeout = INT64_MAX;
97
98 return rel_timeout;
99 }
100
101 static VkResult
anv_bo_sync_wait(struct vk_device * vk_device,uint32_t wait_count,const struct vk_sync_wait * waits,enum vk_sync_wait_flags wait_flags,uint64_t abs_timeout_ns)102 anv_bo_sync_wait(struct vk_device *vk_device,
103 uint32_t wait_count,
104 const struct vk_sync_wait *waits,
105 enum vk_sync_wait_flags wait_flags,
106 uint64_t abs_timeout_ns)
107 {
108 struct anv_device *device = container_of(vk_device, struct anv_device, vk);
109 VkResult result;
110 MESA_TRACE_FUNC();
111
112 uint32_t pending = wait_count;
113 while (pending) {
114 pending = 0;
115 bool signaled = false;
116 for (uint32_t i = 0; i < wait_count; i++) {
117 struct anv_bo_sync *sync = to_anv_bo_sync(waits[i].sync);
118 switch (sync->state) {
119 case ANV_BO_SYNC_STATE_RESET:
120 /* This fence hasn't been submitted yet, we'll catch it the next
121 * time around. Yes, this may mean we dead-loop but, short of
122 * lots of locking and a condition variable, there's not much that
123 * we can do about that.
124 */
125 assert(!(wait_flags & VK_SYNC_WAIT_PENDING));
126 pending++;
127 continue;
128
129 case ANV_BO_SYNC_STATE_SIGNALED:
130 /* This fence is not pending. If waitAll isn't set, we can return
131 * early. Otherwise, we have to keep going.
132 */
133 if (wait_flags & VK_SYNC_WAIT_ANY)
134 return VK_SUCCESS;
135 continue;
136
137 case ANV_BO_SYNC_STATE_SUBMITTED:
138 /* These are the fences we really care about. Go ahead and wait
139 * on it until we hit a timeout.
140 */
141 if (!(wait_flags & VK_SYNC_WAIT_PENDING)) {
142 uint64_t rel_timeout = anv_get_relative_timeout(abs_timeout_ns);
143 result = anv_device_wait(device, sync->bo, rel_timeout);
144 /* This also covers VK_TIMEOUT */
145 if (result != VK_SUCCESS)
146 return result;
147
148 sync->state = ANV_BO_SYNC_STATE_SIGNALED;
149 signaled = true;
150 }
151 if (wait_flags & VK_SYNC_WAIT_ANY)
152 return VK_SUCCESS;
153 break;
154
155 default:
156 unreachable("Invalid BO sync state");
157 }
158 }
159
160 if (pending && !signaled) {
161 /* If we've hit this then someone decided to vkWaitForFences before
162 * they've actually submitted any of them to a queue. This is a
163 * fairly pessimal case, so it's ok to lock here and use a standard
164 * pthreads condition variable.
165 */
166 pthread_mutex_lock(&device->mutex);
167
168 /* It's possible that some of the fences have changed state since the
169 * last time we checked. Now that we have the lock, check for
170 * pending fences again and don't wait if it's changed.
171 */
172 uint32_t now_pending = 0;
173 for (uint32_t i = 0; i < wait_count; i++) {
174 struct anv_bo_sync *sync = to_anv_bo_sync(waits[i].sync);
175 if (sync->state == ANV_BO_SYNC_STATE_RESET)
176 now_pending++;
177 }
178 assert(now_pending <= pending);
179
180 if (now_pending == pending) {
181 struct timespec abstime = {
182 .tv_sec = abs_timeout_ns / NSEC_PER_SEC,
183 .tv_nsec = abs_timeout_ns % NSEC_PER_SEC,
184 };
185
186 ASSERTED int ret;
187 ret = pthread_cond_timedwait(&device->queue_submit,
188 &device->mutex, &abstime);
189 assert(ret != EINVAL);
190 if (os_time_get_nano() >= abs_timeout_ns) {
191 pthread_mutex_unlock(&device->mutex);
192 return VK_TIMEOUT;
193 }
194 }
195
196 pthread_mutex_unlock(&device->mutex);
197 }
198 }
199
200 return VK_SUCCESS;
201 }
202
203 const struct vk_sync_type anv_bo_sync_type = {
204 .size = sizeof(struct anv_bo_sync),
205 .features = VK_SYNC_FEATURE_BINARY |
206 VK_SYNC_FEATURE_GPU_WAIT |
207 VK_SYNC_FEATURE_GPU_MULTI_WAIT |
208 VK_SYNC_FEATURE_CPU_WAIT |
209 VK_SYNC_FEATURE_CPU_RESET |
210 VK_SYNC_FEATURE_WAIT_ANY |
211 VK_SYNC_FEATURE_WAIT_PENDING,
212 .init = anv_bo_sync_init,
213 .finish = anv_bo_sync_finish,
214 .reset = anv_bo_sync_reset,
215 .wait_many = anv_bo_sync_wait,
216 };
217
218 VkResult
anv_create_sync_for_memory(struct vk_device * device,VkDeviceMemory memory,bool signal_memory,struct vk_sync ** sync_out)219 anv_create_sync_for_memory(struct vk_device *device,
220 VkDeviceMemory memory,
221 bool signal_memory,
222 struct vk_sync **sync_out)
223 {
224 ANV_FROM_HANDLE(anv_device_memory, mem, memory);
225 struct anv_bo_sync *bo_sync;
226
227 bo_sync = vk_zalloc(&device->alloc, sizeof(*bo_sync), 8,
228 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
229 if (bo_sync == NULL)
230 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
231
232 bo_sync->sync.type = &anv_bo_sync_type;
233 bo_sync->state = signal_memory ? ANV_BO_SYNC_STATE_RESET :
234 ANV_BO_SYNC_STATE_SUBMITTED;
235 bo_sync->bo = anv_bo_ref(mem->bo);
236
237 *sync_out = &bo_sync->sync;
238
239 return VK_SUCCESS;
240 }
241