1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2
3 #include "rseq-bits-template.h"
4
5 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
6 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
7
8 static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)9 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv,
10 int cpu)
11 {
12 RSEQ_INJECT_C(9)
13
14 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
15 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
16 #ifdef RSEQ_COMPARE_TWICE
17 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
18 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
19 #endif
20 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
21 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
22 RSEQ_INJECT_ASM(3)
23 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
24 RSEQ_INJECT_ASM(4)
25 #ifdef RSEQ_COMPARE_TWICE
26 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
27 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
28 #endif
29 RSEQ_ASM_OP_FINAL_STORE(v, newv, 3)
30 RSEQ_INJECT_ASM(5)
31 RSEQ_ASM_DEFINE_ABORT(4, abort)
32 : /* gcc asm goto does not allow outputs */
33 : [cpu_id] "r" (cpu),
34 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
35 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
36 [v] "m" (*v),
37 [expect] "r" (expect),
38 [newv] "r" (newv)
39 RSEQ_INJECT_INPUT
40 : "memory", RSEQ_ASM_TMP_REG_1
41 RSEQ_INJECT_CLOBBER
42 : abort, cmpfail
43 #ifdef RSEQ_COMPARE_TWICE
44 , error1, error2
45 #endif
46 );
47
48 return 0;
49 abort:
50 RSEQ_INJECT_FAILED
51 return -1;
52 cmpfail:
53 return 1;
54 #ifdef RSEQ_COMPARE_TWICE
55 error1:
56 rseq_bug("cpu_id comparison failed");
57 error2:
58 rseq_bug("expected value comparison failed");
59 #endif
60 }
61
62 static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)63 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
64 off_t voffp, intptr_t *load, int cpu)
65 {
66 RSEQ_INJECT_C(9)
67
68 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
69 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
70 #ifdef RSEQ_COMPARE_TWICE
71 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
72 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
73 #endif
74 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
75 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
76 RSEQ_INJECT_ASM(3)
77 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]")
78 RSEQ_INJECT_ASM(4)
79 #ifdef RSEQ_COMPARE_TWICE
80 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
81 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]")
82 #endif
83 RSEQ_ASM_OP_R_LOAD(v)
84 RSEQ_ASM_OP_R_STORE(load)
85 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
86 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
87 RSEQ_INJECT_ASM(5)
88 RSEQ_ASM_DEFINE_ABORT(4, abort)
89 : /* gcc asm goto does not allow outputs */
90 : [cpu_id] "r" (cpu),
91 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
92 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
93 [v] "m" (*v),
94 [expectnot] "r" (expectnot),
95 [load] "m" (*load),
96 [voffp] "Ir" (voffp)
97 RSEQ_INJECT_INPUT
98 : "memory", RSEQ_ASM_TMP_REG_1
99 RSEQ_INJECT_CLOBBER
100 : abort, cmpfail
101 #ifdef RSEQ_COMPARE_TWICE
102 , error1, error2
103 #endif
104 );
105 return 0;
106 abort:
107 RSEQ_INJECT_FAILED
108 return -1;
109 cmpfail:
110 return 1;
111 #ifdef RSEQ_COMPARE_TWICE
112 error1:
113 rseq_bug("cpu_id comparison failed");
114 error2:
115 rseq_bug("expected value comparison failed");
116 #endif
117 }
118
119 static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)120 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
121 {
122 RSEQ_INJECT_C(9)
123
124 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
125 #ifdef RSEQ_COMPARE_TWICE
126 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
127 #endif
128 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
129 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
130 RSEQ_INJECT_ASM(3)
131 #ifdef RSEQ_COMPARE_TWICE
132 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
133 #endif
134 RSEQ_ASM_OP_R_LOAD(v)
135 RSEQ_ASM_OP_R_ADD(count)
136 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
137 RSEQ_INJECT_ASM(4)
138 RSEQ_ASM_DEFINE_ABORT(4, abort)
139 : /* gcc asm goto does not allow outputs */
140 : [cpu_id] "r" (cpu),
141 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
142 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
143 [v] "m" (*v),
144 [count] "r" (count)
145 RSEQ_INJECT_INPUT
146 : "memory", RSEQ_ASM_TMP_REG_1
147 RSEQ_INJECT_CLOBBER
148 : abort
149 #ifdef RSEQ_COMPARE_TWICE
150 , error1
151 #endif
152 );
153 return 0;
154 abort:
155 RSEQ_INJECT_FAILED
156 return -1;
157 #ifdef RSEQ_COMPARE_TWICE
158 error1:
159 rseq_bug("cpu_id comparison failed");
160 #endif
161 }
162
163 static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)164 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
165 intptr_t *v2, intptr_t expect2,
166 intptr_t newv, int cpu)
167 {
168 RSEQ_INJECT_C(9)
169
170 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
171 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
172 #ifdef RSEQ_COMPARE_TWICE
173 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
174 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
175 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]")
176 #endif
177 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
178 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
179 RSEQ_INJECT_ASM(3)
180 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
181 RSEQ_INJECT_ASM(4)
182 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]")
183 RSEQ_INJECT_ASM(5)
184 #ifdef RSEQ_COMPARE_TWICE
185 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
186 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
187 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]")
188 #endif
189 RSEQ_ASM_OP_FINAL_STORE(v, newv, 3)
190 RSEQ_INJECT_ASM(6)
191 RSEQ_ASM_DEFINE_ABORT(4, abort)
192 : /* gcc asm goto does not allow outputs */
193 : [cpu_id] "r" (cpu),
194 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
195 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
196 [v] "m" (*v),
197 [expect] "r" (expect),
198 [v2] "m" (*v2),
199 [expect2] "r" (expect2),
200 [newv] "r" (newv)
201 RSEQ_INJECT_INPUT
202 : "memory", RSEQ_ASM_TMP_REG_1
203 RSEQ_INJECT_CLOBBER
204 : abort, cmpfail
205 #ifdef RSEQ_COMPARE_TWICE
206 , error1, error2, error3
207 #endif
208 );
209
210 return 0;
211 abort:
212 RSEQ_INJECT_FAILED
213 return -1;
214 cmpfail:
215 return 1;
216 #ifdef RSEQ_COMPARE_TWICE
217 error1:
218 rseq_bug("cpu_id comparison failed");
219 error2:
220 rseq_bug("expected value comparison failed");
221 error3:
222 rseq_bug("2nd expected value comparison failed");
223 #endif
224 }
225
226 #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
227
228 /*
229 * pval = *(ptr+off)
230 * *pval += inc;
231 */
232 static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)233 int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, intptr_t inc,
234 int cpu)
235 {
236 RSEQ_INJECT_C(9)
237
238 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
239 #ifdef RSEQ_COMPARE_TWICE
240 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
241 #endif
242 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
243 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
244 RSEQ_INJECT_ASM(3)
245 #ifdef RSEQ_COMPARE_TWICE
246 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
247 #endif
248 RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, inc, 3)
249 RSEQ_INJECT_ASM(4)
250 RSEQ_ASM_DEFINE_ABORT(4, abort)
251 : /* gcc asm goto does not allow outputs */
252 : [cpu_id] "r" (cpu),
253 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
254 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
255 [ptr] "r" (ptr),
256 [off] "r" (off),
257 [inc] "r" (inc)
258 RSEQ_INJECT_INPUT
259 : "memory", RSEQ_ASM_TMP_REG_1
260 RSEQ_INJECT_CLOBBER
261 : abort
262 #ifdef RSEQ_COMPARE_TWICE
263 , error1
264 #endif
265 );
266 return 0;
267 abort:
268 RSEQ_INJECT_FAILED
269 return -1;
270 #ifdef RSEQ_COMPARE_TWICE
271 error1:
272 rseq_bug("cpu_id comparison failed");
273 #endif
274 }
275
276 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
277 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
278
279 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
280 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
281
282 static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)283 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
284 intptr_t *v2, intptr_t newv2,
285 intptr_t newv, int cpu)
286 {
287 RSEQ_INJECT_C(9)
288
289 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
290 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
291 #ifdef RSEQ_COMPARE_TWICE
292 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
293 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
294 #endif
295 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
296 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
297 RSEQ_INJECT_ASM(3)
298 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
299 RSEQ_INJECT_ASM(4)
300 #ifdef RSEQ_COMPARE_TWICE
301 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
302 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
303 #endif
304 RSEQ_ASM_OP_STORE(v2, newv2)
305 RSEQ_INJECT_ASM(5)
306 #ifdef RSEQ_TEMPLATE_MO_RELEASE
307 RSEQ_ASM_OP_FINAL_STORE_RELEASE(v, newv, 3)
308 #else
309 RSEQ_ASM_OP_FINAL_STORE(v, newv, 3)
310 #endif
311 RSEQ_INJECT_ASM(6)
312 RSEQ_ASM_DEFINE_ABORT(4, abort)
313 : /* gcc asm goto does not allow outputs */
314 : [cpu_id] "r" (cpu),
315 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
316 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
317 [expect] "r" (expect),
318 [v] "m" (*v),
319 [newv] "r" (newv),
320 [v2] "m" (*v2),
321 [newv2] "r" (newv2)
322 RSEQ_INJECT_INPUT
323 : "memory", RSEQ_ASM_TMP_REG_1
324 RSEQ_INJECT_CLOBBER
325 : abort, cmpfail
326 #ifdef RSEQ_COMPARE_TWICE
327 , error1, error2
328 #endif
329 );
330
331 return 0;
332 abort:
333 RSEQ_INJECT_FAILED
334 return -1;
335 cmpfail:
336 return 1;
337 #ifdef RSEQ_COMPARE_TWICE
338 error1:
339 rseq_bug("cpu_id comparison failed");
340 error2:
341 rseq_bug("expected value comparison failed");
342 #endif
343 }
344
345 static inline __always_inline
RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)346 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
347 void *dst, void *src, size_t len,
348 intptr_t newv, int cpu)
349 {
350 RSEQ_INJECT_C(9)
351 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
352 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
353 #ifdef RSEQ_COMPARE_TWICE
354 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
355 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
356 #endif
357 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
358 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
359 RSEQ_INJECT_ASM(3)
360 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
361 RSEQ_INJECT_ASM(4)
362 #ifdef RSEQ_COMPARE_TWICE
363 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
364 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
365 #endif
366 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
367 RSEQ_INJECT_ASM(5)
368 #ifdef RSEQ_TEMPLATE_MO_RELEASE
369 RSEQ_ASM_OP_FINAL_STORE_RELEASE(v, newv, 3)
370 #else
371 RSEQ_ASM_OP_FINAL_STORE(v, newv, 3)
372 #endif
373 RSEQ_INJECT_ASM(6)
374 RSEQ_ASM_DEFINE_ABORT(4, abort)
375 : /* gcc asm goto does not allow outputs */
376 : [cpu_id] "r" (cpu),
377 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
378 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
379 [expect] "r" (expect),
380 [v] "m" (*v),
381 [newv] "r" (newv),
382 [dst] "r" (dst),
383 [src] "r" (src),
384 [len] "r" (len)
385 RSEQ_INJECT_INPUT
386 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
387 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
388 RSEQ_INJECT_CLOBBER
389 : abort, cmpfail
390 #ifdef RSEQ_COMPARE_TWICE
391 , error1, error2
392 #endif
393 );
394
395 return 0;
396 abort:
397 RSEQ_INJECT_FAILED
398 return -1;
399 cmpfail:
400 return 1;
401 #ifdef RSEQ_COMPARE_TWICE
402 error1:
403 rseq_bug("cpu_id comparison failed");
404 error2:
405 rseq_bug("expected value comparison failed");
406 #endif
407 }
408
409 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
410 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
411
412 #include "rseq-bits-reset.h"
413