1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
4 * Copyright (C) 2012 Regents of the University of California
5 * Copyright (C) 2017 SiFive
6 */
7
8 #ifndef _ASM_RISCV_ATOMIC_H
9 #define _ASM_RISCV_ATOMIC_H
10
11 #ifdef CONFIG_GENERIC_ATOMIC64
12 # include <asm-generic/atomic64.h>
13 #else
14 # if (__riscv_xlen < 64)
15 # error "64-bit atomics require XLEN to be at least 64"
16 # endif
17 #endif
18
19 #include <asm/cmpxchg.h>
20
21 #define __atomic_acquire_fence() \
22 __asm__ __volatile__(RISCV_ACQUIRE_BARRIER "" ::: "memory")
23
24 #define __atomic_release_fence() \
25 __asm__ __volatile__(RISCV_RELEASE_BARRIER "" ::: "memory");
26
arch_atomic_read(const atomic_t * v)27 static __always_inline int arch_atomic_read(const atomic_t *v)
28 {
29 return READ_ONCE(v->counter);
30 }
arch_atomic_set(atomic_t * v,int i)31 static __always_inline void arch_atomic_set(atomic_t *v, int i)
32 {
33 WRITE_ONCE(v->counter, i);
34 }
35
36 #ifndef CONFIG_GENERIC_ATOMIC64
37 #define ATOMIC64_INIT(i) { (i) }
arch_atomic64_read(const atomic64_t * v)38 static __always_inline s64 arch_atomic64_read(const atomic64_t *v)
39 {
40 return READ_ONCE(v->counter);
41 }
arch_atomic64_set(atomic64_t * v,s64 i)42 static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
43 {
44 WRITE_ONCE(v->counter, i);
45 }
46 #endif
47
48 /*
49 * First, the atomic ops that have no ordering constraints and therefor don't
50 * have the AQ or RL bits set. These don't return anything, so there's only
51 * one version to worry about.
52 */
53 #define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \
54 static __always_inline \
55 void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
56 { \
57 __asm__ __volatile__ ( \
58 " amo" #asm_op "." #asm_type " zero, %1, %0" \
59 : "+A" (v->counter) \
60 : "r" (I) \
61 : "memory"); \
62 } \
63
64 #ifdef CONFIG_GENERIC_ATOMIC64
65 #define ATOMIC_OPS(op, asm_op, I) \
66 ATOMIC_OP (op, asm_op, I, w, int, )
67 #else
68 #define ATOMIC_OPS(op, asm_op, I) \
69 ATOMIC_OP (op, asm_op, I, w, int, ) \
70 ATOMIC_OP (op, asm_op, I, d, s64, 64)
71 #endif
72
ATOMIC_OPS(add,add,i)73 ATOMIC_OPS(add, add, i)
74 ATOMIC_OPS(sub, add, -i)
75 ATOMIC_OPS(and, and, i)
76 ATOMIC_OPS( or, or, i)
77 ATOMIC_OPS(xor, xor, i)
78
79 #undef ATOMIC_OP
80 #undef ATOMIC_OPS
81
82 /*
83 * Atomic ops that have ordered, relaxed, acquire, and release variants.
84 * There's two flavors of these: the arithmatic ops have both fetch and return
85 * versions, while the logical ops only have fetch versions.
86 */
87 #define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \
88 static __always_inline \
89 c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i, \
90 atomic##prefix##_t *v) \
91 { \
92 register c_type ret; \
93 __asm__ __volatile__ ( \
94 " amo" #asm_op "." #asm_type " %1, %2, %0" \
95 : "+A" (v->counter), "=r" (ret) \
96 : "r" (I) \
97 : "memory"); \
98 return ret; \
99 } \
100 static __always_inline \
101 c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \
102 { \
103 register c_type ret; \
104 __asm__ __volatile__ ( \
105 " amo" #asm_op "." #asm_type ".aqrl %1, %2, %0" \
106 : "+A" (v->counter), "=r" (ret) \
107 : "r" (I) \
108 : "memory"); \
109 return ret; \
110 }
111
112 #define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix) \
113 static __always_inline \
114 c_type arch_atomic##prefix##_##op##_return_relaxed(c_type i, \
115 atomic##prefix##_t *v) \
116 { \
117 return arch_atomic##prefix##_fetch_##op##_relaxed(i, v) c_op I; \
118 } \
119 static __always_inline \
120 c_type arch_atomic##prefix##_##op##_return(c_type i, atomic##prefix##_t *v) \
121 { \
122 return arch_atomic##prefix##_fetch_##op(i, v) c_op I; \
123 }
124
125 #ifdef CONFIG_GENERIC_ATOMIC64
126 #define ATOMIC_OPS(op, asm_op, c_op, I) \
127 ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \
128 ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, )
129 #else
130 #define ATOMIC_OPS(op, asm_op, c_op, I) \
131 ATOMIC_FETCH_OP( op, asm_op, I, w, int, ) \
132 ATOMIC_OP_RETURN(op, asm_op, c_op, I, w, int, ) \
133 ATOMIC_FETCH_OP( op, asm_op, I, d, s64, 64) \
134 ATOMIC_OP_RETURN(op, asm_op, c_op, I, d, s64, 64)
135 #endif
136
137 ATOMIC_OPS(add, add, +, i)
138 ATOMIC_OPS(sub, add, +, -i)
139
140 #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
141 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
142 #define arch_atomic_add_return arch_atomic_add_return
143 #define arch_atomic_sub_return arch_atomic_sub_return
144
145 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
146 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
147 #define arch_atomic_fetch_add arch_atomic_fetch_add
148 #define arch_atomic_fetch_sub arch_atomic_fetch_sub
149
150 #ifndef CONFIG_GENERIC_ATOMIC64
151 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed
152 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed
153 #define arch_atomic64_add_return arch_atomic64_add_return
154 #define arch_atomic64_sub_return arch_atomic64_sub_return
155
156 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed
157 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed
158 #define arch_atomic64_fetch_add arch_atomic64_fetch_add
159 #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
160 #endif
161
162 #undef ATOMIC_OPS
163
164 #ifdef CONFIG_GENERIC_ATOMIC64
165 #define ATOMIC_OPS(op, asm_op, I) \
166 ATOMIC_FETCH_OP(op, asm_op, I, w, int, )
167 #else
168 #define ATOMIC_OPS(op, asm_op, I) \
169 ATOMIC_FETCH_OP(op, asm_op, I, w, int, ) \
170 ATOMIC_FETCH_OP(op, asm_op, I, d, s64, 64)
171 #endif
172
173 ATOMIC_OPS(and, and, i)
174 ATOMIC_OPS( or, or, i)
175 ATOMIC_OPS(xor, xor, i)
176
177 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
178 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
179 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
180 #define arch_atomic_fetch_and arch_atomic_fetch_and
181 #define arch_atomic_fetch_or arch_atomic_fetch_or
182 #define arch_atomic_fetch_xor arch_atomic_fetch_xor
183
184 #ifndef CONFIG_GENERIC_ATOMIC64
185 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
186 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed
187 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
188 #define arch_atomic64_fetch_and arch_atomic64_fetch_and
189 #define arch_atomic64_fetch_or arch_atomic64_fetch_or
190 #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
191 #endif
192
193 #undef ATOMIC_OPS
194
195 #undef ATOMIC_FETCH_OP
196 #undef ATOMIC_OP_RETURN
197
198 #define _arch_atomic_fetch_add_unless(_prev, _rc, counter, _a, _u, sfx) \
199 ({ \
200 __asm__ __volatile__ ( \
201 "0: lr." sfx " %[p], %[c]\n" \
202 " beq %[p], %[u], 1f\n" \
203 " add %[rc], %[p], %[a]\n" \
204 " sc." sfx ".rl %[rc], %[rc], %[c]\n" \
205 " bnez %[rc], 0b\n" \
206 " fence rw, rw\n" \
207 "1:\n" \
208 : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \
209 : [a]"r" (_a), [u]"r" (_u) \
210 : "memory"); \
211 })
212
213 /* This is required to provide a full barrier on success. */
214 static __always_inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
215 {
216 int prev, rc;
217
218 _arch_atomic_fetch_add_unless(prev, rc, v->counter, a, u, "w");
219
220 return prev;
221 }
222 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
223
224 #ifndef CONFIG_GENERIC_ATOMIC64
arch_atomic64_fetch_add_unless(atomic64_t * v,s64 a,s64 u)225 static __always_inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
226 {
227 s64 prev;
228 long rc;
229
230 _arch_atomic_fetch_add_unless(prev, rc, v->counter, a, u, "d");
231
232 return prev;
233 }
234 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
235 #endif
236
237 #define _arch_atomic_inc_unless_negative(_prev, _rc, counter, sfx) \
238 ({ \
239 __asm__ __volatile__ ( \
240 "0: lr." sfx " %[p], %[c]\n" \
241 " bltz %[p], 1f\n" \
242 " addi %[rc], %[p], 1\n" \
243 " sc." sfx ".rl %[rc], %[rc], %[c]\n" \
244 " bnez %[rc], 0b\n" \
245 " fence rw, rw\n" \
246 "1:\n" \
247 : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \
248 : \
249 : "memory"); \
250 })
251
arch_atomic_inc_unless_negative(atomic_t * v)252 static __always_inline bool arch_atomic_inc_unless_negative(atomic_t *v)
253 {
254 int prev, rc;
255
256 _arch_atomic_inc_unless_negative(prev, rc, v->counter, "w");
257
258 return !(prev < 0);
259 }
260
261 #define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
262
263 #define _arch_atomic_dec_unless_positive(_prev, _rc, counter, sfx) \
264 ({ \
265 __asm__ __volatile__ ( \
266 "0: lr." sfx " %[p], %[c]\n" \
267 " bgtz %[p], 1f\n" \
268 " addi %[rc], %[p], -1\n" \
269 " sc." sfx ".rl %[rc], %[rc], %[c]\n" \
270 " bnez %[rc], 0b\n" \
271 " fence rw, rw\n" \
272 "1:\n" \
273 : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \
274 : \
275 : "memory"); \
276 })
277
arch_atomic_dec_unless_positive(atomic_t * v)278 static __always_inline bool arch_atomic_dec_unless_positive(atomic_t *v)
279 {
280 int prev, rc;
281
282 _arch_atomic_dec_unless_positive(prev, rc, v->counter, "w");
283
284 return !(prev > 0);
285 }
286
287 #define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
288
289 #define _arch_atomic_dec_if_positive(_prev, _rc, counter, sfx) \
290 ({ \
291 __asm__ __volatile__ ( \
292 "0: lr." sfx " %[p], %[c]\n" \
293 " addi %[rc], %[p], -1\n" \
294 " bltz %[rc], 1f\n" \
295 " sc." sfx ".rl %[rc], %[rc], %[c]\n" \
296 " bnez %[rc], 0b\n" \
297 " fence rw, rw\n" \
298 "1:\n" \
299 : [p]"=&r" (_prev), [rc]"=&r" (_rc), [c]"+A" (counter) \
300 : \
301 : "memory"); \
302 })
303
arch_atomic_dec_if_positive(atomic_t * v)304 static __always_inline int arch_atomic_dec_if_positive(atomic_t *v)
305 {
306 int prev, rc;
307
308 _arch_atomic_dec_if_positive(prev, rc, v->counter, "w");
309
310 return prev - 1;
311 }
312
313 #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
314
315 #ifndef CONFIG_GENERIC_ATOMIC64
arch_atomic64_inc_unless_negative(atomic64_t * v)316 static __always_inline bool arch_atomic64_inc_unless_negative(atomic64_t *v)
317 {
318 s64 prev;
319 long rc;
320
321 _arch_atomic_inc_unless_negative(prev, rc, v->counter, "d");
322
323 return !(prev < 0);
324 }
325
326 #define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
327
arch_atomic64_dec_unless_positive(atomic64_t * v)328 static __always_inline bool arch_atomic64_dec_unless_positive(atomic64_t *v)
329 {
330 s64 prev;
331 long rc;
332
333 _arch_atomic_dec_unless_positive(prev, rc, v->counter, "d");
334
335 return !(prev > 0);
336 }
337
338 #define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
339
arch_atomic64_dec_if_positive(atomic64_t * v)340 static __always_inline s64 arch_atomic64_dec_if_positive(atomic64_t *v)
341 {
342 s64 prev;
343 long rc;
344
345 _arch_atomic_dec_if_positive(prev, rc, v->counter, "d");
346
347 return prev - 1;
348 }
349
350 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
351 #endif
352
353 #endif /* _ASM_RISCV_ATOMIC_H */
354