1 #ifndef __CB_BDK_ATOMIC_H__
2 #define __CB_BDK_ATOMIC_H__
3 /***********************license start***********************************
4 * Copyright (c) 2003-2017 Cavium Inc. ([email protected]). All rights
5 * reserved.
6 *
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are
10 * met:
11 *
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 *
15 * * Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer in the documentation and/or other materials provided
18 * with the distribution.
19 *
20 * * Neither the name of Cavium Inc. nor the names of
21 * its contributors may be used to endorse or promote products
22 * derived from this software without specific prior written
23 * permission.
24 *
25 * This Software, including technical data, may be subject to U.S. export
26 * control laws, including the U.S. Export Administration Act and its
27 * associated regulations, and may be subject to export or import
28 * regulations in other countries.
29 *
30 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
31 * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR
32 * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT
33 * TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY
34 * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT
35 * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES
36 * OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR
37 * PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT,
38 * QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK
39 * ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
40 ***********************license end**************************************/
41
42 /**
43 * @file
44 *
45 * This file provides atomic operations
46 *
47 * @addtogroup hal
48 * @{
49 */
50
51 /**
52 * Atomically adds a signed value to a 32 bit (aligned) memory location.
53 *
54 * This version does not perform 'sync' operations to enforce memory
55 * operations. This should only be used when there are no memory operation
56 * ordering constraints. (This should NOT be used for reference counting -
57 * use the standard version instead.)
58 *
59 * @param ptr address in memory to add incr to
60 * @param incr amount to increment memory location by (signed)
61 */
bdk_atomic_add32_nosync(int32_t * ptr,int32_t incr)62 static inline void bdk_atomic_add32_nosync(int32_t *ptr, int32_t incr)
63 {
64 /* Atomic add with no ordering */
65 asm volatile ("ldadd %w[i], wzr, [%[b]]"
66 : [r] "+m" (*ptr)
67 : [i] "r" (incr), [b] "r" (ptr)
68 : "memory");
69 }
70
71 /**
72 * Atomically adds a signed value to a 32 bit (aligned) memory location.
73 *
74 * Memory access ordering is enforced before/after the atomic operation,
75 * so no additional 'sync' instructions are required.
76 *
77 *
78 * @param ptr address in memory to add incr to
79 * @param incr amount to increment memory location by (signed)
80 */
bdk_atomic_add32(int32_t * ptr,int32_t incr)81 static inline void bdk_atomic_add32(int32_t *ptr, int32_t incr)
82 {
83 /* Atomic add with acquire and release */
84 asm volatile ("ldaddal %w[i], wzr, [%[b]]"
85 : "+m" (*ptr)
86 : [i] "r" (incr), [b] "r" (ptr)
87 : "memory");
88 }
89
90 /**
91 * Atomically sets a 32 bit (aligned) memory location to a value
92 *
93 * @param ptr address of memory to set
94 * @param value value to set memory location to.
95 */
bdk_atomic_set32(int32_t * ptr,int32_t value)96 static inline void bdk_atomic_set32(int32_t *ptr, int32_t value)
97 {
98 /* Implies a release */
99 asm volatile ("stlr %w[v], [%[b]]"
100 : "+m" (*ptr)
101 : [v] "r" (value), [b] "r" (ptr)
102 : "memory");
103 }
104
105 /**
106 * Returns the current value of a 32 bit (aligned) memory
107 * location.
108 *
109 * @param ptr Address of memory to get
110 * @return Value of the memory
111 */
bdk_atomic_get32(int32_t * ptr)112 static inline int32_t bdk_atomic_get32(int32_t *ptr)
113 {
114 return *(volatile int32_t *)ptr;
115 }
116
117 /**
118 * Atomically adds a signed value to a 64 bit (aligned) memory location.
119 *
120 * This version does not perform 'sync' operations to enforce memory
121 * operations. This should only be used when there are no memory operation
122 * ordering constraints. (This should NOT be used for reference counting -
123 * use the standard version instead.)
124 *
125 * @param ptr address in memory to add incr to
126 * @param incr amount to increment memory location by (signed)
127 */
bdk_atomic_add64_nosync(int64_t * ptr,int64_t incr)128 static inline void bdk_atomic_add64_nosync(int64_t *ptr, int64_t incr)
129 {
130 /* Atomic add with no ordering */
131 asm volatile ("ldadd %x[i], xzr, [%[b]]"
132 : [r] "+m" (*ptr)
133 : [i] "r" (incr), [b] "r" (ptr)
134 : "memory");
135 }
136
137 /**
138 * Atomically adds a signed value to a 64 bit (aligned) memory location.
139 *
140 * Memory access ordering is enforced before/after the atomic operation,
141 * so no additional 'sync' instructions are required.
142 *
143 *
144 * @param ptr address in memory to add incr to
145 * @param incr amount to increment memory location by (signed)
146 */
bdk_atomic_add64(int64_t * ptr,int64_t incr)147 static inline void bdk_atomic_add64(int64_t *ptr, int64_t incr)
148 {
149 /* Atomic add with acquire and release */
150 asm volatile ("ldaddal %x[i], xzr, [%[b]]"
151 : [r] "+m" (*ptr)
152 : [i] "r" (incr), [b] "r" (ptr)
153 : "memory");
154 }
155
156 /**
157 * Atomically sets a 64 bit (aligned) memory location to a value
158 *
159 * @param ptr address of memory to set
160 * @param value value to set memory location to.
161 */
bdk_atomic_set64(int64_t * ptr,int64_t value)162 static inline void bdk_atomic_set64(int64_t *ptr, int64_t value)
163 {
164 /* Implies a release */
165 asm volatile ("stlr %x[v], [%[b]]"
166 : "+m" (*ptr)
167 : [v] "r" (value), [b] "r" (ptr)
168 : "memory");
169 }
170
171 /**
172 * Returns the current value of a 64 bit (aligned) memory
173 * location.
174 *
175 * @param ptr Address of memory to get
176 * @return Value of the memory
177 */
bdk_atomic_get64(int64_t * ptr)178 static inline int64_t bdk_atomic_get64(int64_t *ptr)
179 {
180 return *(volatile int64_t *)ptr;
181 }
182
183 /**
184 * Atomically compares the old value with the value at ptr, and if they match,
185 * stores new_val to ptr.
186 * If *ptr and old don't match, function returns failure immediately.
187 * If *ptr and old match, function spins until *ptr updated to new atomically, or
188 * until *ptr and old no longer match
189 *
190 * Does no memory synchronization.
191 *
192 * @return 1 on success (match and store)
193 * 0 on no match
194 */
195 static inline int bdk_atomic_compare_and_store32_nosync(uint32_t *ptr, uint32_t old_val, uint32_t new_val) __attribute__((always_inline));
bdk_atomic_compare_and_store32_nosync(uint32_t * ptr,uint32_t old_val,uint32_t new_val)196 static inline int bdk_atomic_compare_and_store32_nosync(uint32_t *ptr, uint32_t old_val, uint32_t new_val)
197 {
198 uint32_t val = old_val;
199
200 /* CN88XX pass 1.x has errata AP-22500: GlobalSync request during a multi-cycle ATOMIC stalls forever
201 Don't use compare and swap on these chips */
202
203 asm volatile ("cas %w[o], %w[n], [%[b]]"
204 : [mem] "+m" (*ptr), [o] "+r" (val)
205 : [b] "r" (ptr), [n] "r" (new_val)
206 : );
207
208 return old_val == val;
209 }
210
211 /**
212 * Atomically compares the old value with the value at ptr, and if they match,
213 * stores new_val to ptr.
214 * If *ptr and old don't match, function returns failure immediately.
215 * If *ptr and old match, function spins until *ptr updated to new atomically, or
216 * until *ptr and old no longer match
217 *
218 * Does memory synchronization that is required to use this as a locking primitive.
219 *
220 * @return 1 on success (match and store)
221 * 0 on no match
222 */
223 static inline int bdk_atomic_compare_and_store32(uint32_t *ptr, uint32_t old_val, uint32_t new_val) __attribute__((always_inline));
bdk_atomic_compare_and_store32(uint32_t * ptr,uint32_t old_val,uint32_t new_val)224 static inline int bdk_atomic_compare_and_store32(uint32_t *ptr, uint32_t old_val, uint32_t new_val)
225 {
226 uint32_t val = old_val;
227
228 /* CN88XX pass 1.x has errata AP-22500: GlobalSync request during a multi-cycle ATOMIC stalls forever
229 Don't use compare and swap on these chips */
230
231 asm volatile ("casal %w[o], %w[n], [%[b]]"
232 : [mem] "+m" (*ptr), [o] "+r" (val)
233 : [b] "r" (ptr), [n] "r" (new_val)
234 : );
235
236 return old_val == val;
237 }
238
239 /**
240 * Atomically compares the old value with the value at ptr, and if they match,
241 * stores new_val to ptr.
242 * If *ptr and old don't match, function returns failure immediately.
243 * If *ptr and old match, function spins until *ptr updated to new atomically, or
244 * until *ptr and old no longer match
245 *
246 * Does no memory synchronization.
247 *
248 * @return 1 on success (match and store)
249 * 0 on no match
250 */
251 static inline int bdk_atomic_compare_and_store64_nosync(uint64_t *ptr, uint64_t old_val, uint64_t new_val) __attribute__((always_inline));
bdk_atomic_compare_and_store64_nosync(uint64_t * ptr,uint64_t old_val,uint64_t new_val)252 static inline int bdk_atomic_compare_and_store64_nosync(uint64_t *ptr, uint64_t old_val, uint64_t new_val)
253 {
254 uint32_t val = old_val;
255
256 /* CN88XX pass 1.x has errata AP-22500: GlobalSync request during a multi-cycle ATOMIC stalls forever
257 Don't use compare and swap on these chips */
258
259 asm volatile ("cas %x[o], %x[n], [%[b]]"
260 : [mem] "+m" (*ptr), [o] "+r" (val)
261 : [b] "r" (ptr), [n] "r" (new_val)
262 : );
263
264 return old_val == val;
265 }
266
267 /**
268 * Atomically compares the old value with the value at ptr, and if they match,
269 * stores new_val to ptr.
270 * If *ptr and old don't match, function returns failure immediately.
271 * If *ptr and old match, function spins until *ptr updated to new atomically, or
272 * until *ptr and old no longer match
273 *
274 * Does memory synchronization that is required to use this as a locking primitive.
275 *
276 * @return 1 on success (match and store)
277 * 0 on no match
278 */
279 static inline int bdk_atomic_compare_and_store64(uint64_t *ptr, uint64_t old_val, uint64_t new_val) __attribute__((always_inline));
bdk_atomic_compare_and_store64(uint64_t * ptr,uint64_t old_val,uint64_t new_val)280 static inline int bdk_atomic_compare_and_store64(uint64_t *ptr, uint64_t old_val, uint64_t new_val)
281 {
282 uint32_t val = old_val;
283
284 /* CN88XX pass 1.x has errata AP-22500: GlobalSync request during a multi-cycle ATOMIC stalls forever
285 Don't use compare and swap on these chips */
286
287 asm volatile ("casal %x[o], %x[n], [%[b]]"
288 : [mem] "+m" (*ptr), [o] "+r" (val)
289 : [b] "r" (ptr), [n] "r" (new_val)
290 : );
291 return old_val == val;
292 }
293
294 /**
295 * Atomically adds a signed value to a 64 bit (aligned) memory location,
296 * and returns previous value.
297 *
298 * This version does not perform 'sync' operations to enforce memory
299 * operations. This should only be used when there are no memory operation
300 * ordering constraints. (This should NOT be used for reference counting -
301 * use the standard version instead.)
302 *
303 * @param ptr address in memory to add incr to
304 * @param incr amount to increment memory location by (signed)
305 *
306 * @return Value of memory location before increment
307 */
bdk_atomic_fetch_and_add64_nosync(int64_t * ptr,int64_t incr)308 static inline int64_t bdk_atomic_fetch_and_add64_nosync(int64_t *ptr, int64_t incr)
309 {
310 int64_t result;
311 /* Atomic add with no ordering */
312 asm volatile ("ldadd %x[i], %x[r], [%[b]]"
313 : [r] "=r" (result), "+m" (*ptr)
314 : [i] "r" (incr), [b] "r" (ptr)
315 : "memory");
316 return result;
317 }
318
319 /**
320 * Atomically adds a signed value to a 64 bit (aligned) memory location,
321 * and returns previous value.
322 *
323 * Memory access ordering is enforced before/after the atomic operation,
324 * so no additional 'sync' instructions are required.
325 *
326 * @param ptr address in memory to add incr to
327 * @param incr amount to increment memory location by (signed)
328 *
329 * @return Value of memory location before increment
330 */
bdk_atomic_fetch_and_add64(int64_t * ptr,int64_t incr)331 static inline int64_t bdk_atomic_fetch_and_add64(int64_t *ptr, int64_t incr)
332 {
333 int64_t result;
334 /* Atomic add with acquire/release */
335 asm volatile ("ldaddal %x[i], %x[r], [%[b]]"
336 : [r] "=r" (result), "+m" (*ptr)
337 : [i] "r" (incr), [b] "r" (ptr)
338 : "memory");
339 return result;
340 }
341
342 /**
343 * Atomically adds a signed value to a 32 bit (aligned) memory location,
344 * and returns previous value.
345 *
346 * This version does not perform 'sync' operations to enforce memory
347 * operations. This should only be used when there are no memory operation
348 * ordering constraints. (This should NOT be used for reference counting -
349 * use the standard version instead.)
350 *
351 * @param ptr address in memory to add incr to
352 * @param incr amount to increment memory location by (signed)
353 *
354 * @return Value of memory location before increment
355 */
bdk_atomic_fetch_and_add32_nosync(int32_t * ptr,int32_t incr)356 static inline int32_t bdk_atomic_fetch_and_add32_nosync(int32_t *ptr, int32_t incr)
357 {
358 int32_t result;
359 /* Atomic add with no ordering */
360 asm volatile ("ldadd %w[i], %w[r], [%[b]]"
361 : [r] "=r" (result), "+m" (*ptr)
362 : [i] "r" (incr), [b] "r" (ptr)
363 : "memory");
364 return result;
365 }
366
367 /**
368 * Atomically adds a signed value to a 32 bit (aligned) memory location,
369 * and returns previous value.
370 *
371 * Memory access ordering is enforced before/after the atomic operation,
372 * so no additional 'sync' instructions are required.
373 *
374 * @param ptr address in memory to add incr to
375 * @param incr amount to increment memory location by (signed)
376 *
377 * @return Value of memory location before increment
378 */
bdk_atomic_fetch_and_add32(int32_t * ptr,int32_t incr)379 static inline int32_t bdk_atomic_fetch_and_add32(int32_t *ptr, int32_t incr)
380 {
381 int32_t result;
382 /* Atomic add with acquire/release */
383 asm volatile ("ldaddal %w[i], %w[r], [%[b]]"
384 : [r] "=r" (result), "+m" (*ptr)
385 : [i] "r" (incr), [b] "r" (ptr)
386 : "memory");
387 return result;
388 }
389
390 /**
391 * Atomically set bits in a 64 bit (aligned) memory location,
392 * and returns previous value.
393 *
394 * This version does not perform 'sync' operations to enforce memory
395 * operations. This should only be used when there are no memory operation
396 * ordering constraints.
397 *
398 * @param ptr address in memory
399 * @param mask mask of bits to set
400 *
401 * @return Value of memory location before setting bits
402 */
bdk_atomic_fetch_and_bset64_nosync(uint64_t * ptr,uint64_t mask)403 static inline uint64_t bdk_atomic_fetch_and_bset64_nosync(uint64_t *ptr, uint64_t mask)
404 {
405 uint64_t result;
406 /* Atomic or with no ordering */
407 asm volatile ("ldset %x[i], %x[r], [%[b]]"
408 : [r] "=r" (result), "+m" (*ptr)
409 : [i] "r" (mask), [b] "r" (ptr)
410 : "memory");
411 return result;
412 }
413
414 /**
415 * Atomically set bits in a 32 bit (aligned) memory location,
416 * and returns previous value.
417 *
418 * This version does not perform 'sync' operations to enforce memory
419 * operations. This should only be used when there are no memory operation
420 * ordering constraints.
421 *
422 * @param ptr address in memory
423 * @param mask mask of bits to set
424 *
425 * @return Value of memory location before setting bits
426 */
bdk_atomic_fetch_and_bset32_nosync(uint32_t * ptr,uint32_t mask)427 static inline uint32_t bdk_atomic_fetch_and_bset32_nosync(uint32_t *ptr, uint32_t mask)
428 {
429 uint32_t result;
430 /* Atomic or with no ordering */
431 asm volatile ("ldset %w[i], %w[r], [%[b]]"
432 : [r] "=r" (result), "+m" (*ptr)
433 : [i] "r" (mask), [b] "r" (ptr)
434 : "memory");
435 return result;
436 }
437
438 /**
439 * Atomically clear bits in a 64 bit (aligned) memory location,
440 * and returns previous value.
441 *
442 * This version does not perform 'sync' operations to enforce memory
443 * operations. This should only be used when there are no memory operation
444 * ordering constraints.
445 *
446 * @param ptr address in memory
447 * @param mask mask of bits to clear
448 *
449 * @return Value of memory location before clearing bits
450 */
bdk_atomic_fetch_and_bclr64_nosync(uint64_t * ptr,uint64_t mask)451 static inline uint64_t bdk_atomic_fetch_and_bclr64_nosync(uint64_t *ptr, uint64_t mask)
452 {
453 uint64_t result;
454 /* Atomic and with no ordering */
455 asm volatile ("ldclr %x[i], %x[r], [%[b]]"
456 : [r] "=r" (result), "+m" (*ptr)
457 : [i] "r" (mask), [b] "r" (ptr)
458 : "memory");
459 return result;
460 }
461
462 /**
463 * Atomically clear bits in a 32 bit (aligned) memory location,
464 * and returns previous value.
465 *
466 * This version does not perform 'sync' operations to enforce memory
467 * operations. This should only be used when there are no memory operation
468 * ordering constraints.
469 *
470 * @param ptr address in memory
471 * @param mask mask of bits to clear
472 *
473 * @return Value of memory location before clearing bits
474 */
bdk_atomic_fetch_and_bclr32_nosync(uint32_t * ptr,uint32_t mask)475 static inline uint32_t bdk_atomic_fetch_and_bclr32_nosync(uint32_t *ptr, uint32_t mask)
476 {
477 uint32_t result;
478 /* Atomic and with no ordering */
479 asm volatile ("ldclr %w[i], %w[r], [%[b]]"
480 : [r] "=r" (result), "+m" (*ptr)
481 : [i] "r" (mask), [b] "r" (ptr)
482 : "memory");
483 return result;
484 }
485
486 /** @} */
487 #endif /* !__CB_BDK_ATOMIC_H__ */
488