1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright IBM Corp. 1999, 2009
4 *
5 * Author(s): Martin Schwidefsky <[email protected]>
6 */
7
8 #ifndef __ASM_BARRIER_H
9 #define __ASM_BARRIER_H
10
11 #include <asm/march.h>
12
13 /*
14 * Force strict CPU ordering.
15 * And yes, this is required on UP too when we're talking
16 * to devices.
17 */
18
19 #ifdef MARCH_HAS_Z196_FEATURES
20 /* Fast-BCR without checkpoint synchronization */
21 #define __ASM_BCR_SERIALIZE "bcr 14,0\n"
22 #else
23 #define __ASM_BCR_SERIALIZE "bcr 15,0\n"
24 #endif
25
bcr_serialize(void)26 static __always_inline void bcr_serialize(void)
27 {
28 asm volatile(__ASM_BCR_SERIALIZE : : : "memory");
29 }
30
31 #define __mb() bcr_serialize()
32 #define __rmb() barrier()
33 #define __wmb() barrier()
34 #define __dma_rmb() __mb()
35 #define __dma_wmb() __mb()
36 #define __smp_mb() __mb()
37 #define __smp_rmb() __rmb()
38 #define __smp_wmb() __wmb()
39
40 #define __smp_store_release(p, v) \
41 do { \
42 compiletime_assert_atomic_type(*p); \
43 barrier(); \
44 WRITE_ONCE(*p, v); \
45 } while (0)
46
47 #define __smp_load_acquire(p) \
48 ({ \
49 typeof(*p) ___p1 = READ_ONCE(*p); \
50 compiletime_assert_atomic_type(*p); \
51 barrier(); \
52 ___p1; \
53 })
54
55 #define __smp_mb__before_atomic() barrier()
56 #define __smp_mb__after_atomic() barrier()
57
58 /**
59 * array_index_mask_nospec - generate a mask for array_idx() that is
60 * ~0UL when the bounds check succeeds and 0 otherwise
61 * @index: array element index
62 * @size: number of elements in array
63 */
64 #define array_index_mask_nospec array_index_mask_nospec
array_index_mask_nospec(unsigned long index,unsigned long size)65 static inline unsigned long array_index_mask_nospec(unsigned long index,
66 unsigned long size)
67 {
68 unsigned long mask;
69
70 if (__builtin_constant_p(size) && size > 0) {
71 asm(" clgr %2,%1\n"
72 " slbgr %0,%0\n"
73 :"=d" (mask) : "d" (size-1), "d" (index) :"cc");
74 return mask;
75 }
76 asm(" clgr %1,%2\n"
77 " slbgr %0,%0\n"
78 :"=d" (mask) : "d" (size), "d" (index) :"cc");
79 return ~mask;
80 }
81
82 #include <asm-generic/barrier.h>
83
84 #endif /* __ASM_BARRIER_H */
85