18 static __always_inline u32 __cs_asm(u64 ptr, u32 old, u32 new)  in __cs_asm()  argument
21 		"	cs	%[old],%[new],%[ptr]\n"  in __cs_asm()
23 		: [new] "d" (new)  in __cs_asm()
28 static __always_inline u64 __csg_asm(u64 ptr, u64 old, u64 new)  in __csg_asm()  argument
31 		"	csg	%[old],%[new],%[ptr]\n"  in __csg_asm()
33 		: [new] "d" (new)  in __csg_asm()
38 static inline u8 __arch_cmpxchg1(u64 ptr, u8 old, u8 new)  in __arch_cmpxchg1()  argument
55 		new32.b[i] = new;  in __arch_cmpxchg1()
61 static inline u16 __arch_cmpxchg2(u64 ptr, u16 old, u16 new)  in __arch_cmpxchg2()  argument
78 		new32.b[i] = new;  in __arch_cmpxchg2()
84 static __always_inline u64 __arch_cmpxchg(u64 ptr, u64 old, u64 new, int size)  in __arch_cmpxchg()  argument
87 	case 1:	 return __arch_cmpxchg1(ptr, old & 0xff, new & 0xff);  in __arch_cmpxchg()
88 	case 2:  return __arch_cmpxchg2(ptr, old & 0xffff, new & 0xffff);  in __arch_cmpxchg()
89 	case 4:  return __cs_asm(ptr, old & 0xffffffff, new & 0xffffffff);  in __arch_cmpxchg()
90 	case 8:  return __csg_asm(ptr, old, new);  in __arch_cmpxchg()
110 #define arch_try_cmpxchg(ptr, oldp, new)				\  argument
114 	__typeof__(*(ptr)) __new = (new);				\
155 #define arch_try_cmpxchg(ptr, oldp, new)				\  argument
159 	__typeof__(*(ptr)) __new = (new);				\
179 	u32 mask, old, new;  in __arch_xchg1()  local
185 		new = old & mask;  in __arch_xchg1()
186 		new |= x << shift;  in __arch_xchg1()
187 	} while (!arch_try_cmpxchg((u32 *)ptr, &old, new));  in __arch_xchg1()
194 	u32 mask, old, new;  in __arch_xchg2()  local
200 		new = old & mask;  in __arch_xchg2()
201 		new |= x << shift;  in __arch_xchg2()
202 	} while (!arch_try_cmpxchg((u32 *)ptr, &old, new));  in __arch_xchg2()
241 static __always_inline u128 arch_cmpxchg128(volatile u128 *ptr, u128 old, u128 new)  in arch_cmpxchg128()  argument
244 		"	cdsg	%[old],%[new],%[ptr]\n"  in arch_cmpxchg128()
246 		: [new] "d" (new)  in arch_cmpxchg128()
256 static __always_inline bool arch_try_cmpxchg128(volatile u128 *ptr, u128 *oldp, u128 new)  in arch_try_cmpxchg128()  argument
261 		"	cdsg	%[old],%[new],%[ptr]\n"  in arch_try_cmpxchg128()
263 		: [new] "d" (new)  in arch_try_cmpxchg128()