18 static __always_inline u32 __cs_asm(u64 ptr, u32 old, u32 new)  in __cs_asm()  argument
21 		"	cs	%[old],%[new],%[ptr]\n"  in __cs_asm()
22 		: [old] "+d" (old), [ptr] "+Q" (*(u32 *)ptr)  in __cs_asm()
25 	return old;  in __cs_asm()
28 static __always_inline u64 __csg_asm(u64 ptr, u64 old, u64 new)  in __csg_asm()  argument
31 		"	csg	%[old],%[new],%[ptr]\n"  in __csg_asm()
32 		: [old] "+d" (old), [ptr] "+QS" (*(u64 *)ptr)  in __csg_asm()
35 	return old;  in __csg_asm()
38 static inline u8 __arch_cmpxchg1(u64 ptr, u8 old, u8 new)  in __arch_cmpxchg1()  argument
52 		if (old32.b[i] != old)  in __arch_cmpxchg1()
58 	return old;  in __arch_cmpxchg1()
61 static inline u16 __arch_cmpxchg2(u64 ptr, u16 old, u16 new)  in __arch_cmpxchg2()  argument
75 		if (old32.b[i] != old)  in __arch_cmpxchg2()
81 	return old;  in __arch_cmpxchg2()
84 static __always_inline u64 __arch_cmpxchg(u64 ptr, u64 old, u64 new, int size)  in __arch_cmpxchg()  argument
87 	case 1:	 return __arch_cmpxchg1(ptr, old & 0xff, new & 0xff);  in __arch_cmpxchg()
88 	case 2:  return __arch_cmpxchg2(ptr, old & 0xffff, new & 0xffff);  in __arch_cmpxchg()
89 	case 4:  return __cs_asm(ptr, old & 0xffffffff, new & 0xffffffff);  in __arch_cmpxchg()
90 	case 8:  return __csg_asm(ptr, old, new);  in __arch_cmpxchg()
93 	return old;  in __arch_cmpxchg()
179 	u32 mask, old, new;  in __arch_xchg1()  local
183 	old = READ_ONCE(*(u32 *)ptr);  in __arch_xchg1()
185 		new = old & mask;  in __arch_xchg1()
187 	} while (!arch_try_cmpxchg((u32 *)ptr, &old, new));  in __arch_xchg1()
188 	return old >> shift;  in __arch_xchg1()
194 	u32 mask, old, new;  in __arch_xchg2()  local
198 	old = READ_ONCE(*(u32 *)ptr);  in __arch_xchg2()
200 		new = old & mask;  in __arch_xchg2()
202 	} while (!arch_try_cmpxchg((u32 *)ptr, &old, new));  in __arch_xchg2()
203 	return old >> shift;  in __arch_xchg2()
214 		u32 old = READ_ONCE(*(u32 *)ptr);  in __arch_xchg()  local
217 		} while (!arch_try_cmpxchg((u32 *)ptr, &old, x & 0xffffffff));  in __arch_xchg()
218 		return old;  in __arch_xchg()
221 		u64 old = READ_ONCE(*(u64 *)ptr);  in __arch_xchg()  local
224 		} while (!arch_try_cmpxchg((u64 *)ptr, &old, x));  in __arch_xchg()
225 		return old;  in __arch_xchg()
241 static __always_inline u128 arch_cmpxchg128(volatile u128 *ptr, u128 old, u128 new)  in arch_cmpxchg128()  argument
244 		"	cdsg	%[old],%[new],%[ptr]\n"  in arch_cmpxchg128()
245 		: [old] "+d" (old), [ptr] "+QS" (*ptr)  in arch_cmpxchg128()
248 	return old;  in arch_cmpxchg128()
261 		"	cdsg	%[old],%[new],%[ptr]\n"  in arch_try_cmpxchg128()
262 		: [old] "+d" (*oldp), [ptr] "+QS" (*ptr), "=@cc" (cc)  in arch_try_cmpxchg128()