1// Copyright 2015 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "textflag.h"
7#include "funcdata.h"
8
9// bool armcas(int32 *val, int32 old, int32 new)
10// Atomically:
11//	if(*val == old){
12//		*val = new;
13//		return 1;
14//	}else
15//		return 0;
16//
17// To implement ·cas in sys_$GOOS_arm.s
18// using the native instructions, use:
19//
20//	TEXT ·cas(SB),NOSPLIT,$0
21//		B	·armcas(SB)
22//
23TEXT ·armcas(SB),NOSPLIT,$0-13
24	MOVW	ptr+0(FP), R1
25	MOVW	old+4(FP), R2
26	MOVW	new+8(FP), R3
27casl:
28	LDREX	(R1), R0
29	CMP	R0, R2
30	BNE	casfail
31
32#ifndef GOARM_7
33	MOVB	internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
34	CMP	$0, R11
35	BEQ	2(PC)
36#endif
37	DMB	MB_ISHST
38
39	STREX	R3, (R1), R0
40	CMP	$0, R0
41	BNE	casl
42	MOVW	$1, R0
43
44#ifndef GOARM_7
45	CMP	$0, R11
46	BEQ	2(PC)
47#endif
48	DMB	MB_ISH
49
50	MOVB	R0, ret+12(FP)
51	RET
52casfail:
53	MOVW	$0, R0
54	MOVB	R0, ret+12(FP)
55	RET
56
57// stubs
58
59TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-8
60	B	·Load(SB)
61
62TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-8
63	B	·Load(SB)
64
65TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-8
66	B 	·Load(SB)
67
68TEXT ·Casint32(SB),NOSPLIT,$0-13
69	B	·Cas(SB)
70
71TEXT ·Casint64(SB),NOSPLIT,$-4-21
72	B	·Cas64(SB)
73
74TEXT ·Casuintptr(SB),NOSPLIT,$0-13
75	B	·Cas(SB)
76
77TEXT ·Casp1(SB),NOSPLIT,$0-13
78	B	·Cas(SB)
79
80TEXT ·CasRel(SB),NOSPLIT,$0-13
81	B	·Cas(SB)
82
83TEXT ·Loadint32(SB),NOSPLIT,$0-8
84	B	·Load(SB)
85
86TEXT ·Loadint64(SB),NOSPLIT,$-4-12
87	B	·Load64(SB)
88
89TEXT ·Loaduintptr(SB),NOSPLIT,$0-8
90	B	·Load(SB)
91
92TEXT ·Loaduint(SB),NOSPLIT,$0-8
93	B	·Load(SB)
94
95TEXT ·Storeint32(SB),NOSPLIT,$0-8
96	B	·Store(SB)
97
98TEXT ·Storeint64(SB),NOSPLIT,$0-12
99	B	·Store64(SB)
100
101TEXT ·Storeuintptr(SB),NOSPLIT,$0-8
102	B	·Store(SB)
103
104TEXT ·StorepNoWB(SB),NOSPLIT,$0-8
105	B	·Store(SB)
106
107TEXT ·StoreRel(SB),NOSPLIT,$0-8
108	B	·Store(SB)
109
110TEXT ·StoreReluintptr(SB),NOSPLIT,$0-8
111	B	·Store(SB)
112
113TEXT ·Xaddint32(SB),NOSPLIT,$0-12
114	B	·Xadd(SB)
115
116TEXT ·Xaddint64(SB),NOSPLIT,$-4-20
117	B	·Xadd64(SB)
118
119TEXT ·Xadduintptr(SB),NOSPLIT,$0-12
120	B	·Xadd(SB)
121
122TEXT ·Xchgint32(SB),NOSPLIT,$0-12
123	B	·Xchg(SB)
124
125TEXT ·Xchgint64(SB),NOSPLIT,$-4-20
126	B	·Xchg64(SB)
127
128// 64-bit atomics
129// The native ARM implementations use LDREXD/STREXD, which are
130// available on ARMv6k or later. We use them only on ARMv7.
131// On older ARM, we use Go implementations which simulate 64-bit
132// atomics with locks.
133TEXT armCas64<>(SB),NOSPLIT,$0-21
134	// addr is already in R1
135	MOVW	old_lo+4(FP), R2
136	MOVW	old_hi+8(FP), R3
137	MOVW	new_lo+12(FP), R4
138	MOVW	new_hi+16(FP), R5
139cas64loop:
140	LDREXD	(R1), R6	// loads R6 and R7
141	CMP	R2, R6
142	BNE	cas64fail
143	CMP	R3, R7
144	BNE	cas64fail
145
146	DMB	MB_ISHST
147
148	STREXD	R4, (R1), R0	// stores R4 and R5
149	CMP	$0, R0
150	BNE	cas64loop
151	MOVW	$1, R0
152
153	DMB	MB_ISH
154
155	MOVBU	R0, swapped+20(FP)
156	RET
157cas64fail:
158	MOVW	$0, R0
159	MOVBU	R0, swapped+20(FP)
160	RET
161
162TEXT armXadd64<>(SB),NOSPLIT,$0-20
163	// addr is already in R1
164	MOVW	delta_lo+4(FP), R2
165	MOVW	delta_hi+8(FP), R3
166
167add64loop:
168	LDREXD	(R1), R4	// loads R4 and R5
169	ADD.S	R2, R4
170	ADC	R3, R5
171
172	DMB	MB_ISHST
173
174	STREXD	R4, (R1), R0	// stores R4 and R5
175	CMP	$0, R0
176	BNE	add64loop
177
178	DMB	MB_ISH
179
180	MOVW	R4, new_lo+12(FP)
181	MOVW	R5, new_hi+16(FP)
182	RET
183
184TEXT armXchg64<>(SB),NOSPLIT,$0-20
185	// addr is already in R1
186	MOVW	new_lo+4(FP), R2
187	MOVW	new_hi+8(FP), R3
188
189swap64loop:
190	LDREXD	(R1), R4	// loads R4 and R5
191
192	DMB	MB_ISHST
193
194	STREXD	R2, (R1), R0	// stores R2 and R3
195	CMP	$0, R0
196	BNE	swap64loop
197
198	DMB	MB_ISH
199
200	MOVW	R4, old_lo+12(FP)
201	MOVW	R5, old_hi+16(FP)
202	RET
203
204TEXT armLoad64<>(SB),NOSPLIT,$0-12
205	// addr is already in R1
206
207	LDREXD	(R1), R2	// loads R2 and R3
208	DMB	MB_ISH
209
210	MOVW	R2, val_lo+4(FP)
211	MOVW	R3, val_hi+8(FP)
212	RET
213
214TEXT armStore64<>(SB),NOSPLIT,$0-12
215	// addr is already in R1
216	MOVW	val_lo+4(FP), R2
217	MOVW	val_hi+8(FP), R3
218
219store64loop:
220	LDREXD	(R1), R4	// loads R4 and R5
221
222	DMB	MB_ISHST
223
224	STREXD	R2, (R1), R0	// stores R2 and R3
225	CMP	$0, R0
226	BNE	store64loop
227
228	DMB	MB_ISH
229	RET
230
231// The following functions all panic if their address argument isn't
232// 8-byte aligned. Since we're calling back into Go code to do this,
233// we have to cooperate with stack unwinding. In the normal case, the
234// functions tail-call into the appropriate implementation, which
235// means they must not open a frame. Hence, when they go down the
236// panic path, at that point they push the LR to create a real frame
237// (they don't need to pop it because panic won't return; however, we
238// do need to set the SP delta back).
239
240// Check if R1 is 8-byte aligned, panic if not.
241// Clobbers R2.
242#define CHECK_ALIGN \
243	AND.S	$7, R1, R2 \
244	BEQ 	4(PC) \
245	MOVW.W	R14, -4(R13) /* prepare a real frame */ \
246	BL	·panicUnaligned(SB) \
247	ADD	$4, R13 /* compensate SP delta */
248
249TEXT ·Cas64(SB),NOSPLIT,$-4-21
250	NO_LOCAL_POINTERS
251	MOVW	addr+0(FP), R1
252	CHECK_ALIGN
253
254#ifndef GOARM_7
255	MOVB	internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
256	CMP	$1, R11
257	BEQ	2(PC)
258	JMP	·goCas64(SB)
259#endif
260	JMP	armCas64<>(SB)
261
262TEXT ·Xadd64(SB),NOSPLIT,$-4-20
263	NO_LOCAL_POINTERS
264	MOVW	addr+0(FP), R1
265	CHECK_ALIGN
266
267#ifndef GOARM_7
268	MOVB	internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
269	CMP	$1, R11
270	BEQ	2(PC)
271	JMP	·goXadd64(SB)
272#endif
273	JMP	armXadd64<>(SB)
274
275TEXT ·Xchg64(SB),NOSPLIT,$-4-20
276	NO_LOCAL_POINTERS
277	MOVW	addr+0(FP), R1
278	CHECK_ALIGN
279
280#ifndef GOARM_7
281	MOVB	internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
282	CMP	$1, R11
283	BEQ	2(PC)
284	JMP	·goXchg64(SB)
285#endif
286	JMP	armXchg64<>(SB)
287
288TEXT ·Load64(SB),NOSPLIT,$-4-12
289	NO_LOCAL_POINTERS
290	MOVW	addr+0(FP), R1
291	CHECK_ALIGN
292
293#ifndef GOARM_7
294	MOVB	internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
295	CMP	$1, R11
296	BEQ	2(PC)
297	JMP	·goLoad64(SB)
298#endif
299	JMP	armLoad64<>(SB)
300
301TEXT ·Store64(SB),NOSPLIT,$-4-12
302	NO_LOCAL_POINTERS
303	MOVW	addr+0(FP), R1
304	CHECK_ALIGN
305
306#ifndef GOARM_7
307	MOVB	internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
308	CMP	$1, R11
309	BEQ	2(PC)
310	JMP	·goStore64(SB)
311#endif
312	JMP	armStore64<>(SB)
313