1// Copyright 2009 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package runtime
6
7import (
8	"internal/goexperiment"
9	"internal/runtime/atomic"
10	"unsafe"
11)
12
13// These functions cannot have go:noescape annotations,
14// because while ptr does not escape, new does.
15// If new is marked as not escaping, the compiler will make incorrect
16// escape analysis decisions about the pointer value being stored.
17
18// atomicwb performs a write barrier before an atomic pointer write.
19// The caller should guard the call with "if writeBarrier.enabled".
20//
21// atomicwb should be an internal detail,
22// but widely used packages access it using linkname.
23// Notable members of the hall of shame include:
24//   - github.com/bytedance/gopkg
25//   - github.com/songzhibin97/gkit
26//
27// Do not remove or change the type signature.
28// See go.dev/issue/67401.
29//
30//go:linkname atomicwb
31//go:nosplit
32func atomicwb(ptr *unsafe.Pointer, new unsafe.Pointer) {
33	slot := (*uintptr)(unsafe.Pointer(ptr))
34	buf := getg().m.p.ptr().wbBuf.get2()
35	buf[0] = *slot
36	buf[1] = uintptr(new)
37}
38
39// atomicstorep performs *ptr = new atomically and invokes a write barrier.
40//
41//go:nosplit
42func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) {
43	if writeBarrier.enabled {
44		atomicwb((*unsafe.Pointer)(ptr), new)
45	}
46	if goexperiment.CgoCheck2 {
47		cgoCheckPtrWrite((*unsafe.Pointer)(ptr), new)
48	}
49	atomic.StorepNoWB(noescape(ptr), new)
50}
51
52// atomic_storePointer is the implementation of runtime/internal/UnsafePointer.Store
53// (like StoreNoWB but with the write barrier).
54//
55//go:nosplit
56//go:linkname atomic_storePointer internal/runtime/atomic.storePointer
57func atomic_storePointer(ptr *unsafe.Pointer, new unsafe.Pointer) {
58	atomicstorep(unsafe.Pointer(ptr), new)
59}
60
61// atomic_casPointer is the implementation of runtime/internal/UnsafePointer.CompareAndSwap
62// (like CompareAndSwapNoWB but with the write barrier).
63//
64//go:nosplit
65//go:linkname atomic_casPointer internal/runtime/atomic.casPointer
66func atomic_casPointer(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool {
67	if writeBarrier.enabled {
68		atomicwb(ptr, new)
69	}
70	if goexperiment.CgoCheck2 {
71		cgoCheckPtrWrite(ptr, new)
72	}
73	return atomic.Casp1(ptr, old, new)
74}
75
76// Like above, but implement in terms of sync/atomic's uintptr operations.
77// We cannot just call the runtime routines, because the race detector expects
78// to be able to intercept the sync/atomic forms but not the runtime forms.
79
80//go:linkname sync_atomic_StoreUintptr sync/atomic.StoreUintptr
81func sync_atomic_StoreUintptr(ptr *uintptr, new uintptr)
82
83//go:linkname sync_atomic_StorePointer sync/atomic.StorePointer
84//go:nosplit
85func sync_atomic_StorePointer(ptr *unsafe.Pointer, new unsafe.Pointer) {
86	if writeBarrier.enabled {
87		atomicwb(ptr, new)
88	}
89	if goexperiment.CgoCheck2 {
90		cgoCheckPtrWrite(ptr, new)
91	}
92	sync_atomic_StoreUintptr((*uintptr)(unsafe.Pointer(ptr)), uintptr(new))
93}
94
95//go:linkname sync_atomic_SwapUintptr sync/atomic.SwapUintptr
96func sync_atomic_SwapUintptr(ptr *uintptr, new uintptr) uintptr
97
98//go:linkname sync_atomic_SwapPointer sync/atomic.SwapPointer
99//go:nosplit
100func sync_atomic_SwapPointer(ptr *unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer {
101	if writeBarrier.enabled {
102		atomicwb(ptr, new)
103	}
104	if goexperiment.CgoCheck2 {
105		cgoCheckPtrWrite(ptr, new)
106	}
107	old := unsafe.Pointer(sync_atomic_SwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(new)))
108	return old
109}
110
111//go:linkname sync_atomic_CompareAndSwapUintptr sync/atomic.CompareAndSwapUintptr
112func sync_atomic_CompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool
113
114//go:linkname sync_atomic_CompareAndSwapPointer sync/atomic.CompareAndSwapPointer
115//go:nosplit
116func sync_atomic_CompareAndSwapPointer(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool {
117	if writeBarrier.enabled {
118		atomicwb(ptr, new)
119	}
120	if goexperiment.CgoCheck2 {
121		cgoCheckPtrWrite(ptr, new)
122	}
123	return sync_atomic_CompareAndSwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(old), uintptr(new))
124}
125