1 //
2 //
3 // Copyright 2017 gRPC authors.
4 //
5 // Licensed under the Apache License, Version 2.0 (the "License");
6 // you may not use this file except in compliance with the License.
7 // You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 //
17 //
18
19 #include <grpc/support/port_platform.h>
20
21 #include "src/core/lib/iomgr/lockfree_event.h"
22
23 #include <grpc/support/log.h>
24
25 #include "src/core/lib/debug/trace.h"
26 #include "src/core/lib/gprpp/crash.h"
27 #include "src/core/lib/iomgr/exec_ctx.h"
28
29 extern grpc_core::DebugOnlyTraceFlag grpc_polling_trace;
30
31 // 'state' holds the to call when the fd is readable or writable respectively.
32 // It can contain one of the following values:
33 // kClosureReady : The fd has an I/O event of interest but there is no
34 // closure yet to execute
35
36 // kClosureNotReady : The fd has no I/O event of interest
37
38 // closure ptr : The closure to be executed when the fd has an I/O
39 // event of interest
40
41 // shutdown_error | kShutdownBit :
42 // 'shutdown_error' field ORed with kShutdownBit.
43 // This indicates that the fd is shutdown. Since all
44 // memory allocations are word-aligned, the lower two
45 // bits of the shutdown_error pointer are always 0. So
46 // it is safe to OR these with kShutdownBit
47
48 // Valid state transitions:
49
50 // <closure ptr> <-----3------ kClosureNotReady -----1-------> kClosureReady
51 // | | ^ | ^ | |
52 // | | | | | | |
53 // | +--------------4----------+ 6 +---------2---------------+ |
54 // | | |
55 // | v |
56 // +-----5-------> [shutdown_error | kShutdownBit] <-------7---------+
57
58 // For 1, 4 : See SetReady() function
59 // For 2, 3 : See NotifyOn() function
60 // For 5,6,7: See SetShutdown() function
61
62 namespace grpc_core {
63
LockfreeEvent()64 LockfreeEvent::LockfreeEvent() { InitEvent(); }
65
InitEvent()66 void LockfreeEvent::InitEvent() {
67 // Perform an atomic store to start the state machine.
68
69 // Note carefully that LockfreeEvent *MAY* be used whilst in a destroyed
70 // state, while a file descriptor is on a freelist. In such a state it may
71 // be SetReady'd, and so we need to perform an atomic operation here to
72 // ensure no races
73 gpr_atm_no_barrier_store(&state_, kClosureNotReady);
74 }
75
DestroyEvent()76 void LockfreeEvent::DestroyEvent() {
77 gpr_atm curr;
78 do {
79 curr = gpr_atm_no_barrier_load(&state_);
80 if (curr & kShutdownBit) {
81 internal::StatusFreeHeapPtr(curr & ~kShutdownBit);
82 } else {
83 GPR_ASSERT(curr == kClosureNotReady || curr == kClosureReady);
84 }
85 // we CAS in a shutdown, no error value here. If this event is interacted
86 // with post-deletion (see the note in the constructor) we want the bit
87 // pattern to prevent error retention in a deleted object
88 } while (!gpr_atm_no_barrier_cas(&state_, curr,
89 kShutdownBit /* shutdown, no error */));
90 }
91
NotifyOn(grpc_closure * closure)92 void LockfreeEvent::NotifyOn(grpc_closure* closure) {
93 while (true) {
94 // This load needs to be an acquire load because this can be a shutdown
95 // error that we might need to reference. Adding acquire semantics makes
96 // sure that the shutdown error has been initialized properly before us
97 // referencing it.
98 gpr_atm curr = gpr_atm_acq_load(&state_);
99 if (GRPC_TRACE_FLAG_ENABLED(grpc_polling_trace)) {
100 gpr_log(GPR_DEBUG,
101 "LockfreeEvent::NotifyOn: %p curr=%" PRIxPTR " closure=%p", this,
102 curr, closure);
103 }
104 switch (curr) {
105 case kClosureNotReady: {
106 // kClosureNotReady -> <closure>.
107
108 // We're guaranteed by API that there's an acquire barrier before here,
109 // so there's no need to double-dip and this can be a release-only.
110
111 // The release itself pairs with the acquire half of a set_ready full
112 // barrier.
113 if (gpr_atm_rel_cas(&state_, kClosureNotReady,
114 reinterpret_cast<gpr_atm>(closure))) {
115 return; // Successful. Return
116 }
117
118 break; // retry
119 }
120
121 case kClosureReady: {
122 // Change the state to kClosureNotReady. Schedule the closure if
123 // successful. If not, the state most likely transitioned to shutdown.
124 // We should retry.
125
126 // This can be a no-barrier cas since the state is being transitioned to
127 // kClosureNotReady; set_ready and set_shutdown do not schedule any
128 // closure when transitioning out of CLOSURE_NO_READY state (i.e there
129 // is no other code that needs to 'happen-after' this)
130 if (gpr_atm_no_barrier_cas(&state_, kClosureReady, kClosureNotReady)) {
131 ExecCtx::Run(DEBUG_LOCATION, closure, absl::OkStatus());
132 return; // Successful. Return
133 }
134
135 break; // retry
136 }
137
138 default: {
139 // 'curr' is either a closure or the fd is shutdown(in which case 'curr'
140 // contains a pointer to the shutdown-error). If the fd is shutdown,
141 // schedule the closure with the shutdown error
142 if ((curr & kShutdownBit) > 0) {
143 grpc_error_handle shutdown_err =
144 internal::StatusGetFromHeapPtr(curr & ~kShutdownBit);
145 ExecCtx::Run(
146 DEBUG_LOCATION, closure,
147 GRPC_ERROR_CREATE_REFERENCING("FD Shutdown", &shutdown_err, 1));
148 return;
149 }
150
151 // There is already a closure!. This indicates a bug in the code
152 Crash(
153 "LockfreeEvent::NotifyOn: notify_on called with a previous "
154 "callback still pending");
155 }
156 }
157 }
158
159 GPR_UNREACHABLE_CODE(return);
160 }
161
SetShutdown(grpc_error_handle shutdown_error)162 bool LockfreeEvent::SetShutdown(grpc_error_handle shutdown_error) {
163 intptr_t status_ptr = internal::StatusAllocHeapPtr(shutdown_error);
164 gpr_atm new_state = status_ptr | kShutdownBit;
165
166 while (true) {
167 gpr_atm curr = gpr_atm_no_barrier_load(&state_);
168 if (GRPC_TRACE_FLAG_ENABLED(grpc_polling_trace)) {
169 gpr_log(GPR_DEBUG,
170 "LockfreeEvent::SetShutdown: %p curr=%" PRIxPTR " err=%s",
171 &state_, curr, StatusToString(shutdown_error).c_str());
172 }
173 switch (curr) {
174 case kClosureReady:
175 case kClosureNotReady:
176 // Need a full barrier here so that the initial load in notify_on
177 // doesn't need a barrier
178 if (gpr_atm_full_cas(&state_, curr, new_state)) {
179 return true; // early out
180 }
181 break; // retry
182
183 default: {
184 // 'curr' is either a closure or the fd is already shutdown
185
186 // If fd is already shutdown, we are done
187 if ((curr & kShutdownBit) > 0) {
188 internal::StatusFreeHeapPtr(status_ptr);
189 return false;
190 }
191
192 // Fd is not shutdown. Schedule the closure and move the state to
193 // shutdown state.
194 // Needs an acquire to pair with setting the closure (and get a
195 // happens-after on that edge), and a release to pair with anything
196 // loading the shutdown state.
197 if (gpr_atm_full_cas(&state_, curr, new_state)) {
198 ExecCtx::Run(
199 DEBUG_LOCATION, reinterpret_cast<grpc_closure*>(curr),
200 GRPC_ERROR_CREATE_REFERENCING("FD Shutdown", &shutdown_error, 1));
201 return true;
202 }
203
204 // 'curr' was a closure but now changed to a different state. We will
205 // have to retry
206 break;
207 }
208 }
209 }
210
211 GPR_UNREACHABLE_CODE(return false);
212 }
213
SetReady()214 void LockfreeEvent::SetReady() {
215 while (true) {
216 gpr_atm curr = gpr_atm_no_barrier_load(&state_);
217
218 if (GRPC_TRACE_FLAG_ENABLED(grpc_polling_trace)) {
219 gpr_log(GPR_DEBUG, "LockfreeEvent::SetReady: %p curr=%" PRIxPTR, &state_,
220 curr);
221 }
222
223 switch (curr) {
224 case kClosureReady: {
225 // Already ready. We are done here
226 return;
227 }
228
229 case kClosureNotReady: {
230 // No barrier required as we're transitioning to a state that does not
231 // involve a closure
232 if (gpr_atm_no_barrier_cas(&state_, kClosureNotReady, kClosureReady)) {
233 return; // early out
234 }
235 break; // retry
236 }
237
238 default: {
239 // 'curr' is either a closure or the fd is shutdown
240 if ((curr & kShutdownBit) > 0) {
241 // The fd is shutdown. Do nothing
242 return;
243 }
244 // Full cas: acquire pairs with this cas' release in the event of a
245 // spurious set_ready; release pairs with this or the acquire in
246 // notify_on (or set_shutdown)
247 else if (gpr_atm_full_cas(&state_, curr, kClosureNotReady)) {
248 ExecCtx::Run(DEBUG_LOCATION, reinterpret_cast<grpc_closure*>(curr),
249 absl::OkStatus());
250 return;
251 }
252 // else the state changed again (only possible by either a racing
253 // set_ready or set_shutdown functions. In both these cases, the closure
254 // would have been scheduled for execution. So we are done here
255 return;
256 }
257 }
258 }
259 }
260
261 } // namespace grpc_core
262