1 // Copyright 2022 The gRPC Authors
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 #include <grpc/support/port_platform.h>
15
16 #include "src/core/lib/event_engine/posix_engine/lockfree_event.h"
17
18 #include <atomic>
19 #include <cstdint>
20
21 #include "absl/status/status.h"
22
23 #include <grpc/support/atm.h>
24 #include <grpc/support/log.h>
25
26 #include "src/core/lib/event_engine/posix_engine/event_poller.h"
27 #include "src/core/lib/event_engine/posix_engine/posix_engine_closure.h"
28 #include "src/core/lib/gprpp/crash.h"
29 #include "src/core/lib/gprpp/status_helper.h"
30
31 // 'state' holds the to call when the fd is readable or writable respectively.
32 // It can contain one of the following values:
33 // kClosureReady : The fd has an I/O event of interest but there is no
34 // closure yet to execute
35
36 // kClosureNotReady : The fd has no I/O event of interest
37
38 // closure ptr : The closure to be executed when the fd has an I/O
39 // event of interest
40
41 // shutdown_error | kShutdownBit :
42 // 'shutdown_error' field ORed with kShutdownBit.
43 // This indicates that the fd is shutdown. Since all
44 // memory allocations are word-aligned, the lower two
45 // bits of the shutdown_error pointer are always 0. So
46 // it is safe to OR these with kShutdownBit
47
48 // Valid state transitions:
49
50 // <closure ptr> <-----3------ kClosureNotReady -----1-------> kClosureReady
51 // | | ^ | ^ | |
52 // | | | | | | |
53 // | +--------------4----------+ 6 +---------2---------------+ |
54 // | | |
55 // | v |
56 // +-----5-------> [shutdown_error | kShutdownBit] <-------7---------+
57
58 // For 1, 4 : See SetReady() function
59 // For 2, 3 : See NotifyOn() function
60 // For 5,6,7: See SetShutdown() function
61
62 namespace grpc_event_engine {
63 namespace experimental {
64
InitEvent()65 void LockfreeEvent::InitEvent() {
66 // Perform an atomic store to start the state machine.
67
68 // Note carefully that LockfreeEvent *MAY* be used whilst in a destroyed
69 // state, while a file descriptor is on a freelist. In such a state it may
70 // be SetReady'd, and so we need to perform an atomic operation here to
71 // ensure no races
72 state_.store(kClosureNotReady, std::memory_order_relaxed);
73 }
74
DestroyEvent()75 void LockfreeEvent::DestroyEvent() {
76 intptr_t curr;
77 do {
78 curr = state_.load(std::memory_order_relaxed);
79 if (curr & kShutdownBit) {
80 grpc_core::internal::StatusFreeHeapPtr(curr & ~kShutdownBit);
81 } else {
82 GPR_ASSERT(curr == kClosureNotReady || curr == kClosureReady);
83 }
84 // we CAS in a shutdown, no error value here. If this event is interacted
85 // with post-deletion (see the note in the constructor) we want the bit
86 // pattern to prevent error retention in a deleted object
87 } while (!state_.compare_exchange_strong(curr, kShutdownBit,
88 std::memory_order_acq_rel,
89 std::memory_order_acquire));
90 }
91
NotifyOn(PosixEngineClosure * closure)92 void LockfreeEvent::NotifyOn(PosixEngineClosure* closure) {
93 // This load needs to be an acquire load because this can be a shutdown
94 // error that we might need to reference. Adding acquire semantics makes
95 // sure that the shutdown error has been initialized properly before us
96 // referencing it. The load() needs to be performed only once before entry
97 // into the loop. This is because if any of the compare_exchange_strong
98 // operations inside the loop return false, they automatically update curr
99 // with the new value. So it doesn't need to be loaded again.
100 intptr_t curr = state_.load(std::memory_order_acquire);
101 while (true) {
102 switch (curr) {
103 case kClosureNotReady: {
104 // kClosureNotReady -> <closure>.
105
106 if (state_.compare_exchange_strong(
107 curr, reinterpret_cast<intptr_t>(closure),
108 std::memory_order_acq_rel, std::memory_order_acquire)) {
109 return; // Successful. Return
110 }
111
112 break; // retry
113 }
114
115 case kClosureReady: {
116 // Change the state to kClosureNotReady. Schedule the closure if
117 // successful. If not, the state most likely transitioned to shutdown.
118 // We should retry.
119
120 if (state_.compare_exchange_strong(curr, kClosureNotReady,
121 std::memory_order_acq_rel,
122 std::memory_order_acquire)) {
123 scheduler_->Run(closure);
124 return; // Successful. Return.
125 }
126 break; // retry
127 }
128
129 default: {
130 // 'curr' is either a closure or the fd is shutdown(in which case 'curr'
131 // contains a pointer to the shutdown-error). If the fd is shutdown,
132 // schedule the closure with the shutdown error
133 if ((curr & kShutdownBit) > 0) {
134 absl::Status shutdown_err =
135 grpc_core::internal::StatusGetFromHeapPtr(curr & ~kShutdownBit);
136 closure->SetStatus(shutdown_err);
137 scheduler_->Run(closure);
138 return;
139 }
140
141 // There is already a closure!. This indicates a bug in the code.
142 grpc_core::Crash(
143 "LockfreeEvent::NotifyOn: notify_on called with a previous "
144 "callback still pending");
145 }
146 }
147 }
148
149 GPR_UNREACHABLE_CODE(return);
150 }
151
SetShutdown(absl::Status shutdown_error)152 bool LockfreeEvent::SetShutdown(absl::Status shutdown_error) {
153 intptr_t status_ptr = grpc_core::internal::StatusAllocHeapPtr(shutdown_error);
154 gpr_atm new_state = status_ptr | kShutdownBit;
155 // The load() needs to be performed only once before entry
156 // into the loop. This is because if any of the compare_exchange_strong
157 // operations inside the loop return false, they automatically update curr
158 // with the new value. So it doesn't need to be loaded again.
159 intptr_t curr = state_.load(std::memory_order_acquire);
160
161 while (true) {
162 switch (curr) {
163 case kClosureReady:
164 case kClosureNotReady:
165 // Need a full barrier here so that the initial load in notify_on
166 // doesn't need a barrier
167 if (state_.compare_exchange_strong(curr, new_state,
168 std::memory_order_acq_rel,
169 std::memory_order_acquire)) {
170 return true; // early out
171 }
172 break; // retry
173
174 default: {
175 // 'curr' is either a closure or the fd is already shutdown
176
177 // If fd is already shutdown, we are done.
178 if ((curr & kShutdownBit) > 0) {
179 grpc_core::internal::StatusFreeHeapPtr(status_ptr);
180 return false;
181 }
182
183 // Fd is not shutdown. Schedule the closure and move the state to
184 // shutdown state.
185 // Needs an acquire to pair with setting the closure (and get a
186 // happens-after on that edge), and a release to pair with anything
187 // loading the shutdown state.
188 if (state_.compare_exchange_strong(curr, new_state,
189 std::memory_order_acq_rel,
190 std::memory_order_acquire)) {
191 auto closure = reinterpret_cast<PosixEngineClosure*>(curr);
192 closure->SetStatus(shutdown_error);
193 scheduler_->Run(closure);
194 return true;
195 }
196 // 'curr' was a closure but now changed to a different state. We will
197 // have to retry
198 break;
199 }
200 }
201 }
202 GPR_UNREACHABLE_CODE(return false);
203 }
204
SetReady()205 void LockfreeEvent::SetReady() {
206 // The load() needs to be performed only once before entry
207 // into the loop. This is because if any of the compare_exchange_strong
208 // operations inside the loop return false, they automatically update curr
209 // with the new value. So it doesn't need to be loaded again.
210 intptr_t curr = state_.load(std::memory_order_acquire);
211 while (true) {
212 switch (curr) {
213 case kClosureReady: {
214 // Already ready. We are done here.
215 return;
216 }
217
218 case kClosureNotReady: {
219 if (state_.compare_exchange_strong(curr, kClosureReady,
220 std::memory_order_acq_rel,
221 std::memory_order_acquire)) {
222 return; // early out
223 }
224 break; // retry
225 }
226
227 default: {
228 // 'curr' is either a closure or the fd is shutdown
229 if ((curr & kShutdownBit) > 0) {
230 // The fd is shutdown. Do nothing.
231 return;
232 } else if (state_.compare_exchange_strong(curr, kClosureNotReady,
233 std::memory_order_acq_rel,
234 std::memory_order_acquire)) {
235 // Full cas: acquire pairs with this cas' release in the event of a
236 // spurious set_ready; release pairs with this or the acquire in
237 // notify_on (or set_shutdown)
238 auto closure = reinterpret_cast<PosixEngineClosure*>(curr);
239 closure->SetStatus(absl::OkStatus());
240 scheduler_->Run(closure);
241 return;
242 }
243 // else the state changed again (only possible by either a racing
244 // set_ready or set_shutdown functions. In both these cases, the
245 // closure would have been scheduled for execution. So we are done
246 // here
247 return;
248 }
249 }
250 }
251 }
252
253 } // namespace experimental
254 } // namespace grpc_event_engine
255