1 // Copyright 2020 The Abseil Authors. 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // https://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 #ifndef ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_ 15 #define ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_ 16 17 #include "absl/base/config.h" 18 19 #ifdef _WIN32 20 #include <windows.h> 21 #else 22 #include <sys/time.h> 23 #include <unistd.h> 24 #endif 25 26 #ifdef __linux__ 27 #include <linux/futex.h> 28 #include <sys/syscall.h> 29 #endif 30 31 #include <errno.h> 32 #include <stdio.h> 33 #include <time.h> 34 35 #include <atomic> 36 #include <cstdint> 37 38 #include "absl/base/optimization.h" 39 #include "absl/synchronization/internal/kernel_timeout.h" 40 41 #ifdef ABSL_INTERNAL_HAVE_FUTEX 42 #error ABSL_INTERNAL_HAVE_FUTEX may not be set on the command line 43 #elif defined(__BIONIC__) 44 // Bionic supports all the futex operations we need even when some of the futex 45 // definitions are missing. 46 #define ABSL_INTERNAL_HAVE_FUTEX 47 #elif defined(__linux__) && defined(FUTEX_CLOCK_REALTIME) 48 // FUTEX_CLOCK_REALTIME requires Linux >= 2.6.28. 49 #define ABSL_INTERNAL_HAVE_FUTEX 50 #endif 51 52 #ifdef ABSL_INTERNAL_HAVE_FUTEX 53 54 namespace absl { 55 ABSL_NAMESPACE_BEGIN 56 namespace synchronization_internal { 57 58 // Some Android headers are missing these definitions even though they 59 // support these futex operations. 60 #ifdef __BIONIC__ 61 #ifndef SYS_futex 62 #define SYS_futex __NR_futex 63 #endif 64 #ifndef FUTEX_WAIT_BITSET 65 #define FUTEX_WAIT_BITSET 9 66 #endif 67 #ifndef FUTEX_PRIVATE_FLAG 68 #define FUTEX_PRIVATE_FLAG 128 69 #endif 70 #ifndef FUTEX_CLOCK_REALTIME 71 #define FUTEX_CLOCK_REALTIME 256 72 #endif 73 #ifndef FUTEX_BITSET_MATCH_ANY 74 #define FUTEX_BITSET_MATCH_ANY 0xFFFFFFFF 75 #endif 76 #endif 77 78 #if defined(__NR_futex_time64) && !defined(SYS_futex_time64) 79 #define SYS_futex_time64 __NR_futex_time64 80 #endif 81 82 #if defined(SYS_futex_time64) && !defined(SYS_futex) 83 #define SYS_futex SYS_futex_time64 84 #endif 85 86 class FutexImpl { 87 public: WaitUntil(std::atomic<int32_t> * v,int32_t val,KernelTimeout t)88 static int WaitUntil(std::atomic<int32_t> *v, int32_t val, 89 KernelTimeout t) { 90 long err = 0; // NOLINT(runtime/int) 91 if (t.has_timeout()) { 92 // https://locklessinc.com/articles/futex_cheat_sheet/ 93 // Unlike FUTEX_WAIT, FUTEX_WAIT_BITSET uses absolute time. 94 struct timespec abs_timeout = t.MakeAbsTimespec(); 95 // Atomically check that the futex value is still 0, and if it 96 // is, sleep until abs_timeout or until woken by FUTEX_WAKE. 97 err = syscall( 98 SYS_futex, reinterpret_cast<int32_t *>(v), 99 FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME, val, 100 &abs_timeout, nullptr, FUTEX_BITSET_MATCH_ANY); 101 } else { 102 // Atomically check that the futex value is still 0, and if it 103 // is, sleep until woken by FUTEX_WAKE. 104 err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v), 105 FUTEX_WAIT | FUTEX_PRIVATE_FLAG, val, nullptr); 106 } 107 if (ABSL_PREDICT_FALSE(err != 0)) { 108 return -errno; 109 } 110 return 0; 111 } 112 WaitBitsetAbsoluteTimeout(std::atomic<int32_t> * v,int32_t val,int32_t bits,const struct timespec * abstime)113 static int WaitBitsetAbsoluteTimeout(std::atomic<int32_t> *v, int32_t val, 114 int32_t bits, 115 const struct timespec *abstime) { 116 // NOLINTNEXTLINE(runtime/int) 117 long err = syscall(SYS_futex, reinterpret_cast<int32_t*>(v), 118 FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG, val, abstime, 119 nullptr, bits); 120 if (ABSL_PREDICT_FALSE(err != 0)) { 121 return -errno; 122 } 123 return 0; 124 } 125 Wake(std::atomic<int32_t> * v,int32_t count)126 static int Wake(std::atomic<int32_t> *v, int32_t count) { 127 // NOLINTNEXTLINE(runtime/int) 128 long err = syscall(SYS_futex, reinterpret_cast<int32_t*>(v), 129 FUTEX_WAKE | FUTEX_PRIVATE_FLAG, count); 130 if (ABSL_PREDICT_FALSE(err < 0)) { 131 return -errno; 132 } 133 return 0; 134 } 135 136 // FUTEX_WAKE_BITSET WakeBitset(std::atomic<int32_t> * v,int32_t count,int32_t bits)137 static int WakeBitset(std::atomic<int32_t> *v, int32_t count, int32_t bits) { 138 // NOLINTNEXTLINE(runtime/int) 139 long err = syscall(SYS_futex, reinterpret_cast<int32_t*>(v), 140 FUTEX_WAKE_BITSET | FUTEX_PRIVATE_FLAG, count, nullptr, 141 nullptr, bits); 142 if (ABSL_PREDICT_FALSE(err < 0)) { 143 return -errno; 144 } 145 return 0; 146 } 147 }; 148 149 class Futex : public FutexImpl {}; 150 151 } // namespace synchronization_internal 152 ABSL_NAMESPACE_END 153 } // namespace absl 154 155 #endif // ABSL_INTERNAL_HAVE_FUTEX 156 157 #endif // ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_ 158