1 // Copyright 2017 The Abseil Authors.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // https://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 // Allow dynamic symbol lookup in the kernel VDSO page.
16 //
17 // VDSOSupport -- a class representing kernel VDSO (if present).
18
19 #include "absl/debugging/internal/vdso_support.h"
20
21 #ifdef ABSL_HAVE_VDSO_SUPPORT // defined in vdso_support.h
22
23 #if !defined(__has_include)
24 #define __has_include(header) 0
25 #endif
26
27 #include <errno.h>
28 #include <fcntl.h>
29 #if __has_include(<syscall.h>)
30 #include <syscall.h>
31 #elif __has_include(<sys/syscall.h>)
32 #include <sys/syscall.h>
33 #endif
34 #include <unistd.h>
35
36 #if !defined(__UCLIBC__) && defined(__GLIBC__) && \
37 (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 16))
38 #define ABSL_HAVE_GETAUXVAL
39 #endif
40
41 #ifdef ABSL_HAVE_GETAUXVAL
42 #include <sys/auxv.h>
43 #endif
44
45 #include "absl/base/dynamic_annotations.h"
46 #include "absl/base/internal/raw_logging.h"
47 #include "absl/base/port.h"
48
49 #ifndef AT_SYSINFO_EHDR
50 #define AT_SYSINFO_EHDR 33 // for crosstoolv10
51 #endif
52
53 #if defined(__NetBSD__)
54 using Elf32_auxv_t = Aux32Info;
55 using Elf64_auxv_t = Aux64Info;
56 #endif
57 #if defined(__FreeBSD__)
58 #if defined(__ELF_WORD_SIZE) && __ELF_WORD_SIZE == 64
59 using Elf64_auxv_t = Elf64_Auxinfo;
60 #endif
61 using Elf32_auxv_t = Elf32_Auxinfo;
62 #endif
63
64 namespace absl {
65 ABSL_NAMESPACE_BEGIN
66 namespace debugging_internal {
67
68 ABSL_CONST_INIT
69 std::atomic<const void *> VDSOSupport::vdso_base_(
70 debugging_internal::ElfMemImage::kInvalidBase);
71
72 ABSL_CONST_INIT std::atomic<VDSOSupport::GetCpuFn> VDSOSupport::getcpu_fn_(
73 &InitAndGetCPU);
74
VDSOSupport()75 VDSOSupport::VDSOSupport()
76 // If vdso_base_ is still set to kInvalidBase, we got here
77 // before VDSOSupport::Init has been called. Call it now.
78 : image_(vdso_base_.load(std::memory_order_relaxed) ==
79 debugging_internal::ElfMemImage::kInvalidBase
80 ? Init()
81 : vdso_base_.load(std::memory_order_relaxed)) {}
82
83 // NOTE: we can't use GoogleOnceInit() below, because we can be
84 // called by tcmalloc, and none of the *once* stuff may be functional yet.
85 //
86 // In addition, we hope that the VDSOSupportHelper constructor
87 // causes this code to run before there are any threads, and before
88 // InitGoogle() has executed any chroot or setuid calls.
89 //
90 // Finally, even if there is a race here, it is harmless, because
91 // the operation should be idempotent.
Init()92 const void *VDSOSupport::Init() {
93 const auto kInvalidBase = debugging_internal::ElfMemImage::kInvalidBase;
94 #ifdef ABSL_HAVE_GETAUXVAL
95 if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) {
96 errno = 0;
97 const void *const sysinfo_ehdr =
98 reinterpret_cast<const void *>(getauxval(AT_SYSINFO_EHDR));
99 if (errno == 0) {
100 vdso_base_.store(sysinfo_ehdr, std::memory_order_relaxed);
101 }
102 }
103 #endif // ABSL_HAVE_GETAUXVAL
104 if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) {
105 int fd = open("/proc/self/auxv", O_RDONLY);
106 if (fd == -1) {
107 // Kernel too old to have a VDSO.
108 vdso_base_.store(nullptr, std::memory_order_relaxed);
109 getcpu_fn_.store(&GetCPUViaSyscall, std::memory_order_relaxed);
110 return nullptr;
111 }
112 ElfW(auxv_t) aux;
113 while (read(fd, &aux, sizeof(aux)) == sizeof(aux)) {
114 if (aux.a_type == AT_SYSINFO_EHDR) {
115 #if defined(__NetBSD__)
116 vdso_base_.store(reinterpret_cast<void *>(aux.a_v),
117 std::memory_order_relaxed);
118 #else
119 vdso_base_.store(reinterpret_cast<void *>(aux.a_un.a_val),
120 std::memory_order_relaxed);
121 #endif
122 break;
123 }
124 }
125 close(fd);
126 if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) {
127 // Didn't find AT_SYSINFO_EHDR in auxv[].
128 vdso_base_.store(nullptr, std::memory_order_relaxed);
129 }
130 }
131 GetCpuFn fn = &GetCPUViaSyscall; // default if VDSO not present.
132 if (vdso_base_.load(std::memory_order_relaxed)) {
133 VDSOSupport vdso;
134 SymbolInfo info;
135 if (vdso.LookupSymbol("__vdso_getcpu", "LINUX_2.6", STT_FUNC, &info)) {
136 fn = reinterpret_cast<GetCpuFn>(const_cast<void *>(info.address));
137 }
138 }
139 // Subtle: this code runs outside of any locks; prevent compiler
140 // from assigning to getcpu_fn_ more than once.
141 getcpu_fn_.store(fn, std::memory_order_relaxed);
142 return vdso_base_.load(std::memory_order_relaxed);
143 }
144
SetBase(const void * base)145 const void *VDSOSupport::SetBase(const void *base) {
146 ABSL_RAW_CHECK(base != debugging_internal::ElfMemImage::kInvalidBase,
147 "internal error");
148 const void *old_base = vdso_base_.load(std::memory_order_relaxed);
149 vdso_base_.store(base, std::memory_order_relaxed);
150 image_.Init(base);
151 // Also reset getcpu_fn_, so GetCPU could be tested with simulated VDSO.
152 getcpu_fn_.store(&InitAndGetCPU, std::memory_order_relaxed);
153 return old_base;
154 }
155
LookupSymbol(const char * name,const char * version,int type,SymbolInfo * info) const156 bool VDSOSupport::LookupSymbol(const char *name,
157 const char *version,
158 int type,
159 SymbolInfo *info) const {
160 return image_.LookupSymbol(name, version, type, info);
161 }
162
LookupSymbolByAddress(const void * address,SymbolInfo * info_out) const163 bool VDSOSupport::LookupSymbolByAddress(const void *address,
164 SymbolInfo *info_out) const {
165 return image_.LookupSymbolByAddress(address, info_out);
166 }
167
168 // NOLINT on 'long' because this routine mimics kernel api.
GetCPUViaSyscall(unsigned * cpu,void *,void *)169 long VDSOSupport::GetCPUViaSyscall(unsigned *cpu, // NOLINT(runtime/int)
170 void *, void *) {
171 #ifdef SYS_getcpu
172 return syscall(SYS_getcpu, cpu, nullptr, nullptr);
173 #else
174 // x86_64 never implemented sys_getcpu(), except as a VDSO call.
175 static_cast<void>(cpu); // Avoid an unused argument compiler warning.
176 errno = ENOSYS;
177 return -1;
178 #endif
179 }
180
181 // Use fast __vdso_getcpu if available.
InitAndGetCPU(unsigned * cpu,void * x,void * y)182 long VDSOSupport::InitAndGetCPU(unsigned *cpu, // NOLINT(runtime/int)
183 void *x, void *y) {
184 Init();
185 GetCpuFn fn = getcpu_fn_.load(std::memory_order_relaxed);
186 ABSL_RAW_CHECK(fn != &InitAndGetCPU, "Init() did not set getcpu_fn_");
187 return (*fn)(cpu, x, y);
188 }
189
190 // This function must be very fast, and may be called from very
191 // low level (e.g. tcmalloc). Hence I avoid things like
192 // GoogleOnceInit() and ::operator new.
193 ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY
GetCPU()194 int GetCPU() {
195 unsigned cpu;
196 long ret_code = // NOLINT(runtime/int)
197 (*VDSOSupport::getcpu_fn_)(&cpu, nullptr, nullptr);
198 return ret_code == 0 ? static_cast<int>(cpu) : static_cast<int>(ret_code);
199 }
200
201 } // namespace debugging_internal
202 ABSL_NAMESPACE_END
203 } // namespace absl
204
205 #endif // ABSL_HAVE_VDSO_SUPPORT
206