1 // Copyright 2019 Google LLC
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // https://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 // Implementation of the sandbox2::StackTrace class.
16
17 #include "sandboxed_api/sandbox2/stack_trace.h"
18
19 #include <fcntl.h>
20 #include <sys/stat.h>
21 #include <syscall.h>
22 #include <unistd.h>
23
24 #include <memory>
25 #include <string>
26 #include <utility>
27 #include <vector>
28
29 #include "absl/cleanup/cleanup.h"
30 #include "absl/flags/flag.h"
31 #include "absl/log/check.h"
32 #include "absl/log/log.h"
33 #include "absl/memory/memory.h"
34 #include "absl/status/status.h"
35 #include "absl/status/statusor.h"
36 #include "absl/strings/str_cat.h"
37 #include "absl/strings/string_view.h"
38 #include "absl/strings/strip.h"
39 #include "absl/time/time.h"
40 #include "sandboxed_api/config.h"
41 #include "sandboxed_api/sandbox2/comms.h"
42 #include "sandboxed_api/sandbox2/executor.h"
43 #include "sandboxed_api/sandbox2/limits.h"
44 #include "sandboxed_api/sandbox2/mounts.h"
45 #include "sandboxed_api/sandbox2/namespace.h"
46 #include "sandboxed_api/sandbox2/policy.h"
47 #include "sandboxed_api/sandbox2/policybuilder.h"
48 #include "sandboxed_api/sandbox2/regs.h"
49 #include "sandboxed_api/sandbox2/result.h"
50 #include "sandboxed_api/sandbox2/unwind/unwind.h"
51 #include "sandboxed_api/sandbox2/unwind/unwind.pb.h"
52 #include "sandboxed_api/util/fileops.h"
53 #include "sandboxed_api/util/path.h"
54 #include "sandboxed_api/util/raw_logging.h"
55 #include "sandboxed_api/util/status_macros.h"
56
57 ABSL_FLAG(bool, sandbox_disable_all_stack_traces, false,
58 "Completely disable stack trace collection for sandboxees");
59
60 ABSL_FLAG(bool, sandbox_libunwind_crash_handler, true,
61 "Sandbox libunwind when handling violations (preferred)");
62
63 namespace sandbox2 {
64 namespace {
65
66 namespace file = ::sapi::file;
67 namespace file_util = ::sapi::file_util;
68
69 // Similar to GetStackTrace() but without using the sandbox to isolate
70 // libunwind.
UnsafeGetStackTrace(pid_t pid)71 absl::StatusOr<std::vector<std::string>> UnsafeGetStackTrace(pid_t pid) {
72 LOG(WARNING) << "Using non-sandboxed libunwind";
73 return RunLibUnwindAndSymbolizer(pid, kDefaultMaxFrames);
74 }
75
IsSameFile(const std::string & path,const std::string & other)76 bool IsSameFile(const std::string& path, const std::string& other) {
77 struct stat buf, other_buf;
78 if (stat(path.c_str(), &buf) != 0 || stat(other.c_str(), &other_buf) != 0) {
79 return false;
80 }
81 return buf.st_dev == other_buf.st_dev && buf.st_ino == other_buf.st_ino &&
82 buf.st_mode == other_buf.st_mode &&
83 buf.st_nlink == other_buf.st_nlink && buf.st_uid == other_buf.st_uid &&
84 buf.st_gid == other_buf.st_gid && buf.st_rdev == other_buf.st_rdev &&
85 buf.st_size == other_buf.st_size &&
86 buf.st_blksize == other_buf.st_blksize &&
87 buf.st_blocks == other_buf.st_blocks;
88 }
89
90 } // namespace
91
92 class StackTracePeer {
93 public:
94 static absl::StatusOr<std::unique_ptr<Policy>> GetPolicy(
95 pid_t target_pid, const std::string& maps_file,
96 const std::string& app_path, const std::string& exe_path,
97 const Namespace* ns, bool uses_custom_forkserver);
98
99 static absl::StatusOr<std::vector<std::string>> LaunchLibunwindSandbox(
100 const Regs* regs, const Namespace* ns, bool uses_custom_forkserver,
101 int recursion_depth);
102 };
103
GetPolicy(pid_t target_pid,const std::string & maps_file,const std::string & app_path,const std::string & exe_path,const Namespace * ns,bool uses_custom_forkserver)104 absl::StatusOr<std::unique_ptr<Policy>> StackTracePeer::GetPolicy(
105 pid_t target_pid, const std::string& maps_file, const std::string& app_path,
106 const std::string& exe_path, const Namespace* ns,
107 bool uses_custom_forkserver) {
108 PolicyBuilder builder;
109 if (uses_custom_forkserver) {
110 // Custom forkserver just forks, the binary is loaded outside of the
111 // sandboxee's mount namespace.
112 // Add all possible libraries without the need of parsing the binary
113 // or /proc/pid/maps.
114 for (const auto& library_path : {
115 "/usr/lib64",
116 "/usr/lib",
117 "/lib64",
118 "/lib",
119 }) {
120 if (access(library_path, F_OK) != -1) {
121 VLOG(1) << "Adding library folder '" << library_path << "'";
122 builder.AddDirectory(library_path);
123 } else {
124 VLOG(1) << "Could not add library folder '" << library_path
125 << "' as it does not exist";
126 }
127 }
128 } else {
129 // Use the mounttree of the original executable.
130 CHECK(ns != nullptr);
131 Mounts mounts = ns->mounts();
132 mounts.Remove("/proc").IgnoreError();
133 mounts.Remove(app_path).IgnoreError();
134 builder.SetMounts(std::move(mounts));
135 }
136 builder.AllowOpen()
137 .AllowRead()
138 .AllowWrite()
139 .AllowSyscall(__NR_close)
140 .AllowExit()
141 .AllowHandleSignals()
142 .AllowTcMalloc()
143 .AllowSystemMalloc()
144 // for Comms:RecvFD
145 .AllowSyscall(__NR_recvmsg)
146
147 // libunwind
148 .AllowMmapWithoutExec()
149 .AllowStat()
150 .AllowSyscall(__NR_lseek)
151 #ifdef __NR__llseek
152 .AllowSyscall(__NR__llseek) // Newer glibc on PPC
153 #endif
154 .AllowSyscall(__NR_mincore)
155 .AllowSyscall(__NR_munmap)
156 .AllowPipe()
157
158 // Symbolizer
159 .AllowSyscall(__NR_brk)
160 .AllowTime()
161
162 // Other
163 .AllowDup()
164 .AllowSafeFcntl()
165 .AllowGetPIDs()
166
167 // Required for our ptrace replacement.
168 .TrapPtrace()
169
170 // Add proc maps.
171 .AddFileAt(maps_file,
172 file::JoinPath("/proc", absl::StrCat(target_pid), "maps"))
173 .AddFileAt(maps_file,
174 file::JoinPath("/proc", absl::StrCat(target_pid), "task",
175 absl::StrCat(target_pid), "maps"))
176
177 // Add the binary itself.
178 .AddFileAt(exe_path, app_path)
179 .AllowLlvmCoverage();
180
181 return builder.TryBuild();
182 }
183
184 namespace internal {
185 SandboxPeer::SpawnFn SandboxPeer::spawn_fn_ = nullptr;
186 } // namespace internal
187
LaunchLibunwindSandbox(const Regs * regs,const Namespace * ns,bool uses_custom_forkserver,int recursion_depth)188 absl::StatusOr<std::vector<std::string>> StackTracePeer::LaunchLibunwindSandbox(
189 const Regs* regs, const Namespace* ns, bool uses_custom_forkserver,
190 int recursion_depth) {
191 const pid_t pid = regs->pid();
192
193 sapi::file_util::fileops::FDCloser memory_fd(
194 open(absl::StrCat("/proc/", pid, "/mem").c_str(), O_RDONLY));
195 if (memory_fd.get() == -1) {
196 return absl::InternalError("Opening sandboxee process memory failed");
197 }
198 // Tell executor to use this special internal mode. Using `new` to access a
199 // non-public constructor.
200 auto executor = absl::WrapUnique(new Executor(pid, recursion_depth));
201
202 executor->limits()->set_rlimit_cpu(10).set_walltime_limit(absl::Seconds(5));
203
204 // Temporary directory used to provide files from /proc to the unwind sandbox.
205 char unwind_temp_directory_template[] = "/tmp/.sandbox2_unwind_XXXXXX";
206 char* unwind_temp_directory = mkdtemp(unwind_temp_directory_template);
207 if (!unwind_temp_directory) {
208 return absl::InternalError(
209 "Could not create temporary directory for unwinding");
210 }
211 struct UnwindTempDirectoryCleanup {
212 ~UnwindTempDirectoryCleanup() {
213 file_util::fileops::DeleteRecursively(capture);
214 }
215 char* capture;
216 } cleanup{unwind_temp_directory};
217
218 // Copy over important files from the /proc directory as we can't mount them.
219 const std::string unwind_temp_maps_path =
220 file::JoinPath(unwind_temp_directory, "maps");
221
222 if (!file_util::fileops::CopyFile(
223 file::JoinPath("/proc", absl::StrCat(pid), "maps"),
224 unwind_temp_maps_path, 0400)) {
225 return absl::InternalError("Could not copy maps file");
226 }
227
228 // Get path to the binary.
229 // app_path contains the path like it is also in /proc/pid/maps. It is
230 // relative to the sandboxee's mount namespace. If it is not existing
231 // (anymore) it will have a ' (deleted)' suffix.
232 std::string app_path;
233 std::string proc_pid_exe = file::JoinPath("/proc", absl::StrCat(pid), "exe");
234 if (!file_util::fileops::ReadLinkAbsolute(proc_pid_exe, &app_path)) {
235 return absl::InternalError("Could not obtain absolute path to the binary");
236 }
237
238 std::string exe_path;
239 if (IsSameFile(app_path, proc_pid_exe)) {
240 exe_path = app_path;
241 } else {
242 // The exe_path will have a mountable path of the application, even if it
243 // was removed. Resolve app_path backing file.
244 exe_path = ns ? ns->mounts().ResolvePath(app_path).value_or("") : "";
245 }
246
247 if (exe_path.empty()) {
248 // File was probably removed.
249 LOG(WARNING) << "File was removed, using /proc/pid/exe.";
250 app_path = std::string(absl::StripSuffix(app_path, " (deleted)"));
251 // Create a copy of /proc/pid/exe, mount that one.
252 exe_path = file::JoinPath(unwind_temp_directory, "exe");
253 if (!file_util::fileops::CopyFile(proc_pid_exe, exe_path, 0700)) {
254 return absl::InternalError("Could not copy /proc/pid/exe");
255 }
256 }
257
258 VLOG(1) << "Resolved binary: " << app_path << " / " << exe_path;
259
260 // Add mappings for the binary (as they might not have been added due to the
261 // forkserver).
262 SAPI_ASSIGN_OR_RETURN(
263 std::unique_ptr<Policy> policy,
264 StackTracePeer::GetPolicy(pid, unwind_temp_maps_path, app_path, exe_path,
265 ns, uses_custom_forkserver));
266
267 VLOG(1) << "Running libunwind sandbox";
268 auto sandbox =
269 internal::SandboxPeer::Spawn(std::move(executor), std::move(policy));
270 Comms* comms = sandbox->comms();
271
272 UnwindSetup msg;
273 msg.set_pid(pid);
274 msg.set_regs(reinterpret_cast<const char*>(®s->user_regs_),
275 sizeof(regs->user_regs_));
276 msg.set_default_max_frames(kDefaultMaxFrames);
277
278 absl::Cleanup kill_sandbox = [&sandbox]() {
279 sandbox->Kill();
280 sandbox2::Result result = sandbox->AwaitResult();
281 LOG(INFO) << "Libunwind execution status: " << result.ToString();
282 };
283
284 if (!comms->SendProtoBuf(msg)) {
285 return absl::InternalError("Sending libunwind setup message failed");
286 }
287 if (!comms->SendFD(memory_fd.get())) {
288 return absl::InternalError("Sending sandboxee's memory fd failed");
289 }
290 absl::Status status;
291 if (!comms->RecvStatus(&status)) {
292 return absl::InternalError(
293 "Receiving status from libunwind sandbox failed");
294 }
295 SAPI_RETURN_IF_ERROR(status);
296
297 UnwindResult result;
298 if (!comms->RecvProtoBuf(&result)) {
299 return absl::InternalError("Receiving libunwind result failed");
300 }
301
302 std::move(kill_sandbox).Cancel();
303
304 auto sandbox_result = sandbox->AwaitResult();
305
306 LOG(INFO) << "Libunwind execution status: " << sandbox_result.ToString();
307
308 if (sandbox_result.final_status() != Result::OK) {
309 return absl::InternalError(
310 absl::StrCat("libunwind sandbox did not finish properly: ",
311 sandbox_result.ToString()));
312 }
313
314 return std::vector<std::string>(result.stacktrace().begin(),
315 result.stacktrace().end());
316 }
317
GetStackTrace(const Regs * regs,const Namespace * ns,bool uses_custom_forkserver,int recursion_depth)318 absl::StatusOr<std::vector<std::string>> GetStackTrace(
319 const Regs* regs, const Namespace* ns, bool uses_custom_forkserver,
320 int recursion_depth) {
321 if (absl::GetFlag(FLAGS_sandbox_disable_all_stack_traces)) {
322 return absl::UnavailableError("Stacktraces disabled");
323 }
324 if (!regs) {
325 return absl::InvalidArgumentError(
326 "Could not obtain stacktrace, regs == nullptr");
327 }
328
329 if (!absl::GetFlag(FLAGS_sandbox_libunwind_crash_handler)) {
330 return UnsafeGetStackTrace(regs->pid());
331 }
332
333 // Show a warning if sandboxed libunwind is requested but we're running in
334 // a sanitizer build (= we can't use sandboxed libunwind).
335 if (sapi::sanitizers::IsAny()) {
336 LOG(WARNING) << "Sanitizer build, using non-sandboxed libunwind";
337 return UnsafeGetStackTrace(regs->pid());
338 }
339
340 return StackTracePeer::LaunchLibunwindSandbox(
341 regs, ns, uses_custom_forkserver, recursion_depth);
342 }
343
CompactStackTrace(const std::vector<std::string> & stack_trace)344 std::vector<std::string> CompactStackTrace(
345 const std::vector<std::string>& stack_trace) {
346 std::vector<std::string> compact_trace;
347 compact_trace.reserve(stack_trace.size() / 2);
348 const std::string* prev = nullptr;
349 int seen = 0;
350 auto add_repeats = [&compact_trace](int seen) {
351 if (seen != 0) {
352 compact_trace.push_back(
353 absl::StrCat("(previous frame repeated ", seen, " times)"));
354 }
355 };
356 for (const auto& frame : stack_trace) {
357 if (prev && frame == *prev) {
358 ++seen;
359 } else {
360 prev = &frame;
361 add_repeats(seen);
362 seen = 0;
363 compact_trace.push_back(frame);
364 }
365 }
366 add_repeats(seen);
367 return compact_trace;
368 }
369
370 } // namespace sandbox2
371