1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc. All rights reserved.
3 // https://developers.google.com/protocol-buffers/
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 // * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 // * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 #include <google/protobuf/arenaz_sampler.h>
32
33 #include <atomic>
34 #include <cstdint>
35 #include <limits>
36
37
38 // Must be included last.
39 #include <google/protobuf/port_def.inc>
40
41 namespace google {
42 namespace protobuf {
43 namespace internal {
44
GlobalThreadSafeArenazSampler()45 ThreadSafeArenazSampler& GlobalThreadSafeArenazSampler() {
46 static auto* sampler = new ThreadSafeArenazSampler();
47 return *sampler;
48 }
49
UnsampleSlow(ThreadSafeArenaStats * info)50 void UnsampleSlow(ThreadSafeArenaStats* info) {
51 GlobalThreadSafeArenazSampler().Unregister(info);
52 }
53
54 #if defined(PROTOBUF_ARENAZ_SAMPLE)
55 namespace {
56
57 PROTOBUF_CONSTINIT std::atomic<bool> g_arenaz_enabled{true};
58 PROTOBUF_CONSTINIT std::atomic<int32_t> g_arenaz_sample_parameter{1 << 10};
59 PROTOBUF_THREAD_LOCAL absl::profiling_internal::ExponentialBiased
60 g_exponential_biased_generator;
61
62 } // namespace
63
64 PROTOBUF_THREAD_LOCAL int64_t global_next_sample = 1LL << 10;
65
ThreadSafeArenaStats()66 ThreadSafeArenaStats::ThreadSafeArenaStats() { PrepareForSampling(); }
67 ThreadSafeArenaStats::~ThreadSafeArenaStats() = default;
68
PrepareForSampling()69 void ThreadSafeArenaStats::PrepareForSampling() {
70 num_allocations.store(0, std::memory_order_relaxed);
71 num_resets.store(0, std::memory_order_relaxed);
72 bytes_requested.store(0, std::memory_order_relaxed);
73 bytes_allocated.store(0, std::memory_order_relaxed);
74 bytes_wasted.store(0, std::memory_order_relaxed);
75 max_bytes_allocated.store(0, std::memory_order_relaxed);
76 thread_ids.store(0, std::memory_order_relaxed);
77 // The inliner makes hardcoded skip_count difficult (especially when combined
78 // with LTO). We use the ability to exclude stacks by regex when encoding
79 // instead.
80 depth = absl::GetStackTrace(stack, kMaxStackDepth, /* skip_count= */ 0);
81 }
82
RecordResetSlow(ThreadSafeArenaStats * info)83 void RecordResetSlow(ThreadSafeArenaStats* info) {
84 const size_t max_bytes =
85 info->max_bytes_allocated.load(std::memory_order_relaxed);
86 const size_t allocated_bytes =
87 info->bytes_allocated.load(std::memory_order_relaxed);
88 if (max_bytes < allocated_bytes) {
89 info->max_bytes_allocated.store(allocated_bytes);
90 }
91 info->bytes_requested.store(0, std::memory_order_relaxed);
92 info->bytes_allocated.store(0, std::memory_order_relaxed);
93 info->bytes_wasted.fetch_add(0, std::memory_order_relaxed);
94 info->num_allocations.fetch_add(0, std::memory_order_relaxed);
95 info->num_resets.fetch_add(1, std::memory_order_relaxed);
96 }
97
RecordAllocateSlow(ThreadSafeArenaStats * info,size_t requested,size_t allocated,size_t wasted)98 void RecordAllocateSlow(ThreadSafeArenaStats* info, size_t requested,
99 size_t allocated, size_t wasted) {
100 info->bytes_requested.fetch_add(requested, std::memory_order_relaxed);
101 info->bytes_allocated.fetch_add(allocated, std::memory_order_relaxed);
102 info->bytes_wasted.fetch_add(wasted, std::memory_order_relaxed);
103 info->num_allocations.fetch_add(1, std::memory_order_relaxed);
104 const uint64_t tid = (1ULL << (GetCachedTID() % 63));
105 const uint64_t thread_ids = info->thread_ids.load(std::memory_order_relaxed);
106 if (!(thread_ids & tid)) {
107 info->thread_ids.store(thread_ids | tid, std::memory_order_relaxed);
108 }
109 }
110
SampleSlow(int64_t * next_sample)111 ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) {
112 bool first = *next_sample < 0;
113 *next_sample = g_exponential_biased_generator.GetStride(
114 g_arenaz_sample_parameter.load(std::memory_order_relaxed));
115 // Small values of interval are equivalent to just sampling next time.
116 ABSL_ASSERT(*next_sample >= 1);
117
118 // g_arenaz_enabled can be dynamically flipped, we need to set a threshold low
119 // enough that we will start sampling in a reasonable time, so we just use the
120 // default sampling rate.
121 if (!g_arenaz_enabled.load(std::memory_order_relaxed)) return nullptr;
122 // We will only be negative on our first count, so we should just retry in
123 // that case.
124 if (first) {
125 if (PROTOBUF_PREDICT_TRUE(--*next_sample > 0)) return nullptr;
126 return SampleSlow(next_sample);
127 }
128
129 return GlobalThreadSafeArenazSampler().Register();
130 }
131
SetThreadSafeArenazEnabled(bool enabled)132 void SetThreadSafeArenazEnabled(bool enabled) {
133 g_arenaz_enabled.store(enabled, std::memory_order_release);
134 }
135
SetThreadSafeArenazSampleParameter(int32_t rate)136 void SetThreadSafeArenazSampleParameter(int32_t rate) {
137 if (rate > 0) {
138 g_arenaz_sample_parameter.store(rate, std::memory_order_release);
139 } else {
140 ABSL_RAW_LOG(ERROR, "Invalid thread safe arenaz sample rate: %lld",
141 static_cast<long long>(rate)); // NOLINT(runtime/int)
142 }
143 }
144
SetThreadSafeArenazMaxSamples(int32_t max)145 void SetThreadSafeArenazMaxSamples(int32_t max) {
146 if (max > 0) {
147 GlobalThreadSafeArenazSampler().SetMaxSamples(max);
148 } else {
149 ABSL_RAW_LOG(ERROR, "Invalid thread safe arenaz max samples: %lld",
150 static_cast<long long>(max)); // NOLINT(runtime/int)
151 }
152 }
153
SetThreadSafeArenazGlobalNextSample(int64_t next_sample)154 void SetThreadSafeArenazGlobalNextSample(int64_t next_sample) {
155 if (next_sample >= 0) {
156 global_next_sample = next_sample;
157 } else {
158 ABSL_RAW_LOG(ERROR, "Invalid thread safe arenaz next sample: %lld",
159 static_cast<long long>(next_sample)); // NOLINT(runtime/int)
160 }
161 }
162
163 #else
SampleSlow(int64_t * next_sample)164 ThreadSafeArenaStats* SampleSlow(int64_t* next_sample) {
165 *next_sample = std::numeric_limits<int64_t>::max();
166 return nullptr;
167 }
168
SetThreadSafeArenazEnabled(bool enabled)169 void SetThreadSafeArenazEnabled(bool enabled) {}
SetThreadSafeArenazSampleParameter(int32_t rate)170 void SetThreadSafeArenazSampleParameter(int32_t rate) {}
SetThreadSafeArenazMaxSamples(int32_t max)171 void SetThreadSafeArenazMaxSamples(int32_t max) {}
SetThreadSafeArenazGlobalNextSample(int64_t next_sample)172 void SetThreadSafeArenazGlobalNextSample(int64_t next_sample) {}
173 #endif // defined(PROTOBUF_ARENAZ_SAMPLE)
174
175 } // namespace internal
176 } // namespace protobuf
177 } // namespace google
178