1 // 2 // 3 // Copyright 2023 gRPC authors. 4 // 5 // Licensed under the Apache License, Version 2.0 (the "License"); 6 // you may not use this file except in compliance with the License. 7 // You may obtain a copy of the License at 8 // 9 // http://www.apache.org/licenses/LICENSE-2.0 10 // 11 // Unless required by applicable law or agreed to in writing, software 12 // distributed under the License is distributed on an "AS IS" BASIS, 13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 // See the License for the specific language governing permissions and 15 // limitations under the License. 16 // 17 // 18 19 #ifndef GRPC_SRC_CPP_EXT_OTEL_OTEL_PLUGIN_H 20 #define GRPC_SRC_CPP_EXT_OTEL_OTEL_PLUGIN_H 21 22 #include <grpc/support/port_platform.h> 23 24 #include <stddef.h> 25 #include <stdint.h> 26 27 #include <bitset> 28 #include <memory> 29 #include <string> 30 #include <utility> 31 32 #include "absl/container/flat_hash_map.h" 33 #include "absl/container/flat_hash_set.h" 34 #include "absl/functional/any_invocable.h" 35 #include "absl/strings/string_view.h" 36 #include "absl/types/optional.h" 37 #include "opentelemetry/metrics/async_instruments.h" 38 #include "opentelemetry/metrics/meter_provider.h" 39 #include "opentelemetry/metrics/observer_result.h" 40 #include "opentelemetry/metrics/sync_instruments.h" 41 #include "opentelemetry/nostd/shared_ptr.h" 42 43 #include <grpcpp/ext/otel_plugin.h> 44 45 #include "src/core/lib/channel/channel_args.h" 46 #include "src/core/lib/channel/metrics.h" 47 #include "src/core/lib/transport/metadata_batch.h" 48 49 namespace grpc { 50 namespace internal { 51 52 // An iterable container interface that can be used as a return type for the 53 // OpenTelemetry plugin's label injector. 54 class LabelsIterable { 55 public: 56 virtual ~LabelsIterable() = default; 57 58 // Returns the key-value label at the current position or absl::nullopt if the 59 // iterator has reached the end. 60 virtual absl::optional<std::pair<absl::string_view, absl::string_view>> 61 Next() = 0; 62 63 virtual size_t Size() const = 0; 64 65 // Resets position of iterator to the start. 66 virtual void ResetIteratorPosition() = 0; 67 }; 68 69 // An interface that allows you to add additional labels on the calls traced 70 // through the OpenTelemetry plugin. 71 class LabelsInjector { 72 public: ~LabelsInjector()73 virtual ~LabelsInjector() {} 74 // Read the incoming initial metadata to get the set of labels to be added to 75 // metrics. 76 virtual std::unique_ptr<LabelsIterable> GetLabels( 77 grpc_metadata_batch* incoming_initial_metadata) const = 0; 78 79 // Modify the outgoing initial metadata with metadata information to be sent 80 // to the peer. On the server side, \a labels_from_incoming_metadata returned 81 // from `GetLabels` should be provided as input here. On the client side, this 82 // should be nullptr. 83 virtual void AddLabels( 84 grpc_metadata_batch* outgoing_initial_metadata, 85 LabelsIterable* labels_from_incoming_metadata) const = 0; 86 87 // Adds optional labels to the traced calls. Each entry in the span 88 // corresponds to the CallAttemptTracer::OptionalLabelComponent enum. Returns 89 // false when callback returns false. 90 virtual bool AddOptionalLabels( 91 bool is_client, 92 absl::Span<const grpc_core::RefCountedStringValue> optional_labels, 93 opentelemetry::nostd::function_ref< 94 bool(opentelemetry::nostd::string_view, 95 opentelemetry::common::AttributeValue)> 96 callback) const = 0; 97 98 // Gets the actual size of the optional labels that the Plugin is going to 99 // produce through the AddOptionalLabels method. 100 virtual size_t GetOptionalLabelsSize( 101 bool is_client, 102 absl::Span<const grpc_core::RefCountedStringValue> optional_labels) 103 const = 0; 104 }; 105 106 class InternalOpenTelemetryPluginOption 107 : public grpc::OpenTelemetryPluginOption { 108 public: 109 ~InternalOpenTelemetryPluginOption() override = default; 110 // Determines whether a plugin option is active on a given channel target 111 virtual bool IsActiveOnClientChannel(absl::string_view target) const = 0; 112 // Determines whether a plugin option is active on a given server 113 virtual bool IsActiveOnServer(const grpc_core::ChannelArgs& args) const = 0; 114 // Returns the LabelsInjector used by this plugin option, nullptr if none. 115 virtual const grpc::internal::LabelsInjector* labels_injector() const = 0; 116 }; 117 118 // Tags 119 absl::string_view OpenTelemetryMethodKey(); 120 absl::string_view OpenTelemetryStatusKey(); 121 absl::string_view OpenTelemetryTargetKey(); 122 123 class OpenTelemetryPluginBuilderImpl { 124 public: 125 OpenTelemetryPluginBuilderImpl(); 126 ~OpenTelemetryPluginBuilderImpl(); 127 // If `SetMeterProvider()` is not called, no metrics are collected. 128 OpenTelemetryPluginBuilderImpl& SetMeterProvider( 129 std::shared_ptr<opentelemetry::metrics::MeterProvider> meter_provider); 130 // Methods to manipulate which instruments are enabled in the OpenTelemetry 131 // Stats Plugin. The default set of instruments are - 132 // grpc.client.attempt.started 133 // grpc.client.attempt.duration 134 // grpc.client.attempt.sent_total_compressed_message_size 135 // grpc.client.attempt.rcvd_total_compressed_message_size 136 // grpc.server.call.started 137 // grpc.server.call.duration 138 // grpc.server.call.sent_total_compressed_message_size 139 // grpc.server.call.rcvd_total_compressed_message_size 140 OpenTelemetryPluginBuilderImpl& EnableMetrics( 141 absl::Span<const absl::string_view> metric_names); 142 OpenTelemetryPluginBuilderImpl& DisableMetrics( 143 absl::Span<const absl::string_view> metric_names); 144 OpenTelemetryPluginBuilderImpl& DisableAllMetrics(); 145 // If set, \a server_selector is called per incoming call on the server 146 // to decide whether to collect metrics on that call or not. 147 // TODO(yashkt): We should only need to do this per server connection or even 148 // per server. Change this when we have a ServerTracer. 149 OpenTelemetryPluginBuilderImpl& SetServerSelector( 150 absl::AnyInvocable<bool(const grpc_core::ChannelArgs& /*args*/) const> 151 server_selector); 152 // If set, \a target_attribute_filter is called per channel to decide whether 153 // to record the target attribute on client or to replace it with "other". 154 // This helps reduce the cardinality on metrics in cases where many channels 155 // are created with different targets in the same binary (which might happen 156 // for example, if the channel target string uses IP addresses directly). 157 OpenTelemetryPluginBuilderImpl& SetTargetAttributeFilter( 158 absl::AnyInvocable<bool(absl::string_view /*target*/) const> 159 target_attribute_filter); 160 // If set, \a generic_method_attribute_filter is called per call with a 161 // generic method type to decide whether to record the method name or to 162 // replace it with "other". Non-generic or pre-registered methods remain 163 // unaffected. If not set, by default, generic method names are replaced with 164 // "other" when recording metrics. 165 OpenTelemetryPluginBuilderImpl& SetGenericMethodAttributeFilter( 166 absl::AnyInvocable<bool(absl::string_view /*generic_method*/) const> 167 generic_method_attribute_filter); 168 OpenTelemetryPluginBuilderImpl& AddPluginOption( 169 std::unique_ptr<InternalOpenTelemetryPluginOption> option); 170 // Records \a optional_label_key on all metrics that provide it. 171 OpenTelemetryPluginBuilderImpl& AddOptionalLabel( 172 absl::string_view optional_label_key); 173 // Set scope filter to choose which channels are recorded by this plugin. 174 // Server-side recording remains unaffected. 175 OpenTelemetryPluginBuilderImpl& SetChannelScopeFilter( 176 absl::AnyInvocable< 177 bool(const OpenTelemetryPluginBuilder::ChannelScope& /*scope*/) const> 178 channel_scope_filter); 179 absl::Status BuildAndRegisterGlobal(); 180 TestOnlyEnabledMetrics()181 const absl::flat_hash_set<std::string>& TestOnlyEnabledMetrics() { 182 return metrics_; 183 } 184 185 private: 186 std::shared_ptr<opentelemetry::metrics::MeterProvider> meter_provider_; 187 std::unique_ptr<LabelsInjector> labels_injector_; 188 absl::AnyInvocable<bool(absl::string_view /*target*/) const> 189 target_attribute_filter_; 190 absl::flat_hash_set<std::string> metrics_; 191 absl::AnyInvocable<bool(absl::string_view /*generic_method*/) const> 192 generic_method_attribute_filter_; 193 absl::AnyInvocable<bool(const grpc_core::ChannelArgs& /*args*/) const> 194 server_selector_; 195 std::vector<std::unique_ptr<InternalOpenTelemetryPluginOption>> 196 plugin_options_; 197 std::set<absl::string_view> optional_label_keys_; 198 absl::AnyInvocable<bool( 199 const OpenTelemetryPluginBuilder::ChannelScope& /*scope*/) const> 200 channel_scope_filter_; 201 }; 202 203 class OpenTelemetryPlugin : public grpc_core::StatsPlugin { 204 public: 205 OpenTelemetryPlugin( 206 const absl::flat_hash_set<std::string>& metrics, 207 opentelemetry::nostd::shared_ptr<opentelemetry::metrics::MeterProvider> 208 meter_provider, 209 absl::AnyInvocable<bool(absl::string_view /*target*/) const> 210 target_attribute_filter, 211 absl::AnyInvocable<bool(absl::string_view /*generic_method*/) const> 212 generic_method_attribute_filter, 213 absl::AnyInvocable<bool(const grpc_core::ChannelArgs& /*args*/) const> 214 server_selector, 215 std::vector<std::unique_ptr<InternalOpenTelemetryPluginOption>> 216 plugin_options, 217 const std::set<absl::string_view>& optional_label_keys, 218 absl::AnyInvocable< 219 bool(const OpenTelemetryPluginBuilder::ChannelScope& /*scope*/) const> 220 channel_scope_filter); 221 222 private: 223 class ClientCallTracer; 224 class KeyValueIterable; 225 class NPCMetricsKeyValueIterable; 226 class ServerCallTracer; 227 228 // Creates a convenience wrapper to help iterate over only those plugin 229 // options that are active over a given channel/server. 230 class ActivePluginOptionsView { 231 public: MakeForClient(absl::string_view target,const OpenTelemetryPlugin * otel_plugin)232 static ActivePluginOptionsView MakeForClient( 233 absl::string_view target, const OpenTelemetryPlugin* otel_plugin) { 234 return ActivePluginOptionsView( 235 [target](const InternalOpenTelemetryPluginOption& plugin_option) { 236 return plugin_option.IsActiveOnClientChannel(target); 237 }, 238 otel_plugin); 239 } 240 MakeForServer(const grpc_core::ChannelArgs & args,const OpenTelemetryPlugin * otel_plugin)241 static ActivePluginOptionsView MakeForServer( 242 const grpc_core::ChannelArgs& args, 243 const OpenTelemetryPlugin* otel_plugin) { 244 return ActivePluginOptionsView( 245 [&args](const InternalOpenTelemetryPluginOption& plugin_option) { 246 return plugin_option.IsActiveOnServer(args); 247 }, 248 otel_plugin); 249 } 250 ForEach(absl::FunctionRef<bool (const InternalOpenTelemetryPluginOption &,size_t)> func,const OpenTelemetryPlugin * otel_plugin)251 bool ForEach(absl::FunctionRef< 252 bool(const InternalOpenTelemetryPluginOption&, size_t)> 253 func, 254 const OpenTelemetryPlugin* otel_plugin) const { 255 for (size_t i = 0; i < otel_plugin->plugin_options().size(); ++i) { 256 const auto& plugin_option = otel_plugin->plugin_options()[i]; 257 if (active_mask_[i] && !func(*plugin_option, i)) { 258 return false; 259 } 260 } 261 return true; 262 } 263 264 private: ActivePluginOptionsView(absl::FunctionRef<bool (const InternalOpenTelemetryPluginOption &)> func,const OpenTelemetryPlugin * otel_plugin)265 explicit ActivePluginOptionsView( 266 absl::FunctionRef<bool(const InternalOpenTelemetryPluginOption&)> func, 267 const OpenTelemetryPlugin* otel_plugin) { 268 for (size_t i = 0; i < otel_plugin->plugin_options().size(); ++i) { 269 const auto& plugin_option = otel_plugin->plugin_options()[i]; 270 if (plugin_option != nullptr && func(*plugin_option)) { 271 active_mask_.set(i); 272 } 273 } 274 } 275 276 std::bitset<64> active_mask_; 277 }; 278 279 class ClientScopeConfig : public grpc_core::StatsPlugin::ScopeConfig { 280 public: ClientScopeConfig(const OpenTelemetryPlugin * otel_plugin,const OpenTelemetryPluginBuilder::ChannelScope & scope)281 ClientScopeConfig(const OpenTelemetryPlugin* otel_plugin, 282 const OpenTelemetryPluginBuilder::ChannelScope& scope) 283 : active_plugin_options_view_(ActivePluginOptionsView::MakeForClient( 284 scope.target(), otel_plugin)), 285 filtered_target_( 286 // Use the original target string only if a filter on the 287 // attribute is not registered or if the filter returns true, 288 // otherwise use "other". 289 otel_plugin->target_attribute_filter() == nullptr || 290 otel_plugin->target_attribute_filter()(scope.target()) 291 ? scope.target() 292 : "other") {} 293 active_plugin_options_view()294 const ActivePluginOptionsView& active_plugin_options_view() const { 295 return active_plugin_options_view_; 296 } 297 filtered_target()298 absl::string_view filtered_target() const { return filtered_target_; } 299 300 private: 301 ActivePluginOptionsView active_plugin_options_view_; 302 std::string filtered_target_; 303 }; 304 class ServerScopeConfig : public grpc_core::StatsPlugin::ScopeConfig { 305 public: ServerScopeConfig(const OpenTelemetryPlugin * otel_plugin,const grpc_core::ChannelArgs & args)306 ServerScopeConfig(const OpenTelemetryPlugin* otel_plugin, 307 const grpc_core::ChannelArgs& args) 308 : active_plugin_options_view_( 309 ActivePluginOptionsView::MakeForServer(args, otel_plugin)) {} 310 active_plugin_options_view()311 const ActivePluginOptionsView& active_plugin_options_view() const { 312 return active_plugin_options_view_; 313 } 314 315 private: 316 ActivePluginOptionsView active_plugin_options_view_; 317 }; 318 319 struct ClientMetrics { 320 struct Attempt { 321 std::unique_ptr<opentelemetry::metrics::Counter<uint64_t>> started; 322 std::unique_ptr<opentelemetry::metrics::Histogram<double>> duration; 323 std::unique_ptr<opentelemetry::metrics::Histogram<uint64_t>> 324 sent_total_compressed_message_size; 325 std::unique_ptr<opentelemetry::metrics::Histogram<uint64_t>> 326 rcvd_total_compressed_message_size; 327 } attempt; 328 }; 329 struct ServerMetrics { 330 struct Call { 331 std::unique_ptr<opentelemetry::metrics::Counter<uint64_t>> started; 332 std::unique_ptr<opentelemetry::metrics::Histogram<double>> duration; 333 std::unique_ptr<opentelemetry::metrics::Histogram<uint64_t>> 334 sent_total_compressed_message_size; 335 std::unique_ptr<opentelemetry::metrics::Histogram<uint64_t>> 336 rcvd_total_compressed_message_size; 337 } call; 338 }; 339 340 // This object should be used inline. 341 class CallbackMetricReporter : public grpc_core::CallbackMetricReporter { 342 public: 343 CallbackMetricReporter(OpenTelemetryPlugin* ot_plugin, 344 grpc_core::RegisteredMetricCallback* key) 345 ABSL_EXCLUSIVE_LOCKS_REQUIRED(ot_plugin->mu_); 346 347 void Report( 348 grpc_core::GlobalInstrumentsRegistry::GlobalCallbackInt64GaugeHandle 349 handle, 350 int64_t value, absl::Span<const absl::string_view> label_values, 351 absl::Span<const absl::string_view> optional_values) 352 ABSL_EXCLUSIVE_LOCKS_REQUIRED( 353 CallbackGaugeState<int64_t>::ot_plugin->mu_) override; 354 void Report( 355 grpc_core::GlobalInstrumentsRegistry::GlobalCallbackDoubleGaugeHandle 356 handle, 357 double value, absl::Span<const absl::string_view> label_values, 358 absl::Span<const absl::string_view> optional_values) 359 ABSL_EXCLUSIVE_LOCKS_REQUIRED( 360 CallbackGaugeState<double>::ot_plugin->mu_) override; 361 362 private: 363 OpenTelemetryPlugin* ot_plugin_; 364 grpc_core::RegisteredMetricCallback* key_; 365 }; 366 367 // Returns the string form of \a key 368 static absl::string_view OptionalLabelKeyToString( 369 grpc_core::ClientCallTracer::CallAttemptTracer::OptionalLabelKey key); 370 371 // Returns the OptionalLabelKey form of \a key if \a key is recognized and 372 // is public, absl::nullopt otherwise. 373 static absl::optional< 374 grpc_core::ClientCallTracer::CallAttemptTracer::OptionalLabelKey> 375 OptionalLabelStringToKey(absl::string_view key); 376 377 // StatsPlugin: 378 std::pair<bool, std::shared_ptr<grpc_core::StatsPlugin::ScopeConfig>> 379 IsEnabledForChannel( 380 const OpenTelemetryPluginBuilder::ChannelScope& scope) const override; 381 std::pair<bool, std::shared_ptr<grpc_core::StatsPlugin::ScopeConfig>> 382 IsEnabledForServer(const grpc_core::ChannelArgs& args) const override; 383 void AddCounter( 384 grpc_core::GlobalInstrumentsRegistry::GlobalUInt64CounterHandle handle, 385 uint64_t value, absl::Span<const absl::string_view> label_values, 386 absl::Span<const absl::string_view> optional_values) override; 387 void AddCounter( 388 grpc_core::GlobalInstrumentsRegistry::GlobalDoubleCounterHandle handle, 389 double value, absl::Span<const absl::string_view> label_values, 390 absl::Span<const absl::string_view> optional_values) override; 391 void RecordHistogram( 392 grpc_core::GlobalInstrumentsRegistry::GlobalUInt64HistogramHandle handle, 393 uint64_t value, absl::Span<const absl::string_view> label_values, 394 absl::Span<const absl::string_view> optional_values) override; 395 void RecordHistogram( 396 grpc_core::GlobalInstrumentsRegistry::GlobalDoubleHistogramHandle handle, 397 double value, absl::Span<const absl::string_view> label_values, 398 absl::Span<const absl::string_view> optional_values) override; SetGauge(grpc_core::GlobalInstrumentsRegistry::GlobalInt64GaugeHandle,int64_t,absl::Span<const absl::string_view>,absl::Span<const absl::string_view>)399 void SetGauge( 400 grpc_core::GlobalInstrumentsRegistry::GlobalInt64GaugeHandle /*handle*/, 401 int64_t /*value*/, absl::Span<const absl::string_view> /*label_values*/, 402 absl::Span<const absl::string_view> /*optional_values*/) override {} SetGauge(grpc_core::GlobalInstrumentsRegistry::GlobalDoubleGaugeHandle,double,absl::Span<const absl::string_view>,absl::Span<const absl::string_view>)403 void SetGauge( 404 grpc_core::GlobalInstrumentsRegistry::GlobalDoubleGaugeHandle /*handle*/, 405 double /*value*/, absl::Span<const absl::string_view> /*label_values*/, 406 absl::Span<const absl::string_view> /*optional_values*/) override {} 407 void AddCallback(grpc_core::RegisteredMetricCallback* callback) 408 ABSL_LOCKS_EXCLUDED(mu_) override; 409 void RemoveCallback(grpc_core::RegisteredMetricCallback* callback) 410 ABSL_LOCKS_EXCLUDED(mu_) override; 411 grpc_core::ClientCallTracer* GetClientCallTracer( 412 const grpc_core::Slice& path, bool registered_method, 413 std::shared_ptr<grpc_core::StatsPlugin::ScopeConfig> scope_config) 414 override; 415 grpc_core::ServerCallTracer* GetServerCallTracer( 416 std::shared_ptr<grpc_core::StatsPlugin::ScopeConfig> scope_config) 417 override; 418 419 const absl::AnyInvocable<bool(const grpc_core::ChannelArgs& /*args*/) const>& server_selector()420 server_selector() const { 421 return server_selector_; 422 } 423 const absl::AnyInvocable<bool(absl::string_view /*target*/) const>& target_attribute_filter()424 target_attribute_filter() const { 425 return target_attribute_filter_; 426 } 427 const absl::AnyInvocable<bool(absl::string_view /*generic_method*/) const>& generic_method_attribute_filter()428 generic_method_attribute_filter() const { 429 return generic_method_attribute_filter_; 430 } 431 const std::vector<std::unique_ptr<InternalOpenTelemetryPluginOption>>& plugin_options()432 plugin_options() const { 433 return plugin_options_; 434 } 435 436 template <typename ValueType> 437 struct CallbackGaugeState { 438 // It's possible to set values for multiple sets of labels at the same time 439 // in a single callback. Key is a vector of label values and enabled 440 // optional label values. 441 using Cache = absl::flat_hash_map<std::vector<std::string>, ValueType>; 442 grpc_core::GlobalInstrumentsRegistry::InstrumentID id; 443 opentelemetry::nostd::shared_ptr< 444 opentelemetry::metrics::ObservableInstrument> 445 instrument; 446 bool ot_callback_registered ABSL_GUARDED_BY(ot_plugin->mu_); 447 // instrument1 ----- RegisteredMetricCallback1 448 // x 449 // instrument2 ----- RegisteredMetricCallback2 450 // One instrument can be registered by multiple callbacks. 451 absl::flat_hash_map<grpc_core::RegisteredMetricCallback*, Cache> caches 452 ABSL_GUARDED_BY(ot_plugin->mu_); 453 OpenTelemetryPlugin* ot_plugin; 454 455 static void CallbackGaugeCallback( 456 opentelemetry::metrics::ObserverResult result, void* arg) 457 ABSL_LOCKS_EXCLUDED(ot_plugin->mu_); 458 459 void Observe(opentelemetry::metrics::ObserverResult& result, 460 const Cache& cache) 461 ABSL_EXCLUSIVE_LOCKS_REQUIRED(ot_plugin->mu_); 462 }; 463 464 // Instruments for per-call metrics. 465 ClientMetrics client_; 466 ServerMetrics server_; 467 static constexpr int kOptionalLabelsSizeLimit = 64; 468 using OptionalLabelsBitSet = std::bitset<kOptionalLabelsSizeLimit>; 469 OptionalLabelsBitSet per_call_optional_label_bits_; 470 // Instruments for non-per-call metrics. 471 struct Disabled {}; 472 using Instrument = absl::variant< 473 Disabled, std::unique_ptr<opentelemetry::metrics::Counter<uint64_t>>, 474 std::unique_ptr<opentelemetry::metrics::Counter<double>>, 475 std::unique_ptr<opentelemetry::metrics::Histogram<uint64_t>>, 476 std::unique_ptr<opentelemetry::metrics::Histogram<double>>, 477 std::unique_ptr<CallbackGaugeState<int64_t>>, 478 std::unique_ptr<CallbackGaugeState<double>>>; 479 struct InstrumentData { 480 Instrument instrument; 481 OptionalLabelsBitSet optional_labels_bits; 482 }; 483 std::vector<InstrumentData> instruments_data_; 484 grpc_core::Mutex mu_; 485 absl::flat_hash_map<grpc_core::RegisteredMetricCallback*, 486 grpc_core::Timestamp> 487 callback_timestamps_ ABSL_GUARDED_BY(mu_); 488 opentelemetry::nostd::shared_ptr<opentelemetry::metrics::MeterProvider> 489 meter_provider_; 490 absl::AnyInvocable<bool(const grpc_core::ChannelArgs& /*args*/) const> 491 server_selector_; 492 absl::AnyInvocable<bool(absl::string_view /*target*/) const> 493 target_attribute_filter_; 494 absl::AnyInvocable<bool(absl::string_view /*generic_method*/) const> 495 generic_method_attribute_filter_; 496 std::vector<std::unique_ptr<InternalOpenTelemetryPluginOption>> 497 plugin_options_; 498 absl::AnyInvocable<bool( 499 const OpenTelemetryPluginBuilder::ChannelScope& /*scope*/) const> 500 channel_scope_filter_; 501 }; 502 503 } // namespace internal 504 } // namespace grpc 505 506 #endif // GRPC_SRC_CPP_EXT_OTEL_OTEL_PLUGIN_H 507