1 //
2 //
3 // Copyright 2023 gRPC authors.
4 //
5 // Licensed under the Apache License, Version 2.0 (the "License");
6 // you may not use this file except in compliance with the License.
7 // You may obtain a copy of the License at
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 //
17 //
18
19 #include <grpc/support/port_platform.h>
20
21 #include "src/cpp/ext/csm/metadata_exchange.h"
22
23 #include <stddef.h>
24
25 #include <algorithm>
26 #include <array>
27 #include <cstdint>
28 #include <unordered_map>
29
30 #include "absl/status/statusor.h"
31 #include "absl/strings/escaping.h"
32 #include "absl/strings/str_split.h"
33 #include "absl/strings/string_view.h"
34 #include "absl/strings/strip.h"
35 #include "absl/types/optional.h"
36 #include "absl/types/variant.h"
37 #include "opentelemetry/sdk/resource/semantic_conventions.h"
38 #include "upb/base/string_view.h"
39
40 #include <grpc/slice.h>
41
42 #include "src/core/lib/channel/call_tracer.h"
43 #include "src/core/lib/gprpp/env.h"
44 #include "src/core/lib/gprpp/load_file.h"
45 #include "src/core/lib/iomgr/error.h"
46 #include "src/core/lib/json/json_args.h"
47 #include "src/core/lib/json/json_object_loader.h"
48 #include "src/core/lib/json/json_reader.h"
49 #include "src/core/lib/slice/slice_internal.h"
50 #include "src/cpp/ext/otel/key_value_iterable.h"
51
52 namespace grpc {
53 namespace internal {
54
55 using OptionalLabelKey =
56 grpc_core::ClientCallTracer::CallAttemptTracer::OptionalLabelKey;
57
58 namespace {
59
60 // The keys that will be used in the Metadata Exchange between local and remote.
61 constexpr absl::string_view kMetadataExchangeTypeKey = "type";
62 constexpr absl::string_view kMetadataExchangeWorkloadNameKey = "workload_name";
63 constexpr absl::string_view kMetadataExchangeNamespaceNameKey =
64 "namespace_name";
65 constexpr absl::string_view kMetadataExchangeClusterNameKey = "cluster_name";
66 constexpr absl::string_view kMetadataExchangeLocationKey = "location";
67 constexpr absl::string_view kMetadataExchangeProjectIdKey = "project_id";
68 constexpr absl::string_view kMetadataExchangeCanonicalServiceKey =
69 "canonical_service";
70 // The keys that will be used for the local attributes when recording metrics.
71 constexpr absl::string_view kCanonicalServiceAttribute =
72 "csm.workload_canonical_service";
73 constexpr absl::string_view kMeshIdAttribute = "csm.mesh_id";
74 // The keys that will be used for the peer attributes when recording metrics.
75 constexpr absl::string_view kPeerTypeAttribute = "csm.remote_workload_type";
76 constexpr absl::string_view kPeerWorkloadNameAttribute =
77 "csm.remote_workload_name";
78 constexpr absl::string_view kPeerNamespaceNameAttribute =
79 "csm.remote_workload_namespace_name";
80 constexpr absl::string_view kPeerClusterNameAttribute =
81 "csm.remote_workload_cluster_name";
82 constexpr absl::string_view kPeerLocationAttribute =
83 "csm.remote_workload_location";
84 constexpr absl::string_view kPeerProjectIdAttribute =
85 "csm.remote_workload_project_id";
86 constexpr absl::string_view kPeerCanonicalServiceAttribute =
87 "csm.remote_workload_canonical_service";
88 // Type values used by Google Cloud Resource Detector
89 constexpr absl::string_view kGkeType = "gcp_kubernetes_engine";
90 constexpr absl::string_view kGceType = "gcp_compute_engine";
91
92 // A helper method that decodes the remote metadata \a slice as a protobuf
93 // Struct allocated on \a arena.
DecodeMetadata(grpc_core::Slice slice,upb_Arena * arena)94 google_protobuf_Struct* DecodeMetadata(grpc_core::Slice slice,
95 upb_Arena* arena) {
96 // Treat an empty slice as an invalid metadata value.
97 if (slice.empty()) {
98 return nullptr;
99 }
100 // Decode the slice.
101 std::string decoded_metadata;
102 bool metadata_decoded =
103 absl::Base64Unescape(slice.as_string_view(), &decoded_metadata);
104 if (metadata_decoded) {
105 return google_protobuf_Struct_parse(decoded_metadata.c_str(),
106 decoded_metadata.size(), arena);
107 }
108 return nullptr;
109 }
110
111 // A minimal class for helping with the information we need from the xDS
112 // bootstrap file for GSM Observability reasons.
113 class XdsBootstrapForGSM {
114 public:
115 class Node {
116 public:
id() const117 const std::string& id() const { return id_; }
118
JsonLoader(const grpc_core::JsonArgs &)119 static const grpc_core::JsonLoaderInterface* JsonLoader(
120 const grpc_core::JsonArgs&) {
121 static const auto* loader =
122 grpc_core::JsonObjectLoader<Node>().Field("id", &Node::id_).Finish();
123 return loader;
124 }
125
126 private:
127 std::string id_;
128 };
129
node() const130 const Node& node() const { return node_; }
131
JsonLoader(const grpc_core::JsonArgs &)132 static const grpc_core::JsonLoaderInterface* JsonLoader(
133 const grpc_core::JsonArgs&) {
134 static const auto* loader =
135 grpc_core::JsonObjectLoader<XdsBootstrapForGSM>()
136 .Field("node", &XdsBootstrapForGSM::node_)
137 .Finish();
138 return loader;
139 }
140
141 private:
142 Node node_;
143 };
144
145 // Returns an empty string if no bootstrap config is found.
GetXdsBootstrapContents()146 std::string GetXdsBootstrapContents() {
147 // First, try GRPC_XDS_BOOTSTRAP env var.
148 auto path = grpc_core::GetEnv("GRPC_XDS_BOOTSTRAP");
149 if (path.has_value()) {
150 auto contents = grpc_core::LoadFile(*path, /*add_null_terminator=*/true);
151 if (!contents.ok()) return "";
152 return std::string(contents->as_string_view());
153 }
154 // Next, try GRPC_XDS_BOOTSTRAP_CONFIG env var.
155 auto env_config = grpc_core::GetEnv("GRPC_XDS_BOOTSTRAP_CONFIG");
156 if (env_config.has_value()) {
157 return std::move(*env_config);
158 }
159 // No bootstrap config found.
160 return "";
161 }
162
StringToGcpResourceType(absl::string_view type)163 MeshLabelsIterable::GcpResourceType StringToGcpResourceType(
164 absl::string_view type) {
165 if (type == kGkeType) {
166 return MeshLabelsIterable::GcpResourceType::kGke;
167 } else if (type == kGceType) {
168 return MeshLabelsIterable::GcpResourceType::kGce;
169 }
170 return MeshLabelsIterable::GcpResourceType::kUnknown;
171 }
172
AbslStrToUpbStr(absl::string_view str)173 upb_StringView AbslStrToUpbStr(absl::string_view str) {
174 return upb_StringView_FromDataAndSize(str.data(), str.size());
175 }
176
UpbStrToAbslStr(upb_StringView str)177 absl::string_view UpbStrToAbslStr(upb_StringView str) {
178 return absl::string_view(str.data, str.size);
179 }
180
AddStringKeyValueToStructProto(google_protobuf_Struct * struct_pb,absl::string_view key,absl::string_view value,upb_Arena * arena)181 void AddStringKeyValueToStructProto(google_protobuf_Struct* struct_pb,
182 absl::string_view key,
183 absl::string_view value, upb_Arena* arena) {
184 google_protobuf_Value* value_pb = google_protobuf_Value_new(arena);
185 google_protobuf_Value_set_string_value(value_pb, AbslStrToUpbStr(value));
186 google_protobuf_Struct_fields_set(struct_pb, AbslStrToUpbStr(key), value_pb,
187 arena);
188 }
189
GetStringValueFromAttributeMap(const opentelemetry::sdk::common::AttributeMap & map,absl::string_view key)190 absl::string_view GetStringValueFromAttributeMap(
191 const opentelemetry::sdk::common::AttributeMap& map,
192 absl::string_view key) {
193 const auto& attributes = map.GetAttributes();
194 const auto it = attributes.find(std::string(key));
195 if (it == attributes.end()) {
196 return "unknown";
197 }
198 const auto* string_value = absl::get_if<std::string>(&it->second);
199 if (string_value == nullptr) {
200 return "unknown";
201 }
202 return *string_value;
203 }
204
GetStringValueFromUpbStruct(google_protobuf_Struct * struct_pb,absl::string_view key,upb_Arena * arena)205 absl::string_view GetStringValueFromUpbStruct(google_protobuf_Struct* struct_pb,
206 absl::string_view key,
207 upb_Arena* arena) {
208 if (struct_pb == nullptr) {
209 return "unknown";
210 }
211 google_protobuf_Value* value_pb = google_protobuf_Value_new(arena);
212 bool present = google_protobuf_Struct_fields_get(
213 struct_pb, AbslStrToUpbStr(key), &value_pb);
214 if (present) {
215 if (google_protobuf_Value_has_string_value(value_pb)) {
216 return UpbStrToAbslStr(google_protobuf_Value_string_value(value_pb));
217 }
218 }
219 return "unknown";
220 }
221
222 struct RemoteAttribute {
223 absl::string_view otel_attribute;
224 absl::string_view metadata_attribute;
225 };
226
227 constexpr std::array<RemoteAttribute, 2> kFixedAttributes = {
228 RemoteAttribute{kPeerTypeAttribute, kMetadataExchangeTypeKey},
229 RemoteAttribute{kPeerCanonicalServiceAttribute,
230 kMetadataExchangeCanonicalServiceKey},
231 };
232
233 constexpr std::array<RemoteAttribute, 5> kGkeAttributeList = {
234 RemoteAttribute{kPeerWorkloadNameAttribute,
235 kMetadataExchangeWorkloadNameKey},
236 RemoteAttribute{kPeerNamespaceNameAttribute,
237 kMetadataExchangeNamespaceNameKey},
238 RemoteAttribute{kPeerClusterNameAttribute, kMetadataExchangeClusterNameKey},
239 RemoteAttribute{kPeerLocationAttribute, kMetadataExchangeLocationKey},
240 RemoteAttribute{kPeerProjectIdAttribute, kMetadataExchangeProjectIdKey},
241 };
242
243 constexpr std::array<RemoteAttribute, 3> kGceAttributeList = {
244 RemoteAttribute{kPeerWorkloadNameAttribute,
245 kMetadataExchangeWorkloadNameKey},
246 RemoteAttribute{kPeerLocationAttribute, kMetadataExchangeLocationKey},
247 RemoteAttribute{kPeerProjectIdAttribute, kMetadataExchangeProjectIdKey},
248 };
249
GetAttributesForType(MeshLabelsIterable::GcpResourceType remote_type)250 absl::Span<const RemoteAttribute> GetAttributesForType(
251 MeshLabelsIterable::GcpResourceType remote_type) {
252 switch (remote_type) {
253 case MeshLabelsIterable::GcpResourceType::kGke:
254 return kGkeAttributeList;
255 case MeshLabelsIterable::GcpResourceType::kGce:
256 return kGceAttributeList;
257 default:
258 return {};
259 }
260 }
261
262 absl::optional<std::pair<absl::string_view, absl::string_view>>
NextFromAttributeList(absl::Span<const RemoteAttribute> attributes,size_t start_index,size_t curr,google_protobuf_Struct * decoded_metadata,upb_Arena * arena)263 NextFromAttributeList(absl::Span<const RemoteAttribute> attributes,
264 size_t start_index, size_t curr,
265 google_protobuf_Struct* decoded_metadata,
266 upb_Arena* arena) {
267 GPR_DEBUG_ASSERT(curr >= start_index);
268 const size_t index = curr - start_index;
269 if (index >= attributes.size()) return absl::nullopt;
270 const auto& attribute = attributes[index];
271 return std::make_pair(
272 attribute.otel_attribute,
273 GetStringValueFromUpbStruct(decoded_metadata,
274 attribute.metadata_attribute, arena));
275 }
276
277 } // namespace
278
279 //
280 // MeshLabelsIterable
281 //
282
MeshLabelsIterable(const std::vector<std::pair<absl::string_view,std::string>> & local_labels,grpc_core::Slice remote_metadata)283 MeshLabelsIterable::MeshLabelsIterable(
284 const std::vector<std::pair<absl::string_view, std::string>>& local_labels,
285 grpc_core::Slice remote_metadata)
286 : struct_pb_(DecodeMetadata(std::move(remote_metadata), arena_.ptr())),
287 local_labels_(local_labels),
288 remote_type_(StringToGcpResourceType(GetStringValueFromUpbStruct(
289 struct_pb_, kMetadataExchangeTypeKey, arena_.ptr()))) {}
290
291 absl::optional<std::pair<absl::string_view, absl::string_view>>
Next()292 MeshLabelsIterable::Next() {
293 size_t local_labels_size = local_labels_.size();
294 if (pos_ < local_labels_size) {
295 return local_labels_[pos_++];
296 }
297 const size_t fixed_attribute_end =
298 local_labels_size + kFixedAttributes.size();
299 if (pos_ < fixed_attribute_end) {
300 return NextFromAttributeList(kFixedAttributes, local_labels_size, pos_++,
301 struct_pb_, arena_.ptr());
302 }
303 return NextFromAttributeList(GetAttributesForType(remote_type_),
304 fixed_attribute_end, pos_++, struct_pb_,
305 arena_.ptr());
306 }
307
Size() const308 size_t MeshLabelsIterable::Size() const {
309 return local_labels_.size() + kFixedAttributes.size() +
310 GetAttributesForType(remote_type_).size();
311 }
312
313 // Returns the mesh ID by reading and parsing the bootstrap file. Returns
314 // "unknown" if for some reason, mesh ID could not be figured out.
GetMeshId()315 std::string GetMeshId() {
316 auto json = grpc_core::JsonParse(GetXdsBootstrapContents());
317 if (!json.ok()) {
318 return "unknown";
319 }
320 auto bootstrap = grpc_core::LoadFromJson<XdsBootstrapForGSM>(*json);
321 if (!bootstrap.ok()) {
322 return "unknown";
323 }
324 // The format of the Node ID is -
325 // projects/[GCP Project number]/networks/mesh:[Mesh ID]/nodes/[UUID]
326 std::vector<absl::string_view> parts =
327 absl::StrSplit(bootstrap->node().id(), '/');
328 if (parts.size() != 6) {
329 return "unknown";
330 }
331 absl::string_view mesh_id = parts[3];
332 if (!absl::ConsumePrefix(&mesh_id, "mesh:")) {
333 return "unknown";
334 }
335 return std::string(mesh_id);
336 }
337
338 //
339 // ServiceMeshLabelsInjector
340 //
341
ServiceMeshLabelsInjector(const opentelemetry::sdk::common::AttributeMap & map)342 ServiceMeshLabelsInjector::ServiceMeshLabelsInjector(
343 const opentelemetry::sdk::common::AttributeMap& map) {
344 upb::Arena arena;
345 auto* metadata = google_protobuf_Struct_new(arena.ptr());
346 // Assume kubernetes for now
347 absl::string_view type_value = GetStringValueFromAttributeMap(
348 map, opentelemetry::sdk::resource::SemanticConventions::kCloudPlatform);
349 std::string workload_name_value =
350 grpc_core::GetEnv("CSM_WORKLOAD_NAME").value_or("unknown");
351 absl::string_view namespace_value = GetStringValueFromAttributeMap(
352 map,
353 opentelemetry::sdk::resource::SemanticConventions::kK8sNamespaceName);
354 absl::string_view cluster_name_value = GetStringValueFromAttributeMap(
355 map, opentelemetry::sdk::resource::SemanticConventions::kK8sClusterName);
356 absl::string_view location_value = GetStringValueFromAttributeMap(
357 map, opentelemetry::sdk::resource::SemanticConventions::
358 kCloudAvailabilityZone); // if zonal
359 if (location_value == "unknown") {
360 location_value = GetStringValueFromAttributeMap(
361 map, opentelemetry::sdk::resource::SemanticConventions::
362 kCloudRegion); // if regional
363 }
364 absl::string_view project_id_value = GetStringValueFromAttributeMap(
365 map, opentelemetry::sdk::resource::SemanticConventions::kCloudAccountId);
366 std::string canonical_service_value =
367 grpc_core::GetEnv("CSM_CANONICAL_SERVICE_NAME").value_or("unknown");
368 // Create metadata to be sent over wire.
369 AddStringKeyValueToStructProto(metadata, kMetadataExchangeTypeKey, type_value,
370 arena.ptr());
371 AddStringKeyValueToStructProto(metadata, kMetadataExchangeCanonicalServiceKey,
372 canonical_service_value, arena.ptr());
373 if (type_value == kGkeType) {
374 AddStringKeyValueToStructProto(metadata, kMetadataExchangeWorkloadNameKey,
375 workload_name_value, arena.ptr());
376 AddStringKeyValueToStructProto(metadata, kMetadataExchangeNamespaceNameKey,
377 namespace_value, arena.ptr());
378 AddStringKeyValueToStructProto(metadata, kMetadataExchangeClusterNameKey,
379 cluster_name_value, arena.ptr());
380 AddStringKeyValueToStructProto(metadata, kMetadataExchangeLocationKey,
381 location_value, arena.ptr());
382 AddStringKeyValueToStructProto(metadata, kMetadataExchangeProjectIdKey,
383 project_id_value, arena.ptr());
384 } else if (type_value == kGceType) {
385 AddStringKeyValueToStructProto(metadata, kMetadataExchangeWorkloadNameKey,
386 workload_name_value, arena.ptr());
387 AddStringKeyValueToStructProto(metadata, kMetadataExchangeLocationKey,
388 location_value, arena.ptr());
389 AddStringKeyValueToStructProto(metadata, kMetadataExchangeProjectIdKey,
390 project_id_value, arena.ptr());
391 }
392
393 size_t output_length;
394 char* output =
395 google_protobuf_Struct_serialize(metadata, arena.ptr(), &output_length);
396 serialized_labels_to_send_ = grpc_core::Slice::FromCopiedString(
397 absl::Base64Escape(absl::string_view(output, output_length)));
398 // Fill up local labels map. The rest we get from the detected Resource and
399 // from the peer.
400 local_labels_.emplace_back(kCanonicalServiceAttribute,
401 canonical_service_value);
402 local_labels_.emplace_back(kMeshIdAttribute, GetMeshId());
403 }
404
GetLabels(grpc_metadata_batch * incoming_initial_metadata) const405 std::unique_ptr<LabelsIterable> ServiceMeshLabelsInjector::GetLabels(
406 grpc_metadata_batch* incoming_initial_metadata) const {
407 auto peer_metadata =
408 incoming_initial_metadata->Take(grpc_core::XEnvoyPeerMetadata());
409 return std::make_unique<MeshLabelsIterable>(
410 local_labels_, peer_metadata.has_value() ? *std::move(peer_metadata)
411 : grpc_core::Slice());
412 }
413
AddLabels(grpc_metadata_batch * outgoing_initial_metadata,LabelsIterable * labels_from_incoming_metadata) const414 void ServiceMeshLabelsInjector::AddLabels(
415 grpc_metadata_batch* outgoing_initial_metadata,
416 LabelsIterable* labels_from_incoming_metadata) const {
417 // On the server, if the labels from incoming metadata did not have a
418 // non-empty base64 encoded "x-envoy-peer-metadata", do not perform metadata
419 // exchange.
420 if (labels_from_incoming_metadata != nullptr &&
421 !static_cast<MeshLabelsIterable*>(labels_from_incoming_metadata)
422 ->GotRemoteLabels()) {
423 return;
424 }
425 outgoing_initial_metadata->Set(grpc_core::XEnvoyPeerMetadata(),
426 serialized_labels_to_send_.Ref());
427 }
428
AddOptionalLabels(bool is_client,absl::Span<const grpc_core::RefCountedStringValue> optional_labels,opentelemetry::nostd::function_ref<bool (opentelemetry::nostd::string_view,opentelemetry::common::AttributeValue)> callback) const429 bool ServiceMeshLabelsInjector::AddOptionalLabels(
430 bool is_client,
431 absl::Span<const grpc_core::RefCountedStringValue> optional_labels,
432 opentelemetry::nostd::function_ref<
433 bool(opentelemetry::nostd::string_view,
434 opentelemetry::common::AttributeValue)>
435 callback) const {
436 if (!is_client) {
437 // Currently the CSM optional labels are only set on client.
438 return true;
439 }
440 // Performs JSON label name format to CSM Observability Metric spec format
441 // conversion.
442 absl::string_view service_name =
443 optional_labels[static_cast<size_t>(
444 grpc_core::ClientCallTracer::CallAttemptTracer::
445 OptionalLabelKey::kXdsServiceName)]
446 .as_string_view();
447 absl::string_view service_namespace =
448 optional_labels[static_cast<size_t>(
449 grpc_core::ClientCallTracer::CallAttemptTracer::
450 OptionalLabelKey::kXdsServiceNamespace)]
451 .as_string_view();
452 return callback("csm.service_name",
453 service_name.empty()
454 ? "unknown"
455 : AbslStrViewToOpenTelemetryStrView(service_name)) &&
456 callback("csm.service_namespace_name",
457 service_namespace.empty()
458 ? "unknown"
459 : AbslStrViewToOpenTelemetryStrView(service_namespace));
460 }
461
462 } // namespace internal
463 } // namespace grpc
464