1 /* 2 * Copyright (C) 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_PROTO_TRACE_TOKENIZER_H_ 18 #define SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_PROTO_TRACE_TOKENIZER_H_ 19 20 #include <algorithm> 21 #include <cstddef> 22 #include <cstdint> 23 #include <optional> 24 #include <utility> 25 26 #include "perfetto/base/logging.h" 27 #include "perfetto/base/status.h" 28 #include "perfetto/protozero/field.h" 29 #include "perfetto/protozero/proto_utils.h" 30 #include "perfetto/public/compiler.h" 31 #include "perfetto/trace_processor/trace_blob_view.h" 32 #include "protos/perfetto/trace/trace_packet.pbzero.h" 33 #include "src/trace_processor/util/gzip_utils.h" 34 35 #include "protos/perfetto/trace/trace.pbzero.h" 36 #include "src/trace_processor/util/status_macros.h" 37 #include "src/trace_processor/util/trace_blob_view_reader.h" 38 39 namespace perfetto::trace_processor { 40 41 // Reads a protobuf trace in chunks and extracts boundaries of trace packets 42 // with their timestamps. 43 class ProtoTraceTokenizer { 44 public: 45 ProtoTraceTokenizer(); 46 47 template <typename Callback = base::Status(TraceBlobView)> Tokenize(TraceBlobView tbv,Callback callback)48 base::Status Tokenize(TraceBlobView tbv, Callback callback) { 49 reader_.PushBack(std::move(tbv)); 50 51 for (;;) { 52 size_t start_offset = reader_.start_offset(); 53 size_t avail = reader_.avail(); 54 55 // The header must be at least 2 bytes (1 byte for tag, 1 byte for 56 // size/varint) and can be at most 20 bytes (10 bytes for tag + 10 bytes 57 // for size/varint). 58 const size_t kMinHeaderBytes = 2; 59 const size_t kMaxHeaderBytes = 20; 60 std::optional<TraceBlobView> header = reader_.SliceOff( 61 start_offset, 62 std::min(std::max(avail, kMinHeaderBytes), kMaxHeaderBytes)); 63 64 // This means that kMinHeaderBytes was not available. Just wait for the 65 // next round. 66 if (PERFETTO_UNLIKELY(!header)) { 67 return base::OkStatus(); 68 } 69 70 uint64_t tag; 71 const uint8_t* tag_start = header->data(); 72 const uint8_t* tag_end = protozero::proto_utils::ParseVarInt( 73 tag_start, header->data() + header->size(), &tag); 74 75 if (PERFETTO_UNLIKELY(tag_end == tag_start)) { 76 return header->size() < kMaxHeaderBytes 77 ? base::OkStatus() 78 : base::ErrStatus("Failed to parse tag"); 79 } 80 81 if (PERFETTO_UNLIKELY(tag != kTracePacketTag)) { 82 // Other field. Skip. 83 auto field_type = static_cast<uint8_t>(tag & 0b111); 84 switch (field_type) { 85 case static_cast<uint8_t>( 86 protozero::proto_utils::ProtoWireType::kVarInt): { 87 uint64_t varint; 88 const uint8_t* varint_start = tag_end; 89 const uint8_t* varint_end = protozero::proto_utils::ParseVarInt( 90 tag_end, header->data() + header->size(), &varint); 91 if (PERFETTO_UNLIKELY(varint_end == varint_start)) { 92 return header->size() < kMaxHeaderBytes 93 ? base::OkStatus() 94 : base::ErrStatus("Failed to skip varint"); 95 } 96 PERFETTO_CHECK(reader_.PopFrontBytes( 97 static_cast<size_t>(varint_end - tag_start))); 98 continue; 99 } 100 case static_cast<uint8_t>( 101 protozero::proto_utils::ProtoWireType::kLengthDelimited): { 102 uint64_t varint; 103 const uint8_t* varint_start = tag_end; 104 const uint8_t* varint_end = protozero::proto_utils::ParseVarInt( 105 tag_end, header->data() + header->size(), &varint); 106 if (PERFETTO_UNLIKELY(varint_end == varint_start)) { 107 return header->size() < kMaxHeaderBytes 108 ? base::OkStatus() 109 : base::ErrStatus("Failed to skip delimited"); 110 } 111 112 size_t size_incl_header = 113 static_cast<size_t>(varint_end - tag_start) + varint; 114 if (size_incl_header > avail) { 115 return base::OkStatus(); 116 } 117 PERFETTO_CHECK(reader_.PopFrontBytes(size_incl_header)); 118 continue; 119 } 120 case static_cast<uint8_t>( 121 protozero::proto_utils::ProtoWireType::kFixed64): { 122 size_t size_incl_header = 123 static_cast<size_t>(tag_end - tag_start) + 8; 124 if (size_incl_header > avail) { 125 return base::OkStatus(); 126 } 127 PERFETTO_CHECK(reader_.PopFrontBytes(size_incl_header)); 128 continue; 129 } 130 case static_cast<uint8_t>( 131 protozero::proto_utils::ProtoWireType::kFixed32): { 132 size_t size_incl_header = 133 static_cast<size_t>(tag_end - tag_start) + 4; 134 if (size_incl_header > avail) { 135 return base::OkStatus(); 136 } 137 PERFETTO_CHECK(reader_.PopFrontBytes(size_incl_header)); 138 continue; 139 } 140 default: 141 return base::ErrStatus("Unknown field type"); 142 } 143 } 144 145 uint64_t field_size; 146 const uint8_t* size_start = tag_end; 147 const uint8_t* size_end = protozero::proto_utils::ParseVarInt( 148 size_start, header->data() + header->size(), &field_size); 149 150 // If we had less than the maximum number of header bytes, it's possible 151 // that we just need more to actually parse. Otherwise, this is an error. 152 if (PERFETTO_UNLIKELY(size_start == size_end)) { 153 return header->size() < kMaxHeaderBytes 154 ? base::OkStatus() 155 : base::ErrStatus("Failed to parse TracePacket size"); 156 } 157 158 // Empty packets can legitimately happen if the producer ends up emitting 159 // no data: just ignore them. 160 auto hdr_size = static_cast<size_t>(size_end - header->data()); 161 if (PERFETTO_UNLIKELY(field_size == 0)) { 162 PERFETTO_CHECK(reader_.PopFrontBytes(hdr_size)); 163 continue; 164 } 165 166 // If there's not enough bytes in the reader, then we cannot do anymore. 167 size_t size_incl_header = hdr_size + field_size; 168 if (size_incl_header > avail) { 169 return base::OkStatus(); 170 } 171 172 auto packet = reader_.SliceOff(start_offset + hdr_size, field_size); 173 PERFETTO_CHECK(packet); 174 PERFETTO_CHECK(reader_.PopFrontBytes(hdr_size + field_size)); 175 protos::pbzero::TracePacket::Decoder decoder(packet->data(), 176 packet->length()); 177 if (!decoder.has_compressed_packets()) { 178 RETURN_IF_ERROR(callback(std::move(*packet))); 179 continue; 180 } 181 182 if (!util::IsGzipSupported()) { 183 return base::ErrStatus( 184 "Cannot decode compressed packets. Zlib not enabled"); 185 } 186 187 protozero::ConstBytes field = decoder.compressed_packets(); 188 TraceBlobView compressed_packets = packet->slice(field.data, field.size); 189 TraceBlobView packets; 190 RETURN_IF_ERROR(Decompress(std::move(compressed_packets), &packets)); 191 192 const uint8_t* start = packets.data(); 193 const uint8_t* end = packets.data() + packets.length(); 194 const uint8_t* ptr = start; 195 while ((end - ptr) > 2) { 196 const uint8_t* packet_outer = ptr; 197 if (PERFETTO_UNLIKELY(*ptr != kTracePacketTag)) { 198 return base::ErrStatus("Expected TracePacket tag"); 199 } 200 uint64_t packet_size = 0; 201 ptr = protozero::proto_utils::ParseVarInt(++ptr, end, &packet_size); 202 const uint8_t* packet_start = ptr; 203 ptr += packet_size; 204 if (PERFETTO_UNLIKELY((ptr - packet_outer) < 2 || ptr > end)) { 205 return base::ErrStatus("Invalid packet size"); 206 } 207 TraceBlobView sliced = 208 packets.slice(packet_start, static_cast<size_t>(packet_size)); 209 RETURN_IF_ERROR(callback(std::move(sliced))); 210 } 211 } 212 } 213 214 private: 215 static constexpr uint8_t kTracePacketTag = 216 protozero::proto_utils::MakeTagLengthDelimited( 217 protos::pbzero::Trace::kPacketFieldNumber); 218 219 base::Status Decompress(TraceBlobView input, TraceBlobView* output); 220 221 // Used to glue together trace packets that span across two (or more) 222 // Parse() boundaries. 223 util::TraceBlobViewReader reader_; 224 225 // Allows support for compressed trace packets. 226 util::GzipDecompressor decompressor_; 227 }; 228 229 } // namespace perfetto::trace_processor 230 231 #endif // SRC_TRACE_PROCESSOR_IMPORTERS_PROTO_PROTO_TRACE_TOKENIZER_H_ 232