1#!/usr/bin/env python3 2# Copyright 2020 The Pigweed Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); you may not 5# use this file except in compliance with the License. You may obtain a copy of 6# the License at 7# 8# https://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13# License for the specific language governing permissions and limitations under 14# the License. 15""" 16Generates json trace files viewable using chrome://tracing from binary 17trace files. 18 19Example usage: 20python pw_trace_tokenized/py/trace_tokenized.py -i trace.bin -o trace.json 21out/pw_strict_host_clang_debug/obj/pw_trace_tokenized/bin/trace_tokenized_example_basic 22""" # pylint: disable=line-too-long 23# pylint: enable=line-too-long 24 25from enum import IntEnum 26import argparse 27import logging 28import struct 29import sys 30from pw_tokenizer import database, tokens 31from pw_trace import trace 32 33_LOG = logging.getLogger('pw_trace_tokenizer') 34 35 36def varint_decode(encoded): 37 # Taken from pw_tokenizer.decode._decode_signed_integer 38 count = 0 39 result = 0 40 shift = 0 41 for byte in encoded: 42 count += 1 43 result |= (byte & 0x7F) << shift 44 if not byte & 0x80: 45 return result, count 46 47 shift += 7 48 if shift >= 64: 49 break # Error 50 return None 51 52 53# Token string: "event_type|flag|module|group|label|<optional DATA_FMT>" 54class TokenIdx(IntEnum): 55 EVENT_TYPE = 0 56 FLAG = 1 57 MODULE = 2 58 GROUP = 3 59 LABEL = 4 60 DATA_FMT = 5 # optional 61 62 63def get_trace_type(type_str): 64 if type_str == "PW_TRACE_EVENT_TYPE_INSTANT": 65 return trace.TraceType.INSTANTANEOUS 66 if type_str == "PW_TRACE_EVENT_TYPE_INSTANT_GROUP": 67 return trace.TraceType.INSTANTANEOUS_GROUP 68 if type_str == "PW_TRACE_EVENT_TYPE_ASYNC_START": 69 return trace.TraceType.ASYNC_START 70 if type_str == "PW_TRACE_EVENT_TYPE_ASYNC_STEP": 71 return trace.TraceType.ASYNC_STEP 72 if type_str == "PW_TRACE_EVENT_TYPE_ASYNC_END": 73 return trace.TraceType.ASYNC_END 74 if type_str == "PW_TRACE_EVENT_TYPE_DURATION_START": 75 return trace.TraceType.DURATION_START 76 if type_str == "PW_TRACE_EVENT_TYPE_DURATION_END": 77 return trace.TraceType.DURATION_END 78 if type_str == "PW_TRACE_EVENT_TYPE_DURATION_GROUP_START": 79 return trace.TraceType.DURATION_GROUP_START 80 if type_str == "PW_TRACE_EVENT_TYPE_DURATION_GROUP_END": 81 return trace.TraceType.DURATION_GROUP_END 82 return trace.TraceType.INVALID 83 84 85def has_trace_id(token_string): 86 token_values = token_string.split("|") 87 return trace.event_has_trace_id(token_values[TokenIdx.EVENT_TYPE]) 88 89 90def has_data(token_string): 91 token_values = token_string.split("|") 92 return len(token_values) > TokenIdx.DATA_FMT 93 94 95def create_trace_event(token_string, timestamp_us, trace_id, data): 96 token_values = token_string.split("|") 97 return trace.TraceEvent( 98 event_type=get_trace_type(token_values[TokenIdx.EVENT_TYPE]), 99 module=token_values[TokenIdx.MODULE], 100 label=token_values[TokenIdx.LABEL], 101 timestamp_us=timestamp_us, 102 group=token_values[TokenIdx.GROUP], 103 trace_id=trace_id, 104 flags=token_values[TokenIdx.FLAG], 105 has_data=has_data(token_string), 106 data_fmt=( 107 token_values[TokenIdx.DATA_FMT] if has_data(token_string) else "" 108 ), 109 data=data if has_data(token_string) else b'', 110 ) 111 112 113def parse_trace_event(buffer, db, last_time, ticks_per_second): 114 """Parse a single trace event from bytes""" 115 us_per_tick = 1000000 / ticks_per_second 116 idx = 0 117 # Read token 118 token = struct.unpack('I', buffer[idx : idx + 4])[0] 119 idx += 4 120 121 # Decode token 122 if len(db.token_to_entries[token]) == 0: 123 _LOG.error("token not found: %08x", token) 124 return None 125 126 token_string = str(db.token_to_entries[token][0]) 127 128 # Read time 129 time_delta, time_bytes = varint_decode(buffer[idx:]) 130 timestamp_us = last_time + us_per_tick * time_delta 131 idx += time_bytes 132 133 # Trace ID 134 trace_id = None 135 if has_trace_id(token_string) and idx < len(buffer): 136 trace_id, trace_id_bytes = varint_decode(buffer[idx:]) 137 idx += trace_id_bytes 138 139 # Data 140 data = None 141 if has_data(token_string) and idx < len(buffer): 142 data = buffer[idx:] 143 144 # Create trace event 145 return create_trace_event(token_string, timestamp_us, trace_id, data) 146 147 148def get_trace_events( 149 databases, raw_trace_data, ticks_per_second, time_offset: int 150): 151 """Handles the decoding traces.""" 152 153 db = tokens.Database.merged(*databases) 154 last_timestamp = time_offset 155 events = [] 156 idx = 0 157 158 while idx + 1 < len(raw_trace_data): 159 # Read size 160 size = int(raw_trace_data[idx]) 161 if idx + size > len(raw_trace_data): 162 _LOG.error("incomplete file") 163 break 164 165 event = parse_trace_event( 166 raw_trace_data[idx + 1 : idx + 1 + size], 167 db, 168 last_timestamp, 169 ticks_per_second, 170 ) 171 if event: 172 last_timestamp = event.timestamp_us 173 events.append(event) 174 idx = idx + size + 1 175 return events 176 177 178def get_trace_data_from_file(input_file_name): 179 """Handles the decoding traces.""" 180 with open(input_file_name, "rb") as input_file: 181 return input_file.read() 182 return None 183 184 185def save_trace_file(trace_lines, file_name): 186 """Handles generating the trace file.""" 187 with open(file_name, 'w') as output_file: 188 output_file.write("[") 189 for line in trace_lines: 190 output_file.write("%s,\n" % line) 191 output_file.write("{}]") 192 193 194def get_trace_events_from_file( 195 databases, input_file_name, ticks_per_second, time_offset: int 196): 197 """Get trace events from a file.""" 198 raw_trace_data = get_trace_data_from_file(input_file_name) 199 return get_trace_events( 200 databases, raw_trace_data, ticks_per_second, time_offset 201 ) 202 203 204def _parse_args(): 205 """Parse and return command line arguments.""" 206 207 parser = argparse.ArgumentParser( 208 description=__doc__, 209 formatter_class=argparse.RawDescriptionHelpFormatter, 210 ) 211 parser.add_argument( 212 'databases', 213 nargs='+', 214 action=database.LoadTokenDatabases, 215 help='Databases (ELF, binary, or CSV) to use to lookup tokens.', 216 ) 217 parser.add_argument( 218 '-i', 219 '--input', 220 dest='input_file', 221 help='The binary trace input file, generated using trace_to_file.h.', 222 ) 223 parser.add_argument( 224 '-o', 225 '--output', 226 dest='output_file', 227 help=('The json file to which to write the output.'), 228 ) 229 parser.add_argument( 230 '-t', 231 '--ticks_per_second', 232 type=int, 233 dest='ticks_per_second', 234 default=1000, 235 help=('The clock rate of the trace events (Default 1000).'), 236 ) 237 parser.add_argument( 238 '--time_offset', 239 type=int, 240 dest='time_offset', 241 default=0, 242 help=('Time offset (us) of the trace events (Default 0).'), 243 ) 244 245 return parser.parse_args() 246 247 248def _main(args): 249 events = get_trace_events_from_file( 250 args.databases, args.input_file, args.ticks_per_second, args.time_offset 251 ) 252 json_lines = trace.generate_trace_json(events) 253 save_trace_file(json_lines, args.output_file) 254 255 256if __name__ == '__main__': 257 if sys.version_info[0] < 3: 258 sys.exit('ERROR: The detokenizer command line tools require Python 3.') 259 _main(_parse_args()) 260