1#!/usr/bin/env python3 2# Copyright (C) 2022 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# disibuted under the License is disibuted on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16from __future__ import absolute_import 17from __future__ import division 18from __future__ import print_function 19 20import argparse 21import sys 22import json 23from typing import Any, List, Dict 24 25INTRODUCTION = ''' 26# PerfettoSQL standard library 27*This page documents the PerfettoSQL standard library.* 28 29## Introduction 30The PerfettoSQL standard library is a repository of tables, views, functions 31and macros, contributed by domain experts, which make querying traces easier 32Its design is heavily inspired by standard libraries in languages like Python, 33C++ and Java. 34 35Some of the purposes of the standard library include: 361) Acting as a way of sharing and commonly written queries without needing 37to copy/paste large amounts of SQL. 382) Raising the abstraction level when exposing data in the trace. Many 39modules in the standard library convert low-level trace concepts 40e.g. slices, tracks and into concepts developers may be more familar with 41e.g. for Android developers: app startups, binder transactions etc. 42 43Standard library modules can be included as follows: 44``` 45-- Include all tables/views/functions from the android.startup.startups 46-- module in the standard library. 47INCLUDE PERFETTO MODULE android.startup.startups; 48 49-- Use the android_startups table defined in the android.startup.startups 50-- module. 51SELECT * 52FROM android_startups; 53``` 54 55Prelude is a special module is automatically included. It contains key helper 56tables, views and functions which are universally useful. 57 58More information on importing modules is available in the 59[syntax documentation](/docs/analysis/perfetto-sql-syntax#including-perfettosql-modules) 60for the `INCLUDE PERFETTO MODULE` statement. 61 62<!-- TODO(b/290185551): talk about experimental module and contributions. --> 63''' 64 65 66def _escape(desc: str) -> str: 67 """Escapes special characters in a markdown table.""" 68 return desc.replace('|', '\\|') 69 70 71def _md_table_header(cols: List[str]) -> str: 72 col_str = ' | '.join(cols) + '\n' 73 lines = ['-' * len(col) for col in cols] 74 underlines = ' | '.join(lines) 75 return col_str + underlines 76 77 78def _md_rolldown(summary: str, content: str) -> str: 79 return f"""<details> 80 <summary style="cursor: pointer;">{summary}</summary> 81 82 {content} 83 84 </details> 85 """ 86 87 88def _bold(s: str) -> str: 89 return f"<strong>{s}</strong>" 90 91 92class ModuleMd: 93 """Responsible for module level markdown generation.""" 94 95 def __init__(self, package_name: str, module_dict: Dict): 96 self.module_name = module_dict['module_name'] 97 self.include_str = self.module_name if package_name != 'prelude' else 'N/A' 98 self.objs, self.funs, self.view_funs, self.macros = [], [], [], [] 99 100 # Views/tables 101 for data in module_dict['data_objects']: 102 if not data['cols']: 103 continue 104 105 obj_summary = ( 106 f'''{_bold(data['name'])}. {data['summary_desc']}\n''' 107 ) 108 content = [f"{data['type']}"] 109 if (data['summary_desc'] != data['desc']): 110 content.append(data['desc']) 111 112 table = [_md_table_header(['Column', 'Type', 'Description'])] 113 for info in data['cols']: 114 name = info["name"] 115 table.append( 116 f'{name} | {info["type"]} | {_escape(info["desc"])}') 117 content.append('\n\n') 118 content.append('\n'.join(table)) 119 120 self.objs.append(_md_rolldown(obj_summary, '\n'.join(content))) 121 122 self.objs.append('\n\n') 123 124 # Functions 125 for d in module_dict['functions']: 126 summary = f'''{_bold(d['name'])} -> {d['return_type']}. {d['summary_desc']}\n\n''' 127 content = [] 128 if (d['summary_desc'] != d['desc']): 129 content.append(d['desc']) 130 131 content.append( 132 f"Returns {d['return_type']}: {d['return_desc']}\n\n") 133 if d['args']: 134 content.append(_md_table_header(['Argument', 'Type', 'Description'])) 135 for arg_dict in d['args']: 136 content.append( 137 f'''{arg_dict['name']} | {arg_dict['type']} | {_escape(arg_dict['desc'])}''' 138 ) 139 140 self.funs.append(_md_rolldown(summary, '\n'.join(content))) 141 self.funs.append('\n\n') 142 143 # Table functions 144 for data in module_dict['table_functions']: 145 obj_summary = f'''{_bold(data['name'])}. {data['summary_desc']}\n\n''' 146 content = [] 147 if (data['summary_desc'] != data['desc']): 148 content.append(data['desc']) 149 150 if data['args']: 151 args_table = [_md_table_header(['Argument', 'Type', 'Description'])] 152 for arg_dict in data['args']: 153 args_table.append( 154 f'''{arg_dict['name']} | {arg_dict['type']} | {_escape(arg_dict['desc'])}''' 155 ) 156 content.append('\n'.join(args_table)) 157 content.append('\n\n') 158 159 content.append(_md_table_header(['Column', 'Type', 'Description'])) 160 for column in data['cols']: 161 content.append( 162 f'{column["name"]} | {column["type"]} | {column["desc"]}') 163 164 self.view_funs.append(_md_rolldown(obj_summary, '\n'.join(content))) 165 self.view_funs.append('\n\n') 166 167 # Macros 168 for data in module_dict['macros']: 169 obj_summary = f'''{_bold(data['name'])}. {data['summary_desc']}\n\n''' 170 content = [] 171 if (data['summary_desc'] != data['desc']): 172 content.append(data['desc']) 173 174 content.append( 175 f'''Returns: {data['return_type']}, {data['return_desc']}\n\n''') 176 if data['args']: 177 table = [_md_table_header(['Argument', 'Type', 'Description'])] 178 for arg_dict in data['args']: 179 table.append( 180 f'''{arg_dict['name']} | {arg_dict['type']} | {_escape(arg_dict['desc'])}''' 181 ) 182 content.append('\n'.join(table)) 183 184 self.macros.append(_md_rolldown(obj_summary, '\n'.join(content))) 185 self.macros.append('\n\n') 186 187 188class PackageMd: 189 """Responsible for package level markdown generation.""" 190 191 def __init__(self, package_name: str, module_files: List[Dict[str, 192 Any]]) -> None: 193 self.package_name = package_name 194 self.modules_md = sorted( 195 [ModuleMd(package_name, file_dict) for file_dict in module_files], 196 key=lambda x: x.module_name) 197 198 def get_prelude_description(self) -> str: 199 if not self.package_name == 'prelude': 200 raise ValueError("Only callable on prelude module") 201 202 lines = [] 203 lines.append(f'## Package: {self.package_name}') 204 205 # Prelude is a special module which is automatically imported and doesn't 206 # have any include keys. 207 objs = '\n'.join(obj for module in self.modules_md for obj in module.objs) 208 if objs: 209 lines.append('#### Views/Tables') 210 lines.append(objs) 211 212 funs = '\n'.join(fun for module in self.modules_md for fun in module.funs) 213 if funs: 214 lines.append('#### Functions') 215 lines.append(funs) 216 217 table_funs = '\n'.join( 218 view_fun for module in self.modules_md for view_fun in module.view_funs) 219 if table_funs: 220 lines.append('#### Table Functions') 221 lines.append(table_funs) 222 223 macros = '\n'.join( 224 macro for module in self.modules_md for macro in module.macros) 225 if macros: 226 lines.append('#### Macros') 227 lines.append(macros) 228 229 return '\n'.join(lines) 230 231 def get_md(self) -> str: 232 if not self.modules_md: 233 return '' 234 235 if self.package_name == 'prelude': 236 raise ValueError("Can't be called with prelude module") 237 238 lines = [] 239 lines.append(f'## Package: {self.package_name}') 240 241 for file in self.modules_md: 242 if not any((file.objs, file.funs, file.view_funs, file.macros)): 243 continue 244 245 lines.append(f'### {file.module_name}') 246 if file.objs: 247 lines.append('#### Views/Tables') 248 lines.append('\n'.join(file.objs)) 249 if file.funs: 250 lines.append('#### Functions') 251 lines.append('\n'.join(file.funs)) 252 if file.view_funs: 253 lines.append('#### Table Functions') 254 lines.append('\n'.join(file.view_funs)) 255 if file.macros: 256 lines.append('#### Macros') 257 lines.append('\n'.join(file.macros)) 258 259 return '\n'.join(lines) 260 261 def is_empty(self) -> bool: 262 for file in self.modules_md: 263 if any((file.objs, file.funs, file.view_funs, file.macros)): 264 return False 265 return True 266 267 268def main(): 269 parser = argparse.ArgumentParser() 270 parser.add_argument('--input', required=True) 271 parser.add_argument('--output', required=True) 272 args = parser.parse_args() 273 274 with open(args.input) as f: 275 stdlib_json = json.load(f) 276 277 # Fetch the modules from json documentation. 278 packages: Dict[str, PackageMd] = {} 279 for package in stdlib_json: 280 package_name = package["name"] 281 modules = package["modules"] 282 # Remove 'common' when it has been removed from the code. 283 if package_name not in ['deprecated', 'common']: 284 package = PackageMd(package_name, modules) 285 if (not package.is_empty()): 286 packages[package_name] = package 287 288 prelude = packages.pop('prelude') 289 290 with open(args.output, 'w') as f: 291 f.write(INTRODUCTION) 292 f.write(prelude.get_prelude_description()) 293 f.write('\n') 294 f.write('\n'.join(module.get_md() for module in packages.values())) 295 296 return 0 297 298 299if __name__ == '__main__': 300 sys.exit(main()) 301