1#!/usr/bin/env python3 2# Copyright (C) 2021 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16# This tool checks that every create (table|view) is prefixed by 17# drop (table|view). 18 19from __future__ import absolute_import 20from __future__ import division 21from __future__ import print_function 22 23import os 24import sys 25from typing import Dict, Tuple, List 26 27ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 28sys.path.append(os.path.join(ROOT_DIR)) 29 30from python.generators.sql_processing.utils import check_banned_create_view_as 31from python.generators.sql_processing.utils import check_banned_words 32from python.generators.sql_processing.utils import match_pattern 33from python.generators.sql_processing.utils import DROP_TABLE_VIEW_PATTERN 34from python.generators.sql_processing.utils import CREATE_TABLE_VIEW_PATTERN 35from python.generators.sql_processing.utils import CREATE_TABLE_AS_PATTERN 36 37 38def check_if_create_table_allowlisted( 39 sql: str, filename: str, stdlib_path: str, 40 allowlist: Dict[str, List[str]]) -> List[str]: 41 errors = [] 42 for _, matches in match_pattern(CREATE_TABLE_AS_PATTERN, sql).items(): 43 name = matches[0] 44 # Normalize paths before checking presence in the allowlist so it will 45 # work on Windows for the Chrome stdlib presubmit. 46 allowlist_normpath = dict( 47 (os.path.normpath(path), tables) for path, tables in allowlist.items()) 48 allowlist_key = os.path.normpath(filename[len(stdlib_path):]) 49 if allowlist_key not in allowlist_normpath: 50 errors.append(f"CREATE TABLE '{name}' is deprecated. " 51 "Use CREATE PERFETTO TABLE instead.\n" 52 f"Offending file: {filename}\n") 53 continue 54 if name not in allowlist_normpath[allowlist_key]: 55 errors.append( 56 f"Table '{name}' uses CREATE TABLE which is deprecated " 57 "and this table is not allowlisted. Use CREATE PERFETTO TABLE.\n" 58 f"Offending file: {filename}\n") 59 return errors 60 61# Allowlist path are relative to the metrics root. 62CREATE_TABLE_ALLOWLIST = { 63 ('/android' 64 '/android_blocking_calls_cuj_metric.sql'): [ 65 'android_cujs', 'relevant_binder_calls_with_names', 66 'android_blocking_calls_cuj_calls' 67 ], 68 ('/android' 69 '/android_blocking_calls_unagg.sql'): [ 70 'filtered_processes_with_non_zero_blocking_calls', 'process_info', 71 'android_blocking_calls_unagg_calls' 72 ], 73 '/android/jank/cujs.sql': ['android_jank_cuj'], 74 '/chrome/gesture_flow_event.sql': [ 75 '{{prefix}}_latency_info_flow_step_filtered' 76 ], 77 '/chrome/gesture_jank.sql': [ 78 '{{prefix}}_jank_maybe_null_prev_and_next_without_precompute' 79 ], 80 '/experimental/frame_times.sql': ['DisplayCompositorPresentationEvents'], 81} 82 83 84def match_create_table_pattern_to_dict( 85 sql: str, pattern: str) -> Dict[str, Tuple[int, str]]: 86 res = {} 87 for line_num, matches in match_pattern(pattern, sql).items(): 88 res[matches[3]] = [line_num, str(matches[2])] 89 return res 90 91 92def match_drop_view_pattern_to_dict(sql: str, 93 pattern: str) -> Dict[str, Tuple[int, str]]: 94 res = {} 95 for line_num, matches in match_pattern(pattern, sql).items(): 96 res[matches[1]] = [line_num, str(matches[0])] 97 return res 98 99 100def check(path: str, metrics_sources: str) -> List[str]: 101 errors = [] 102 with open(path) as f: 103 sql = f.read() 104 105 # Check that each function/macro is using "CREATE OR REPLACE" 106 lines = [l.strip() for l in sql.split('\n')] 107 for line in lines: 108 if line.startswith('--'): 109 continue 110 if 'create perfetto function' in line.casefold(): 111 errors.append( 112 f'Use "CREATE OR REPLACE PERFETTO FUNCTION" in Perfetto metrics, ' 113 f'to prevent the file from crashing if the metric is rerun.\n' 114 f'Offending file: {path}\n') 115 if 'create perfetto macro' in line.casefold(): 116 errors.append( 117 f'Use "CREATE OR REPLACE PERFETTO MACRO" in Perfetto metrics, to ' 118 f'prevent the file from crashing if the metric is rerun.\n' 119 f'Offending file: {path}\n') 120 121 # Check that CREATE VIEW/TABLE has a matching DROP VIEW/TABLE before it. 122 create_table_view_dir = match_create_table_pattern_to_dict( 123 sql, CREATE_TABLE_VIEW_PATTERN) 124 drop_table_view_dir = match_drop_view_pattern_to_dict( 125 sql, DROP_TABLE_VIEW_PATTERN) 126 errors += check_if_create_table_allowlisted( 127 sql, 128 path.split(ROOT_DIR)[1], 129 metrics_sources.split(ROOT_DIR)[1], CREATE_TABLE_ALLOWLIST) 130 errors += check_banned_create_view_as(sql) 131 for name, [line, type] in create_table_view_dir.items(): 132 if name not in drop_table_view_dir: 133 errors.append(f'Missing DROP before CREATE {type.upper()} "{name}"\n' 134 f'Offending file: {path}\n') 135 continue 136 drop_line, drop_type = drop_table_view_dir[name] 137 if drop_line > line: 138 errors.append(f'DROP has to be before CREATE {type.upper()} "{name}"\n' 139 f'Offending file: {path}\n') 140 continue 141 if drop_type != type: 142 errors.append(f'DROP type doesnt match CREATE {type.upper()} "{name}"\n' 143 f'Offending file: {path}\n') 144 145 errors += check_banned_words(sql) 146 return errors 147 148 149def main(): 150 errors = [] 151 metrics_sources = os.path.join(ROOT_DIR, 'src', 'trace_processor', 'metrics', 152 'sql') 153 for root, _, files in os.walk(metrics_sources, topdown=True): 154 for f in files: 155 path = os.path.join(root, f) 156 if path.endswith('.sql'): 157 errors += check(path, metrics_sources) 158 159 if errors: 160 sys.stderr.write("\n".join(errors)) 161 sys.stderr.write("\n") 162 return 0 if not errors else 1 163 164 165if __name__ == '__main__': 166 sys.exit(main()) 167