1#! /usr/bin/env python3 2# 3# Copyright 2018 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17from __future__ import print_function 18 19"""Tool for comparing 2 LTP projects to find added / removed tests & testsuites. 20 21The tool can be run in two separate steps by dumping the list of tests as a json 22file, i.e.: 23 24$ git checkout master 25$ compare_ltp_projects.py --ltp-old . --ltp-old-json ./ltp_old.json 26$ git checkout mymergebranch 27$ compare_ltp_projects.py --ltp-old-json ./ltp_old.json --ltp-new . 28""" 29 30import os 31import argparse 32import os.path 33import sys 34import json 35 36def scan_tests(ltp_root, suite): 37 ''' Find all tests that are run as part of given test suite in LTP. 38 39 Args: 40 ltp_root: Path ot the LTP project. 41 suite: Name of testsuite. 42 43 Returns: 44 List of tests names that are run as part of the given test suite. 45 ''' 46 47 tests = [] 48 if not suite: 49 return tests 50 51 runtest_dir = ltp_root + os.path.sep + 'runtest' 52 test_suiteFile = runtest_dir + os.path.sep + suite 53 if not os.path.isfile(test_suiteFile): 54 print('No tests defined for suite {}',format(suite)) 55 return tests 56 57 with open(test_suiteFile) as f: 58 for line in f: 59 line = line.strip() 60 if line and not line.startswith('#'): 61 tests.append(line.split()[0]) 62 f.close() 63 return tests 64 65def scan_test_suites(ltp_root, scenario): 66 ''' Find all testuites and tests run as part of given LTP scenario 67 68 Args: 69 ltp_root: Path to the LTP project. 70 scenario: name of the scenario (found in ltp_root/scenario_groups). E.g. "vts" 71 72 Returns: 73 List of testsuite names that are run as part of given scenario (e.g. vts). 74 If scenario is not specified, return all testsuites. 75 ''' 76 77 runtest_dir = ltp_root + os.path.sep + 'runtest' 78 if not os.path.isdir(runtest_dir): 79 print('Invalid ltp_root {}, runtest directory doesnt exist'.format(ltp_root)) 80 sys.exit(2) 81 82 test_suites = [] 83 if scenario: 84 scenarioFile = ltp_root + os.path.sep + 'scenario_groups' + os.path.sep + scenario 85 if not os.path.isfile(scenarioFile): 86 return test_suites 87 with open(scenarioFile) as f: 88 for line in f: 89 line = line.strip() 90 if not line.startswith('#'): 91 test_suites.append(line) 92 return test_suites 93 94 runtest_dir = ltp_root + os.path.sep + 'runtest' 95 test_suites = [f for f in os.listdir(runtest_dir) if os.path.isfile(runtest_dir + os.path.sep + f)] 96 97 return test_suites 98 99def scan_ltp(ltp_root, scenario): 100 ''' scan LTP project and return all tests and testuites present. 101 102 Args: 103 ltp_root: Path to the LTP project. 104 scenario: specific scenario we want to scan. e.g. vts 105 106 Returns: 107 Dictionary of all LTP test names keyed by testsuite they are run as part of. 108 E.g. 109 { 110 'mem': ['oom01', 'mem02',...], 111 'syscalls': ['open01', 'read02',...], 112 ... 113 } 114 ''' 115 116 if not os.path.isdir(ltp_root): 117 print('ltp_root {} does not exist'.format(ltp_root)) 118 sys.exit(1) 119 120 test_suites = scan_test_suites(ltp_root, scenario) 121 if not test_suites: 122 print('No Testsuites found for scenario {}'.format(scenario)) 123 sys.exit(3) 124 125 ltp_tests = {} 126 for suite in test_suites: 127 ltp_tests[suite] = scan_tests(ltp_root, suite) 128 return ltp_tests 129 130def show_diff(ltp_tests_1, ltp_tests_2): 131 ''' Find and print diff between testcases in 2 LTP project checkouts. 132 133 Args: 134 ltp_tests_1: dictionary of tests keyed by test suite names 135 ltp_tests_2: dictionary of tests keyed by test suite names 136 ''' 137 test_suites1 = set(ltp_tests_1.keys()) 138 test_suites2 = set(ltp_tests_2.keys()) 139 140 # Generate lists of deleted, added and common test suites between 141 # LTP1 & LTP2 142 deleted_test_suites = sorted(test_suites1.difference(test_suites2)) 143 added_test_suites = sorted(test_suites2.difference(test_suites1)) 144 common_test_suites = test_suites1.intersection(test_suites2) 145 146 deleted_tests = [] 147 added_tests = [] 148 for suite in common_test_suites: 149 tests1 = set(ltp_tests_1[suite]) 150 tests2 = set(ltp_tests_2[suite]) 151 152 exclusive_test1 = tests1.difference(tests2) 153 exclusive_test2 = tests2.difference(tests1) 154 for e in exclusive_test1: 155 deleted_tests.append(suite + '.' + e) 156 for e in exclusive_test2: 157 added_tests.append(suite + '.' + e) 158 deleted_tests = sorted(deleted_tests) 159 added_tests = sorted(added_tests) 160 print_columns(added_test_suites, deleted_test_suites, added_tests, deleted_tests) 161 162 163def print_columns(added_test_suites, deleted_test_suites, added_tests, deleted_tests): 164 DEFAULT_WIDTH = 8 165 # find the maximum width of a test suite name or test name 166 # we have to print to decide the alignment. 167 if not deleted_test_suites: 168 width_suites = DEFAULT_WIDTH 169 else: 170 width_suites = max([len(x) for x in deleted_test_suites]) 171 172 if not deleted_tests: 173 width_tests = DEFAULT_WIDTH 174 else: 175 width_tests = max([len(x) for x in deleted_tests]) 176 width = max(width_suites, width_tests); 177 178 # total rows we have to print 179 total_rows = max(len(deleted_test_suites), len(added_test_suites)) 180 added_text = 'Added ({})'.format(len(added_test_suites)) 181 deleted_text = 'Deleted ({})'.format(len(deleted_test_suites)) 182 if total_rows > 0: 183 print('{:*^{len}}'.format(' Tests Suites ', len=width*2+2)) 184 print('{:<{len}} {:<{len}}'.format(added_text, deleted_text, len=width)) 185 for i in range(total_rows): 186 print('{:<{len}} {:<{len}}'.format('' if i >= len(added_test_suites) else str(added_test_suites[i]), 187 '' if i >= len(deleted_test_suites) else str(deleted_test_suites[i]), len=width)) 188 189 print('') 190 # total rows we have to print 191 total_rows = max(len(deleted_tests), len(added_tests)) 192 added_text = 'Added ({})'.format(len(added_tests)) 193 deleted_text = 'Deleted ({})'.format(len(deleted_tests)) 194 if total_rows: 195 print('{:*^{len}}'.format(' Tests ', len=width*2+2)) 196 print('{:^{len}} {:^{len}}'.format(added_text, deleted_text, len=width)) 197 for i in range(total_rows): 198 print('{:<{len}} {:<{len}}'.format('' if i >= len(added_tests) else str(added_tests[i]), 199 '' if i >= len(deleted_tests) else str(deleted_tests[i]), len=width)) 200def main(): 201 arg_parser = argparse.ArgumentParser( 202 description='Diff 2 LTP projects for supported test cases') 203 arg_parser.add_argument('-o', '--ltp-old', 204 dest='ltp_old', 205 help="LTP Root Directory before merge") 206 arg_parser.add_argument('-oj', '--ltp-old-json', 207 dest='ltp_old_json', 208 default='ltp_old.json', 209 help="Old LTP parsed directory in json format") 210 arg_parser.add_argument('-n', '--ltp-new', 211 dest='ltp_new', 212 help="LTP Root Directory after merge") 213 arg_parser.add_argument('-nj', '--ltp-new-json', 214 dest='ltp_new_json', 215 default='ltp_new.json', 216 help="New LTP parsed directory in json format") 217 arg_parser.add_argument('--scenario', default=None, 218 dest='scenario', 219 help="LTP scenario to list tests for") 220 args = arg_parser.parse_args() 221 222 # Find tests in the "old" directory or read the json output from a prior run 223 if args.ltp_old: 224 ltp_tests1 = scan_ltp(args.ltp_old, args.scenario) 225 elif args.ltp_old_json and os.path.isfile(args.ltp_old_json): 226 with open(args.ltp_old_json) as f: 227 ltp_tests1 = json.load(f) 228 else: 229 ltp_tests1 = None 230 231 # Do the same for the "new" directory 232 if args.ltp_new: 233 ltp_tests2 = scan_ltp(args.ltp_new, args.scenario) 234 elif args.ltp_new_json and os.path.isfile(args.ltp_new_json): 235 with open(args.ltp_new_json) as f: 236 ltp_tests2 = json.load(f) 237 else: 238 ltp_tests2 = None 239 240 if ltp_tests1 and ltp_tests2: 241 # Show the difference of the two directories if both are present 242 show_diff(ltp_tests1, ltp_tests2) 243 elif ltp_tests1: 244 # If just the old directory was read, dump it as json 245 with open(args.ltp_old_json, 'w') as f: 246 json.dump(ltp_tests1, f) 247 elif ltp_tests2: 248 # If just the new directory was read, dump it as json 249 with open(args.ltp_new_json, 'w') as f: 250 json.dump(ltp_tests2, f) 251 252if __name__ == '__main__': 253 main() 254