1#!/usr/bin/env vpython3 2# Copyright 2017 The PDFium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Generates a coverage report for given tests. 6 7Requires that 'use_clang_coverage = true' is set in args.gn. 8Prefers that 'is_component_build = false' is set in args.gn. 9""" 10 11import argparse 12from collections import namedtuple 13import fnmatch 14import os 15import pprint 16import subprocess 17import sys 18 19# Add parent dir to avoid having to set PYTHONPATH. 20sys.path.append( 21 os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir))) 22 23import common 24 25# 'binary' is the file that is to be run for the test. 26# 'use_test_runner' indicates if 'binary' depends on test_runner.py and thus 27# requires special handling. 28# 'opt_args' are optional arguments to pass to the test 'binary'. 29TestSpec = namedtuple('TestSpec', 'binary, use_test_runner, opt_args') 30 31# All of the coverage tests that the script knows how to run. 32COVERAGE_TESTS = { 33 'pdfium_unittests': 34 TestSpec('pdfium_unittests', False, []), 35 'pdfium_embeddertests': 36 TestSpec('pdfium_embeddertests', False, []), 37 'corpus_tests': 38 TestSpec('run_corpus_tests.py', True, []), 39 'corpus_tests_javascript_disabled': 40 TestSpec('run_corpus_tests.py', True, ['--disable-javascript']), 41 'corpus_tests_xfa_disabled': 42 TestSpec('run_corpus_tests.py', True, ['--disable-xfa']), 43 'corpus_tests_render_oneshot': 44 TestSpec('run_corpus_tests.py', True, ['--render-oneshot']), 45 'corpus_tests_reverse_byte_order': 46 TestSpec('run_corpus_tests.py', True, ['--reverse-byte-order']), 47 'javascript_tests': 48 TestSpec('run_javascript_tests.py', True, []), 49 'javascript_tests_javascript_disabled': 50 TestSpec('run_javascript_tests.py', True, ['--disable-javascript']), 51 'javascript_tests_xfa_disabled': 52 TestSpec('run_javascript_tests.py', True, ['--disable-xfa']), 53 'pixel_tests': 54 TestSpec('run_pixel_tests.py', True, []), 55 'pixel_tests_javascript_disabled': 56 TestSpec('run_pixel_tests.py', True, ['--disable-javascript']), 57 'pixel_tests_xfa_disabled': 58 TestSpec('run_pixel_tests.py', True, ['--disable-xfa']), 59 'pixel_tests_render_oneshot': 60 TestSpec('run_pixel_tests.py', True, ['--render-oneshot']), 61 'pixel_tests_reverse_byte_order': 62 TestSpec('run_pixel_tests.py', True, ['--reverse-byte-order']), 63} 64 65 66class CoverageExecutor: 67 68 def __init__(self, parser, args): 69 """Initialize executor based on the current script environment 70 71 Args: 72 parser: argparse.ArgumentParser for handling improper inputs. 73 args: Dictionary of arguments passed into the calling script. 74 """ 75 self.dry_run = args['dry_run'] 76 self.verbose = args['verbose'] 77 78 self.source_directory = args['source_directory'] 79 if not os.path.isdir(self.source_directory): 80 parser.error("'%s' needs to be a directory" % self.source_directory) 81 82 self.llvm_directory = os.path.join(self.source_directory, 'third_party', 83 'llvm-build', 'Release+Asserts', 'bin') 84 if not os.path.isdir(self.llvm_directory): 85 parser.error("Cannot find LLVM bin directory , expected it to be in '%s'" 86 % self.llvm_directory) 87 88 self.build_directory = args['build_directory'] 89 if not os.path.isdir(self.build_directory): 90 parser.error("'%s' needs to be a directory" % self.build_directory) 91 92 (self.coverage_tests, 93 self.build_targets) = self.calculate_coverage_tests(args) 94 if not self.coverage_tests: 95 parser.error( 96 'No valid tests in set to be run. This is likely due to bad command ' 97 'line arguments') 98 99 if not common.GetBooleanGnArg('use_clang_coverage', self.build_directory, 100 self.verbose): 101 parser.error( 102 'use_clang_coverage does not appear to be set to true for build, but ' 103 'is needed') 104 105 self.use_goma = common.GetBooleanGnArg('use_goma', self.build_directory, 106 self.verbose) 107 108 self.output_directory = args['output_directory'] 109 if not os.path.exists(self.output_directory): 110 if not self.dry_run: 111 os.makedirs(self.output_directory) 112 elif not os.path.isdir(self.output_directory): 113 parser.error('%s exists, but is not a directory' % self.output_directory) 114 elif len(os.listdir(self.output_directory)) > 0: 115 parser.error('%s is not empty, cowardly refusing to continue' % 116 self.output_directory) 117 118 self.prof_data = os.path.join(self.output_directory, 'pdfium.profdata') 119 120 def check_output(self, args, dry_run=False, env=None): 121 """Dry run aware wrapper of subprocess.check_output()""" 122 if dry_run: 123 print("Would have run '%s'" % ' '.join(args)) 124 return '' 125 126 output = subprocess.check_output(args, env=env) 127 128 if self.verbose: 129 print("check_output(%s) returned '%s'" % (args, output)) 130 return output 131 132 def call(self, args, dry_run=False, env=None): 133 """Dry run aware wrapper of subprocess.call()""" 134 if dry_run: 135 print("Would have run '%s'" % ' '.join(args)) 136 return 0 137 138 output = subprocess.call(args, env=env) 139 140 if self.verbose: 141 print('call(%s) returned %s' % (args, output)) 142 return output 143 144 def call_silent(self, args, dry_run=False, env=None): 145 """Dry run aware wrapper of subprocess.call() that eats output from call""" 146 if dry_run: 147 print("Would have run '%s'" % ' '.join(args)) 148 return 0 149 150 with open(os.devnull, 'w') as f: 151 output = subprocess.call(args, env=env, stdout=f) 152 153 if self.verbose: 154 print('call_silent(%s) returned %s' % (args, output)) 155 return output 156 157 def calculate_coverage_tests(self, args): 158 """Determine which tests should be run.""" 159 testing_tools_directory = os.path.join(self.source_directory, 'testing', 160 'tools') 161 tests = args['tests'] if args['tests'] else COVERAGE_TESTS.keys() 162 coverage_tests = {} 163 build_targets = set() 164 for name in tests: 165 test_spec = COVERAGE_TESTS[name] 166 if test_spec.use_test_runner: 167 binary_path = os.path.join(testing_tools_directory, test_spec.binary) 168 build_targets.add('pdfium_diff') 169 build_targets.add('pdfium_test') 170 else: 171 binary_path = os.path.join(self.build_directory, test_spec.binary) 172 build_targets.add(name) 173 coverage_tests[name] = TestSpec(binary_path, test_spec.use_test_runner, 174 test_spec.opt_args) 175 176 build_targets = list(build_targets) 177 178 return coverage_tests, build_targets 179 180 def build_binaries(self): 181 """Build all the binaries that are going to be needed for coverage 182 generation.""" 183 call_args = ['ninja'] 184 if self.use_goma: 185 call_args += ['-j', '250'] 186 call_args += ['-C', self.build_directory] 187 call_args += self.build_targets 188 return self.call(call_args, dry_run=self.dry_run) == 0 189 190 def generate_coverage(self, name, spec): 191 """Generate the coverage data for a test 192 193 Args: 194 name: Name associated with the test to be run. This is used as a label 195 in the coverage data, so should be unique across all of the tests 196 being run. 197 spec: Tuple containing the TestSpec. 198 """ 199 if self.verbose: 200 print("Generating coverage for test '%s', using data '%s'" % (name, spec)) 201 if not os.path.exists(spec.binary): 202 print('Unable to generate coverage for %s, since it appears to not exist' 203 ' @ %s' % (name, spec.binary)) 204 return False 205 206 binary_args = [spec.binary] 207 if spec.opt_args: 208 binary_args.extend(spec.opt_args) 209 profile_pattern_string = '%8m' 210 expected_profraw_file = '%s.%s.profraw' % (name, profile_pattern_string) 211 expected_profraw_path = os.path.join(self.output_directory, 212 expected_profraw_file) 213 214 env = { 215 'LLVM_PROFILE_FILE': expected_profraw_path, 216 'PATH': os.getenv('PATH') + os.pathsep + self.llvm_directory 217 } 218 219 if spec.use_test_runner: 220 # Test runner performs multi-threading in the wrapper script, not the test 221 # binary, so need to limit the number of instances of the binary being run 222 # to the max value in LLVM_PROFILE_FILE, which is 8. 223 binary_args.extend(['-j', '8', '--build-dir', self.build_directory]) 224 if self.call(binary_args, dry_run=self.dry_run, env=env) and self.verbose: 225 print('Running %s appears to have failed, which might affect ' 226 'results' % spec.binary) 227 228 return True 229 230 def merge_raw_coverage_results(self): 231 """Merge raw coverage data sets into one one file for report generation.""" 232 llvm_profdata_bin = os.path.join(self.llvm_directory, 'llvm-profdata') 233 234 raw_data = [] 235 raw_data_pattern = '*.profraw' 236 for file_name in os.listdir(self.output_directory): 237 if fnmatch.fnmatch(file_name, raw_data_pattern): 238 raw_data.append(os.path.join(self.output_directory, file_name)) 239 240 return self.call( 241 [llvm_profdata_bin, 'merge', '-o', self.prof_data, '-sparse=true'] + 242 raw_data) == 0 243 244 def generate_html_report(self): 245 """Generate HTML report by calling upstream coverage.py""" 246 coverage_bin = os.path.join(self.source_directory, 'tools', 'code_coverage', 247 'coverage.py') 248 report_directory = os.path.join(self.output_directory, 'HTML') 249 250 coverage_args = ['-p', self.prof_data] 251 coverage_args += ['-b', self.build_directory] 252 coverage_args += ['-o', report_directory] 253 coverage_args += self.build_targets 254 255 # Only analyze the directories of interest. 256 coverage_args += ['-f', 'core'] 257 coverage_args += ['-f', 'fpdfsdk'] 258 coverage_args += ['-f', 'fxbarcode'] 259 coverage_args += ['-f', 'fxjs'] 260 coverage_args += ['-f', 'public'] 261 coverage_args += ['-f', 'samples'] 262 coverage_args += ['-f', 'xfa'] 263 264 # Ignore test files. 265 coverage_args += ['-i', '.*test.*'] 266 267 # Component view is only useful for Chromium 268 coverage_args += ['--no-component-view'] 269 270 return self.call([coverage_bin] + coverage_args) == 0 271 272 def run(self): 273 """Setup environment, execute the tests and generate coverage report""" 274 if not self.fetch_profiling_tools(): 275 print('Unable to fetch profiling tools') 276 return False 277 278 if not self.build_binaries(): 279 print('Failed to successfully build binaries') 280 return False 281 282 for name in self.coverage_tests: 283 if not self.generate_coverage(name, self.coverage_tests[name]): 284 print('Failed to successfully generate coverage data') 285 return False 286 287 if not self.merge_raw_coverage_results(): 288 print('Failed to successfully merge raw coverage results') 289 return False 290 291 if not self.generate_html_report(): 292 print('Failed to successfully generate HTML report') 293 return False 294 295 return True 296 297 def fetch_profiling_tools(self): 298 """Call coverage.py with no args to ensure profiling tools are present.""" 299 return self.call_silent( 300 os.path.join(self.source_directory, 'tools', 'code_coverage', 301 'coverage.py')) == 0 302 303 304def main(): 305 parser = argparse.ArgumentParser() 306 parser.formatter_class = argparse.RawDescriptionHelpFormatter 307 parser.description = 'Generates a coverage report for given tests.' 308 309 parser.add_argument( 310 '-s', 311 '--source_directory', 312 help='Location of PDFium source directory, defaults to CWD', 313 default=os.getcwd()) 314 build_default = os.path.join('out', 'Coverage') 315 parser.add_argument( 316 '-b', 317 '--build_directory', 318 help= 319 'Location of PDFium build directory with coverage enabled, defaults to ' 320 '%s under CWD' % build_default, 321 default=os.path.join(os.getcwd(), build_default)) 322 output_default = 'coverage_report' 323 parser.add_argument( 324 '-o', 325 '--output_directory', 326 help='Location to write out coverage report to, defaults to %s under CWD ' 327 % output_default, 328 default=os.path.join(os.getcwd(), output_default)) 329 parser.add_argument( 330 '-n', 331 '--dry-run', 332 help='Output commands instead of executing them', 333 action='store_true') 334 parser.add_argument( 335 '-v', 336 '--verbose', 337 help='Output additional diagnostic information', 338 action='store_true') 339 parser.add_argument( 340 'tests', 341 help='Tests to be run, defaults to all. Valid entries are %s' % 342 COVERAGE_TESTS.keys(), 343 nargs='*') 344 345 args = vars(parser.parse_args()) 346 if args['verbose']: 347 pprint.pprint(args) 348 349 executor = CoverageExecutor(parser, args) 350 if executor.run(): 351 return 0 352 return 1 353 354 355if __name__ == '__main__': 356 sys.exit(main()) 357