1#!/usr/bin/env python3 2# Copyright 2016 The PDFium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import argparse 7from dataclasses import dataclass, field 8from datetime import timedelta 9from io import BytesIO 10import multiprocessing 11import os 12import re 13import shutil 14import subprocess 15import sys 16import time 17 18import common 19import pdfium_root 20import pngdiffer 21from skia_gold import skia_gold 22import suppressor 23 24pdfium_root.add_source_directory_to_import_path(os.path.join('build', 'util')) 25from lib.results import result_sink, result_types 26 27 28# Arbitrary timestamp, expressed in seconds since the epoch, used to make sure 29# that tests that depend on the current time are stable. Happens to be the 30# timestamp of the first commit to repo, 2014/5/9 17:48:50. 31TEST_SEED_TIME = "1399672130" 32 33# List of test types that should run text tests instead of pixel tests. 34TEXT_TESTS = ['javascript'] 35 36# Timeout (in seconds) for individual test commands. 37# TODO(crbug.com/pdfium/1967): array_buffer.in is slow under MSan, so need a 38# very generous 5 minute timeout for now. 39TEST_TIMEOUT = timedelta(minutes=5).total_seconds() 40 41 42class TestRunner: 43 44 def __init__(self, dirname): 45 # Currently the only used directories are corpus, javascript, and pixel, 46 # which all correspond directly to the type for the test being run. In the 47 # future if there are tests that don't have this clean correspondence, then 48 # an argument for the type will need to be added. 49 self.per_process_config = _PerProcessConfig( 50 test_dir=dirname, test_type=dirname) 51 52 @property 53 def options(self): 54 return self.per_process_config.options 55 56 def IsSkiaGoldEnabled(self): 57 return (self.options.run_skia_gold and 58 not self.per_process_config.test_type in TEXT_TESTS) 59 60 def IsExecutionSuppressed(self, input_path): 61 return self.per_process_state.test_suppressor.IsExecutionSuppressed( 62 input_path) 63 64 def IsResultSuppressed(self, input_filename): 65 return self.per_process_state.test_suppressor.IsResultSuppressed( 66 input_filename) 67 68 def HandleResult(self, test_case, test_result): 69 input_filename = os.path.basename(test_case.input_path) 70 71 test_result.status = self._SuppressStatus(input_filename, 72 test_result.status) 73 if test_result.status == result_types.UNKNOWN: 74 self.result_suppressed_cases.append(input_filename) 75 self.surprises.append(test_case.input_path) 76 elif test_result.status == result_types.SKIP: 77 self.result_suppressed_cases.append(input_filename) 78 elif not test_result.IsPass(): 79 self.failures.append(test_case.input_path) 80 81 for artifact in test_result.image_artifacts: 82 if artifact.skia_gold_status == result_types.PASS: 83 if self.IsResultSuppressed(artifact.image_path): 84 self.skia_gold_unexpected_successes.append(artifact.GetSkiaGoldId()) 85 else: 86 self.skia_gold_successes.append(artifact.GetSkiaGoldId()) 87 elif artifact.skia_gold_status == result_types.FAIL: 88 self.skia_gold_failures.append(artifact.GetSkiaGoldId()) 89 90 # Log test result. 91 print(f'{test_result.status}: {test_result.test_id}') 92 if not test_result.IsPass(): 93 if test_result.reason: 94 print(f'Failure reason: {test_result.reason}') 95 if test_result.log: 96 print(f'Test output:\n{test_result.log}') 97 for artifact in test_result.image_artifacts: 98 if artifact.skia_gold_status == result_types.FAIL: 99 print(f'Failed Skia Gold: {artifact.image_path}') 100 if artifact.image_diff: 101 print(f'Failed image diff: {artifact.image_diff.reason}') 102 103 # Report test result to ResultDB. 104 if self.resultdb: 105 only_artifacts = None 106 only_failure_reason = test_result.reason 107 if len(test_result.image_artifacts) == 1: 108 only = test_result.image_artifacts[0] 109 only_artifacts = only.GetDiffArtifacts() 110 if only.GetDiffReason(): 111 only_failure_reason += f': {only.GetDiffReason()}' 112 self.resultdb.Post( 113 test_id=test_result.test_id, 114 status=test_result.status, 115 duration=test_result.duration_milliseconds, 116 test_log=test_result.log, 117 test_file=None, 118 artifacts=only_artifacts, 119 failure_reason=only_failure_reason) 120 121 # Milo only supports a single diff per test, so if we have multiple pages, 122 # report each page as its own "test." 123 if len(test_result.image_artifacts) > 1: 124 for page, artifact in enumerate(test_result.image_artifacts): 125 self.resultdb.Post( 126 test_id=f'{test_result.test_id}/{page}', 127 status=self._SuppressArtifactStatus(test_result, 128 artifact.GetDiffStatus()), 129 duration=None, 130 test_log=None, 131 test_file=None, 132 artifacts=artifact.GetDiffArtifacts(), 133 failure_reason=artifact.GetDiffReason()) 134 135 def _SuppressStatus(self, input_filename, status): 136 if not self.IsResultSuppressed(input_filename): 137 return status 138 139 if status == result_types.PASS: 140 # There isn't an actual status for succeeded-but-ignored, so use the 141 # "abort" status to differentiate this from failed-but-ignored. 142 # 143 # Note that this appears as a preliminary failure in Gerrit. 144 return result_types.UNKNOWN 145 146 # There isn't an actual status for failed-but-ignored, so use the "skip" 147 # status to differentiate this from succeeded-but-ignored. 148 return result_types.SKIP 149 150 def _SuppressArtifactStatus(self, test_result, status): 151 if status != result_types.FAIL: 152 return status 153 154 if test_result.status != result_types.SKIP: 155 return status 156 157 return result_types.SKIP 158 159 def Run(self): 160 # Running a test defines a number of attributes on the fly. 161 # pylint: disable=attribute-defined-outside-init 162 163 relative_test_dir = self.per_process_config.test_dir 164 if relative_test_dir != 'corpus': 165 relative_test_dir = os.path.join('resources', relative_test_dir) 166 167 parser = argparse.ArgumentParser() 168 169 parser.add_argument( 170 '--build-dir', 171 default=os.path.join('out', 'Debug'), 172 help='relative path from the base source directory') 173 174 parser.add_argument( 175 '-j', 176 default=multiprocessing.cpu_count(), 177 dest='num_workers', 178 type=int, 179 help='run NUM_WORKERS jobs in parallel') 180 181 parser.add_argument( 182 '--disable-javascript', 183 action='store_true', 184 help='Prevents JavaScript from executing in PDF files.') 185 186 parser.add_argument( 187 '--disable-xfa', 188 action='store_true', 189 help='Prevents processing XFA forms.') 190 191 parser.add_argument( 192 '--render-oneshot', 193 action='store_true', 194 help='Sets whether to use the oneshot renderer.') 195 196 parser.add_argument( 197 '--run-skia-gold', 198 action='store_true', 199 default=False, 200 help='When flag is on, skia gold tests will be run.') 201 202 parser.add_argument( 203 '--regenerate_expected', 204 action='store_true', 205 help='Regenerates expected images. For each failing image diff, this ' 206 'will regenerate the most specific expected image file that exists. ' 207 'This also will suggest removals of unnecessary expected image files ' 208 'by renaming them with an additional ".bak" extension, although these ' 209 'removals should be reviewed manually. Use "git clean" to quickly deal ' 210 'with any ".bak" files.') 211 212 parser.add_argument( 213 '--reverse-byte-order', 214 action='store_true', 215 help='Run image-based tests using --reverse-byte-order.') 216 217 parser.add_argument( 218 '--ignore_errors', 219 action='store_true', 220 help='Prevents the return value from being non-zero ' 221 'when image comparison fails.') 222 223 parser.add_argument( 224 '--use-renderer', 225 choices=('agg', 'gdi', 'skia'), 226 help='Forces the renderer to use.') 227 228 parser.add_argument( 229 'inputted_file_paths', 230 nargs='*', 231 help='Path to test files to run, relative to ' 232 f'testing/{relative_test_dir}. If omitted, runs all test files under ' 233 f'testing/{relative_test_dir}.', 234 metavar='relative/test/path') 235 236 skia_gold.add_skia_gold_args(parser) 237 238 self.per_process_config.options = parser.parse_args() 239 240 finder = self.per_process_config.NewFinder() 241 pdfium_test_path = self.per_process_config.GetPdfiumTestPath(finder) 242 if not os.path.exists(pdfium_test_path): 243 print(f"FAILURE: Can't find test executable '{pdfium_test_path}'") 244 print('Use --build-dir to specify its location.') 245 return 1 246 247 error_message = self.per_process_config.InitializeFeatures(pdfium_test_path) 248 if error_message: 249 print('FAILURE:', error_message) 250 return 1 251 252 self.per_process_state = _PerProcessState(self.per_process_config) 253 shutil.rmtree(self.per_process_state.working_dir, ignore_errors=True) 254 os.makedirs(self.per_process_state.working_dir) 255 256 error_message = self.per_process_state.image_differ.CheckMissingTools( 257 self.options.regenerate_expected) 258 if error_message: 259 print('FAILURE:', error_message) 260 return 1 261 262 self.resultdb = result_sink.TryInitClient() 263 if self.resultdb: 264 print('Detected ResultSink environment') 265 266 # Collect test cases. 267 walk_from_dir = finder.TestingDir(relative_test_dir) 268 269 self.test_cases = TestCaseManager() 270 self.execution_suppressed_cases = [] 271 input_file_re = re.compile('^.+[.](in|pdf)$') 272 if self.options.inputted_file_paths: 273 for file_name in self.options.inputted_file_paths: 274 input_path = os.path.join(walk_from_dir, file_name) 275 if not os.path.isfile(input_path): 276 print(f"Can't find test file '{file_name}'") 277 return 1 278 279 self.test_cases.NewTestCase(input_path) 280 else: 281 for file_dir, _, filename_list in os.walk(walk_from_dir): 282 for input_filename in filename_list: 283 if input_file_re.match(input_filename): 284 input_path = os.path.join(file_dir, input_filename) 285 if self.IsExecutionSuppressed(input_path): 286 self.execution_suppressed_cases.append(input_path) 287 continue 288 if not os.path.isfile(input_path): 289 continue 290 291 self.test_cases.NewTestCase(input_path) 292 293 # Execute test cases. 294 self.failures = [] 295 self.surprises = [] 296 self.skia_gold_successes = [] 297 self.skia_gold_unexpected_successes = [] 298 self.skia_gold_failures = [] 299 self.result_suppressed_cases = [] 300 301 if self.IsSkiaGoldEnabled(): 302 assert self.options.gold_output_dir 303 # Clear out and create top level gold output directory before starting 304 skia_gold.clear_gold_output_dir(self.options.gold_output_dir) 305 306 with multiprocessing.Pool( 307 processes=self.options.num_workers, 308 initializer=_InitializePerProcessState, 309 initargs=[self.per_process_config]) as pool: 310 if self.per_process_config.test_type in TEXT_TESTS: 311 test_function = _RunTextTest 312 else: 313 test_function = _RunPixelTest 314 for result in pool.imap(test_function, self.test_cases): 315 self.HandleResult(self.test_cases.GetTestCase(result.test_id), result) 316 317 # Report test results. 318 if self.surprises: 319 self.surprises.sort() 320 print('\nUnexpected Successes:') 321 for surprise in self.surprises: 322 print(surprise) 323 324 if self.failures: 325 self.failures.sort() 326 print('\nSummary of Failures:') 327 for failure in self.failures: 328 print(failure) 329 330 if self.skia_gold_unexpected_successes: 331 self.skia_gold_unexpected_successes.sort() 332 print('\nUnexpected Skia Gold Successes:') 333 for surprise in self.skia_gold_unexpected_successes: 334 print(surprise) 335 336 if self.skia_gold_failures: 337 self.skia_gold_failures.sort() 338 print('\nSummary of Skia Gold Failures:') 339 for failure in self.skia_gold_failures: 340 print(failure) 341 342 self._PrintSummary() 343 344 if self.failures: 345 if not self.options.ignore_errors: 346 return 1 347 348 return 0 349 350 def _PrintSummary(self): 351 number_test_cases = len(self.test_cases) 352 number_failures = len(self.failures) 353 number_suppressed = len(self.result_suppressed_cases) 354 number_successes = number_test_cases - number_failures - number_suppressed 355 number_surprises = len(self.surprises) 356 print('\nTest cases executed:', number_test_cases) 357 print(' Successes:', number_successes) 358 print(' Suppressed:', number_suppressed) 359 print(' Surprises:', number_surprises) 360 print(' Failures:', number_failures) 361 if self.IsSkiaGoldEnabled(): 362 number_gold_failures = len(self.skia_gold_failures) 363 number_gold_successes = len(self.skia_gold_successes) 364 number_gold_surprises = len(self.skia_gold_unexpected_successes) 365 number_total_gold_tests = sum( 366 [number_gold_failures, number_gold_successes, number_gold_surprises]) 367 print('\nSkia Gold Test cases executed:', number_total_gold_tests) 368 print(' Skia Gold Successes:', number_gold_successes) 369 print(' Skia Gold Surprises:', number_gold_surprises) 370 print(' Skia Gold Failures:', number_gold_failures) 371 skia_tester = self.per_process_state.GetSkiaGoldTester() 372 if self.skia_gold_failures and skia_tester.IsTryjobRun(): 373 cl_triage_link = skia_tester.GetCLTriageLink() 374 print(' Triage link for CL:', cl_triage_link) 375 skia_tester.WriteCLTriageLink(cl_triage_link) 376 print() 377 print('Test cases not executed:', len(self.execution_suppressed_cases)) 378 379 def SetDeleteOutputOnSuccess(self, new_value): 380 """Set whether to delete generated output if the test passes.""" 381 self.per_process_config.delete_output_on_success = new_value 382 383 def SetEnforceExpectedImages(self, new_value): 384 """Set whether to enforce that each test case provide an expected image.""" 385 self.per_process_config.enforce_expected_images = new_value 386 387 388def _RunTextTest(test_case): 389 """Runs a text test case.""" 390 test_case_runner = _TestCaseRunner(test_case) 391 with test_case_runner: 392 test_case_runner.test_result = test_case_runner.GenerateAndTest( 393 test_case_runner.TestText) 394 return test_case_runner.test_result 395 396 397def _RunPixelTest(test_case): 398 """Runs a pixel test case.""" 399 test_case_runner = _TestCaseRunner(test_case) 400 with test_case_runner: 401 test_case_runner.test_result = test_case_runner.GenerateAndTest( 402 test_case_runner.TestPixel) 403 return test_case_runner.test_result 404 405 406# `_PerProcessState` singleton. This is initialized when creating the 407# `multiprocessing.Pool()`. `TestRunner.Run()` creates its own separate 408# instance of `_PerProcessState` as well. 409_per_process_state = None 410 411 412def _InitializePerProcessState(config): 413 """Initializes the `_per_process_state` singleton.""" 414 global _per_process_state 415 assert not _per_process_state 416 _per_process_state = _PerProcessState(config) 417 418 419@dataclass 420class _PerProcessConfig: 421 """Configuration for initializing `_PerProcessState`. 422 423 Attributes: 424 test_dir: The name of the test directory. 425 test_type: The test type. 426 delete_output_on_success: Whether to delete output on success. 427 enforce_expected_images: Whether to enforce expected images. 428 options: The dictionary of command line options. 429 features: The set of features supported by `pdfium_test`. 430 rendering_option: The renderer to use (agg, gdi, or skia). 431 """ 432 test_dir: str 433 test_type: str 434 delete_output_on_success: bool = False 435 enforce_expected_images: bool = False 436 options: dict = None 437 features: set = None 438 rendering_option: str = None 439 440 def NewFinder(self): 441 return common.DirectoryFinder(self.options.build_dir) 442 443 def GetPdfiumTestPath(self, finder): 444 return finder.ExecutablePath('pdfium_test') 445 446 def InitializeFeatures(self, pdfium_test_path): 447 output = subprocess.check_output([pdfium_test_path, '--show-config'], 448 timeout=TEST_TIMEOUT) 449 self.features = set(output.decode('utf-8').strip().split(',')) 450 451 if 'SKIA' in self.features: 452 self.rendering_option = 'skia' 453 else: 454 self.rendering_option = 'agg' 455 456 if self.options.use_renderer == 'agg': 457 self.rendering_option = 'agg' 458 elif self.options.use_renderer == 'gdi': 459 if 'GDI' not in self.features: 460 return 'pdfium_test does not support the GDI renderer' 461 self.rendering_option = 'gdi' 462 elif self.options.use_renderer == 'skia': 463 if 'SKIA' not in self.features: 464 return 'pdfium_test does not support the Skia renderer' 465 self.rendering_option = 'skia' 466 467 return None 468 469 470class _PerProcessState: 471 """State defined per process.""" 472 473 def __init__(self, config): 474 self.test_dir = config.test_dir 475 self.test_type = config.test_type 476 self.delete_output_on_success = config.delete_output_on_success 477 self.enforce_expected_images = config.enforce_expected_images 478 self.options = config.options 479 self.features = config.features 480 481 finder = config.NewFinder() 482 self.pdfium_test_path = config.GetPdfiumTestPath(finder) 483 self.fixup_path = finder.ScriptPath('fixup_pdf_template.py') 484 self.text_diff_path = finder.ScriptPath('text_diff.py') 485 self.font_dir = os.path.join(finder.TestingDir(), 'resources', 'fonts') 486 self.third_party_font_dir = finder.ThirdPartyFontsDir() 487 488 self.source_dir = finder.TestingDir() 489 self.working_dir = finder.WorkingDir(os.path.join('testing', self.test_dir)) 490 491 self.test_suppressor = suppressor.Suppressor( 492 finder, self.features, self.options.disable_javascript, 493 self.options.disable_xfa, config.rendering_option) 494 self.image_differ = pngdiffer.PNGDiffer(finder, 495 self.options.reverse_byte_order, 496 config.rendering_option) 497 498 self.process_name = multiprocessing.current_process().name 499 self.skia_tester = None 500 501 def __getstate__(self): 502 raise RuntimeError('Cannot pickle per-process state') 503 504 def GetSkiaGoldTester(self): 505 """Gets the `SkiaGoldTester` singleton for this worker.""" 506 if not self.skia_tester: 507 self.skia_tester = skia_gold.SkiaGoldTester( 508 source_type=self.test_type, 509 skia_gold_args=self.options, 510 process_name=self.process_name) 511 return self.skia_tester 512 513 514class _TestCaseRunner: 515 """Runner for a single test case.""" 516 517 def __init__(self, test_case): 518 self.test_case = test_case 519 self.test_result = None 520 self.duration_start = 0 521 522 self.source_dir, self.input_filename = os.path.split( 523 self.test_case.input_path) 524 self.pdf_path = os.path.join(self.working_dir, f'{self.test_id}.pdf') 525 self.actual_images = None 526 527 def __enter__(self): 528 self.duration_start = time.perf_counter_ns() 529 return self 530 531 def __exit__(self, exc_type, exc_value, traceback): 532 if not self.test_result: 533 self.test_result = self.test_case.NewResult( 534 result_types.UNKNOWN, reason='No test result recorded') 535 duration = time.perf_counter_ns() - self.duration_start 536 self.test_result.duration_milliseconds = duration * 1e-6 537 538 @property 539 def options(self): 540 return _per_process_state.options 541 542 @property 543 def test_id(self): 544 return self.test_case.test_id 545 546 @property 547 def working_dir(self): 548 return _per_process_state.working_dir 549 550 def IsResultSuppressed(self): 551 return _per_process_state.test_suppressor.IsResultSuppressed( 552 self.input_filename) 553 554 def IsImageDiffSuppressed(self): 555 return _per_process_state.test_suppressor.IsImageDiffSuppressed( 556 self.input_filename) 557 558 def GetImageMatchingAlgorithm(self): 559 return _per_process_state.test_suppressor.GetImageMatchingAlgorithm( 560 self.input_filename) 561 562 def RunCommand(self, command, stdout=None): 563 """Runs a test command. 564 565 Args: 566 command: The list of command arguments. 567 stdout: Optional `file`-like object to send standard output. 568 569 Returns: 570 The test result. 571 """ 572 573 # Standard output and error are directed to the test log. If `stdout` was 574 # provided, redirect standard output to it instead. 575 if stdout: 576 assert stdout != subprocess.PIPE 577 try: 578 stdout.fileno() 579 except OSError: 580 # `stdout` doesn't have a file descriptor, so it can't be passed to 581 # `subprocess.run()` directly. 582 original_stdout = stdout 583 stdout = subprocess.PIPE 584 stderr = subprocess.PIPE 585 else: 586 stdout = subprocess.PIPE 587 stderr = subprocess.STDOUT 588 589 test_result = self.test_case.NewResult(result_types.PASS) 590 try: 591 run_result = subprocess.run( 592 command, 593 stdout=stdout, 594 stderr=stderr, 595 timeout=TEST_TIMEOUT, 596 check=False) 597 if run_result.returncode != 0: 598 test_result.status = result_types.FAIL 599 test_result.reason = 'Command {} exited with code {}'.format( 600 run_result.args, run_result.returncode) 601 except subprocess.TimeoutExpired as timeout_expired: 602 run_result = timeout_expired 603 test_result.status = result_types.TIMEOUT 604 test_result.reason = 'Command {} timed out'.format(run_result.cmd) 605 606 if stdout == subprocess.PIPE and stderr == subprocess.PIPE: 607 # Copy captured standard output, if any, to the original `stdout`. 608 if run_result.stdout: 609 original_stdout.write(run_result.stdout) 610 611 if not test_result.IsPass(): 612 # On failure, report captured output to the test log. 613 if stderr == subprocess.STDOUT: 614 test_result.log = run_result.stdout 615 else: 616 test_result.log = run_result.stderr 617 test_result.log = test_result.log.decode(errors='backslashreplace') 618 return test_result 619 620 def GenerateAndTest(self, test_function): 621 """Generate test input and run pdfium_test.""" 622 test_result = self.Generate() 623 if not test_result.IsPass(): 624 return test_result 625 626 return test_function() 627 628 def _RegenerateIfNeeded(self): 629 if not self.options.regenerate_expected: 630 return 631 if self.IsResultSuppressed() or self.IsImageDiffSuppressed(): 632 return 633 _per_process_state.image_differ.Regenerate( 634 self.input_filename, 635 self.source_dir, 636 self.working_dir, 637 image_matching_algorithm=self.GetImageMatchingAlgorithm()) 638 639 def Generate(self): 640 input_event_path = os.path.join(self.source_dir, f'{self.test_id}.evt') 641 if os.path.exists(input_event_path): 642 output_event_path = f'{os.path.splitext(self.pdf_path)[0]}.evt' 643 shutil.copyfile(input_event_path, output_event_path) 644 645 template_path = os.path.join(self.source_dir, f'{self.test_id}.in') 646 if not os.path.exists(template_path): 647 if os.path.exists(self.test_case.input_path): 648 shutil.copyfile(self.test_case.input_path, self.pdf_path) 649 return self.test_case.NewResult(result_types.PASS) 650 651 return self.RunCommand([ 652 sys.executable, _per_process_state.fixup_path, 653 f'--output-dir={self.working_dir}', template_path 654 ]) 655 656 def TestText(self): 657 txt_path = os.path.join(self.working_dir, f'{self.test_id}.txt') 658 with open(txt_path, 'w') as outfile: 659 cmd_to_run = [ 660 _per_process_state.pdfium_test_path, '--send-events', 661 f'--time={TEST_SEED_TIME}' 662 ] 663 664 if self.options.disable_javascript: 665 cmd_to_run.append('--disable-javascript') 666 667 if self.options.disable_xfa: 668 cmd_to_run.append('--disable-xfa') 669 670 cmd_to_run.append(self.pdf_path) 671 test_result = self.RunCommand(cmd_to_run, stdout=outfile) 672 if not test_result.IsPass(): 673 return test_result 674 675 # If the expected file does not exist, the output is expected to be empty. 676 expected_txt_path = os.path.join(self.source_dir, 677 f'{self.test_id}_expected.txt') 678 if not os.path.exists(expected_txt_path): 679 return self._VerifyEmptyText(txt_path) 680 681 # If JavaScript is disabled, the output should be empty. 682 # However, if the test is suppressed and JavaScript is disabled, do not 683 # verify that the text is empty so the suppressed test does not surprise. 684 if self.options.disable_javascript and not self.IsResultSuppressed(): 685 return self._VerifyEmptyText(txt_path) 686 687 return self.RunCommand([ 688 sys.executable, _per_process_state.text_diff_path, expected_txt_path, 689 txt_path 690 ]) 691 692 def _VerifyEmptyText(self, txt_path): 693 with open(txt_path, "rb") as txt_file: 694 txt_data = txt_file.read() 695 696 if txt_data: 697 return self.test_case.NewResult( 698 result_types.FAIL, 699 log=txt_data.decode(errors='backslashreplace'), 700 reason=f'{txt_path} should be empty') 701 702 return self.test_case.NewResult(result_types.PASS) 703 704 # TODO(crbug.com/pdfium/1656): Remove when ready to fully switch over to 705 # Skia Gold 706 def TestPixel(self): 707 # Remove any existing generated images from previous runs. 708 self.actual_images = _per_process_state.image_differ.GetActualFiles( 709 self.input_filename, self.source_dir, self.working_dir) 710 self._CleanupPixelTest() 711 712 # Generate images. 713 cmd_to_run = [ 714 _per_process_state.pdfium_test_path, '--send-events', '--png', '--md5', 715 f'--time={TEST_SEED_TIME}' 716 ] 717 718 if 'use_ahem' in self.source_dir or 'use_symbolneu' in self.source_dir: 719 cmd_to_run.append(f'--font-dir={_per_process_state.font_dir}') 720 else: 721 cmd_to_run.append(f'--font-dir={_per_process_state.third_party_font_dir}') 722 cmd_to_run.append('--croscore-font-names') 723 724 if self.options.disable_javascript: 725 cmd_to_run.append('--disable-javascript') 726 727 if self.options.disable_xfa: 728 cmd_to_run.append('--disable-xfa') 729 730 if self.options.render_oneshot: 731 cmd_to_run.append('--render-oneshot') 732 733 if self.options.reverse_byte_order: 734 cmd_to_run.append('--reverse-byte-order') 735 736 if self.options.use_renderer: 737 cmd_to_run.append(f'--use-renderer={self.options.use_renderer}') 738 739 cmd_to_run.append(self.pdf_path) 740 741 with BytesIO() as command_output: 742 test_result = self.RunCommand(cmd_to_run, stdout=command_output) 743 if not test_result.IsPass(): 744 return test_result 745 746 test_result.image_artifacts = [] 747 for line in command_output.getvalue().splitlines(): 748 # Expect this format: MD5:<path to image file>:<hexadecimal MD5 hash> 749 line = bytes.decode(line).strip() 750 if line.startswith('MD5:'): 751 image_path, md5_hash = line[4:].rsplit(':', 1) 752 test_result.image_artifacts.append( 753 self._NewImageArtifact( 754 image_path=image_path.strip(), md5_hash=md5_hash.strip())) 755 756 if self.actual_images: 757 image_diffs = _per_process_state.image_differ.ComputeDifferences( 758 self.input_filename, 759 self.source_dir, 760 self.working_dir, 761 image_matching_algorithm=self.GetImageMatchingAlgorithm()) 762 if image_diffs: 763 test_result.status = result_types.FAIL 764 test_result.reason = 'Images differ' 765 766 # Merge image diffs into test result. 767 diff_map = {} 768 diff_log = [] 769 for diff in image_diffs: 770 diff_map[diff.actual_path] = diff 771 diff_log.append(f'{os.path.basename(diff.actual_path)} vs. ') 772 if diff.expected_path: 773 diff_log.append(f'{os.path.basename(diff.expected_path)}\n') 774 else: 775 diff_log.append('missing expected file\n') 776 777 for artifact in test_result.image_artifacts: 778 artifact.image_diff = diff_map.get(artifact.image_path) 779 test_result.log = ''.join(diff_log) 780 781 elif _per_process_state.enforce_expected_images: 782 if not self.IsImageDiffSuppressed(): 783 test_result.status = result_types.FAIL 784 test_result.reason = 'Missing expected images' 785 786 if not test_result.IsPass(): 787 self._RegenerateIfNeeded() 788 return test_result 789 790 if _per_process_state.delete_output_on_success: 791 self._CleanupPixelTest() 792 return test_result 793 794 def _NewImageArtifact(self, *, image_path, md5_hash): 795 artifact = ImageArtifact(image_path=image_path, md5_hash=md5_hash) 796 797 if self.options.run_skia_gold: 798 if _per_process_state.GetSkiaGoldTester().UploadTestResultToSkiaGold( 799 artifact.GetSkiaGoldId(), artifact.image_path): 800 artifact.skia_gold_status = result_types.PASS 801 else: 802 artifact.skia_gold_status = result_types.FAIL 803 804 return artifact 805 806 def _CleanupPixelTest(self): 807 for image_file in self.actual_images: 808 if os.path.exists(image_file): 809 os.remove(image_file) 810 811 812@dataclass 813class TestCase: 814 """Description of a test case to run. 815 816 Attributes: 817 test_id: A unique identifier for the test. 818 input_path: The absolute path to the test file. 819 """ 820 test_id: str 821 input_path: str 822 823 def NewResult(self, status, **kwargs): 824 """Derives a new test result corresponding to this test case.""" 825 return TestResult(test_id=self.test_id, status=status, **kwargs) 826 827 828@dataclass 829class TestResult: 830 """Results from running a test case. 831 832 Attributes: 833 test_id: The corresponding test case ID. 834 status: The overall `result_types` status. 835 duration_milliseconds: Test time in milliseconds. 836 log: Optional log of the test's output. 837 image_artfacts: Optional list of image artifacts. 838 reason: Optional reason why the test failed. 839 """ 840 test_id: str 841 status: str 842 duration_milliseconds: float = None 843 log: str = None 844 image_artifacts: list = field(default_factory=list) 845 reason: str = None 846 847 def IsPass(self): 848 """Whether the test passed.""" 849 return self.status == result_types.PASS 850 851 852@dataclass 853class ImageArtifact: 854 """Image artifact for a test result. 855 856 Attributes: 857 image_path: The absolute path to the image file. 858 md5_hash: The MD5 hash of the pixel buffer. 859 skia_gold_status: Optional Skia Gold status. 860 image_diff: Optional image diff. 861 """ 862 image_path: str 863 md5_hash: str 864 skia_gold_status: str = None 865 image_diff: pngdiffer.ImageDiff = None 866 867 def GetSkiaGoldId(self): 868 # The output filename without image extension becomes the test ID. For 869 # example, "/path/to/.../testing/corpus/example_005.pdf.0.png" becomes 870 # "example_005.pdf.0". 871 return _GetTestId(os.path.basename(self.image_path)) 872 873 def GetDiffStatus(self): 874 return result_types.FAIL if self.image_diff else result_types.PASS 875 876 def GetDiffReason(self): 877 return self.image_diff.reason if self.image_diff else None 878 879 def GetDiffArtifacts(self): 880 if not self.image_diff: 881 return None 882 if not self.image_diff.expected_path or not self.image_diff.diff_path: 883 return None 884 return { 885 'actual_image': 886 _GetArtifactFromFilePath(self.image_path), 887 'expected_image': 888 _GetArtifactFromFilePath(self.image_diff.expected_path), 889 'image_diff': 890 _GetArtifactFromFilePath(self.image_diff.diff_path) 891 } 892 893 894class TestCaseManager: 895 """Manages a collection of test cases.""" 896 897 def __init__(self): 898 self.test_cases = {} 899 900 def __len__(self): 901 return len(self.test_cases) 902 903 def __iter__(self): 904 return iter(self.test_cases.values()) 905 906 def NewTestCase(self, input_path, **kwargs): 907 """Creates and registers a new test case.""" 908 input_basename = os.path.basename(input_path) 909 test_id = _GetTestId(input_basename) 910 if test_id in self.test_cases: 911 raise ValueError( 912 f'Test ID "{test_id}" derived from "{input_basename}" must be unique') 913 914 test_case = TestCase(test_id=test_id, input_path=input_path, **kwargs) 915 self.test_cases[test_id] = test_case 916 return test_case 917 918 def GetTestCase(self, test_id): 919 """Looks up a test case previously registered by `NewTestCase()`.""" 920 return self.test_cases[test_id] 921 922 923def _GetTestId(input_basename): 924 """Constructs a test ID by stripping the last extension from the basename.""" 925 return os.path.splitext(input_basename)[0] 926 927 928def _GetArtifactFromFilePath(file_path): 929 """Constructs a ResultSink artifact from a file path.""" 930 return {'filePath': file_path} 931
