1# Copyright 2021 The PDFium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import os 6import logging 7import shlex 8import shutil 9 10from . import pdfium_skia_gold_properties 11from . import pdfium_skia_gold_session_manager 12 13GS_BUCKET = 'skia-pdfium-gm' 14 15 16def _ParseKeyValuePairs(kv_str): 17 """ 18 Parses a string of the type 'key1 value1 key2 value2' into a dict. 19 """ 20 kv_pairs = shlex.split(kv_str) 21 if len(kv_pairs) % 2: 22 raise ValueError('Uneven number of key/value pairs. Got %s' % kv_str) 23 return {kv_pairs[i]: kv_pairs[i + 1] for i in range(0, len(kv_pairs), 2)} 24 25 26def add_skia_gold_args(parser): 27 group = parser.add_argument_group('Skia Gold Arguments') 28 group.add_argument( 29 '--git-revision', help='Revision being tested.', default=None) 30 group.add_argument( 31 '--gerrit-issue', 32 help='For Skia Gold integration. Gerrit issue ID.', 33 default='') 34 group.add_argument( 35 '--gerrit-patchset', 36 help='For Skia Gold integration. Gerrit patch set number.', 37 default='') 38 group.add_argument( 39 '--buildbucket-id', 40 help='For Skia Gold integration. Buildbucket build ID.', 41 default='') 42 group.add_argument( 43 '--bypass-skia-gold-functionality', 44 action='store_true', 45 default=False, 46 help='Bypass all interaction with Skia Gold, effectively disabling the ' 47 'image comparison portion of any tests that use Gold. Only meant to ' 48 'be used in case a Gold outage occurs and cannot be fixed quickly.') 49 local_group = group.add_mutually_exclusive_group() 50 local_group.add_argument( 51 '--local-pixel-tests', 52 action='store_true', 53 default=None, 54 help='Specifies to run the test harness in local run mode or not. When ' 55 'run in local mode, uploading to Gold is disabled and links to ' 56 'help with local debugging are output. Running in local mode also ' 57 'implies --no-luci-auth. If both this and --no-local-pixel-tests are ' 58 'left unset, the test harness will attempt to detect whether it is ' 59 'running on a workstation or not and set this option accordingly.') 60 local_group.add_argument( 61 '--no-local-pixel-tests', 62 action='store_false', 63 dest='local_pixel_tests', 64 help='Specifies to run the test harness in non-local (bot) mode. When ' 65 'run in this mode, data is actually uploaded to Gold and triage links ' 66 'arge generated. If both this and --local-pixel-tests are left unset, ' 67 'the test harness will attempt to detect whether it is running on a ' 68 'workstation or not and set this option accordingly.') 69 group.add_argument( 70 '--no-luci-auth', 71 action='store_true', 72 default=False, 73 help='Don\'t use the service account provided by LUCI for ' 74 'authentication for Skia Gold, instead relying on gsutil to be ' 75 'pre-authenticated. Meant for testing locally instead of on the bots.') 76 77 group.add_argument( 78 '--gold_key', 79 default='', 80 dest="gold_key", 81 help='Key value pairs of config data such like the hardware/software ' 82 'configuration the image was produced on.') 83 group.add_argument( 84 '--gold_output_dir', 85 default='', 86 dest="gold_output_dir", 87 help='Path to the dir where diff output image files are saved, ' 88 'if running locally. If this is a tryjob run, will contain link to skia ' 89 'gold CL triage link. Required with --run-skia-gold.') 90 91 92def clear_gold_output_dir(output_dir): 93 # make sure the output directory exists and is empty. 94 if os.path.exists(output_dir): 95 shutil.rmtree(output_dir, ignore_errors=True) 96 os.makedirs(output_dir) 97 98 99class SkiaGoldTester: 100 101 def __init__(self, source_type, skia_gold_args, process_name=None): 102 """ 103 source_type: source_type (=corpus) field used for all results. 104 skia_gold_args: Parsed arguments from argparse.ArgumentParser. 105 process_name: Unique name of current process, if multiprocessing is on. 106 """ 107 self._source_type = source_type 108 self._output_dir = skia_gold_args.gold_output_dir 109 # goldctl is not thread safe, so each process needs its own directory 110 if process_name: 111 self._output_dir = os.path.join(skia_gold_args.gold_output_dir, 112 process_name) 113 clear_gold_output_dir(self._output_dir) 114 self._keys = _ParseKeyValuePairs(skia_gold_args.gold_key) 115 self._skia_gold_args = skia_gold_args 116 self._skia_gold_session_manager = None 117 self._skia_gold_properties = None 118 119 def WriteCLTriageLink(self, link): 120 # pdfium recipe will read from this file and display the link in the step 121 # presentation 122 assert isinstance(link, str) 123 output_file_name = os.path.join(self._output_dir, 'cl_triage_link.txt') 124 if os.path.exists(output_file_name): 125 os.remove(output_file_name) 126 with open(output_file_name, 'wb') as outfile: 127 outfile.write(link.encode('utf8')) 128 129 def GetSkiaGoldProperties(self): 130 if not self._skia_gold_properties: 131 if self._skia_gold_args.local_pixel_tests is None: 132 self._skia_gold_args.local_pixel_tests = 'SWARMING_SERVER' \ 133 not in os.environ 134 135 self._skia_gold_properties = pdfium_skia_gold_properties\ 136 .PDFiumSkiaGoldProperties(self._skia_gold_args) 137 return self._skia_gold_properties 138 139 def GetSkiaGoldSessionManager(self): 140 if not self._skia_gold_session_manager: 141 self._skia_gold_session_manager = pdfium_skia_gold_session_manager\ 142 .PDFiumSkiaGoldSessionManager(self._output_dir, 143 self.GetSkiaGoldProperties()) 144 return self._skia_gold_session_manager 145 146 def IsTryjobRun(self): 147 return self.GetSkiaGoldProperties().IsTryjobRun() 148 149 def GetCLTriageLink(self): 150 return 'https://pdfium-gold.skia.org/search?issue={issue}&crs=gerrit&'\ 151 'corpus={source_type}'.format( 152 issue=self.GetSkiaGoldProperties().issue, source_type=self._source_type) 153 154 def UploadTestResultToSkiaGold(self, image_name, image_path): 155 gold_properties = self.GetSkiaGoldProperties() 156 use_luci = not (gold_properties.local_pixel_tests or 157 gold_properties.no_luci_auth) 158 gold_session = self.GetSkiaGoldSessionManager()\ 159 .GetSkiaGoldSession(self._keys, corpus=self._source_type, 160 bucket=GS_BUCKET) 161 162 status, error = gold_session.RunComparison( 163 name=image_name, png_file=image_path, use_luci=use_luci) 164 165 status_codes =\ 166 self.GetSkiaGoldSessionManager().GetSessionClass().StatusCodes 167 if status == status_codes.SUCCESS: 168 return True 169 if status == status_codes.AUTH_FAILURE: 170 logging.error('Gold authentication failed with output %s', error) 171 elif status == status_codes.INIT_FAILURE: 172 logging.error('Gold initialization failed with output %s', error) 173 elif status == status_codes.COMPARISON_FAILURE_REMOTE: 174 logging.error('Remote comparison failed. See outputted triage links.') 175 elif status == status_codes.COMPARISON_FAILURE_LOCAL: 176 logging.error('Local comparison failed. Local diff files:') 177 _OutputLocalDiffFiles(gold_session, image_name) 178 print() 179 elif status == status_codes.LOCAL_DIFF_FAILURE: 180 logging.error( 181 'Local comparison failed and an error occurred during diff ' 182 'generation: %s', error) 183 # There might be some files, so try outputting them. 184 logging.error('Local diff files:') 185 _OutputLocalDiffFiles(gold_session, image_name) 186 print() 187 else: 188 logging.error( 189 'Given unhandled SkiaGoldSession StatusCode %s with error %s', status, 190 error) 191 192 return False 193 194 195def _OutputLocalDiffFiles(gold_session, image_name): 196 """Logs the local diff image files from the given SkiaGoldSession. 197 198 Args: 199 gold_session: A skia_gold_session.SkiaGoldSession instance to pull files 200 from. 201 image_name: A string containing the name of the image/test that was 202 compared. 203 """ 204 given_file = gold_session.GetGivenImageLink(image_name) 205 closest_file = gold_session.GetClosestImageLink(image_name) 206 diff_file = gold_session.GetDiffImageLink(image_name) 207 failure_message = 'Unable to retrieve link' 208 logging.error('Generated image for %s: %s', image_name, given_file or 209 failure_message) 210 logging.error('Closest image for %s: %s', image_name, closest_file or 211 failure_message) 212 logging.error('Diff image for %s: %s', image_name, diff_file or 213 failure_message) 214