xref: /aosp_15_r20/external/cronet/testing/scripts/common.py (revision 6777b5387eb2ff775bb5750e3f5d96f37fb7352b)
1*6777b538SAndroid Build Coastguard Worker# Copyright 2014 The Chromium Authors
2*6777b538SAndroid Build Coastguard Worker# Use of this source code is governed by a BSD-style license that can be
3*6777b538SAndroid Build Coastguard Worker# found in the LICENSE file.
4*6777b538SAndroid Build Coastguard Worker
5*6777b538SAndroid Build Coastguard Workerfrom __future__ import print_function
6*6777b538SAndroid Build Coastguard Workerimport argparse
7*6777b538SAndroid Build Coastguard Workerimport codecs
8*6777b538SAndroid Build Coastguard Workerimport contextlib
9*6777b538SAndroid Build Coastguard Workerimport json
10*6777b538SAndroid Build Coastguard Workerimport os
11*6777b538SAndroid Build Coastguard Workerimport logging
12*6777b538SAndroid Build Coastguard Workerimport platform
13*6777b538SAndroid Build Coastguard Workerimport subprocess
14*6777b538SAndroid Build Coastguard Workerimport sys
15*6777b538SAndroid Build Coastguard Workerimport tempfile
16*6777b538SAndroid Build Coastguard Workerimport time
17*6777b538SAndroid Build Coastguard Workerimport traceback
18*6777b538SAndroid Build Coastguard Worker
19*6777b538SAndroid Build Coastguard Workerlogging.basicConfig(level=logging.INFO)
20*6777b538SAndroid Build Coastguard Worker
21*6777b538SAndroid Build Coastguard Worker# Add src/testing/ into sys.path for importing xvfb and test_env.
22*6777b538SAndroid Build Coastguard Workersys.path.append(
23*6777b538SAndroid Build Coastguard Worker    os.path.abspath(os.path.join(os.path.dirname(__file__), os.path.pardir)))
24*6777b538SAndroid Build Coastguard Workerimport test_env
25*6777b538SAndroid Build Coastguard Workerif sys.platform.startswith('linux'):
26*6777b538SAndroid Build Coastguard Worker  import xvfb
27*6777b538SAndroid Build Coastguard Worker
28*6777b538SAndroid Build Coastguard Worker
29*6777b538SAndroid Build Coastguard WorkerSCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
30*6777b538SAndroid Build Coastguard WorkerSRC_DIR = os.path.abspath(
31*6777b538SAndroid Build Coastguard Worker    os.path.join(SCRIPT_DIR, os.path.pardir, os.path.pardir))
32*6777b538SAndroid Build Coastguard Worker
33*6777b538SAndroid Build Coastguard Worker# Use result_sink.py in //build/util/lib/results/ for uploading the
34*6777b538SAndroid Build Coastguard Worker# results of non-isolated script tests.
35*6777b538SAndroid Build Coastguard WorkerBUILD_UTIL_DIR = os.path.join(SRC_DIR, 'build', 'util')
36*6777b538SAndroid Build Coastguard Workersys.path.insert(0, BUILD_UTIL_DIR)
37*6777b538SAndroid Build Coastguard Workertry:
38*6777b538SAndroid Build Coastguard Worker  from lib.results import result_sink
39*6777b538SAndroid Build Coastguard Worker  from lib.results import result_types
40*6777b538SAndroid Build Coastguard Workerexcept ImportError:
41*6777b538SAndroid Build Coastguard Worker  # Some build-time scripts import this file and run into issues with
42*6777b538SAndroid Build Coastguard Worker  # result_sink's dependency on requests since we can't depend on vpython
43*6777b538SAndroid Build Coastguard Worker  # during build-time. So silently swallow the error in that case.
44*6777b538SAndroid Build Coastguard Worker  result_sink = None
45*6777b538SAndroid Build Coastguard Worker
46*6777b538SAndroid Build Coastguard Worker# run_web_tests.py returns the number of failures as the return
47*6777b538SAndroid Build Coastguard Worker# code, but caps the return code at 101 to avoid overflow or colliding
48*6777b538SAndroid Build Coastguard Worker# with reserved values from the shell.
49*6777b538SAndroid Build Coastguard WorkerMAX_FAILURES_EXIT_STATUS = 101
50*6777b538SAndroid Build Coastguard Worker
51*6777b538SAndroid Build Coastguard Worker
52*6777b538SAndroid Build Coastguard Worker# Exit code to indicate infrastructure issue.
53*6777b538SAndroid Build Coastguard WorkerINFRA_FAILURE_EXIT_CODE = 87
54*6777b538SAndroid Build Coastguard Worker
55*6777b538SAndroid Build Coastguard Worker
56*6777b538SAndroid Build Coastguard Worker# ACL might be explicitly set or inherited.
57*6777b538SAndroid Build Coastguard WorkerCORRECT_ACL_VARIANTS = [
58*6777b538SAndroid Build Coastguard Worker    'APPLICATION PACKAGE AUTHORITY' \
59*6777b538SAndroid Build Coastguard Worker    '\\ALL RESTRICTED APPLICATION PACKAGES:(OI)(CI)(RX)', \
60*6777b538SAndroid Build Coastguard Worker    'APPLICATION PACKAGE AUTHORITY' \
61*6777b538SAndroid Build Coastguard Worker    '\\ALL RESTRICTED APPLICATION PACKAGES:(I)(OI)(CI)(RX)'
62*6777b538SAndroid Build Coastguard Worker]
63*6777b538SAndroid Build Coastguard Worker
64*6777b538SAndroid Build Coastguard Worker# pylint: disable=useless-object-inheritance
65*6777b538SAndroid Build Coastguard Worker
66*6777b538SAndroid Build Coastguard Worker
67*6777b538SAndroid Build Coastguard Workerdef set_lpac_acls(acl_dir, is_test_script=False):
68*6777b538SAndroid Build Coastguard Worker  """Sets LPAC ACLs on a directory. Windows 10 only."""
69*6777b538SAndroid Build Coastguard Worker  if platform.release() != '10':
70*6777b538SAndroid Build Coastguard Worker    return
71*6777b538SAndroid Build Coastguard Worker  try:
72*6777b538SAndroid Build Coastguard Worker    existing_acls = subprocess.check_output(['icacls', acl_dir],
73*6777b538SAndroid Build Coastguard Worker                                            stderr=subprocess.STDOUT,
74*6777b538SAndroid Build Coastguard Worker                                            universal_newlines=True)
75*6777b538SAndroid Build Coastguard Worker  except subprocess.CalledProcessError as e:
76*6777b538SAndroid Build Coastguard Worker    logging.error('Failed to retrieve existing ACLs for directory %s', acl_dir)
77*6777b538SAndroid Build Coastguard Worker    logging.error('Command output: %s', e.output)
78*6777b538SAndroid Build Coastguard Worker    sys.exit(e.returncode)
79*6777b538SAndroid Build Coastguard Worker  acls_correct = False
80*6777b538SAndroid Build Coastguard Worker  for acl in CORRECT_ACL_VARIANTS:
81*6777b538SAndroid Build Coastguard Worker    if acl in existing_acls:
82*6777b538SAndroid Build Coastguard Worker      acls_correct = True
83*6777b538SAndroid Build Coastguard Worker  if not acls_correct:
84*6777b538SAndroid Build Coastguard Worker    try:
85*6777b538SAndroid Build Coastguard Worker      existing_acls = subprocess.check_output(
86*6777b538SAndroid Build Coastguard Worker          ['icacls', acl_dir, '/grant', '*S-1-15-2-2:(OI)(CI)(RX)'],
87*6777b538SAndroid Build Coastguard Worker          stderr=subprocess.STDOUT)
88*6777b538SAndroid Build Coastguard Worker    except subprocess.CalledProcessError as e:
89*6777b538SAndroid Build Coastguard Worker      logging.error(
90*6777b538SAndroid Build Coastguard Worker          'Failed to retrieve existing ACLs for directory %s', acl_dir)
91*6777b538SAndroid Build Coastguard Worker      logging.error('Command output: %s', e.output)
92*6777b538SAndroid Build Coastguard Worker      sys.exit(e.returncode)
93*6777b538SAndroid Build Coastguard Worker  if not is_test_script:
94*6777b538SAndroid Build Coastguard Worker    return
95*6777b538SAndroid Build Coastguard Worker  # Bots running on luci use hardlinks that do not have correct ACLs so these
96*6777b538SAndroid Build Coastguard Worker  # must be manually overridden here.
97*6777b538SAndroid Build Coastguard Worker  with temporary_file() as tempfile_path:
98*6777b538SAndroid Build Coastguard Worker    subprocess.check_output(
99*6777b538SAndroid Build Coastguard Worker        ['icacls', acl_dir, '/save', tempfile_path, '/t', '/q', '/c'],
100*6777b538SAndroid Build Coastguard Worker        stderr=subprocess.STDOUT)
101*6777b538SAndroid Build Coastguard Worker    # ACL files look like this, e.g. for c:\a\b\c\d\Release_x64
102*6777b538SAndroid Build Coastguard Worker    #
103*6777b538SAndroid Build Coastguard Worker    # Release_x64
104*6777b538SAndroid Build Coastguard Worker    # D:AI(A;OICI;0x1200a9;;;S-1-15-2-2)(A;OICIID;FA;;;BA)
105*6777b538SAndroid Build Coastguard Worker    # Release_x64\icudtl_extra.dat
106*6777b538SAndroid Build Coastguard Worker    # D:AI(A;ID;0x1200a9;;;S-1-15-2-2)(A;ID;FA;;;BA)(A;ID;0x1301bf;;;BU)
107*6777b538SAndroid Build Coastguard Worker    with codecs.open(tempfile_path, encoding='utf_16_le') as aclfile:
108*6777b538SAndroid Build Coastguard Worker      for filename in aclfile:
109*6777b538SAndroid Build Coastguard Worker        acl = next(aclfile).strip()
110*6777b538SAndroid Build Coastguard Worker        full_filename = os.path.abspath(
111*6777b538SAndroid Build Coastguard Worker            os.path.join(acl_dir, os.pardir, filename.strip()))
112*6777b538SAndroid Build Coastguard Worker        if 'S-1-15-2-2' in acl:
113*6777b538SAndroid Build Coastguard Worker          continue
114*6777b538SAndroid Build Coastguard Worker        if os.path.isdir(full_filename):
115*6777b538SAndroid Build Coastguard Worker          continue
116*6777b538SAndroid Build Coastguard Worker        subprocess.check_output(
117*6777b538SAndroid Build Coastguard Worker            ['icacls', full_filename, '/grant', '*S-1-15-2-2:(RX)'],
118*6777b538SAndroid Build Coastguard Worker            stderr=subprocess.STDOUT)
119*6777b538SAndroid Build Coastguard Worker
120*6777b538SAndroid Build Coastguard Worker
121*6777b538SAndroid Build Coastguard Workerdef run_script(argv, funcs):
122*6777b538SAndroid Build Coastguard Worker  def parse_json(path):
123*6777b538SAndroid Build Coastguard Worker    with open(path) as f:
124*6777b538SAndroid Build Coastguard Worker      return json.load(f)
125*6777b538SAndroid Build Coastguard Worker  parser = argparse.ArgumentParser()
126*6777b538SAndroid Build Coastguard Worker  # TODO(phajdan.jr): Make build-config-fs required after passing it in recipe.
127*6777b538SAndroid Build Coastguard Worker  parser.add_argument('--build-config-fs')
128*6777b538SAndroid Build Coastguard Worker  parser.add_argument('--paths', type=parse_json, default={})
129*6777b538SAndroid Build Coastguard Worker  # Properties describe the environment of the build, and are the same per
130*6777b538SAndroid Build Coastguard Worker  # script invocation.
131*6777b538SAndroid Build Coastguard Worker  parser.add_argument('--properties', type=parse_json, default={})
132*6777b538SAndroid Build Coastguard Worker  # Args contains per-invocation arguments that potentially change the
133*6777b538SAndroid Build Coastguard Worker  # behavior of the script.
134*6777b538SAndroid Build Coastguard Worker  parser.add_argument('--args', type=parse_json, default=[])
135*6777b538SAndroid Build Coastguard Worker
136*6777b538SAndroid Build Coastguard Worker  subparsers = parser.add_subparsers()
137*6777b538SAndroid Build Coastguard Worker
138*6777b538SAndroid Build Coastguard Worker  run_parser = subparsers.add_parser('run')
139*6777b538SAndroid Build Coastguard Worker  run_parser.add_argument(
140*6777b538SAndroid Build Coastguard Worker      '--output', type=argparse.FileType('w'), required=True)
141*6777b538SAndroid Build Coastguard Worker  run_parser.add_argument('--filter-file', type=argparse.FileType('r'))
142*6777b538SAndroid Build Coastguard Worker  run_parser.set_defaults(func=funcs['run'])
143*6777b538SAndroid Build Coastguard Worker
144*6777b538SAndroid Build Coastguard Worker  run_parser = subparsers.add_parser('compile_targets')
145*6777b538SAndroid Build Coastguard Worker  run_parser.add_argument(
146*6777b538SAndroid Build Coastguard Worker      '--output', type=argparse.FileType('w'), required=True)
147*6777b538SAndroid Build Coastguard Worker  run_parser.set_defaults(func=funcs['compile_targets'])
148*6777b538SAndroid Build Coastguard Worker
149*6777b538SAndroid Build Coastguard Worker  args = parser.parse_args(argv)
150*6777b538SAndroid Build Coastguard Worker  return args.func(args)
151*6777b538SAndroid Build Coastguard Worker
152*6777b538SAndroid Build Coastguard Worker
153*6777b538SAndroid Build Coastguard Workerdef run_command(argv, env=None, cwd=None):
154*6777b538SAndroid Build Coastguard Worker  print('Running %r in %r (env: %r)' % (argv, cwd, env), file=sys.stderr)
155*6777b538SAndroid Build Coastguard Worker  rc = test_env.run_command(argv, env=env, cwd=cwd)
156*6777b538SAndroid Build Coastguard Worker  print('Command %r returned exit code %d' % (argv, rc), file=sys.stderr)
157*6777b538SAndroid Build Coastguard Worker  return rc
158*6777b538SAndroid Build Coastguard Worker
159*6777b538SAndroid Build Coastguard Worker
160*6777b538SAndroid Build Coastguard Worker@contextlib.contextmanager
161*6777b538SAndroid Build Coastguard Workerdef temporary_file():
162*6777b538SAndroid Build Coastguard Worker  fd, path = tempfile.mkstemp()
163*6777b538SAndroid Build Coastguard Worker  os.close(fd)
164*6777b538SAndroid Build Coastguard Worker  try:
165*6777b538SAndroid Build Coastguard Worker    yield path
166*6777b538SAndroid Build Coastguard Worker  finally:
167*6777b538SAndroid Build Coastguard Worker    os.remove(path)
168*6777b538SAndroid Build Coastguard Worker
169*6777b538SAndroid Build Coastguard Worker
170*6777b538SAndroid Build Coastguard Workerdef record_local_script_results(name, output_fd, failures, valid):
171*6777b538SAndroid Build Coastguard Worker  """Records to a local json file and to RDB the results of the script test.
172*6777b538SAndroid Build Coastguard Worker
173*6777b538SAndroid Build Coastguard Worker  For legacy reasons, local script tests (ie: script tests that run
174*6777b538SAndroid Build Coastguard Worker  locally and that don't conform to the isolated-test API) are expected to
175*6777b538SAndroid Build Coastguard Worker  record their results using a specific format. This method encapsulates
176*6777b538SAndroid Build Coastguard Worker  that format and also uploads those results to Result DB.
177*6777b538SAndroid Build Coastguard Worker
178*6777b538SAndroid Build Coastguard Worker  Args:
179*6777b538SAndroid Build Coastguard Worker    name: Name of the script test.
180*6777b538SAndroid Build Coastguard Worker    output_fd: A .write()-supporting file descriptor to write results to.
181*6777b538SAndroid Build Coastguard Worker    failures: List of strings representing test failures.
182*6777b538SAndroid Build Coastguard Worker    valid: Whether the results are valid.
183*6777b538SAndroid Build Coastguard Worker  """
184*6777b538SAndroid Build Coastguard Worker  local_script_results = {
185*6777b538SAndroid Build Coastguard Worker      'valid': valid,
186*6777b538SAndroid Build Coastguard Worker      'failures': failures
187*6777b538SAndroid Build Coastguard Worker  }
188*6777b538SAndroid Build Coastguard Worker  json.dump(local_script_results, output_fd)
189*6777b538SAndroid Build Coastguard Worker
190*6777b538SAndroid Build Coastguard Worker  if not result_sink:
191*6777b538SAndroid Build Coastguard Worker    return
192*6777b538SAndroid Build Coastguard Worker  result_sink_client = result_sink.TryInitClient()
193*6777b538SAndroid Build Coastguard Worker  if not result_sink_client:
194*6777b538SAndroid Build Coastguard Worker    return
195*6777b538SAndroid Build Coastguard Worker  status = result_types.PASS
196*6777b538SAndroid Build Coastguard Worker  if not valid:
197*6777b538SAndroid Build Coastguard Worker    status = result_types.UNKNOWN
198*6777b538SAndroid Build Coastguard Worker  elif failures:
199*6777b538SAndroid Build Coastguard Worker    status = result_types.FAIL
200*6777b538SAndroid Build Coastguard Worker  test_log = '\n'.join(failures)
201*6777b538SAndroid Build Coastguard Worker  result_sink_client.Post(name, status, None, test_log, None)
202*6777b538SAndroid Build Coastguard Worker
203*6777b538SAndroid Build Coastguard Worker
204*6777b538SAndroid Build Coastguard Workerdef parse_common_test_results(json_results, test_separator='/'):
205*6777b538SAndroid Build Coastguard Worker  def convert_trie_to_flat_paths(trie, prefix=None):
206*6777b538SAndroid Build Coastguard Worker    # Also see blinkpy.web_tests.layout_package.json_results_generator
207*6777b538SAndroid Build Coastguard Worker    result = {}
208*6777b538SAndroid Build Coastguard Worker    for name, data in trie.items():
209*6777b538SAndroid Build Coastguard Worker      if prefix:
210*6777b538SAndroid Build Coastguard Worker        name = prefix + test_separator + name
211*6777b538SAndroid Build Coastguard Worker      if len(data) and not 'actual' in data and not 'expected' in data:
212*6777b538SAndroid Build Coastguard Worker        result.update(convert_trie_to_flat_paths(data, name))
213*6777b538SAndroid Build Coastguard Worker      else:
214*6777b538SAndroid Build Coastguard Worker        result[name] = data
215*6777b538SAndroid Build Coastguard Worker    return result
216*6777b538SAndroid Build Coastguard Worker
217*6777b538SAndroid Build Coastguard Worker  results = {
218*6777b538SAndroid Build Coastguard Worker    'passes': {},
219*6777b538SAndroid Build Coastguard Worker    'unexpected_passes': {},
220*6777b538SAndroid Build Coastguard Worker    'failures': {},
221*6777b538SAndroid Build Coastguard Worker    'unexpected_failures': {},
222*6777b538SAndroid Build Coastguard Worker    'flakes': {},
223*6777b538SAndroid Build Coastguard Worker    'unexpected_flakes': {},
224*6777b538SAndroid Build Coastguard Worker  }
225*6777b538SAndroid Build Coastguard Worker
226*6777b538SAndroid Build Coastguard Worker  # TODO(dpranke): crbug.com/357866 - we should simplify the handling of
227*6777b538SAndroid Build Coastguard Worker  # both the return code and parsing the actual results, below.
228*6777b538SAndroid Build Coastguard Worker
229*6777b538SAndroid Build Coastguard Worker  passing_statuses = ('PASS', 'SLOW', 'NEEDSREBASELINE')
230*6777b538SAndroid Build Coastguard Worker
231*6777b538SAndroid Build Coastguard Worker  for test, result in convert_trie_to_flat_paths(
232*6777b538SAndroid Build Coastguard Worker      json_results['tests']).items():
233*6777b538SAndroid Build Coastguard Worker    key = 'unexpected_' if result.get('is_unexpected') else ''
234*6777b538SAndroid Build Coastguard Worker    data = result['actual']
235*6777b538SAndroid Build Coastguard Worker    actual_results = data.split()
236*6777b538SAndroid Build Coastguard Worker    last_result = actual_results[-1]
237*6777b538SAndroid Build Coastguard Worker    expected_results = result['expected'].split()
238*6777b538SAndroid Build Coastguard Worker
239*6777b538SAndroid Build Coastguard Worker    if (len(actual_results) > 1 and
240*6777b538SAndroid Build Coastguard Worker        (last_result in expected_results or last_result in passing_statuses)):
241*6777b538SAndroid Build Coastguard Worker      key += 'flakes'
242*6777b538SAndroid Build Coastguard Worker    elif last_result in passing_statuses:
243*6777b538SAndroid Build Coastguard Worker      key += 'passes'
244*6777b538SAndroid Build Coastguard Worker      # TODO(dpranke): crbug.com/357867 ...  Why are we assigning result
245*6777b538SAndroid Build Coastguard Worker      # instead of actual_result here. Do we even need these things to be
246*6777b538SAndroid Build Coastguard Worker      # hashes, or just lists?
247*6777b538SAndroid Build Coastguard Worker      data = result
248*6777b538SAndroid Build Coastguard Worker    else:
249*6777b538SAndroid Build Coastguard Worker      key += 'failures'
250*6777b538SAndroid Build Coastguard Worker    results[key][test] = data
251*6777b538SAndroid Build Coastguard Worker
252*6777b538SAndroid Build Coastguard Worker  return results
253*6777b538SAndroid Build Coastguard Worker
254*6777b538SAndroid Build Coastguard Worker
255*6777b538SAndroid Build Coastguard Workerdef write_interrupted_test_results_to(filepath, test_start_time):
256*6777b538SAndroid Build Coastguard Worker  """Writes a test results JSON file* to filepath.
257*6777b538SAndroid Build Coastguard Worker
258*6777b538SAndroid Build Coastguard Worker  This JSON file is formatted to explain that something went wrong.
259*6777b538SAndroid Build Coastguard Worker
260*6777b538SAndroid Build Coastguard Worker  *src/docs/testing/json_test_results_format.md
261*6777b538SAndroid Build Coastguard Worker
262*6777b538SAndroid Build Coastguard Worker  Args:
263*6777b538SAndroid Build Coastguard Worker    filepath: A path to a file to write the output to.
264*6777b538SAndroid Build Coastguard Worker    test_start_time: The start time of the test run expressed as a
265*6777b538SAndroid Build Coastguard Worker      floating-point offset in seconds from the UNIX epoch.
266*6777b538SAndroid Build Coastguard Worker  """
267*6777b538SAndroid Build Coastguard Worker  with open(filepath, 'w') as fh:
268*6777b538SAndroid Build Coastguard Worker    output = {
269*6777b538SAndroid Build Coastguard Worker        'interrupted': True,
270*6777b538SAndroid Build Coastguard Worker        'num_failures_by_type': {},
271*6777b538SAndroid Build Coastguard Worker        'seconds_since_epoch': test_start_time,
272*6777b538SAndroid Build Coastguard Worker        'tests': {},
273*6777b538SAndroid Build Coastguard Worker        'version': 3,
274*6777b538SAndroid Build Coastguard Worker    }
275*6777b538SAndroid Build Coastguard Worker    json.dump(output, fh)
276*6777b538SAndroid Build Coastguard Worker
277*6777b538SAndroid Build Coastguard Worker
278*6777b538SAndroid Build Coastguard Workerdef get_gtest_summary_passes(output):
279*6777b538SAndroid Build Coastguard Worker  """Returns a mapping of test to boolean indicating if the test passed.
280*6777b538SAndroid Build Coastguard Worker
281*6777b538SAndroid Build Coastguard Worker  Only partially parses the format. This code is based on code in tools/build,
282*6777b538SAndroid Build Coastguard Worker  specifically
283*6777b538SAndroid Build Coastguard Worker  https://chromium.googlesource.com/chromium/tools/build/+/17fef98756c5f250b20bf716829a0004857235ff/scripts/slave/recipe_modules/test_utils/util.py#189
284*6777b538SAndroid Build Coastguard Worker  """
285*6777b538SAndroid Build Coastguard Worker  if not output:
286*6777b538SAndroid Build Coastguard Worker    return {}
287*6777b538SAndroid Build Coastguard Worker
288*6777b538SAndroid Build Coastguard Worker  mapping = {}
289*6777b538SAndroid Build Coastguard Worker
290*6777b538SAndroid Build Coastguard Worker  for cur_iteration_data in output.get('per_iteration_data', []):
291*6777b538SAndroid Build Coastguard Worker    for test_fullname, results in cur_iteration_data.items():
292*6777b538SAndroid Build Coastguard Worker      # Results is a list with one entry per test try. Last one is the final
293*6777b538SAndroid Build Coastguard Worker      # result.
294*6777b538SAndroid Build Coastguard Worker      last_result = results[-1]
295*6777b538SAndroid Build Coastguard Worker
296*6777b538SAndroid Build Coastguard Worker      if last_result['status'] == 'SUCCESS':
297*6777b538SAndroid Build Coastguard Worker        mapping[test_fullname] = True
298*6777b538SAndroid Build Coastguard Worker      elif last_result['status'] != 'SKIPPED':
299*6777b538SAndroid Build Coastguard Worker        mapping[test_fullname] = False
300*6777b538SAndroid Build Coastguard Worker
301*6777b538SAndroid Build Coastguard Worker  return mapping
302*6777b538SAndroid Build Coastguard Worker
303*6777b538SAndroid Build Coastguard Worker
304*6777b538SAndroid Build Coastguard Workerdef extract_filter_list(filter_list):
305*6777b538SAndroid Build Coastguard Worker  """Helper for isolated script test wrappers. Parses the
306*6777b538SAndroid Build Coastguard Worker  --isolated-script-test-filter command line argument. Currently, double-colon
307*6777b538SAndroid Build Coastguard Worker  ('::') is used as the separator between test names, because a single colon may
308*6777b538SAndroid Build Coastguard Worker  be used in the names of perf benchmarks, which contain URLs.
309*6777b538SAndroid Build Coastguard Worker  """
310*6777b538SAndroid Build Coastguard Worker  return filter_list.split('::')
311*6777b538SAndroid Build Coastguard Worker
312*6777b538SAndroid Build Coastguard Worker
313*6777b538SAndroid Build Coastguard Workerdef add_emulator_args(parser):
314*6777b538SAndroid Build Coastguard Worker    parser.add_argument(
315*6777b538SAndroid Build Coastguard Worker        '--avd-config',
316*6777b538SAndroid Build Coastguard Worker        type=os.path.realpath,
317*6777b538SAndroid Build Coastguard Worker        help=('Path to the avd config. Required for Android products. '
318*6777b538SAndroid Build Coastguard Worker              '(See //tools/android/avd/proto for message definition '
319*6777b538SAndroid Build Coastguard Worker              'and existing *.textpb files.)'))
320*6777b538SAndroid Build Coastguard Worker    parser.add_argument(
321*6777b538SAndroid Build Coastguard Worker        '--emulator-window',
322*6777b538SAndroid Build Coastguard Worker        action='store_true',
323*6777b538SAndroid Build Coastguard Worker        default=False,
324*6777b538SAndroid Build Coastguard Worker        help='Enable graphical window display on the emulator.')
325*6777b538SAndroid Build Coastguard Worker
326*6777b538SAndroid Build Coastguard Worker
327*6777b538SAndroid Build Coastguard Workerclass BaseIsolatedScriptArgsAdapter:
328*6777b538SAndroid Build Coastguard Worker  """The base class for all script adapters that need to translate flags
329*6777b538SAndroid Build Coastguard Worker  set by isolated script test contract into the specific test script's flags.
330*6777b538SAndroid Build Coastguard Worker  """
331*6777b538SAndroid Build Coastguard Worker
332*6777b538SAndroid Build Coastguard Worker  def __init__(self):
333*6777b538SAndroid Build Coastguard Worker    self._parser = argparse.ArgumentParser()
334*6777b538SAndroid Build Coastguard Worker    self._options = None
335*6777b538SAndroid Build Coastguard Worker    self._rest_args = None
336*6777b538SAndroid Build Coastguard Worker    self._script_writes_output_json = None
337*6777b538SAndroid Build Coastguard Worker    self._parser.add_argument(
338*6777b538SAndroid Build Coastguard Worker        '--isolated-outdir', type=str,
339*6777b538SAndroid Build Coastguard Worker        required=False,
340*6777b538SAndroid Build Coastguard Worker        help='value of $ISOLATED_OUTDIR from swarming task')
341*6777b538SAndroid Build Coastguard Worker    self._parser.add_argument(
342*6777b538SAndroid Build Coastguard Worker        '--isolated-script-test-output', type=os.path.abspath,
343*6777b538SAndroid Build Coastguard Worker        required=False,
344*6777b538SAndroid Build Coastguard Worker        help='path to write test results JSON object to')
345*6777b538SAndroid Build Coastguard Worker    self._parser.add_argument(
346*6777b538SAndroid Build Coastguard Worker        '--isolated-script-test-filter', type=str,
347*6777b538SAndroid Build Coastguard Worker        required=False)
348*6777b538SAndroid Build Coastguard Worker    self._parser.add_argument(
349*6777b538SAndroid Build Coastguard Worker        '--isolated-script-test-repeat', type=int,
350*6777b538SAndroid Build Coastguard Worker        required=False)
351*6777b538SAndroid Build Coastguard Worker    self._parser.add_argument(
352*6777b538SAndroid Build Coastguard Worker        '--isolated-script-test-launcher-retry-limit', type=int,
353*6777b538SAndroid Build Coastguard Worker        required=False)
354*6777b538SAndroid Build Coastguard Worker    self._parser.add_argument(
355*6777b538SAndroid Build Coastguard Worker        '--isolated-script-test-also-run-disabled-tests',
356*6777b538SAndroid Build Coastguard Worker        default=False, action='store_true', required=False)
357*6777b538SAndroid Build Coastguard Worker
358*6777b538SAndroid Build Coastguard Worker    self._parser.add_argument(
359*6777b538SAndroid Build Coastguard Worker        '--xvfb',
360*6777b538SAndroid Build Coastguard Worker        help='start xvfb. Ignored on unsupported platforms',
361*6777b538SAndroid Build Coastguard Worker        action='store_true')
362*6777b538SAndroid Build Coastguard Worker    # Used to create the correct subclass.
363*6777b538SAndroid Build Coastguard Worker    self._parser.add_argument(
364*6777b538SAndroid Build Coastguard Worker        '--script-type', choices=['isolated', 'typ', 'bare'],
365*6777b538SAndroid Build Coastguard Worker        help='Which script adapter to use')
366*6777b538SAndroid Build Coastguard Worker
367*6777b538SAndroid Build Coastguard Worker    # Arguments that are ignored, but added here because it's easier to ignore
368*6777b538SAndroid Build Coastguard Worker    # them to to update bot configs to not pass them.
369*6777b538SAndroid Build Coastguard Worker    self._parser.add_argument('--isolated-script-test-chartjson-output')
370*6777b538SAndroid Build Coastguard Worker    self._parser.add_argument('--isolated-script-test-perf-output')
371*6777b538SAndroid Build Coastguard Worker
372*6777b538SAndroid Build Coastguard Worker  def parse_args(self, args=None):
373*6777b538SAndroid Build Coastguard Worker    self._options, self._rest_args = self._parser.parse_known_args(args)
374*6777b538SAndroid Build Coastguard Worker
375*6777b538SAndroid Build Coastguard Worker  @property
376*6777b538SAndroid Build Coastguard Worker  def parser(self):
377*6777b538SAndroid Build Coastguard Worker    return self._parser
378*6777b538SAndroid Build Coastguard Worker
379*6777b538SAndroid Build Coastguard Worker  @property
380*6777b538SAndroid Build Coastguard Worker  def options(self):
381*6777b538SAndroid Build Coastguard Worker    return self._options
382*6777b538SAndroid Build Coastguard Worker
383*6777b538SAndroid Build Coastguard Worker  @property
384*6777b538SAndroid Build Coastguard Worker  def rest_args(self):
385*6777b538SAndroid Build Coastguard Worker    return self._rest_args
386*6777b538SAndroid Build Coastguard Worker
387*6777b538SAndroid Build Coastguard Worker  def generate_test_output_args(self, output):
388*6777b538SAndroid Build Coastguard Worker    del output  # unused
389*6777b538SAndroid Build Coastguard Worker    return []
390*6777b538SAndroid Build Coastguard Worker
391*6777b538SAndroid Build Coastguard Worker  def generate_test_filter_args(self, test_filter_str):
392*6777b538SAndroid Build Coastguard Worker    del test_filter_str  # unused
393*6777b538SAndroid Build Coastguard Worker    raise RuntimeError('Flag not supported.')
394*6777b538SAndroid Build Coastguard Worker
395*6777b538SAndroid Build Coastguard Worker  def generate_test_repeat_args(self, repeat_count):
396*6777b538SAndroid Build Coastguard Worker    del repeat_count  # unused
397*6777b538SAndroid Build Coastguard Worker    raise RuntimeError('Flag not supported.')
398*6777b538SAndroid Build Coastguard Worker
399*6777b538SAndroid Build Coastguard Worker  def generate_test_launcher_retry_limit_args(self, retry_limit):
400*6777b538SAndroid Build Coastguard Worker    del retry_limit  # unused
401*6777b538SAndroid Build Coastguard Worker    raise RuntimeError('Flag not supported.')
402*6777b538SAndroid Build Coastguard Worker
403*6777b538SAndroid Build Coastguard Worker  def generate_sharding_args(self, total_shards, shard_index):
404*6777b538SAndroid Build Coastguard Worker    del total_shards, shard_index  # unused
405*6777b538SAndroid Build Coastguard Worker    raise RuntimeError('Flag not supported.')
406*6777b538SAndroid Build Coastguard Worker
407*6777b538SAndroid Build Coastguard Worker  def generate_test_also_run_disabled_tests_args(self):
408*6777b538SAndroid Build Coastguard Worker    raise RuntimeError('Flag not supported.')
409*6777b538SAndroid Build Coastguard Worker
410*6777b538SAndroid Build Coastguard Worker  def select_python_executable(self):
411*6777b538SAndroid Build Coastguard Worker    return sys.executable
412*6777b538SAndroid Build Coastguard Worker
413*6777b538SAndroid Build Coastguard Worker  def generate_isolated_script_cmd(self):
414*6777b538SAndroid Build Coastguard Worker    isolated_script_cmd = [ self.select_python_executable() ] + self.rest_args
415*6777b538SAndroid Build Coastguard Worker
416*6777b538SAndroid Build Coastguard Worker    if self.options.isolated_script_test_output:
417*6777b538SAndroid Build Coastguard Worker      output_args = self.generate_test_output_args(
418*6777b538SAndroid Build Coastguard Worker          self.options.isolated_script_test_output)
419*6777b538SAndroid Build Coastguard Worker      self._script_writes_output_json = bool(output_args)
420*6777b538SAndroid Build Coastguard Worker      isolated_script_cmd += output_args
421*6777b538SAndroid Build Coastguard Worker
422*6777b538SAndroid Build Coastguard Worker    # Augment test filter args if needed
423*6777b538SAndroid Build Coastguard Worker    if self.options.isolated_script_test_filter:
424*6777b538SAndroid Build Coastguard Worker      isolated_script_cmd += self.generate_test_filter_args(
425*6777b538SAndroid Build Coastguard Worker          self.options.isolated_script_test_filter)
426*6777b538SAndroid Build Coastguard Worker
427*6777b538SAndroid Build Coastguard Worker    # Augment test repeat if needed
428*6777b538SAndroid Build Coastguard Worker    if self.options.isolated_script_test_repeat is not None:
429*6777b538SAndroid Build Coastguard Worker      isolated_script_cmd += self.generate_test_repeat_args(
430*6777b538SAndroid Build Coastguard Worker          self.options.isolated_script_test_repeat)
431*6777b538SAndroid Build Coastguard Worker
432*6777b538SAndroid Build Coastguard Worker    # Augment test launcher retry limit args if needed
433*6777b538SAndroid Build Coastguard Worker    if self.options.isolated_script_test_launcher_retry_limit is not None:
434*6777b538SAndroid Build Coastguard Worker      isolated_script_cmd += self.generate_test_launcher_retry_limit_args(
435*6777b538SAndroid Build Coastguard Worker          self.options.isolated_script_test_launcher_retry_limit)
436*6777b538SAndroid Build Coastguard Worker
437*6777b538SAndroid Build Coastguard Worker    # Augment test also run disable tests args if needed
438*6777b538SAndroid Build Coastguard Worker    if self.options.isolated_script_test_also_run_disabled_tests:
439*6777b538SAndroid Build Coastguard Worker      isolated_script_cmd += self.generate_test_also_run_disabled_tests_args()
440*6777b538SAndroid Build Coastguard Worker
441*6777b538SAndroid Build Coastguard Worker    # Augment shard args if needed
442*6777b538SAndroid Build Coastguard Worker    env = os.environ.copy()
443*6777b538SAndroid Build Coastguard Worker
444*6777b538SAndroid Build Coastguard Worker    total_shards = None
445*6777b538SAndroid Build Coastguard Worker    shard_index = None
446*6777b538SAndroid Build Coastguard Worker
447*6777b538SAndroid Build Coastguard Worker    if 'GTEST_TOTAL_SHARDS' in env:
448*6777b538SAndroid Build Coastguard Worker      total_shards = int(env['GTEST_TOTAL_SHARDS'])
449*6777b538SAndroid Build Coastguard Worker    if 'GTEST_SHARD_INDEX' in env:
450*6777b538SAndroid Build Coastguard Worker      shard_index = int(env['GTEST_SHARD_INDEX'])
451*6777b538SAndroid Build Coastguard Worker    if total_shards is not None and shard_index is not None:
452*6777b538SAndroid Build Coastguard Worker      isolated_script_cmd += self.generate_sharding_args(
453*6777b538SAndroid Build Coastguard Worker          total_shards, shard_index)
454*6777b538SAndroid Build Coastguard Worker
455*6777b538SAndroid Build Coastguard Worker    return isolated_script_cmd
456*6777b538SAndroid Build Coastguard Worker
457*6777b538SAndroid Build Coastguard Worker  def clean_up_after_test_run(self):
458*6777b538SAndroid Build Coastguard Worker    pass
459*6777b538SAndroid Build Coastguard Worker
460*6777b538SAndroid Build Coastguard Worker  def do_pre_test_run_tasks(self):
461*6777b538SAndroid Build Coastguard Worker    pass
462*6777b538SAndroid Build Coastguard Worker
463*6777b538SAndroid Build Coastguard Worker  def do_post_test_run_tasks(self):
464*6777b538SAndroid Build Coastguard Worker    pass
465*6777b538SAndroid Build Coastguard Worker
466*6777b538SAndroid Build Coastguard Worker  def _write_simple_test_results(self, start_time, exit_code):
467*6777b538SAndroid Build Coastguard Worker    if exit_code is None:
468*6777b538SAndroid Build Coastguard Worker      failure_type = 'CRASH'
469*6777b538SAndroid Build Coastguard Worker    elif exit_code == 0:
470*6777b538SAndroid Build Coastguard Worker      failure_type = 'PASS'
471*6777b538SAndroid Build Coastguard Worker    else:
472*6777b538SAndroid Build Coastguard Worker      failure_type = 'FAIL'
473*6777b538SAndroid Build Coastguard Worker
474*6777b538SAndroid Build Coastguard Worker    test_name = os.path.basename(self._rest_args[0])
475*6777b538SAndroid Build Coastguard Worker    # See //docs/testing/json_test_results_format.md
476*6777b538SAndroid Build Coastguard Worker    results_json = {
477*6777b538SAndroid Build Coastguard Worker        'version': 3,
478*6777b538SAndroid Build Coastguard Worker        'interrupted': False,
479*6777b538SAndroid Build Coastguard Worker        'num_failures_by_type': { failure_type: 1 },
480*6777b538SAndroid Build Coastguard Worker        'path_delimiter': '/',
481*6777b538SAndroid Build Coastguard Worker        'seconds_since_epoch': start_time,
482*6777b538SAndroid Build Coastguard Worker        'tests': {
483*6777b538SAndroid Build Coastguard Worker            test_name: {
484*6777b538SAndroid Build Coastguard Worker              'expected': 'PASS',
485*6777b538SAndroid Build Coastguard Worker              'actual': failure_type,
486*6777b538SAndroid Build Coastguard Worker              'time': time.time() - start_time,
487*6777b538SAndroid Build Coastguard Worker            },
488*6777b538SAndroid Build Coastguard Worker        },
489*6777b538SAndroid Build Coastguard Worker    }
490*6777b538SAndroid Build Coastguard Worker    with open(self.options.isolated_script_test_output, 'w') as fp:
491*6777b538SAndroid Build Coastguard Worker      json.dump(results_json, fp)
492*6777b538SAndroid Build Coastguard Worker
493*6777b538SAndroid Build Coastguard Worker
494*6777b538SAndroid Build Coastguard Worker  def run_test(self, cwd=None):
495*6777b538SAndroid Build Coastguard Worker    self.parse_args()
496*6777b538SAndroid Build Coastguard Worker    cmd = self.generate_isolated_script_cmd()
497*6777b538SAndroid Build Coastguard Worker
498*6777b538SAndroid Build Coastguard Worker    self.do_pre_test_run_tasks()
499*6777b538SAndroid Build Coastguard Worker
500*6777b538SAndroid Build Coastguard Worker    env = os.environ.copy()
501*6777b538SAndroid Build Coastguard Worker
502*6777b538SAndroid Build Coastguard Worker    env['CHROME_HEADLESS'] = '1'
503*6777b538SAndroid Build Coastguard Worker    print('Running command: %s\nwith env: %r' % (
504*6777b538SAndroid Build Coastguard Worker        ' '.join(cmd), env))
505*6777b538SAndroid Build Coastguard Worker    sys.stdout.flush()
506*6777b538SAndroid Build Coastguard Worker    start_time = time.time()
507*6777b538SAndroid Build Coastguard Worker    try:
508*6777b538SAndroid Build Coastguard Worker      if self.options.xvfb and sys.platform.startswith('linux'):
509*6777b538SAndroid Build Coastguard Worker        exit_code = xvfb.run_executable(cmd, env, cwd=cwd)
510*6777b538SAndroid Build Coastguard Worker      else:
511*6777b538SAndroid Build Coastguard Worker        exit_code = test_env.run_command(cmd, env=env, cwd=cwd, log=False)
512*6777b538SAndroid Build Coastguard Worker      print('Command returned exit code %d' % exit_code)
513*6777b538SAndroid Build Coastguard Worker      sys.stdout.flush()
514*6777b538SAndroid Build Coastguard Worker      self.do_post_test_run_tasks()
515*6777b538SAndroid Build Coastguard Worker    except Exception:
516*6777b538SAndroid Build Coastguard Worker      traceback.print_exc()
517*6777b538SAndroid Build Coastguard Worker      exit_code = None
518*6777b538SAndroid Build Coastguard Worker    finally:
519*6777b538SAndroid Build Coastguard Worker      self.clean_up_after_test_run()
520*6777b538SAndroid Build Coastguard Worker
521*6777b538SAndroid Build Coastguard Worker    if (self.options.isolated_script_test_output
522*6777b538SAndroid Build Coastguard Worker        and not self._script_writes_output_json):
523*6777b538SAndroid Build Coastguard Worker      self._write_simple_test_results(start_time, exit_code)
524*6777b538SAndroid Build Coastguard Worker
525*6777b538SAndroid Build Coastguard Worker    return exit_code if exit_code is not None else 2
526