1 #!/usr/bin/python3 -u
48         # client side test control, as saved in old Autotest paths.
50         # server side test control, as saved in old Autotest paths.
64     parser.add_option("-m", help="Send mail for FAILED tests",
66     parser.add_option("-r", help="Reparse the results of a job",
68     parser.add_option("-o", help="Parse a single results directory",
70     parser.add_option("-l", help=("Levels of subdirectories to include "
73     parser.add_option("-n", help="No blocking on an existing parse",
75     parser.add_option("-s", help="Database server hostname",
77     parser.add_option("-u", help="Database username", dest="db_user",
79     parser.add_option("-p", help="Database password", dest="db_pass",
81     parser.add_option("-d", help="Database name", dest="db_name",
83     parser.add_option("--dry-run", help="Do not actually commit any results.",
86             "--detach", action="store_true",
90     parser.add_option("--write-pidfile",
94     parser.add_option("--record-duration",
98     parser.add_option("--suite-report",
104     parser.add_option("--datastore-creds",
106                             "The path to gcloud datastore credentials file, "
113             "--export-to-gcloud-path",
115                   "The path to export_to_gcloud script. Please find "
116                   "chromite path on your server. The script is under "
121     parser.add_option("--disable-perf-upload",
143     @param testname: String representing the test name.
144     @param status: String representing the test status.
149     format_string = "%-12s %-20s %-12s %-10s %s"
166                                                 "Test name", "FAIL/WARN",
180     job's test results, such that the consumers of the tko database
198     test_idx| job_idx | test              | ... | invalid | invalidates_test_idx
224     orig_tests = tko_models.Test.objects.filter(job__job_idx=orig_job_idx)
225     retry_tests = tko_models.Test.objects.filter(job__job_idx=retry_job_idx)
230     # Maintain a dictionary that maps (test, subdir) to original tests.
231     # Note that within the scope of a job, (test, subdir) uniquelly
232     # identifies a test run, but 'test' does not.
233     # In a control file, one could run the same test with different
238     #    (test='example_Fail.Error', subdir='example_Fail.Error.subdir_1')
239     #    (test='example_Fail.Error', subdir='example_Fail.Error.subdir_2')
240     invalidated_tests = {(orig_test.test, orig_test.subdir): orig_test
243         # It is possible that (retry.test, retry.subdir) doesn't exist
248         orig_test = invalidated_tests.get((retry.test, retry.subdir), None)
255 def _throttle_result_size(path):  argument
256     """Limit the total size of test results for the given path.
258     @param path: Path of the result directory.
263                 path)
266     max_result_size_KB = _max_result_size_from_control(path)
271         result_utils.execute(path, max_result_size_KB)
275                 (path, traceback.format_exc()))
278 def _max_result_size_from_control(path):  argument
284         control = os.path.join(path, control_file)
285         if not os.path.exists(control):
309     @param filename: the serialized binary destination path.
317 def parse_one(db, pid_file_manager, jobname, path, parse_options):  argument
323                     e.g. '1234-chromeos-test/host1'
324     @param path: The path to the results to be parsed.
334     tko_utils.dprint("\nScanning %s (%s)" % (jobname, path))
341     job_keyval = models.job.read_keyval(path)
345     job = parser.make_job(path)
346     tko_utils.dprint("+ Parsing dir=%s, jobname=%s" % (path, jobname))
347     status_log_path = _find_status_log_path(path)
381     # _throttle_result_size(path)
383     # Record test result size to job_keyvals
386             path, log=tko_utils.dprint)
388                      (time.time()-start_time))
396     for test in job.tests:
397         if not test.subdir:
400                          % (test.testname, test.subdir, test.status,
401                             test.reason))
402         if test.status not in ('GOOD', 'WARN'):
406                 jobname, test.kernel.base, test.subdir,
407                 test.status, test.reason))
423             for test in job.tests:
424                 perf_uploader.upload_test(job, test, jobname)
460     binary_file_name = os.path.join(path, "job.serialize")
472         # Check if we should not offload this test's results.
475             gs_instructions_file = os.path.join(
476                     path, constants.GS_OFFLOADER_INSTRUCTIONS)
478             if os.path.exists(gs_instructions_file):
504     for test in job.tests:
505         db.insert_test(job, test)
508 def _find_status_log_path(path):  argument
509     if os.path.exists(os.path.join(path, "status.log")):
510         return os.path.join(path, "status.log")
511     if os.path.exists(os.path.join(path, "status")):
512         return os.path.join(path, "status")
524     for test in tests:
525         if test not in already_added:
526             already_added.add(test)
527             job.tests.append(test)
536     raw_old_tests = db.select("test_idx,subdir,test", "tko_tests",
539         old_tests = dict(((test, subdir), test_idx)
540                             for test_idx, subdir, test in raw_old_tests)
544     for test in job.tests:
545         test_idx = old_tests.pop((test.testname, test.subdir), None)
547             test.test_idx = test_idx
549             tko_utils.dprint("! Reparse returned new test "
551                                 (test.testname, test.subdir))
566 def _get_job_subdirs(path):  argument
568     Returns a list of job subdirectories at path. Returns None if the test
572     machine_list = os.path.join(path, ".machines")
573     if os.path.exists(machine_list):
577                                if os.path.exists(os.path.join(path, subdir)))
582     contents = set(os.listdir(path))
585                   os.path.isdir(os.path.join(path, sub)))
593 def parse_leaf_path(db, pid_file_manager, path, level, parse_options):  argument
594     """Parse a leaf path.
598     @param path: The path to the results to be parsed.
602     @returns: The job name of the parsed job, e.g. '123-chromeos-test/host1'
604     job_elements = path.split("/")[-level:]
606     db.run_with_retry(parse_one, db, pid_file_manager, jobname, path,
611 def parse_path(db, pid_file_manager, path, level, parse_options):  argument
612     """Parse a path
616     @param path: The path to the results to be parsed.
621               set(['123-chromeos-test/host1', '123-chromeos-test/host2'])
624     job_subdirs = _get_job_subdirs(path)
626         # parse status.log in current directory, if it exists. multi-machine
629         if os.path.exists(os.path.join(path, 'status.log')):
630             new_job = parse_leaf_path(db, pid_file_manager, path, level,
633         # multi-machine job
635             jobpath = os.path.join(path, subdir)
641         new_job = parse_leaf_path(db, pid_file_manager, path, level,
684     results_dir = os.path.abspath(args[0])
685     assert os.path.exists(results_dir)
705             jobs_list = [os.path.join(results_dir, subdir)
714         for path in jobs_list:
715             lockfile = open(os.path.join(path, ".parse.lock"), "w")
729                 new_jobs = parse_path(db, pid_file_manager, path, options.level,
748     @param test_results_dir: path to test results dir.
755     config_dir = os.path.join(test_results_dir, os.pardir)
774         tko_utils.dprint("No side_effects.Config found in %s - "