1""" 2This script takes a pytest CSV file produced by pytest --csv foo.csv 3and summarizes it into a more minimal CSV that is good for uploading 4to Google Sheets. We have been using this with dynamic shapes to 5understand how many tests fail when we turn on dynamic shapes. If 6you have a test suite with a lot of skips or xfails, if force the 7tests to run anyway, this can help you understand what the actual 8errors things are failing with are. 9 10The resulting csv is written to stdout. An easy way to get the csv 11onto your local file system is to send it to GitHub Gist: 12 13 $ python scripts/analysis/format_test_csv.py foo.csv | gh gist create - 14 15See also scripts/analysis/run_test_csv.sh 16""" 17 18import argparse 19import csv 20import subprocess 21import sys 22 23 24parser = argparse.ArgumentParser( 25 description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter 26) 27parser.add_argument("--log-url", type=str, default="", help="URL of raw logs") 28parser.add_argument("file", help="pytest CSV file to format") 29args = parser.parse_args() 30 31out = csv.writer(sys.stdout, dialect="excel") 32hash = subprocess.check_output( 33 "git rev-parse HEAD".split(" "), encoding="utf-8" 34).rstrip() 35 36out.writerow([hash, args.log_url, ""]) 37 38with open(args.file) as f: 39 reader = csv.DictReader(f) 40 for row in reader: 41 if row["status"] not in {"failed", "error"}: 42 continue 43 msg = row["message"].split("\n")[0] 44 msg.replace( 45 " - erroring out! It's likely that this is caused by data-dependent control flow or similar.", 46 "", 47 ) 48 msg.replace("\t", " ") 49 # Feel free to edit this; the idea is to remove prefixes that are 50 # just gooping up the resulting spreadsheet output 51 name = row["name"].replace("test_make_fx_symbolic_exhaustive_", "") 52 out.writerow([name, msg, ""]) 53