1#!/bin/bash 2 3# This is a very simple script that runs all three benchmark suites 4# and then runs a log parsing script to get the logs into a csv form 5# and uploads the result to GitHub Gist for easy copy paste into Google 6# Spreadsheet. 7# 8# Useful flag sets: 9# 10# Run models that are skipped in dynamic CI only, to update skips 11# ./run_all.sh --training --backend aot_eager --dynamic-ci-skips-only 12# 13# Run CI models with dynamic shapes, training, aot_eager 14# ./run_all.sh --training --backend aot_eager --dynamic-shapes --ci 15# 16# Run CI models with dynamic shapes, inference, inductor 17# ./run_all.sh --backend inductor --dynamic-shapes --ci 18# 19# WARNING: this will silently clobber .csv and .log files in your CWD! 20 21set -x 22 23# Some QoL for people running this script on Meta servers 24if getent hosts fwdproxy; then 25 export https_proxy=http://fwdproxy:8080 http_proxy=http://fwdproxy:8080 no_proxy=.fbcdn.net,.facebook.com,.thefacebook.com,.tfbnw.net,.fb.com,.fburl.com,.facebook.net,.sb.fbsbx.com,localhost 26fi 27 28# Feel free to edit these, but we expect most users not to need to modify this 29BASE_FLAGS=( --accuracy --explain --timing --print-graph-breaks ) 30DATE="$(date)" 31WORK="$PWD" 32 33cd "$(dirname "$BASH_SOURCE")"/../.. 34 35python benchmarks/dynamo/benchmarks.py --output "$WORK"/benchmarks.csv "${BASE_FLAGS[@]}" "$@" 2>&1 | tee "$WORK"/sweep.log 36gh gist create -d "Sweep logs for $(git rev-parse --abbrev-ref HEAD) $* - $(git rev-parse HEAD) $DATE" "$WORK"/sweep.log | tee -a "$WORK"/sweep.log 37python benchmarks/dynamo/parse_logs.py "$WORK"/sweep.log > "$WORK"/final.csv 38gh gist create "$WORK"/final.csv 39