1#!/bin/bash 2 3SCRIPT_PARENT_DIR=$(dirname "${BASH_SOURCE[0]}") 4 5# shellcheck source=.ci/pytorch/common.sh 6source "$SCRIPT_PARENT_DIR/common.sh" 7 8cd .ci/pytorch/perf_test 9 10echo "Running CPU perf test for PyTorch..." 11 12pip install -q awscli 13 14# Set multipart_threshold to be sufficiently high, so that `aws s3 cp` is not a multipart read 15# More info at https://github.com/aws/aws-cli/issues/2321 16aws configure set default.s3.multipart_threshold 5GB 17UPSTREAM_DEFAULT_BRANCH="$(git remote show https://github.com/pytorch/pytorch.git | awk '/HEAD branch/ {print $NF}')" 18 19if [[ "$COMMIT_SOURCE" == "$UPSTREAM_DEFAULT_BRANCH" ]]; then 20 # Get current default branch commit hash 21 DEFAULT_BRANCH_COMMIT_ID=$(git log --format="%H" -n 1) 22 export DEFAULT_BRANCH_COMMIT_ID 23fi 24 25# Find the default branch commit to test against 26git remote add upstream https://github.com/pytorch/pytorch.git 27git fetch upstream 28IFS=$'\n' 29while IFS='' read -r commit_id; do 30 if aws s3 ls s3://ossci-perf-test/pytorch/cpu_runtime/"${commit_id}".json; then 31 LATEST_TESTED_COMMIT=${commit_id} 32 break 33 fi 34done < <(git rev-list upstream/"$UPSTREAM_DEFAULT_BRANCH") 35aws s3 cp s3://ossci-perf-test/pytorch/cpu_runtime/"${LATEST_TESTED_COMMIT}".json cpu_runtime.json 36 37if [[ "$COMMIT_SOURCE" == "$UPSTREAM_DEFAULT_BRANCH" ]]; then 38 # Prepare new baseline file 39 cp cpu_runtime.json new_cpu_runtime.json 40 python update_commit_hash.py new_cpu_runtime.json "${DEFAULT_BRANCH_COMMIT_ID}" 41fi 42 43# Include tests 44# shellcheck source=./perf_test/test_cpu_speed_mini_sequence_labeler.sh 45. ./test_cpu_speed_mini_sequence_labeler.sh 46# shellcheck source=./perf_test/test_cpu_speed_mnist.sh 47. ./test_cpu_speed_mnist.sh 48# shellcheck source=./perf_test/test_cpu_speed_torch.sh 49. ./test_cpu_speed_torch.sh 50# shellcheck source=./perf_test/test_cpu_speed_torch_tensor.sh 51. ./test_cpu_speed_torch_tensor.sh 52 53# Run tests 54export TEST_MODE="compare_with_baseline" 55if [[ "$COMMIT_SOURCE" == "$UPSTREAM_DEFAULT_BRANCH" ]]; then 56 export TEST_MODE="compare_and_update" 57fi 58 59# Operator tests 60run_test test_cpu_speed_torch ${TEST_MODE} 61run_test test_cpu_speed_torch_tensor ${TEST_MODE} 62 63# Sample model tests 64run_test test_cpu_speed_mini_sequence_labeler 20 ${TEST_MODE} 65run_test test_cpu_speed_mnist 20 ${TEST_MODE} 66 67if [[ "$COMMIT_SOURCE" == "$UPSTREAM_DEFAULT_BRANCH" ]]; then 68 # This could cause race condition if we are testing the same default branch commit twice, 69 # but the chance of them executing this line at the same time is low. 70 aws s3 cp new_cpu_runtime.json s3://ossci-perf-test/pytorch/cpu_runtime/"${DEFAULT_BRANCH_COMMIT_ID}".json --acl public-read 71fi 72