1#!/usr/bin/env bash 2 3set -euo pipefail 4 5PACKAGE_TYPE=${PACKAGE_TYPE:-conda} 6 7PKG_DIR=${PKG_DIR:-/tmp/workspace/final_pkgs} 8 9# Designates whether to submit as a release candidate or a nightly build 10# Value should be `test` when uploading release candidates 11# currently set within `designate_upload_channel` 12UPLOAD_CHANNEL=${UPLOAD_CHANNEL:-nightly} 13# Designates what subfolder to put packages into 14UPLOAD_SUBFOLDER=${UPLOAD_SUBFOLDER:-} 15UPLOAD_BUCKET="s3://pytorch" 16BACKUP_BUCKET="s3://pytorch-backup" 17BUILD_NAME=${BUILD_NAME:-} 18 19DRY_RUN=${DRY_RUN:-enabled} 20# Don't actually do work unless explicit 21ANACONDA="true anaconda" 22AWS_S3_CP="aws s3 cp --dryrun" 23if [[ "${DRY_RUN}" = "disabled" ]]; then 24 ANACONDA="anaconda" 25 AWS_S3_CP="aws s3 cp" 26fi 27 28if [[ "${USE_SPLIT_BUILD:-false}" == "true" ]]; then 29 UPLOAD_SUBFOLDER="${UPLOAD_SUBFOLDER}_pypi_pkg" 30fi 31 32# this is special build with all dependencies packaged 33if [[ ${BUILD_NAME} == *-full* ]]; then 34 UPLOAD_SUBFOLDER="${UPLOAD_SUBFOLDER}_full" 35fi 36 37# Sleep 2 minutes between retries for conda upload 38retry () { 39 "$@" || (sleep 5m && "$@") || (sleep 5m && "$@") || (sleep 5m && "$@") || (sleep 5m && "$@") 40} 41 42do_backup() { 43 local backup_dir 44 backup_dir=$1 45 ( 46 pushd /tmp/workspace 47 set -x 48 ${AWS_S3_CP} --recursive . "${BACKUP_BUCKET}/${CIRCLE_TAG}/${backup_dir}/" 49 ) 50} 51 52conda_upload() { 53 ( 54 set -x 55 retry \ 56 ${ANACONDA} \ 57 upload \ 58 ${PKG_DIR}/*.tar.bz2 \ 59 -u "pytorch-${UPLOAD_CHANNEL}" \ 60 --label main \ 61 --no-progress \ 62 --force 63 ) 64} 65 66s3_upload() { 67 local extension 68 local pkg_type 69 extension="$1" 70 pkg_type="$2" 71 s3_root_dir="${UPLOAD_BUCKET}/${pkg_type}/${UPLOAD_CHANNEL}" 72 if [[ -z ${UPLOAD_SUBFOLDER:-} ]]; then 73 s3_upload_dir="${s3_root_dir}/" 74 else 75 s3_upload_dir="${s3_root_dir}/${UPLOAD_SUBFOLDER}/" 76 fi 77 ( 78 for pkg in ${PKG_DIR}/*.${extension}; do 79 ( 80 set -x 81 ${AWS_S3_CP} --no-progress --acl public-read "${pkg}" "${s3_upload_dir}" 82 ) 83 done 84 ) 85} 86 87# Install dependencies (should be a no-op if previously installed) 88conda install -yq anaconda-client 89pip install -q awscli 90 91case "${PACKAGE_TYPE}" in 92 conda) 93 conda_upload 94 for conda_archive in ${PKG_DIR}/*.tar.bz2; do 95 # Fetch platform (eg. win-64, linux-64, etc.) from index file because 96 # there's no actual conda command to read this 97 subdir=$(\ 98 tar -xOf "${conda_archive}" info/index.json \ 99 | grep subdir \ 100 | cut -d ':' -f2 \ 101 | sed -e 's/[[:space:]]//' -e 's/"//g' -e 's/,//' \ 102 ) 103 BACKUP_DIR="conda/${subdir}" 104 done 105 ;; 106 libtorch) 107 s3_upload "zip" "libtorch" 108 BACKUP_DIR="libtorch/${UPLOAD_CHANNEL}/${UPLOAD_SUBFOLDER}" 109 ;; 110 # wheel can either refer to wheel/manywheel 111 *wheel) 112 s3_upload "whl" "whl" 113 BACKUP_DIR="whl/${UPLOAD_CHANNEL}/${UPLOAD_SUBFOLDER}" 114 ;; 115 *) 116 echo "ERROR: unknown package type: ${PACKAGE_TYPE}" 117 exit 1 118 ;; 119esac 120 121# CIRCLE_TAG is defined by upstream circleci, 122# this can be changed to recognize tagged versions 123if [[ -n "${CIRCLE_TAG:-}" ]]; then 124 do_backup "${BACKUP_DIR}" 125fi 126