1#!/bin/bash 2# Copyright 2018 The gRPC Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16set -ex 17 18shopt -s nullglob 19 20cd "$(dirname "$0")/../../.." 21 22GRPC_VERSION=$(grep -e "^ *version: " build_handwritten.yaml | head -n 1 | sed 's/.*: //') 23 24INPUT_ARTIFACTS=$KOKORO_GFILE_DIR/github/grpc/artifacts 25INDEX_FILENAME=index.xml 26 27BUILD_ID=${KOKORO_BUILD_ID:-$(uuidgen)} 28BUILD_BRANCH_NAME=master 29BUILD_GIT_COMMIT=${KOKORO_GIT_COMMIT:-unknown} 30BUILD_TIMESTAMP=$(date -Iseconds) 31BUILD_RELPATH=$(date "+%Y/%m")/$BUILD_GIT_COMMIT-$BUILD_ID/ 32 33GCS_ROOT=gs://packages.grpc.io/ 34GCS_ARCHIVE_PREFIX=archive/ 35GCS_ARCHIVE_ROOT=$GCS_ROOT$GCS_ARCHIVE_PREFIX 36GCS_INDEX=$GCS_ROOT$INDEX_FILENAME 37 38LOCAL_STAGING_TEMPDIR=$(mktemp -d) 39LOCAL_BUILD_ROOT=$LOCAL_STAGING_TEMPDIR/$BUILD_RELPATH 40LOCAL_BUILD_INDEX=$LOCAL_BUILD_ROOT$INDEX_FILENAME 41 42mkdir -p "$LOCAL_BUILD_ROOT" 43 44find "$INPUT_ARTIFACTS" -type f 45 46# protoc Plugins 47PROTOC_PLUGINS_ZIPPED_PACKAGES=$(mktemp -d) 48for zip_dir in protoc_windows_{x86,x64} 49do 50 zip -jr "$PROTOC_PLUGINS_ZIPPED_PACKAGES/grpc-$zip_dir-$GRPC_VERSION.zip" "$INPUT_ARTIFACTS/$zip_dir/"* 51done 52for tar_dir in protoc_linux_{x86,x64} protoc_macos_x64 53do 54 chmod +x "$INPUT_ARTIFACTS/$tar_dir"/* 55 tar -cvzf "$PROTOC_PLUGINS_ZIPPED_PACKAGES/grpc-$tar_dir-$GRPC_VERSION.tar.gz" -C "$INPUT_ARTIFACTS/$tar_dir" . 56done 57 58PROTOC_PACKAGES=( 59 "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/grpc-protoc_windows_{x86,x64}-"$GRPC_VERSION.zip" 60 "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/grpc-protoc_linux_{x86,x64}-"$GRPC_VERSION.tar.gz" 61 "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/grpc-protoc_macos_x64-"$GRPC_VERSION.tar.gz" 62) 63 64# C# 65UNZIPPED_CSHARP_PACKAGES=$(mktemp -d) 66# the "_multiplatform" suffix is to fix https://github.com/grpc/grpc/issues/32179 67unzip "$INPUT_ARTIFACTS/csharp_nugets_windows_dotnetcli_multiplatform.zip" -d "$UNZIPPED_CSHARP_PACKAGES" 68CSHARP_PACKAGES=( 69 "$UNZIPPED_CSHARP_PACKAGES"/* 70 "$INPUT_ARTIFACTS"/grpc_unity_package.[0-9]*.zip 71) 72 73# Python 74PYTHON_GRPCIO_PACKAGES=( 75 "$INPUT_ARTIFACTS"/grpcio-[0-9]*.tar.gz 76 "$INPUT_ARTIFACTS"/grpcio-[0-9]*.whl 77 "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio-[0-9]*.whl 78) 79PYTHON_GRPCIO_TOOLS_PACKAGES=( 80 "$INPUT_ARTIFACTS"/grpcio-tools-[0-9]*.tar.gz 81 "$INPUT_ARTIFACTS"/grpcio_tools-[0-9]*.whl 82 "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio_tools-[0-9]*.whl 83) 84PYTHON_GRPCIO_HEALTH_CHECKING_PACKAGES=( 85 "$INPUT_ARTIFACTS"/grpcio-health-checking-[0-9]*.tar.gz 86) 87PYTHON_GRPCIO_REFLECTION_PACKAGES=( 88 "$INPUT_ARTIFACTS"/grpcio-reflection-[0-9]*.tar.gz 89) 90PYTHON_GRPCIO_TESTING_PACKAGES=( 91 "$INPUT_ARTIFACTS"/grpcio-testing-[0-9]*.tar.gz 92) 93 94# PHP 95PHP_PACKAGES=( 96 "$INPUT_ARTIFACTS"/grpc-[0-9]*.tgz 97) 98 99# Ruby 100RUBY_PACKAGES=( 101 "$INPUT_ARTIFACTS"/grpc-[0-9]*.gem 102 "$INPUT_ARTIFACTS"/grpc-tools-[0-9]*.gem 103) 104 105function add_to_manifest() { 106 local artifact_type=$1 107 local artifact_file=$2 108 local artifact_prefix=$3 109 local artifact_name 110 artifact_name=$(basename "$artifact_file") 111 local artifact_size 112 artifact_size=$(stat -c%s "$artifact_file") 113 local artifact_sha256 114 artifact_sha256=$(openssl sha256 -r "$artifact_file" | cut -d " " -f 1) 115 local artifact_target=$LOCAL_BUILD_ROOT/$artifact_type/$artifact_prefix 116 mkdir -p "$artifact_target" 117 cp "$artifact_file" "$artifact_target" 118 cat <<EOF 119 <artifact name='$artifact_name' 120 type='$artifact_type' 121 path='$artifact_type/$artifact_prefix$artifact_name' 122 size='$artifact_size' 123 sha256='$artifact_sha256' /> 124EOF 125} 126 127{ 128 cat <<EOF 129<?xml version="1.0"?> 130<?xml-stylesheet href="/web-assets/build-201807.xsl" type="text/xsl"?> 131<build id='$BUILD_ID' timestamp='$BUILD_TIMESTAMP' version="201807"> 132 <metadata> 133 <project>gRPC</project> 134 <repository>https://github.com/grpc/grpc</repository> 135 <branch>$BUILD_BRANCH_NAME</branch> 136 <commit>$BUILD_GIT_COMMIT</commit> 137 </metadata> 138 <artifacts> 139EOF 140 141 for pkg in "${PROTOC_PACKAGES[@]}"; do add_to_manifest protoc "$pkg"; done 142 for pkg in "${CSHARP_PACKAGES[@]}"; do add_to_manifest csharp "$pkg"; done 143 for pkg in "${PHP_PACKAGES[@]}"; do add_to_manifest php "$pkg"; done 144 for pkg in "${PYTHON_GRPCIO_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio/; done 145 for pkg in "${PYTHON_GRPCIO_TOOLS_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-tools/; done 146 for pkg in "${PYTHON_GRPCIO_HEALTH_CHECKING_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-health-checking/; done 147 for pkg in "${PYTHON_GRPCIO_REFLECTION_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-reflection/; done 148 for pkg in "${PYTHON_GRPCIO_TESTING_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-testing/; done 149 for pkg in "${RUBY_PACKAGES[@]}"; do add_to_manifest ruby "$pkg"; done 150 151 cat <<EOF 152 </artifacts> 153</build> 154EOF 155}> "$LOCAL_BUILD_INDEX" 156 157LOCAL_BUILD_INDEX_SIZE=$(stat -c%s "$LOCAL_BUILD_INDEX") 158LOCAL_BUILD_INDEX_SHA256=$(openssl sha256 -r "$LOCAL_BUILD_INDEX" | cut -d " " -f 1) 159 160OLD_INDEX=$(mktemp) 161NEW_INDEX=$(mktemp) 162 163# Download the current /index.xml into $OLD_INDEX 164gsutil cp "$GCS_INDEX" "$OLD_INDEX" 165 166{ 167 # we want to add an entry as the first child under <builds> tag 168 # we can get by without a real XML parser by rewriting the header, 169 # injecting our new tag, and then dumping the rest of the file as is. 170 cat <<EOF 171<?xml version="1.0"?> 172<?xml-stylesheet href="/web-assets/home.xsl" type="text/xsl"?> 173<packages> 174 <builds> 175 <build id='$BUILD_ID' 176 timestamp='$BUILD_TIMESTAMP' 177 branch='$BUILD_BRANCH_NAME' 178 commit='$BUILD_GIT_COMMIT' 179 path='$GCS_ARCHIVE_PREFIX$BUILD_RELPATH$INDEX_FILENAME' 180 size='$LOCAL_BUILD_INDEX_SIZE' 181 sha256='$LOCAL_BUILD_INDEX_SHA256' /> 182EOF 183 tail --lines=+5 "$OLD_INDEX" 184}> "$NEW_INDEX" 185 186 187function generate_directory_index() 188{ 189 local target_dir=$1 190 local current_directory_name 191 current_directory_name=$(basename "$target_dir") 192 cat <<EOF 193<!DOCTYPE html> 194<html xmlns="http://www.w3.org/1999/xhtml" lang="" xml:lang=""> 195 <head> 196 <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> 197 <title>Index of $current_directory_name - packages.grpc.io</title> 198 <link rel="stylesheet" type="text/css" href="/web-assets/dirindex.css" /> 199 </head> 200 <body> 201 <h1>Index of <a href="#"><code>$current_directory_name</code></a></h1> 202 <ul> 203 <li><a href="#">.</a></li> 204 <li><a href="..">..</a></li> 205EOF 206 207( 208 cd "$target_dir" 209 find * -maxdepth 0 -type d -print | sort | while read -r line 210 do 211 echo " <li><a href='$line/'>$line/</a></li>" 212 done 213 find * -maxdepth 0 -type f -print | sort | while read -r line 214 do 215 echo " <li><a href='$line'>$line</a></li>" 216 done 217) 218 219cat <<EOF 220 </ul> 221 </body> 222</html> 223EOF 224} 225 226# Upload the current build artifacts 227gsutil -m cp -r "$LOCAL_STAGING_TEMPDIR/${BUILD_RELPATH%%/*}" "$GCS_ARCHIVE_ROOT" 228# Upload directory indices for subdirectories 229( 230 cd "$LOCAL_BUILD_ROOT" 231 find * -type d | while read -r directory 232 do 233 generate_directory_index "$directory" | gsutil -h 'Content-Type:text/html' cp - "$GCS_ARCHIVE_ROOT$BUILD_RELPATH$directory/$INDEX_FILENAME" 234 done 235) 236# Upload the new /index.xml 237gsutil -h "Content-Type:application/xml" cp "$NEW_INDEX" "$GCS_INDEX" 238 239# Upload C# nugets to the dev nuget feed 240pushd "$UNZIPPED_CSHARP_PACKAGES" 241docker pull mcr.microsoft.com/dotnet/core/sdk:2.1 242for nugetfile in *.nupkg 243do 244 echo "Going to push $nugetfile" 245 # use nuget from a docker container to push the nupkg 246 set +x # IMPORTANT: avoid revealing the nuget api key by the command echo 247 docker run -v "$(pwd):/nugets:ro" --rm=true mcr.microsoft.com/dotnet/core/sdk:2.1 bash -c "dotnet nuget push /nugets/$nugetfile -k $(cat ${KOKORO_GFILE_DIR}/artifactory_grpc_nuget_dev_api_key) --source https://grpc.jfrog.io/grpc/api/nuget/v3/grpc-nuget-dev" 248 set -ex 249done 250popd 251