1*55e87721SMatt Gilbride#!/bin/bash 2*55e87721SMatt Gilbride# Copyright 2020 Google LLC 3*55e87721SMatt Gilbride# 4*55e87721SMatt Gilbride# Licensed under the Apache License, Version 2.0 (the "License"); 5*55e87721SMatt Gilbride# you may not use this file except in compliance with the License. 6*55e87721SMatt Gilbride# You may obtain a copy of the License at 7*55e87721SMatt Gilbride# 8*55e87721SMatt Gilbride# http://www.apache.org/licenses/LICENSE-2.0 9*55e87721SMatt Gilbride# 10*55e87721SMatt Gilbride# Unless required by applicable law or agreed to in writing, software 11*55e87721SMatt Gilbride# distributed under the License is distributed on an "AS IS" BASIS, 12*55e87721SMatt Gilbride# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13*55e87721SMatt Gilbride# See the License for the specific language governing permissions and 14*55e87721SMatt Gilbride# limitations under the License. 15*55e87721SMatt Gilbride 16*55e87721SMatt Gilbride# TODO: remove java-core once we figure out how setup_cloud understands Maven's 17*55e87721SMatt Gilbride# "--also-make-dependents" option. https://github.com/googleapis/google-cloud-java/issues/9088 18*55e87721SMatt Gilbrideexcluded_modules=('gapic-libraries-bom' 'google-cloud-jar-parent' 'google-cloud-pom-parent' 'java-core') 19*55e87721SMatt Gilbride 20*55e87721SMatt Gilbridefunction retry_with_backoff { 21*55e87721SMatt Gilbride attempts_left=$1 22*55e87721SMatt Gilbride sleep_seconds=$2 23*55e87721SMatt Gilbride shift 2 24*55e87721SMatt Gilbride command=$@ 25*55e87721SMatt Gilbride 26*55e87721SMatt Gilbride # store current flag state 27*55e87721SMatt Gilbride flags=$- 28*55e87721SMatt Gilbride 29*55e87721SMatt Gilbride # allow a failures to continue 30*55e87721SMatt Gilbride set +e 31*55e87721SMatt Gilbride unset IFS 32*55e87721SMatt Gilbride ${command} 33*55e87721SMatt Gilbride exit_code=$? 34*55e87721SMatt Gilbride 35*55e87721SMatt Gilbride # restore "e" flag 36*55e87721SMatt Gilbride if [[ ${flags} =~ e ]]; then 37*55e87721SMatt Gilbride set -e 38*55e87721SMatt Gilbride else 39*55e87721SMatt Gilbride set +e 40*55e87721SMatt Gilbride fi 41*55e87721SMatt Gilbride 42*55e87721SMatt Gilbride if [[ $exit_code == 0 ]]; then 43*55e87721SMatt Gilbride return 0 44*55e87721SMatt Gilbride fi 45*55e87721SMatt Gilbride 46*55e87721SMatt Gilbride # failure 47*55e87721SMatt Gilbride if [[ ${attempts_left} -gt 0 ]]; then 48*55e87721SMatt Gilbride echo "failure (${exit_code}), sleeping ${sleep_seconds}..." 49*55e87721SMatt Gilbride sleep ${sleep_seconds} 50*55e87721SMatt Gilbride new_attempts=$((${attempts_left} - 1)) 51*55e87721SMatt Gilbride new_sleep=$((${sleep_seconds} * 2)) 52*55e87721SMatt Gilbride retry_with_backoff ${new_attempts} ${new_sleep} ${command} 53*55e87721SMatt Gilbride fi 54*55e87721SMatt Gilbride 55*55e87721SMatt Gilbride return $exit_code 56*55e87721SMatt Gilbride} 57*55e87721SMatt Gilbride 58*55e87721SMatt Gilbride## Helper functions 59*55e87721SMatt Gilbridefunction now() { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n'; } 60*55e87721SMatt Gilbridefunction msg() { println "$*" >&2; } 61*55e87721SMatt Gilbridefunction println() { printf '%s\n' "$(now) $*"; } 62*55e87721SMatt Gilbride 63*55e87721SMatt Gilbridefunction setup_application_credentials() { 64*55e87721SMatt Gilbride # if GOOGLE_APPLICATION_CREDENTIALS is specified as a relative path, prepend Kokoro root directory onto it 65*55e87721SMatt Gilbride if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" && "${GOOGLE_APPLICATION_CREDENTIALS}" != /* ]]; then 66*55e87721SMatt Gilbride export GOOGLE_APPLICATION_CREDENTIALS=$(realpath "${KOKORO_GFILE_DIR}/${GOOGLE_APPLICATION_CREDENTIALS}") 67*55e87721SMatt Gilbride fi 68*55e87721SMatt Gilbride} 69*55e87721SMatt Gilbride 70*55e87721SMatt Gilbridefunction setup_cloud() { 71*55e87721SMatt Gilbride gcloud config set project "$GOOGLE_CLOUD_PROJECT" 72*55e87721SMatt Gilbride 73*55e87721SMatt Gilbride terraform -version && 74*55e87721SMatt Gilbride source ./.cloud/helpers/init.sh "$1" && 75*55e87721SMatt Gilbride source ./.cloud/helpers/plan.sh "$1" && 76*55e87721SMatt Gilbride source ./.cloud/helpers/apply.sh && 77*55e87721SMatt Gilbride source ./.cloud/helpers/populate-env.sh 78*55e87721SMatt Gilbride 79*55e87721SMatt Gilbride destroy() { 80*55e87721SMatt Gilbride arguments=$? 81*55e87721SMatt Gilbride time source ./.cloud/helpers/destroy.sh 82*55e87721SMatt Gilbride exit $arguments 83*55e87721SMatt Gilbride } 84*55e87721SMatt Gilbride trap destroy EXIT 85*55e87721SMatt Gilbride} 86*55e87721SMatt Gilbride 87*55e87721SMatt Gilbridefunction generate_modified_modules_list() { 88*55e87721SMatt Gilbride # Find the files changed from when the PR branched to the last commit 89*55e87721SMatt Gilbride # Filter for java modules and get all the unique elements 90*55e87721SMatt Gilbride # grep returns 1 (error code) and exits the pipeline if there is no match 91*55e87721SMatt Gilbride # If there is no match, it will return true so the rest of the commands can run 92*55e87721SMatt Gilbride git config --global --add safe.directory $(realpath .) 93*55e87721SMatt Gilbride modified_files=$(git diff --name-only "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}") 94*55e87721SMatt Gilbride printf "Modified files:\n%s\n" "${modified_files}" 95*55e87721SMatt Gilbride 96*55e87721SMatt Gilbride # Generate the list of valid maven modules 97*55e87721SMatt Gilbride maven_modules_list=$(mvn help:evaluate -Dexpression=project.modules | grep '<.*>.*</.*>' | sed -e 's/<.*>\(.*\)<\/.*>/\1/g') 98*55e87721SMatt Gilbride maven_modules=() 99*55e87721SMatt Gilbride for module in $maven_modules_list; do 100*55e87721SMatt Gilbride if [[ ! " ${excluded_modules[*]} " =~ " ${module} " ]]; then 101*55e87721SMatt Gilbride maven_modules+=("${module}") 102*55e87721SMatt Gilbride fi 103*55e87721SMatt Gilbride done 104*55e87721SMatt Gilbride 105*55e87721SMatt Gilbride modified_module_list=() 106*55e87721SMatt Gilbride # If either parent pom.xml is touched, run ITs on all the modules 107*55e87721SMatt Gilbride parent_pom_modified=$(echo "${modified_files}" | grep -E '^google-cloud-(pom|jar)-parent/pom.xml$' || true) 108*55e87721SMatt Gilbride shared_dependencies_modified=$(echo "${modified_files}" | grep -E '^java-shared-dependencies' || true) 109*55e87721SMatt Gilbride if [[ (-n $parent_pom_modified) || (-n $shared_dependencies_modified) || ("${TEST_ALL_MODULES}" == "true") ]]; then 110*55e87721SMatt Gilbride modified_module_list=(${maven_modules[*]}) 111*55e87721SMatt Gilbride echo "Testing the entire monorepo" 112*55e87721SMatt Gilbride else 113*55e87721SMatt Gilbride modules=$(echo "${modified_files}" | grep -E 'java-.*' || true) 114*55e87721SMatt Gilbride printf "Files in java modules:\n%s\n" "${modules}" 115*55e87721SMatt Gilbride if [[ -n $modules ]]; then 116*55e87721SMatt Gilbride modules=$(echo "${modules}" | cut -d '/' -f1 | sort -u) 117*55e87721SMatt Gilbride for module in $modules; do 118*55e87721SMatt Gilbride if [[ ! " ${excluded_modules[*]} " =~ " ${module} " && " ${maven_modules[*]} " =~ " ${module} " ]]; then 119*55e87721SMatt Gilbride modified_module_list+=("${module}") 120*55e87721SMatt Gilbride fi 121*55e87721SMatt Gilbride done 122*55e87721SMatt Gilbride else 123*55e87721SMatt Gilbride echo "Found no changes in the java modules" 124*55e87721SMatt Gilbride fi 125*55e87721SMatt Gilbride fi 126*55e87721SMatt Gilbride} 127*55e87721SMatt Gilbride 128*55e87721SMatt Gilbridefunction run_integration_tests() { 129*55e87721SMatt Gilbride printf "Running Integration Tests for:\n%s\n" "$1" 130*55e87721SMatt Gilbride # --also-make-dependents to run other modules that use the affected module 131*55e87721SMatt Gilbride mvn -B ${INTEGRATION_TEST_ARGS} \ 132*55e87721SMatt Gilbride -pl "$1" \ 133*55e87721SMatt Gilbride --also-make-dependents \ 134*55e87721SMatt Gilbride -ntp \ 135*55e87721SMatt Gilbride -Penable-integration-tests \ 136*55e87721SMatt Gilbride -DtrimStackTrace=false \ 137*55e87721SMatt Gilbride -Dclirr.skip=true \ 138*55e87721SMatt Gilbride -Denforcer.skip=true \ 139*55e87721SMatt Gilbride -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \ 140*55e87721SMatt Gilbride -Dcheckstyle.skip=true \ 141*55e87721SMatt Gilbride -Dflatten.skip=true \ 142*55e87721SMatt Gilbride -Danimal.sniffer.skip=true \ 143*55e87721SMatt Gilbride -Djacoco.skip=true \ 144*55e87721SMatt Gilbride -DskipUnitTests=true \ 145*55e87721SMatt Gilbride -Dmaven.wagon.http.retryHandler.count=5 \ 146*55e87721SMatt Gilbride -fae \ 147*55e87721SMatt Gilbride -T 1C \ 148*55e87721SMatt Gilbride verify 149*55e87721SMatt Gilbride 150*55e87721SMatt Gilbride RETURN_CODE=$? 151*55e87721SMatt Gilbride printf "Finished Integration Tests for:\n%s\n" "$1" 152*55e87721SMatt Gilbride} 153*55e87721SMatt Gilbride 154*55e87721SMatt Gilbridefunction run_graalvm_tests() { 155*55e87721SMatt Gilbride printf "Running GraalVM ITs on:\n%s\n" "$1" 156*55e87721SMatt Gilbride 157*55e87721SMatt Gilbride mvn -B ${INTEGRATION_TEST_ARGS} \ 158*55e87721SMatt Gilbride -pl "$1" \ 159*55e87721SMatt Gilbride --also-make-dependents \ 160*55e87721SMatt Gilbride -ntp \ 161*55e87721SMatt Gilbride -DtrimStackTrace=false \ 162*55e87721SMatt Gilbride -Dclirr.skip=true \ 163*55e87721SMatt Gilbride -Denforcer.skip=true \ 164*55e87721SMatt Gilbride -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \ 165*55e87721SMatt Gilbride -Dcheckstyle.skip=true \ 166*55e87721SMatt Gilbride -Dflatten.skip=true \ 167*55e87721SMatt Gilbride -Danimal.sniffer.skip=true \ 168*55e87721SMatt Gilbride -Pnative \ 169*55e87721SMatt Gilbride -fae \ 170*55e87721SMatt Gilbride test 171*55e87721SMatt Gilbride 172*55e87721SMatt Gilbride RETURN_CODE=$? 173*55e87721SMatt Gilbride printf "Finished Unit and Integration Tests for GraalVM:\n%s\n" "$1" 174*55e87721SMatt Gilbride} 175*55e87721SMatt Gilbride 176*55e87721SMatt Gilbridefunction generate_graalvm_modules_list() { 177*55e87721SMatt Gilbride modules_assigned_list=() 178*55e87721SMatt Gilbride generate_modified_modules_list 179*55e87721SMatt Gilbride if [[ "${TEST_ALL_MODULES}" == "true" ]]; then 180*55e87721SMatt Gilbride # Assign the modules to modules_assigned_list (based on num jobs) 181*55e87721SMatt Gilbride num=0 182*55e87721SMatt Gilbride for module in "${modified_module_list[@]}"; do 183*55e87721SMatt Gilbride # Add 1 as JOB_NUMBER is 1-indexed instead of 0-indexed 184*55e87721SMatt Gilbride mod_num=$((num % NUM_JOBS + 1)) 185*55e87721SMatt Gilbride # Spaces are intentionally added -- Query is regex and array elements are space separated 186*55e87721SMatt Gilbride # It tries to match the *exact* `module` text 187*55e87721SMatt Gilbride if [[ ! " ${excluded_modules[*]} " =~ " ${module} " ]] && [[ $mod_num -eq $JOB_NUMBER ]]; then 188*55e87721SMatt Gilbride modules_assigned_list+=("${module}") 189*55e87721SMatt Gilbride fi 190*55e87721SMatt Gilbride num=$((num + 1)) 191*55e87721SMatt Gilbride done 192*55e87721SMatt Gilbride elif [[ ${#modified_module_list[@]} -gt 0 ]]; then 193*55e87721SMatt Gilbride # MAVEN_MODULES ENV_VAR is expecting comma delimited string (similar to mvn -pl) 194*55e87721SMatt Gilbride # This will get all the modules and put all the elements into an array 195*55e87721SMatt Gilbride maven_modules_list=($(echo "${MAVEN_MODULES}" | tr ',' ' ')) 196*55e87721SMatt Gilbride for maven_module in "${maven_modules_list[@]}"; do 197*55e87721SMatt Gilbride # Check that the modified_module_list contains a module from MAVEN_MODULES 198*55e87721SMatt Gilbride # Spaces are intentionally added -- Query is regex and array elements are space separated 199*55e87721SMatt Gilbride # It tries to match the *exact* `maven_module` text 200*55e87721SMatt Gilbride if [[ " ${modified_module_list[*]} " =~ " ${maven_module} " ]]; then 201*55e87721SMatt Gilbride modules_assigned_list+=("${module}") 202*55e87721SMatt Gilbride fi 203*55e87721SMatt Gilbride done 204*55e87721SMatt Gilbride fi 205*55e87721SMatt Gilbride module_list=$( 206*55e87721SMatt Gilbride IFS=, 207*55e87721SMatt Gilbride echo "${modules_assigned_list[*]}" 208*55e87721SMatt Gilbride ) 209*55e87721SMatt Gilbride} 210*55e87721SMatt Gilbride 211*55e87721SMatt Gilbridefunction install_modules() { 212*55e87721SMatt Gilbride retry_with_backoff 3 10 \ 213*55e87721SMatt Gilbride mvn -B \ 214*55e87721SMatt Gilbride -ntp \ 215*55e87721SMatt Gilbride -DtrimStackTrace=false \ 216*55e87721SMatt Gilbride -Dclirr.skip=true \ 217*55e87721SMatt Gilbride -Denforcer.skip=true \ 218*55e87721SMatt Gilbride -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \ 219*55e87721SMatt Gilbride -Dcheckstyle.skip=true \ 220*55e87721SMatt Gilbride -Dflatten.skip=true \ 221*55e87721SMatt Gilbride -Danimal.sniffer.skip=true \ 222*55e87721SMatt Gilbride -DskipTests=true \ 223*55e87721SMatt Gilbride -Djacoco.skip=true \ 224*55e87721SMatt Gilbride -T 1C \ 225*55e87721SMatt Gilbride install 226*55e87721SMatt Gilbride} 227