1#!/bin/bash 2# Copyright 2020 Google LLC 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16# TODO: remove java-core once we figure out how setup_cloud understands Maven's 17# "--also-make-dependents" option. https://github.com/googleapis/google-cloud-java/issues/9088 18excluded_modules=('gapic-libraries-bom' 'google-cloud-jar-parent' 'google-cloud-pom-parent' 'java-core') 19 20function retry_with_backoff { 21 attempts_left=$1 22 sleep_seconds=$2 23 shift 2 24 command=$@ 25 26 # store current flag state 27 flags=$- 28 29 # allow a failures to continue 30 set +e 31 unset IFS 32 ${command} 33 exit_code=$? 34 35 # restore "e" flag 36 if [[ ${flags} =~ e ]]; then 37 set -e 38 else 39 set +e 40 fi 41 42 if [[ $exit_code == 0 ]]; then 43 return 0 44 fi 45 46 # failure 47 if [[ ${attempts_left} -gt 0 ]]; then 48 echo "failure (${exit_code}), sleeping ${sleep_seconds}..." 49 sleep ${sleep_seconds} 50 new_attempts=$((${attempts_left} - 1)) 51 new_sleep=$((${sleep_seconds} * 2)) 52 retry_with_backoff ${new_attempts} ${new_sleep} ${command} 53 fi 54 55 return $exit_code 56} 57 58## Helper functions 59function now() { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n'; } 60function msg() { println "$*" >&2; } 61function println() { printf '%s\n' "$(now) $*"; } 62 63function setup_application_credentials() { 64 # if GOOGLE_APPLICATION_CREDENTIALS is specified as a relative path, prepend Kokoro root directory onto it 65 if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" && "${GOOGLE_APPLICATION_CREDENTIALS}" != /* ]]; then 66 export GOOGLE_APPLICATION_CREDENTIALS=$(realpath "${KOKORO_GFILE_DIR}/${GOOGLE_APPLICATION_CREDENTIALS}") 67 fi 68} 69 70function setup_cloud() { 71 gcloud config set project "$GOOGLE_CLOUD_PROJECT" 72 73 terraform -version && 74 source ./.cloud/helpers/init.sh "$1" && 75 source ./.cloud/helpers/plan.sh "$1" && 76 source ./.cloud/helpers/apply.sh && 77 source ./.cloud/helpers/populate-env.sh 78 79 destroy() { 80 arguments=$? 81 time source ./.cloud/helpers/destroy.sh 82 exit $arguments 83 } 84 trap destroy EXIT 85} 86 87function generate_modified_modules_list() { 88 # Find the files changed from when the PR branched to the last commit 89 # Filter for java modules and get all the unique elements 90 # grep returns 1 (error code) and exits the pipeline if there is no match 91 # If there is no match, it will return true so the rest of the commands can run 92 git config --global --add safe.directory $(realpath .) 93 modified_files=$(git diff --name-only "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}") 94 printf "Modified files:\n%s\n" "${modified_files}" 95 96 # Generate the list of valid maven modules 97 maven_modules_list=$(mvn help:evaluate -Dexpression=project.modules | grep '<.*>.*</.*>' | sed -e 's/<.*>\(.*\)<\/.*>/\1/g') 98 maven_modules=() 99 for module in $maven_modules_list; do 100 if [[ ! " ${excluded_modules[*]} " =~ " ${module} " ]]; then 101 maven_modules+=("${module}") 102 fi 103 done 104 105 modified_module_list=() 106 # If either parent pom.xml is touched, run ITs on all the modules 107 parent_pom_modified=$(echo "${modified_files}" | grep -E '^google-cloud-(pom|jar)-parent/pom.xml$' || true) 108 shared_dependencies_modified=$(echo "${modified_files}" | grep -E '^java-shared-dependencies' || true) 109 if [[ (-n $parent_pom_modified) || (-n $shared_dependencies_modified) || ("${TEST_ALL_MODULES}" == "true") ]]; then 110 modified_module_list=(${maven_modules[*]}) 111 echo "Testing the entire monorepo" 112 else 113 modules=$(echo "${modified_files}" | grep -E 'java-.*' || true) 114 printf "Files in java modules:\n%s\n" "${modules}" 115 if [[ -n $modules ]]; then 116 modules=$(echo "${modules}" | cut -d '/' -f1 | sort -u) 117 for module in $modules; do 118 if [[ ! " ${excluded_modules[*]} " =~ " ${module} " && " ${maven_modules[*]} " =~ " ${module} " ]]; then 119 modified_module_list+=("${module}") 120 fi 121 done 122 else 123 echo "Found no changes in the java modules" 124 fi 125 fi 126} 127 128function run_integration_tests() { 129 printf "Running Integration Tests for:\n%s\n" "$1" 130 # --also-make-dependents to run other modules that use the affected module 131 mvn -B ${INTEGRATION_TEST_ARGS} \ 132 -pl "$1" \ 133 --also-make-dependents \ 134 -ntp \ 135 -Penable-integration-tests \ 136 -DtrimStackTrace=false \ 137 -Dclirr.skip=true \ 138 -Denforcer.skip=true \ 139 -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \ 140 -Dcheckstyle.skip=true \ 141 -Dflatten.skip=true \ 142 -Danimal.sniffer.skip=true \ 143 -Djacoco.skip=true \ 144 -DskipUnitTests=true \ 145 -Dmaven.wagon.http.retryHandler.count=5 \ 146 -fae \ 147 -T 1C \ 148 verify 149 150 RETURN_CODE=$? 151 printf "Finished Integration Tests for:\n%s\n" "$1" 152} 153 154function run_graalvm_tests() { 155 printf "Running GraalVM ITs on:\n%s\n" "$1" 156 157 mvn -B ${INTEGRATION_TEST_ARGS} \ 158 -pl "$1" \ 159 --also-make-dependents \ 160 -ntp \ 161 -DtrimStackTrace=false \ 162 -Dclirr.skip=true \ 163 -Denforcer.skip=true \ 164 -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \ 165 -Dcheckstyle.skip=true \ 166 -Dflatten.skip=true \ 167 -Danimal.sniffer.skip=true \ 168 -Pnative \ 169 -fae \ 170 test 171 172 RETURN_CODE=$? 173 printf "Finished Unit and Integration Tests for GraalVM:\n%s\n" "$1" 174} 175 176function generate_graalvm_modules_list() { 177 modules_assigned_list=() 178 generate_modified_modules_list 179 if [[ "${TEST_ALL_MODULES}" == "true" ]]; then 180 # Assign the modules to modules_assigned_list (based on num jobs) 181 num=0 182 for module in "${modified_module_list[@]}"; do 183 # Add 1 as JOB_NUMBER is 1-indexed instead of 0-indexed 184 mod_num=$((num % NUM_JOBS + 1)) 185 # Spaces are intentionally added -- Query is regex and array elements are space separated 186 # It tries to match the *exact* `module` text 187 if [[ ! " ${excluded_modules[*]} " =~ " ${module} " ]] && [[ $mod_num -eq $JOB_NUMBER ]]; then 188 modules_assigned_list+=("${module}") 189 fi 190 num=$((num + 1)) 191 done 192 elif [[ ${#modified_module_list[@]} -gt 0 ]]; then 193 # MAVEN_MODULES ENV_VAR is expecting comma delimited string (similar to mvn -pl) 194 # This will get all the modules and put all the elements into an array 195 maven_modules_list=($(echo "${MAVEN_MODULES}" | tr ',' ' ')) 196 for maven_module in "${maven_modules_list[@]}"; do 197 # Check that the modified_module_list contains a module from MAVEN_MODULES 198 # Spaces are intentionally added -- Query is regex and array elements are space separated 199 # It tries to match the *exact* `maven_module` text 200 if [[ " ${modified_module_list[*]} " =~ " ${maven_module} " ]]; then 201 modules_assigned_list+=("${module}") 202 fi 203 done 204 fi 205 module_list=$( 206 IFS=, 207 echo "${modules_assigned_list[*]}" 208 ) 209} 210 211function install_modules() { 212 retry_with_backoff 3 10 \ 213 mvn -B \ 214 -ntp \ 215 -DtrimStackTrace=false \ 216 -Dclirr.skip=true \ 217 -Denforcer.skip=true \ 218 -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \ 219 -Dcheckstyle.skip=true \ 220 -Dflatten.skip=true \ 221 -Danimal.sniffer.skip=true \ 222 -DskipTests=true \ 223 -Djacoco.skip=true \ 224 -T 1C \ 225 install 226} 227