xref: /aosp_15_r20/external/google-cloud-java/.kokoro/common.sh (revision 55e87721aa1bc457b326496a7ca40f3ea1a63287)
1*55e87721SMatt Gilbride#!/bin/bash
2*55e87721SMatt Gilbride# Copyright 2020 Google LLC
3*55e87721SMatt Gilbride#
4*55e87721SMatt Gilbride# Licensed under the Apache License, Version 2.0 (the "License");
5*55e87721SMatt Gilbride# you may not use this file except in compliance with the License.
6*55e87721SMatt Gilbride# You may obtain a copy of the License at
7*55e87721SMatt Gilbride#
8*55e87721SMatt Gilbride#      http://www.apache.org/licenses/LICENSE-2.0
9*55e87721SMatt Gilbride#
10*55e87721SMatt Gilbride# Unless required by applicable law or agreed to in writing, software
11*55e87721SMatt Gilbride# distributed under the License is distributed on an "AS IS" BASIS,
12*55e87721SMatt Gilbride# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13*55e87721SMatt Gilbride# See the License for the specific language governing permissions and
14*55e87721SMatt Gilbride# limitations under the License.
15*55e87721SMatt Gilbride
16*55e87721SMatt Gilbride# TODO: remove java-core once we figure out how setup_cloud understands Maven's
17*55e87721SMatt Gilbride# "--also-make-dependents" option. https://github.com/googleapis/google-cloud-java/issues/9088
18*55e87721SMatt Gilbrideexcluded_modules=('gapic-libraries-bom' 'google-cloud-jar-parent' 'google-cloud-pom-parent' 'java-core')
19*55e87721SMatt Gilbride
20*55e87721SMatt Gilbridefunction retry_with_backoff {
21*55e87721SMatt Gilbride  attempts_left=$1
22*55e87721SMatt Gilbride  sleep_seconds=$2
23*55e87721SMatt Gilbride  shift 2
24*55e87721SMatt Gilbride  command=$@
25*55e87721SMatt Gilbride
26*55e87721SMatt Gilbride  # store current flag state
27*55e87721SMatt Gilbride  flags=$-
28*55e87721SMatt Gilbride
29*55e87721SMatt Gilbride  # allow a failures to continue
30*55e87721SMatt Gilbride  set +e
31*55e87721SMatt Gilbride  unset IFS
32*55e87721SMatt Gilbride  ${command}
33*55e87721SMatt Gilbride  exit_code=$?
34*55e87721SMatt Gilbride
35*55e87721SMatt Gilbride  # restore "e" flag
36*55e87721SMatt Gilbride  if [[ ${flags} =~ e ]]; then
37*55e87721SMatt Gilbride    set -e
38*55e87721SMatt Gilbride  else
39*55e87721SMatt Gilbride    set +e
40*55e87721SMatt Gilbride  fi
41*55e87721SMatt Gilbride
42*55e87721SMatt Gilbride  if [[ $exit_code == 0 ]]; then
43*55e87721SMatt Gilbride    return 0
44*55e87721SMatt Gilbride  fi
45*55e87721SMatt Gilbride
46*55e87721SMatt Gilbride  # failure
47*55e87721SMatt Gilbride  if [[ ${attempts_left} -gt 0 ]]; then
48*55e87721SMatt Gilbride    echo "failure (${exit_code}), sleeping ${sleep_seconds}..."
49*55e87721SMatt Gilbride    sleep ${sleep_seconds}
50*55e87721SMatt Gilbride    new_attempts=$((${attempts_left} - 1))
51*55e87721SMatt Gilbride    new_sleep=$((${sleep_seconds} * 2))
52*55e87721SMatt Gilbride    retry_with_backoff ${new_attempts} ${new_sleep} ${command}
53*55e87721SMatt Gilbride  fi
54*55e87721SMatt Gilbride
55*55e87721SMatt Gilbride  return $exit_code
56*55e87721SMatt Gilbride}
57*55e87721SMatt Gilbride
58*55e87721SMatt Gilbride## Helper functions
59*55e87721SMatt Gilbridefunction now() { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n'; }
60*55e87721SMatt Gilbridefunction msg() { println "$*" >&2; }
61*55e87721SMatt Gilbridefunction println() { printf '%s\n' "$(now) $*"; }
62*55e87721SMatt Gilbride
63*55e87721SMatt Gilbridefunction setup_application_credentials() {
64*55e87721SMatt Gilbride  # if GOOGLE_APPLICATION_CREDENTIALS is specified as a relative path, prepend Kokoro root directory onto it
65*55e87721SMatt Gilbride  if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" && "${GOOGLE_APPLICATION_CREDENTIALS}" != /* ]]; then
66*55e87721SMatt Gilbride    export GOOGLE_APPLICATION_CREDENTIALS=$(realpath "${KOKORO_GFILE_DIR}/${GOOGLE_APPLICATION_CREDENTIALS}")
67*55e87721SMatt Gilbride  fi
68*55e87721SMatt Gilbride}
69*55e87721SMatt Gilbride
70*55e87721SMatt Gilbridefunction setup_cloud() {
71*55e87721SMatt Gilbride  gcloud config set project "$GOOGLE_CLOUD_PROJECT"
72*55e87721SMatt Gilbride
73*55e87721SMatt Gilbride  terraform -version &&
74*55e87721SMatt Gilbride    source ./.cloud/helpers/init.sh "$1" &&
75*55e87721SMatt Gilbride    source ./.cloud/helpers/plan.sh "$1" &&
76*55e87721SMatt Gilbride    source ./.cloud/helpers/apply.sh &&
77*55e87721SMatt Gilbride    source ./.cloud/helpers/populate-env.sh
78*55e87721SMatt Gilbride
79*55e87721SMatt Gilbride  destroy() {
80*55e87721SMatt Gilbride    arguments=$?
81*55e87721SMatt Gilbride    time source ./.cloud/helpers/destroy.sh
82*55e87721SMatt Gilbride    exit $arguments
83*55e87721SMatt Gilbride  }
84*55e87721SMatt Gilbride  trap destroy EXIT
85*55e87721SMatt Gilbride}
86*55e87721SMatt Gilbride
87*55e87721SMatt Gilbridefunction generate_modified_modules_list() {
88*55e87721SMatt Gilbride  # Find the files changed from when the PR branched to the last commit
89*55e87721SMatt Gilbride  # Filter for java modules and get all the unique elements
90*55e87721SMatt Gilbride  # grep returns 1 (error code) and exits the pipeline if there is no match
91*55e87721SMatt Gilbride  # If there is no match, it will return true so the rest of the commands can run
92*55e87721SMatt Gilbride  git config --global --add safe.directory $(realpath .)
93*55e87721SMatt Gilbride  modified_files=$(git diff --name-only "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}")
94*55e87721SMatt Gilbride  printf "Modified files:\n%s\n" "${modified_files}"
95*55e87721SMatt Gilbride
96*55e87721SMatt Gilbride  # Generate the list of valid maven modules
97*55e87721SMatt Gilbride  maven_modules_list=$(mvn help:evaluate -Dexpression=project.modules | grep '<.*>.*</.*>' | sed -e 's/<.*>\(.*\)<\/.*>/\1/g')
98*55e87721SMatt Gilbride  maven_modules=()
99*55e87721SMatt Gilbride  for module in $maven_modules_list; do
100*55e87721SMatt Gilbride    if [[ ! " ${excluded_modules[*]} " =~ " ${module} " ]]; then
101*55e87721SMatt Gilbride      maven_modules+=("${module}")
102*55e87721SMatt Gilbride    fi
103*55e87721SMatt Gilbride  done
104*55e87721SMatt Gilbride
105*55e87721SMatt Gilbride  modified_module_list=()
106*55e87721SMatt Gilbride  # If either parent pom.xml is touched, run ITs on all the modules
107*55e87721SMatt Gilbride  parent_pom_modified=$(echo "${modified_files}" | grep -E '^google-cloud-(pom|jar)-parent/pom.xml$' || true)
108*55e87721SMatt Gilbride  shared_dependencies_modified=$(echo "${modified_files}" | grep -E '^java-shared-dependencies' || true)
109*55e87721SMatt Gilbride  if [[ (-n $parent_pom_modified) || (-n $shared_dependencies_modified) || ("${TEST_ALL_MODULES}" == "true") ]]; then
110*55e87721SMatt Gilbride    modified_module_list=(${maven_modules[*]})
111*55e87721SMatt Gilbride    echo "Testing the entire monorepo"
112*55e87721SMatt Gilbride  else
113*55e87721SMatt Gilbride    modules=$(echo "${modified_files}" | grep -E 'java-.*' || true)
114*55e87721SMatt Gilbride    printf "Files in java modules:\n%s\n" "${modules}"
115*55e87721SMatt Gilbride    if [[ -n $modules ]]; then
116*55e87721SMatt Gilbride      modules=$(echo "${modules}" | cut -d '/' -f1 | sort -u)
117*55e87721SMatt Gilbride      for module in $modules; do
118*55e87721SMatt Gilbride        if [[ ! " ${excluded_modules[*]} " =~ " ${module} " && " ${maven_modules[*]} " =~ " ${module} " ]]; then
119*55e87721SMatt Gilbride          modified_module_list+=("${module}")
120*55e87721SMatt Gilbride        fi
121*55e87721SMatt Gilbride      done
122*55e87721SMatt Gilbride    else
123*55e87721SMatt Gilbride      echo "Found no changes in the java modules"
124*55e87721SMatt Gilbride    fi
125*55e87721SMatt Gilbride  fi
126*55e87721SMatt Gilbride}
127*55e87721SMatt Gilbride
128*55e87721SMatt Gilbridefunction run_integration_tests() {
129*55e87721SMatt Gilbride  printf "Running Integration Tests for:\n%s\n" "$1"
130*55e87721SMatt Gilbride  # --also-make-dependents to run other modules that use the affected module
131*55e87721SMatt Gilbride  mvn -B ${INTEGRATION_TEST_ARGS} \
132*55e87721SMatt Gilbride    -pl "$1" \
133*55e87721SMatt Gilbride    --also-make-dependents \
134*55e87721SMatt Gilbride    -ntp \
135*55e87721SMatt Gilbride    -Penable-integration-tests \
136*55e87721SMatt Gilbride    -DtrimStackTrace=false \
137*55e87721SMatt Gilbride    -Dclirr.skip=true \
138*55e87721SMatt Gilbride    -Denforcer.skip=true \
139*55e87721SMatt Gilbride    -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
140*55e87721SMatt Gilbride    -Dcheckstyle.skip=true \
141*55e87721SMatt Gilbride    -Dflatten.skip=true \
142*55e87721SMatt Gilbride    -Danimal.sniffer.skip=true \
143*55e87721SMatt Gilbride    -Djacoco.skip=true \
144*55e87721SMatt Gilbride    -DskipUnitTests=true \
145*55e87721SMatt Gilbride    -Dmaven.wagon.http.retryHandler.count=5 \
146*55e87721SMatt Gilbride    -fae \
147*55e87721SMatt Gilbride    -T 1C \
148*55e87721SMatt Gilbride    verify
149*55e87721SMatt Gilbride
150*55e87721SMatt Gilbride  RETURN_CODE=$?
151*55e87721SMatt Gilbride  printf "Finished Integration Tests for:\n%s\n" "$1"
152*55e87721SMatt Gilbride}
153*55e87721SMatt Gilbride
154*55e87721SMatt Gilbridefunction run_graalvm_tests() {
155*55e87721SMatt Gilbride  printf "Running GraalVM ITs on:\n%s\n" "$1"
156*55e87721SMatt Gilbride
157*55e87721SMatt Gilbride  mvn -B ${INTEGRATION_TEST_ARGS} \
158*55e87721SMatt Gilbride    -pl "$1" \
159*55e87721SMatt Gilbride    --also-make-dependents \
160*55e87721SMatt Gilbride    -ntp \
161*55e87721SMatt Gilbride    -DtrimStackTrace=false \
162*55e87721SMatt Gilbride    -Dclirr.skip=true \
163*55e87721SMatt Gilbride    -Denforcer.skip=true \
164*55e87721SMatt Gilbride    -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
165*55e87721SMatt Gilbride    -Dcheckstyle.skip=true \
166*55e87721SMatt Gilbride    -Dflatten.skip=true \
167*55e87721SMatt Gilbride    -Danimal.sniffer.skip=true \
168*55e87721SMatt Gilbride    -Pnative \
169*55e87721SMatt Gilbride    -fae \
170*55e87721SMatt Gilbride    test
171*55e87721SMatt Gilbride
172*55e87721SMatt Gilbride  RETURN_CODE=$?
173*55e87721SMatt Gilbride  printf "Finished Unit and Integration Tests for GraalVM:\n%s\n" "$1"
174*55e87721SMatt Gilbride}
175*55e87721SMatt Gilbride
176*55e87721SMatt Gilbridefunction generate_graalvm_modules_list() {
177*55e87721SMatt Gilbride  modules_assigned_list=()
178*55e87721SMatt Gilbride  generate_modified_modules_list
179*55e87721SMatt Gilbride  if [[ "${TEST_ALL_MODULES}" == "true" ]]; then
180*55e87721SMatt Gilbride    # Assign the modules to modules_assigned_list (based on num jobs)
181*55e87721SMatt Gilbride    num=0
182*55e87721SMatt Gilbride    for module in "${modified_module_list[@]}"; do
183*55e87721SMatt Gilbride      # Add 1 as JOB_NUMBER is 1-indexed instead of 0-indexed
184*55e87721SMatt Gilbride      mod_num=$((num % NUM_JOBS + 1))
185*55e87721SMatt Gilbride      # Spaces are intentionally added -- Query is regex and array elements are space separated
186*55e87721SMatt Gilbride      # It tries to match the *exact* `module` text
187*55e87721SMatt Gilbride      if [[ ! " ${excluded_modules[*]} " =~ " ${module} " ]] && [[ $mod_num -eq $JOB_NUMBER ]]; then
188*55e87721SMatt Gilbride        modules_assigned_list+=("${module}")
189*55e87721SMatt Gilbride      fi
190*55e87721SMatt Gilbride      num=$((num + 1))
191*55e87721SMatt Gilbride    done
192*55e87721SMatt Gilbride  elif [[ ${#modified_module_list[@]} -gt 0 ]]; then
193*55e87721SMatt Gilbride    # MAVEN_MODULES ENV_VAR is expecting comma delimited string (similar to mvn -pl)
194*55e87721SMatt Gilbride    # This will get all the modules and put all the elements into an array
195*55e87721SMatt Gilbride    maven_modules_list=($(echo "${MAVEN_MODULES}" | tr ',' ' '))
196*55e87721SMatt Gilbride    for maven_module in "${maven_modules_list[@]}"; do
197*55e87721SMatt Gilbride      # Check that the modified_module_list contains a module from MAVEN_MODULES
198*55e87721SMatt Gilbride      # Spaces are intentionally added -- Query is regex and array elements are space separated
199*55e87721SMatt Gilbride      # It tries to match the *exact* `maven_module` text
200*55e87721SMatt Gilbride      if [[ " ${modified_module_list[*]} " =~ " ${maven_module} " ]]; then
201*55e87721SMatt Gilbride        modules_assigned_list+=("${module}")
202*55e87721SMatt Gilbride      fi
203*55e87721SMatt Gilbride    done
204*55e87721SMatt Gilbride  fi
205*55e87721SMatt Gilbride  module_list=$(
206*55e87721SMatt Gilbride    IFS=,
207*55e87721SMatt Gilbride    echo "${modules_assigned_list[*]}"
208*55e87721SMatt Gilbride  )
209*55e87721SMatt Gilbride}
210*55e87721SMatt Gilbride
211*55e87721SMatt Gilbridefunction install_modules() {
212*55e87721SMatt Gilbride  retry_with_backoff 3 10 \
213*55e87721SMatt Gilbride    mvn -B \
214*55e87721SMatt Gilbride    -ntp \
215*55e87721SMatt Gilbride    -DtrimStackTrace=false \
216*55e87721SMatt Gilbride    -Dclirr.skip=true \
217*55e87721SMatt Gilbride    -Denforcer.skip=true \
218*55e87721SMatt Gilbride    -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
219*55e87721SMatt Gilbride    -Dcheckstyle.skip=true \
220*55e87721SMatt Gilbride    -Dflatten.skip=true \
221*55e87721SMatt Gilbride    -Danimal.sniffer.skip=true \
222*55e87721SMatt Gilbride    -DskipTests=true \
223*55e87721SMatt Gilbride    -Djacoco.skip=true \
224*55e87721SMatt Gilbride    -T 1C \
225*55e87721SMatt Gilbride    install
226*55e87721SMatt Gilbride}
227