xref: /aosp_15_r20/external/google-cloud-java/.kokoro/common.sh (revision 55e87721aa1bc457b326496a7ca40f3ea1a63287)

#!/bin/bash
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# TODO: remove java-core once we figure out how setup_cloud understands Maven's
# "--also-make-dependents" option. https://github.com/googleapis/google-cloud-java/issues/9088
excluded_modules=('gapic-libraries-bom' 'google-cloud-jar-parent' 'google-cloud-pom-parent' 'java-core')

function retry_with_backoff {
  attempts_left=$1
  sleep_seconds=$2
  shift 2
  command=$@

  # store current flag state
  flags=$-

  # allow a failures to continue
  set +e
  unset IFS
  ${command}
  exit_code=$?

  # restore "e" flag
  if [[ ${flags} =~ e ]]; then
    set -e
  else
    set +e
  fi

  if [[ $exit_code == 0 ]]; then
    return 0
  fi

  # failure
  if [[ ${attempts_left} -gt 0 ]]; then
    echo "failure (${exit_code}), sleeping ${sleep_seconds}..."
    sleep ${sleep_seconds}
    new_attempts=$((${attempts_left} - 1))
    new_sleep=$((${sleep_seconds} * 2))
    retry_with_backoff ${new_attempts} ${new_sleep} ${command}
  fi

  return $exit_code
}

## Helper functions
function now() { date +"%Y-%m-%d %H:%M:%S" | tr -d '\n'; }
function msg() { println "$*" >&2; }
function println() { printf '%s\n' "$(now) $*"; }

function setup_application_credentials() {
  # if GOOGLE_APPLICATION_CREDENTIALS is specified as a relative path, prepend Kokoro root directory onto it
  if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" && "${GOOGLE_APPLICATION_CREDENTIALS}" != /* ]]; then
    export GOOGLE_APPLICATION_CREDENTIALS=$(realpath "${KOKORO_GFILE_DIR}/${GOOGLE_APPLICATION_CREDENTIALS}")
  fi
}

function setup_cloud() {
  gcloud config set project "$GOOGLE_CLOUD_PROJECT"

  terraform -version &&
    source ./.cloud/helpers/init.sh "$1" &&
    source ./.cloud/helpers/plan.sh "$1" &&
    source ./.cloud/helpers/apply.sh &&
    source ./.cloud/helpers/populate-env.sh

  destroy() {
    arguments=$?
    time source ./.cloud/helpers/destroy.sh
    exit $arguments
  }
  trap destroy EXIT
}

function generate_modified_modules_list() {
  # Find the files changed from when the PR branched to the last commit
  # Filter for java modules and get all the unique elements
  # grep returns 1 (error code) and exits the pipeline if there is no match
  # If there is no match, it will return true so the rest of the commands can run
  git config --global --add safe.directory $(realpath .)
  modified_files=$(git diff --name-only "${KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH}...${KOKORO_GITHUB_PULL_REQUEST_COMMIT}")
  printf "Modified files:\n%s\n" "${modified_files}"

  # Generate the list of valid maven modules
  maven_modules_list=$(mvn help:evaluate -Dexpression=project.modules | grep '<.*>.*</.*>' | sed -e 's/<.*>\(.*\)<\/.*>/\1/g')
  maven_modules=()
  for module in $maven_modules_list; do
    if [[ ! " ${excluded_modules[*]} " =~ " ${module} " ]]; then
      maven_modules+=("${module}")
    fi
  done

  modified_module_list=()
  # If either parent pom.xml is touched, run ITs on all the modules
  parent_pom_modified=$(echo "${modified_files}" | grep -E '^google-cloud-(pom|jar)-parent/pom.xml$' || true)
  shared_dependencies_modified=$(echo "${modified_files}" | grep -E '^java-shared-dependencies' || true)
  if [[ (-n $parent_pom_modified) || (-n $shared_dependencies_modified) || ("${TEST_ALL_MODULES}" == "true") ]]; then
    modified_module_list=(${maven_modules[*]})
    echo "Testing the entire monorepo"
  else
    modules=$(echo "${modified_files}" | grep -E 'java-.*' || true)
    printf "Files in java modules:\n%s\n" "${modules}"
    if [[ -n $modules ]]; then
      modules=$(echo "${modules}" | cut -d '/' -f1 | sort -u)
      for module in $modules; do
        if [[ ! " ${excluded_modules[*]} " =~ " ${module} " && " ${maven_modules[*]} " =~ " ${module} " ]]; then
          modified_module_list+=("${module}")
        fi
      done
    else
      echo "Found no changes in the java modules"
    fi
  fi
}

function run_integration_tests() {
  printf "Running Integration Tests for:\n%s\n" "$1"
  # --also-make-dependents to run other modules that use the affected module
  mvn -B ${INTEGRATION_TEST_ARGS} \
    -pl "$1" \
    --also-make-dependents \
    -ntp \
    -Penable-integration-tests \
    -DtrimStackTrace=false \
    -Dclirr.skip=true \
    -Denforcer.skip=true \
    -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
    -Dcheckstyle.skip=true \
    -Dflatten.skip=true \
    -Danimal.sniffer.skip=true \
    -Djacoco.skip=true \
    -DskipUnitTests=true \
    -Dmaven.wagon.http.retryHandler.count=5 \
    -fae \
    -T 1C \
    verify

  RETURN_CODE=$?
  printf "Finished Integration Tests for:\n%s\n" "$1"
}

function run_graalvm_tests() {
  printf "Running GraalVM ITs on:\n%s\n" "$1"

  mvn -B ${INTEGRATION_TEST_ARGS} \
    -pl "$1" \
    --also-make-dependents \
    -ntp \
    -DtrimStackTrace=false \
    -Dclirr.skip=true \
    -Denforcer.skip=true \
    -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
    -Dcheckstyle.skip=true \
    -Dflatten.skip=true \
    -Danimal.sniffer.skip=true \
    -Pnative \
    -fae \
    test

  RETURN_CODE=$?
  printf "Finished Unit and Integration Tests for GraalVM:\n%s\n" "$1"
}

function generate_graalvm_modules_list() {
  modules_assigned_list=()
  generate_modified_modules_list
  if [[ "${TEST_ALL_MODULES}" == "true" ]]; then
    # Assign the modules to modules_assigned_list (based on num jobs)
    num=0
    for module in "${modified_module_list[@]}"; do
      # Add 1 as JOB_NUMBER is 1-indexed instead of 0-indexed
      mod_num=$((num % NUM_JOBS + 1))
      # Spaces are intentionally added -- Query is regex and array elements are space separated
      # It tries to match the *exact* `module` text
      if [[ ! " ${excluded_modules[*]} " =~ " ${module} " ]] && [[ $mod_num -eq $JOB_NUMBER ]]; then
        modules_assigned_list+=("${module}")
      fi
      num=$((num + 1))
    done
  elif [[ ${#modified_module_list[@]} -gt 0 ]]; then
    # MAVEN_MODULES ENV_VAR is expecting comma delimited string (similar to mvn -pl)
    # This will get all the modules and put all the elements into an array
    maven_modules_list=($(echo "${MAVEN_MODULES}" | tr ',' ' '))
    for maven_module in "${maven_modules_list[@]}"; do
      # Check that the modified_module_list contains a module from MAVEN_MODULES
      # Spaces are intentionally added -- Query is regex and array elements are space separated
      # It tries to match the *exact* `maven_module` text
      if [[ " ${modified_module_list[*]} " =~ " ${maven_module} " ]]; then
        modules_assigned_list+=("${module}")
      fi
    done
  fi
  module_list=$(
    IFS=,
    echo "${modules_assigned_list[*]}"
  )
}

function install_modules() {
  retry_with_backoff 3 10 \
    mvn -B \
    -ntp \
    -DtrimStackTrace=false \
    -Dclirr.skip=true \
    -Denforcer.skip=true \
    -Dorg.slf4j.simpleLogger.showDateTime=true -Dorg.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss:SSS \
    -Dcheckstyle.skip=true \
    -Dflatten.skip=true \
    -Danimal.sniffer.skip=true \
    -DskipTests=true \
    -Djacoco.skip=true \
    -T 1C \
    install
}