1#!/usr/bin/env bash 2# Copyright 2020 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# TODO(sergiitk): move to grpc/grpc when implementing support of other languages 16set -eo pipefail 17 18# Constants 19readonly PYTHON_VERSION="${PYTHON_VERSION:-3.9}" 20# Test driver 21readonly TEST_DRIVER_REPO_NAME="grpc" 22readonly TEST_DRIVER_REPO_URL="https://github.com/${TEST_DRIVER_REPO_OWNER:-grpc}/grpc.git" 23readonly TEST_DRIVER_BRANCH="${TEST_DRIVER_BRANCH:-master}" 24readonly TEST_DRIVER_PATH="tools/run_tests/xds_k8s_test_driver" 25readonly TEST_DRIVER_PROTOS_PATH="src/proto/grpc/testing" 26readonly FORCE_TESTING_VERSION="${FORCE_TESTING_VERSION:-}" 27 28# GKE cluster identifiers. 29readonly GKE_CLUSTER_PSM_LB="psm-lb" 30readonly GKE_CLUSTER_PSM_SECURITY="psm-security" 31readonly GKE_CLUSTER_PSM_BASIC="psm-basic" 32 33####################################### 34# Determines the cluster name and zone based on the given cluster identifier. 35# Globals: 36# GKE_CLUSTER_NAME: Set to reflect the cluster name to use 37# GKE_CLUSTER_ZONE: Set to reflect the cluster zone to use. 38# Arguments: 39# The cluster identifier 40# Outputs: 41# Writes the output to stdout, stderr 42####################################### 43activate_gke_cluster() { 44 case $1 in 45 GKE_CLUSTER_PSM_LB) 46 GKE_CLUSTER_NAME="psm-interop-lb-primary" 47 GKE_CLUSTER_ZONE="us-central1-a" 48 ;; 49 GKE_CLUSTER_PSM_SECURITY) 50 GKE_CLUSTER_NAME="psm-interop-security" 51 GKE_CLUSTER_ZONE="us-central1-a" 52 ;; 53 GKE_CLUSTER_PSM_BASIC) 54 GKE_CLUSTER_NAME="interop-test-psm-basic" 55 GKE_CLUSTER_ZONE="us-central1-c" 56 ;; 57 *) 58 echo "Unknown GKE cluster: ${1}" 59 exit 1 60 ;; 61 esac 62 echo "Activated GKE cluster: GKE_CLUSTER_NAME=${GKE_CLUSTER_NAME} GKE_CLUSTER_ZONE=${GKE_CLUSTER_ZONE}" 63} 64 65####################################### 66# Determines the secondary cluster name and zone based on the given cluster 67# identifier. 68# Globals: 69# GKE_CLUSTER_NAME: Set to reflect the cluster name to use 70# GKE_CLUSTER_ZONE: Set to reflect the cluster zone to use. 71# Arguments: 72# The cluster identifier 73# Outputs: 74# Writes the output to stdout, stderr 75####################################### 76activate_secondary_gke_cluster() { 77 case $1 in 78 GKE_CLUSTER_PSM_LB) 79 SECONDARY_GKE_CLUSTER_NAME="psm-interop-lb-secondary" 80 SECONDARY_GKE_CLUSTER_ZONE="us-west1-b" 81 ;; 82 *) 83 echo "Unknown secondary GKE cluster: ${1}" 84 exit 1 85 ;; 86 esac 87 echo "Activated secondary GKE cluster: GKE_CLUSTER_NAME=${GKE_CLUSTER_NAME} GKE_CLUSTER_ZONE=${GKE_CLUSTER_ZONE}" 88} 89 90####################################### 91# Run command end report its exit code. Doesn't exit on non-zero exit code. 92# Globals: 93# None 94# Arguments: 95# Command to execute 96# Outputs: 97# Writes the output of given command to stdout, stderr 98####################################### 99run_ignore_exit_code() { 100 local exit_code=-1 101 "$@" || exit_code=$? 102 echo "Exit code: ${exit_code}" 103} 104 105####################################### 106# Parses information about git repository at given path to global variables. 107# Globals: 108# GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build 109# GIT_COMMIT: Populated with the SHA-1 of git commit being built 110# GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built 111# Arguments: 112# Git source dir 113####################################### 114parse_src_repo_git_info() { 115 local src_dir="${SRC_DIR:?SRC_DIR must be set}" 116 readonly GIT_ORIGIN_URL=$(git -C "${src_dir}" remote get-url origin) 117 readonly GIT_COMMIT=$(git -C "${src_dir}" rev-parse HEAD) 118 readonly GIT_COMMIT_SHORT=$(git -C "${src_dir}" rev-parse --short HEAD) 119} 120 121 122####################################### 123# Checks if the given string is a version branch. 124# Version branches: "master", "v1.47.x" 125# NOT version branches: "v1.47.0", "1.47.x", "", "dev", "main" 126# Arguments: 127# Version to test 128####################################### 129is_version_branch() { 130 if [ $# -eq 0 ]; then 131 echo "Usage is_version_branch VERSION" 132 false 133 return 134 fi 135 if [[ $1 == "master" ]]; then 136 true 137 return 138 fi 139 # Do not inline version_regex: keep it a string to avoid issues with escaping chars in ~= expr. 140 local version_regex='^v[0-9]+\.[0-9]+\.x$' 141 [[ "${1}" =~ $version_regex ]] 142} 143 144####################################### 145# List GCR image tags matching given tag name. 146# Arguments: 147# Image name 148# Tag name 149# Outputs: 150# Writes the table with the list of found tags to stdout. 151# If no tags found, the output is an empty string. 152####################################### 153gcloud_gcr_list_image_tags() { 154 gcloud container images list-tags --format="table[box](tags,digest,timestamp.date())" --filter="tags:$2" "$1" 155} 156 157####################################### 158# A helper to execute `gcloud -q components update`. 159# Arguments: 160# None 161# Outputs: 162# Writes the output of `gcloud` command to stdout, stderr 163####################################### 164gcloud_update() { 165 echo "Update gcloud components:" 166 gcloud -q components update 167} 168 169####################################### 170# Create kube context authenticated with GKE cluster, saves context name. 171# to KUBE_CONTEXT 172# Globals: 173# GKE_CLUSTER_NAME 174# GKE_CLUSTER_ZONE 175# KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access 176# SECONDARY_KUBE_CONTEXT: Populated with name of kubectl context with secondary GKE cluster access, if any 177# Arguments: 178# None 179# Outputs: 180# Writes the output of `gcloud` command to stdout, stderr 181# Writes authorization info $HOME/.kube/config 182####################################### 183gcloud_get_cluster_credentials() { 184 if [[ -n "${SECONDARY_GKE_CLUSTER_NAME}" && -n "${SECONDARY_GKE_CLUSTER_ZONE}" ]]; then 185 gcloud container clusters get-credentials "${SECONDARY_GKE_CLUSTER_NAME}" --zone "${SECONDARY_GKE_CLUSTER_ZONE}" 186 readonly SECONDARY_KUBE_CONTEXT="$(kubectl config current-context)" 187 else 188 readonly SECONDARY_KUBE_CONTEXT="" 189 fi 190 gcloud container clusters get-credentials "${GKE_CLUSTER_NAME}" --zone "${GKE_CLUSTER_ZONE}" 191 readonly KUBE_CONTEXT="$(kubectl config current-context)" 192} 193 194####################################### 195# Clone the source code of the test driver to $TEST_DRIVER_REPO_DIR, unless 196# given folder exists. 197# Globals: 198# TEST_DRIVER_REPO_URL 199# TEST_DRIVER_BRANCH 200# TEST_DRIVER_REPO_DIR: path to the repo containing the test driver 201# TEST_DRIVER_REPO_DIR_USE_EXISTING: set non-empty value to use exiting 202# clone of the driver repo located at $TEST_DRIVER_REPO_DIR. 203# Useful for debugging the build script locally. 204# Arguments: 205# None 206# Outputs: 207# Writes the output of `git` command to stdout, stderr 208# Writes driver source code to $TEST_DRIVER_REPO_DIR 209####################################### 210test_driver_get_source() { 211 if [[ -n "${TEST_DRIVER_REPO_DIR_USE_EXISTING}" && -d "${TEST_DRIVER_REPO_DIR}" ]]; then 212 echo "Using exiting driver directory: ${TEST_DRIVER_REPO_DIR}." 213 else 214 echo "Cloning driver to ${TEST_DRIVER_REPO_URL} branch ${TEST_DRIVER_BRANCH} to ${TEST_DRIVER_REPO_DIR}" 215 git clone -b "${TEST_DRIVER_BRANCH}" --depth=1 "${TEST_DRIVER_REPO_URL}" "${TEST_DRIVER_REPO_DIR}" 216 fi 217} 218 219####################################### 220# Install Python modules from required in $TEST_DRIVER_FULL_DIR/requirements.lock 221# to Python virtual environment. Creates and activates Python venv if necessary. 222# Globals: 223# TEST_DRIVER_FULL_DIR 224# PYTHON_VERSION 225# Arguments: 226# None 227# Outputs: 228# Writes the output of `python`, `pip` commands to stdout, stderr 229# Writes the list of installed modules to stdout 230####################################### 231test_driver_pip_install() { 232 echo "Install python dependencies" 233 cd "${TEST_DRIVER_FULL_DIR}" 234 235 # Create and activate virtual environment unless already using one 236 if [[ -z "${VIRTUAL_ENV}" ]]; then 237 local venv_dir="${TEST_DRIVER_FULL_DIR}/venv" 238 if [[ -d "${venv_dir}" ]]; then 239 echo "Found python virtual environment directory: ${venv_dir}" 240 else 241 echo "Creating python virtual environment: ${venv_dir}" 242 "python${PYTHON_VERSION}" -m venv "${venv_dir}" 243 fi 244 # Intentional: No need to check python venv activate script. 245 # shellcheck source=/dev/null 246 source "${venv_dir}/bin/activate" 247 fi 248 249 python3 -m pip install -r requirements.lock 250 echo "Installed Python packages:" 251 python3 -m pip list 252} 253 254####################################### 255# Compile proto-files needed for the test driver 256# Globals: 257# TEST_DRIVER_REPO_DIR 258# TEST_DRIVER_FULL_DIR 259# TEST_DRIVER_PROTOS_PATH 260# Arguments: 261# None 262# Outputs: 263# Writes the output of `python -m grpc_tools.protoc` to stdout, stderr 264# Writes the list if compiled python code to stdout 265# Writes compiled python code with proto messages and grpc services to 266# $TEST_DRIVER_FULL_DIR/src/proto 267####################################### 268test_driver_compile_protos() { 269 declare -a protos 270 protos=( 271 "${TEST_DRIVER_PROTOS_PATH}/test.proto" 272 "${TEST_DRIVER_PROTOS_PATH}/messages.proto" 273 "${TEST_DRIVER_PROTOS_PATH}/empty.proto" 274 ) 275 echo "Generate python code from grpc.testing protos: ${protos[*]}" 276 cd "${TEST_DRIVER_REPO_DIR}" 277 python3 -m grpc_tools.protoc \ 278 --proto_path=. \ 279 --python_out="${TEST_DRIVER_FULL_DIR}" \ 280 --grpc_python_out="${TEST_DRIVER_FULL_DIR}" \ 281 "${protos[@]}" 282 local protos_out_dir="${TEST_DRIVER_FULL_DIR}/${TEST_DRIVER_PROTOS_PATH}" 283 echo "Generated files ${protos_out_dir}:" 284 ls -Fl "${protos_out_dir}" 285} 286 287####################################### 288# Installs the test driver and it's requirements. 289# https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#installation 290# Globals: 291# TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing 292# the test driver 293# TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code 294# Arguments: 295# The directory for test driver's source code 296# Outputs: 297# Writes the output to stdout, stderr 298####################################### 299test_driver_install() { 300 readonly TEST_DRIVER_REPO_DIR="${1:?Usage test_driver_install TEST_DRIVER_REPO_DIR}" 301 readonly TEST_DRIVER_FULL_DIR="${TEST_DRIVER_REPO_DIR}/${TEST_DRIVER_PATH}" 302 test_driver_get_source 303 test_driver_pip_install 304 test_driver_compile_protos 305} 306 307####################################### 308# Outputs Kokoro image version and Ubuntu's lsb_release 309# Arguments: 310# None 311# Outputs: 312# Writes the output to stdout 313####################################### 314kokoro_print_version() { 315 echo "Kokoro VM version:" 316 if [[ -f /VERSION ]]; then 317 cat /VERSION 318 fi 319 run_ignore_exit_code lsb_release -a 320} 321 322####################################### 323# Report extra information about the job via sponge properties. 324# Globals: 325# KOKORO_ARTIFACTS_DIR 326# GIT_ORIGIN_URL 327# GIT_COMMIT_SHORT 328# TESTGRID_EXCLUDE 329# Arguments: 330# None 331# Outputs: 332# Writes the output to stdout 333# Writes job properties to $KOKORO_ARTIFACTS_DIR/custom_sponge_config.csv 334####################################### 335kokoro_write_sponge_properties() { 336 # CSV format: "property_name","property_value" 337 # Bump TESTS_FORMAT_VERSION when reported test name changed enough to when it 338 # makes more sense to discard previous test results from a testgrid board. 339 # Use GIT_ORIGIN_URL to exclude test runs executed against repo forks from 340 # testgrid reports. 341 cat >"${KOKORO_ARTIFACTS_DIR}/custom_sponge_config.csv" <<EOF 342TESTS_FORMAT_VERSION,2 343TESTGRID_EXCLUDE,${TESTGRID_EXCLUDE:-0} 344GIT_ORIGIN_URL,${GIT_ORIGIN_URL:?GIT_ORIGIN_URL must be set} 345GIT_COMMIT_SHORT,${GIT_COMMIT_SHORT:?GIT_COMMIT_SHORT must be set} 346EOF 347 echo "Sponge properties:" 348 cat "${KOKORO_ARTIFACTS_DIR}/custom_sponge_config.csv" 349} 350 351####################################### 352# Configure Python virtual environment on Kokoro VM. 353# Arguments: 354# None 355# Outputs: 356# Writes the output of `pyenv` commands to stdout 357####################################### 358kokoro_setup_python_virtual_environment() { 359 # Kokoro provides pyenv, so use it instead of `python -m venv` 360 echo "Setup pyenv environment" 361 eval "$(pyenv init -)" 362 eval "$(pyenv virtualenv-init -)" 363 py_latest_patch="$(pyenv versions --bare --skip-aliases | grep -E "^${PYTHON_VERSION}\.[0-9]{1,2}$" | sort --version-sort | tail -n 1)" 364 echo "Activating python ${py_latest_patch} virtual environment" 365 pyenv virtualenv --without-pip "${py_latest_patch}" k8s_xds_test_runner 366 pyenv local k8s_xds_test_runner 367 pyenv activate k8s_xds_test_runner 368 python3 -m ensurepip 369 # pip is fixed to 21.0.1 due to issue https://github.com/pypa/pip/pull/9835 370 # internal details: b/186411224 371 # TODO(sergiitk): revert https://github.com/grpc/grpc/pull/26087 when 21.1.1 released 372 python3 -m pip install -U pip==21.0.1 373 python3 -m pip --version 374} 375 376####################################### 377# Determines the version branch under test from Kokoro environment. 378# Globals: 379# KOKORO_JOB_NAME 380# KOKORO_BUILD_INITIATOR 381# FORCE_TESTING_VERSION: Forces the testing version to be something else. 382# TESTING_VERSION: Populated with the version branch under test, 383# f.e. master, dev, v1.42.x. 384# Outputs: 385# Sets TESTING_VERSION global variable. 386####################################### 387kokoro_get_testing_version() { 388 # All grpc kokoro jobs names structured to have the version identifier in the third position: 389 # - grpc/core/master/linux/... 390 # - grpc/core/v1.42.x/branch/linux/... 391 # - grpc/java/v1.47.x/branch/... 392 # - grpc/go/v1.47.x/branch/... 393 # - grpc/node/v1.6.x/... 394 local version_from_job_name 395 version_from_job_name=$(echo "${KOKORO_JOB_NAME}" | cut -d '/' -f3) 396 397 if [[ -n "${FORCE_TESTING_VERSION}" ]]; then 398 # Allows to override the testing version, and force tagging the built 399 # images, if necessary. 400 readonly TESTING_VERSION="${FORCE_TESTING_VERSION}" 401 elif [[ "${KOKORO_BUILD_INITIATOR:-anonymous}" != "kokoro" ]]; then 402 # If not initiated by Kokoro, it's a dev branch. 403 # This allows to know later down the line that the built image doesn't need 404 # to be tagged, and avoid overriding an actual versioned image used in tests 405 # (e.g. v1.42.x, master) with a dev build. 406 if [[ -n "${version_from_job_name}" ]]; then 407 readonly TESTING_VERSION="dev-${version_from_job_name}" 408 else 409 readonly TESTING_VERSION="dev" 410 fi 411 else 412 readonly TESTING_VERSION="${version_from_job_name}" 413 fi 414} 415 416####################################### 417# Installs and configures the test driver on Kokoro VM. 418# Globals: 419# KOKORO_ARTIFACTS_DIR 420# KOKORO_JOB_NAME 421# TEST_DRIVER_REPO_NAME 422# TESTING_VERSION: Populated with the version branch under test, f.e. v1.42.x, master 423# SRC_DIR: Populated with absolute path to the source repo on Kokoro VM 424# TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing 425# the test driver 426# TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code 427# TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile 428# TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report 429# KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access 430# SECONDARY_KUBE_CONTEXT: Populated with name of kubectl context with secondary GKE cluster access, if any 431# GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build 432# GIT_COMMIT: Populated with the SHA-1 of git commit being built 433# GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built 434# Arguments: 435# The name of github repository being built 436# Outputs: 437# Writes the output to stdout, stderr, files 438####################################### 439kokoro_setup_test_driver() { 440 local src_repository_name="${1:?Usage kokoro_setup_test_driver GITHUB_REPOSITORY_NAME}" 441 # Capture Kokoro VM version info in the log. 442 kokoro_print_version 443 444 # Get testing version from the job name. 445 kokoro_get_testing_version 446 447 # Kokoro clones repo to ${KOKORO_ARTIFACTS_DIR}/github/${GITHUB_REPOSITORY} 448 local github_root="${KOKORO_ARTIFACTS_DIR}/github" 449 readonly SRC_DIR="${github_root}/${src_repository_name}" 450 local test_driver_repo_dir 451 test_driver_repo_dir="${TEST_DRIVER_REPO_DIR:-$(mktemp -d)/${TEST_DRIVER_REPO_NAME}}" 452 parse_src_repo_git_info SRC_DIR 453 kokoro_write_sponge_properties 454 kokoro_setup_python_virtual_environment 455 456 # gcloud requires python, so this should be executed after pyenv setup 457 gcloud_update 458 gcloud_get_cluster_credentials 459 test_driver_install "${test_driver_repo_dir}" 460 # shellcheck disable=SC2034 # Used in the main script 461 readonly TEST_DRIVER_FLAGFILE="config/grpc-testing.cfg" 462 # Test artifacts dir: xml reports, logs, etc. 463 local artifacts_dir="${KOKORO_ARTIFACTS_DIR}/artifacts" 464 # Folders after $artifacts_dir reported as target name 465 readonly TEST_XML_OUTPUT_DIR="${artifacts_dir}/${KOKORO_JOB_NAME}" 466 mkdir -p "${artifacts_dir}" "${TEST_XML_OUTPUT_DIR}" 467} 468 469####################################### 470# Installs and configures the test driver for testing build script locally. 471# Globals: 472# TEST_DRIVER_REPO_NAME 473# TEST_DRIVER_REPO_DIR: Unless provided, populated with a temporary dir with 474# the path to the test driver repo 475# SRC_DIR: Populated with absolute path to the source repo 476# KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access 477# TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile 478# TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report 479# GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build 480# GIT_COMMIT: Populated with the SHA-1 of git commit being built 481# GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built 482# SECONDARY_KUBE_CONTEXT: Populated with name of kubectl context with secondary GKE cluster access, if any 483# Arguments: 484# The path to the folder containing the build script 485# Outputs: 486# Writes the output to stdout, stderr, files 487####################################### 488local_setup_test_driver() { 489 local script_dir="${1:?Usage: local_setup_test_driver SCRIPT_DIR}" 490 readonly SRC_DIR="$(git -C "${script_dir}" rev-parse --show-toplevel)" 491 parse_src_repo_git_info "${SRC_DIR}" 492 readonly KUBE_CONTEXT="${KUBE_CONTEXT:-$(kubectl config current-context)}" 493 readonly SECONDARY_KUBE_CONTEXT="${SECONDARY_KUBE_CONTEXT}" 494 495 # Never override docker image for local runs, unless explicitly forced. 496 if [[ -n "${FORCE_TESTING_VERSION}" ]]; then 497 readonly TESTING_VERSION="${FORCE_TESTING_VERSION}" 498 else 499 readonly TESTING_VERSION="dev" 500 fi 501 502 local test_driver_repo_dir 503 test_driver_repo_dir="${TEST_DRIVER_REPO_DIR:-$(mktemp -d)/${TEST_DRIVER_REPO_NAME}}" 504 test_driver_install "${test_driver_repo_dir}" 505 506 # shellcheck disable=SC2034 # Used in the main script 507 readonly TEST_DRIVER_FLAGFILE="config/local-dev.cfg" 508 # Test out 509 readonly TEST_XML_OUTPUT_DIR="${TEST_DRIVER_FULL_DIR}/out" 510 mkdir -p "${TEST_XML_OUTPUT_DIR}" 511} 512 513####################################### 514# Tag and push the given Docker image 515# Arguments: 516# The Docker image name 517# The Docker image original tag name 518# The Docker image new tag name 519# Outputs: 520# Writes the output to stdout, stderr, files 521####################################### 522tag_and_push_docker_image() { 523 local image_name="$1" 524 local from_tag="$2" 525 local to_tag="$3" 526 527 docker tag "${image_name}:${from_tag}" "${image_name}:${to_tag}" 528 docker push "${image_name}:${to_tag}" 529} 530