1#!/bin/bash 2# Copyright 2017 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# Source this rc script to prepare the environment for macos builds 16 17# Print basic info about the mac worker 18echo "kokoro pool: \"$KOKORO_JOB_POOL\"" 19echo "OS: $(sw_vers -productName) $(sw_vers -productVersion) $(sw_vers -buildVersion)" 20echo "CPU type: $(sysctl -n machdep.cpu.brand_string)" 21echo "CPU: $(sysctl -n machdep.cpu.vendor) $(uname -m) Family $(sysctl -n machdep.cpu.family) Model $(sysctl -n machdep.cpu.brand_string)" 22echo "CPU Cores: $(sysctl -n hw.ncpu)" 23echo "Memory: $(sysctl -n hw.memsize)" 24echo "Kokoro image version: $(cat /VERSION)" 25 26# Info on disk usage and mounted volumes 27mount 28df -h /tmpfs / 29 30# show original open file limit values 31launchctl limit maxfiles 32ulimit -a 33 34# WARNING: TRY TO KEEP THIS FILE AS LEAN AS POSSIBLE 35# The rules: 36# - think twice before adding installer that takes long time (builds are already slow) 37# - do not add any steps that are flaky (builds will become flaky) 38# - try adding a dependency under a language-specific section first (reduces latency and increases build stability) 39# - only add stuff that you absolutely need for your builds to work (add comment to explain why its needed) 40 41# Disable HOMEBREW update to avoid new updates which potentially have problems. 42# Brew packages installed when Kokoro image was built tend to have less conflict. 43export HOMEBREW_NO_AUTO_UPDATE=1 44 45# Dump the brew configuration for debugging just in case. Check "Core tap HEAD" field 46# because it should be the same as below unless it's been updated. 47# - Kokoro MacOS Mojave: 27fa87c94a6cf7be40fc8f8fc96bc7c387b7781e 48brew config 49 50# Add GCP credentials for BQ access 51pip install --user google-api-python-client oauth2client six==1.16.0 52export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json 53DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" 54 55# If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests 56if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ]; then 57 export RUN_TESTS_FLAGS="--filter_pr_tests --base_branch origin/$KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH $RUN_TESTS_FLAGS" 58fi 59 60if [ "${PREPARE_BUILD_INSTALL_DEPS_RUBY}" == "true" ] 61then 62 # Workaround for https://github.com/rvm/rvm/issues/5133 63 export CURL_CA_BUNDLE=$(pwd)/etc/roots.pem 64 65 # Fetch keys per https://rvm.io/rvm/install 66 gpg_recv_keys_success=0 67 for ((i=0;i<5;i++)); do 68 # Use the Ubuntu keyserver instead of pool.sks-keyservers.net because sks-keyservers is now deprecated. 69 GPG_KEYSERVER_ADDRESS="keyserver.ubuntu.com" 70 gpg --keyserver "hkp://${GPG_KEYSERVER_ADDRESS}" --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB \ 71 && gpg_recv_keys_success=1 72 [[ "$gpg_recv_keys_success" == 1 ]] && break 73 sleep 3 74 done 75 [[ "$gpg_recv_keys_success" == 1 ]] || exit 1 76 # fix .rvm directory ownership on kokoro monterey image 77 sudo chown -R "${USER}" ~/.rvm 78 rvm get stable # Per https://stackoverflow.com/questions/65477613/rvm-where-is-ruby-3-0-0 79 # stop echoing bash commands temporarily to prevent rvm from polluting the logs 80 set +x 81 source $HOME/.rvm/scripts/rvm 82 83 RUBY_VERSION=3.2.0 84 echo "Installing ruby-${RUBY_VERSION}" 85 time rvm install "ruby-${RUBY_VERSION}" 86 echo "Setting default ruby version." 87 rvm use "$RUBY_VERSION" --default 88 echo "Updating osx-ssl-certs." 89 gem install bundler -v 2.4 90 rvm osx-ssl-certs status all 91 rvm osx-ssl-certs update all 92 93 # restore echo 94 set -x 95fi 96 97if [ "${PREPARE_BUILD_INSTALL_DEPS_OBJC}" == "true" ] 98then 99 # cocoapods 100 export LANG=en_US.UTF-8 101 # use "sudo" to avoid permission error on kokoro monterey image 102 time sudo gem install cocoapods --version 1.11.3 --no-document --user-install 103 # pre-fetch cocoapods master repo's most recent commit only 104 mkdir -p ~/.cocoapods/repos 105 time git clone --depth 1 https://github.com/CocoaPods/Specs.git ~/.cocoapods/repos/master 106 107 # Needed for ios-binary-size 108 time pip install --user -r $DIR/requirements.macos.txt 109 110 # Store intermediate build files of ObjC tests into /tmpfs 111 # TODO(jtattermusch): this has likely been done to avoid running 112 # out of disk space when running ios-binary-size tests, but 113 # it would be good to revisit. 114 mkdir /tmpfs/Build-ios-binary-size 115 ln -s /tmpfs/Build-ios-binary-size src/objective-c/examples/Sample/Build 116 mkdir -p /tmpfs/DerivedData 117 rm -rf ~/Library/Developer/Xcode/DerivedData 118 mkdir -p ~/Library/Developer/Xcode 119 ln -s /tmpfs/DerivedData ~/Library/Developer/Xcode/DerivedData 120fi 121 122if [ "${PREPARE_BUILD_INSTALL_DEPS_PYTHON}" == "true" ] 123then 124 # python 125 time pip install --user -r $DIR/requirements.macos.txt 126 time pip install --user --upgrade virtualenv Mako tox setuptools==44.1.1 twisted 127 128 # Force reinstall of python3.7 to upgrade its patch version. 129 # Kokoro mac workers come with python 3.7.3 and that version 130 # suffers from https://bugs.python.org/issue22490. 131 # See https://github.com/grpc/grpc/pull/28963. 132 time curl -O https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg 133 echo "bf54a14eef23467991e8c7a88c7307762e484c024a94ec1ee292ac1db3d41fc9 python-3.7.9-macosx10.9.pkg" > /tmp/python_installer_checksum.sha256 134 shasum -c /tmp/python_installer_checksum.sha256 135 time sudo installer -pkg ./python-3.7.9-macosx10.9.pkg -target / 136 python3.7 --version 137 138 # Force reinstall of python3.8 to upgrade its patch version. 139 # Kokoro mac workers come with python 3.8.2 and that version 140 # suffers from https://bugs.python.org/issue22490. 141 # See https://github.com/grpc/grpc/pull/28963. 142 time curl -O https://www.python.org/ftp/python/3.8.10/python-3.8.10-macosx10.9.pkg 143 echo "4c65bc7534d5f07edacbe0fbd609b5734dbf3ac02f5444f9bd97963d589d8afd python-3.8.10-macosx10.9.pkg" > /tmp/python_installer_checksum.sha256 144 shasum -c /tmp/python_installer_checksum.sha256 145 time sudo installer -pkg ./python-3.8.10-macosx10.9.pkg -target / 146 python3.8 --version 147 148 # Install Python 3.9 if it doesn't exist 149 if [ ! -f "/usr/local/bin/python3.9" ]; then 150 time curl -O https://www.python.org/ftp/python/3.9.10/python-3.9.10-macosx10.9.pkg 151 echo "732bc5c95ae127dfb6fb1bcf683509ad20c558152b63b8d5f651246f6bdfc8da python-3.9.10-macosx10.9.pkg" > /tmp/python_installer_checksum.sha256 152 shasum -c /tmp/python_installer_checksum.sha256 153 time sudo installer -pkg ./python-3.9.10-macosx10.9.pkg -target / 154 fi 155 156 # Install Python 3.10 if it doesn't exist 157 if [ ! -f "/usr/local/bin/python3.10" ]; then 158 time curl -O https://www.python.org/ftp/python/3.10.5/python-3.10.5-macos11.pkg 159 echo "568c69b4361af1faf0ae35c4cac7236c1a332f5c python-3.10.5-macos11.pkg" > /tmp/python_installer_checksum.sha256 160 shasum -c /tmp/python_installer_checksum.sha256 161 time sudo installer -pkg ./python-3.10.5-macos11.pkg -target / 162 fi 163 164 # Install Python 3.11 if it doesn't exist 165 if [ ! -f "/usr/local/bin/python3.11" ]; then 166 time curl -O https://www.python.org/ftp/python/3.11.0/python-3.11.0rc1-macos11.pkg 167 echo "eec3f817797b1d61f48f29b96ac0443ea19f3170 python-3.11.0rc1-macos11.pkg" > /tmp/python_installer_checksum.sha256 168 shasum -c /tmp/python_installer_checksum.sha256 169 time sudo installer -pkg ./python-3.11.0rc1-macos11.pkg -target / 170 fi 171fi 172 173if [ "${PREPARE_BUILD_INSTALL_DEPS_CSHARP}" == "true" ] 174then 175 # Disable some unwanted dotnet options 176 export NUGET_XMLDOC_MODE=skip 177 export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true 178 export DOTNET_CLI_TELEMETRY_OPTOUT=true 179 180 # Installed versions should be kept in sync with 181 # templates/tools/dockerfile/csharp_dotnetcli_deps.include 182 time curl -O https://download.visualstudio.microsoft.com/download/pr/e0fe8c99-e33c-4d75-bd4e-2478ed3ee35a/ff06e47afc7c13bdbbaa50a9713ac772/dotnet-sdk-3.1.415-osx-x64.pkg 183 time sudo installer -pkg ./dotnet-sdk-3.1.415-osx-x64.pkg -target / 184 185 time curl -O https://download.visualstudio.microsoft.com/download/pr/14a45451-4cc9-48e1-af69-0aff75891d09/ff6e83986a2a9a535015fb3104a90a1b/dotnet-sdk-6.0.100-osx-x64.pkg 186 time sudo installer -pkg ./dotnet-sdk-6.0.100-osx-x64.pkg -target / 187fi 188 189if [ "${PREPARE_BUILD_INSTALL_DEPS_PHP}" == "true" ] 190then 191 # php7.4 should already be installed on kokoro monterey image, so this should be instantaneous. 192 time brew install php@7.4 || true 193 export LDFLAGS="-L/usr/local/opt/php@7.4/lib ${LDFLAGS}" 194 export CPPFLAGS="-I/usr/local/opt/php@7.4/include ${CPPFLAGS}" 195 export PATH="/usr/local/opt/php@7.4/bin:/usr/local/opt/php@7.4/sbin:${PATH}" 196 197 # the exit code from "brew install [email protected]" is useless 198 # so instead we check if PHP was indeed installed successfully. 199 # Failing early is better than cryptic errors later in the build process. 200 /usr/local/opt/php@7.4/bin/php --version 201 202 # Workaround for https://github.com/Homebrew/homebrew-core/issues/41081 203 mkdir -p /usr/local/lib/php/pecl 204fi 205 206# TODO(jtattermusch): better debugging of clock skew, remove once not needed 207date 208 209time git submodule update --init 210