1#!/bin/bash 2# Copyright 2016 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16set -ex 17 18cd "$(dirname "$0")/../../.." 19 20export GRPC_PYTHON_BUILD_WITH_CYTHON=1 21export PYTHON=${PYTHON:-python} 22export AUDITWHEEL=${AUDITWHEEL:-auditwheel} 23 24# activate ccache if desired 25# shellcheck disable=SC1091 26source tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc 27 28# Needed for building binary distribution wheels -- bdist_wheel 29"${PYTHON}" -m pip install --upgrade wheel 30 31if [ "$GRPC_SKIP_PIP_CYTHON_UPGRADE" == "" ] 32then 33 # Install Cython to avoid source wheel build failure. 34 # This only needs to be done when not running under docker (=on MacOS) 35 # since the docker images used for building python wheels 36 # already have a new-enough version of cython pre-installed. 37 # Any installation step is a potential source of breakages, 38 # so we are trying to perform as few download-and-install operations 39 # as possible. 40 "${PYTHON}" -m pip install --upgrade cython 41fi 42 43# Allow build_ext to build C/C++ files in parallel 44# by enabling a monkeypatch. It speeds up the build a lot. 45# Use externally provided GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS value if set. 46export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=${GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS:-2} 47 48mkdir -p "${ARTIFACTS_OUT}" 49ARTIFACT_DIR="$PWD/${ARTIFACTS_OUT}" 50 51# check whether we are crosscompiling. AUDITWHEEL_ARCH is set by the dockcross docker image. 52if [ "$AUDITWHEEL_ARCH" == "aarch64" ] 53then 54 # when crosscompiling for aarch64, --plat-name needs to be set explicitly 55 # to end up with correctly named wheel file 56 # the value should be manylinuxABC_ARCH and dockcross docker image 57 # conveniently provides the value in the AUDITWHEEL_PLAT env 58 WHEEL_PLAT_NAME_FLAG="--plat-name=$AUDITWHEEL_PLAT" 59 60 # override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix 61 GRPC_PYTHON_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-aarch64-linux-gnu.so"))')" 62 export GRPC_PYTHON_OVERRIDE_EXT_SUFFIX 63 64 # since we're crosscompiling, we need to explicitly choose the right platform for boringssl assembly optimizations 65 export GRPC_BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM="linux-aarch64" 66fi 67 68# check whether we are crosscompiling. AUDITWHEEL_ARCH is set by the dockcross docker image. 69if [ "$AUDITWHEEL_ARCH" == "armv7l" ] 70then 71 # when crosscompiling for arm, --plat-name needs to be set explicitly 72 # to end up with correctly named wheel file 73 # our dockcross-based docker image onveniently provides the value in the AUDITWHEEL_PLAT env 74 WHEEL_PLAT_NAME_FLAG="--plat-name=$AUDITWHEEL_PLAT" 75 76 # override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix 77 GRPC_PYTHON_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-arm-linux-gnueabihf.so"))')" 78 export GRPC_PYTHON_OVERRIDE_EXT_SUFFIX 79 80 # since we're crosscompiling, we need to explicitly choose the right platform for boringssl assembly optimizations 81 export GRPC_BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM="linux-arm" 82fi 83 84# Build the source distribution first because MANIFEST.in cannot override 85# exclusion of built shared objects among package resources (for some 86# inexplicable reason). 87${SETARCH_CMD} "${PYTHON}" setup.py sdist 88 89# Wheel has a bug where directories don't get excluded. 90# https://bitbucket.org/pypa/wheel/issues/99/cannot-exclude-directory 91# shellcheck disable=SC2086 92${SETARCH_CMD} "${PYTHON}" setup.py bdist_wheel $WHEEL_PLAT_NAME_FLAG 93 94GRPCIO_STRIP_TEMPDIR=$(mktemp -d) 95GRPCIO_TAR_GZ_LIST=( dist/grpcio-*.tar.gz ) 96GRPCIO_TAR_GZ=${GRPCIO_TAR_GZ_LIST[0]} 97GRPCIO_STRIPPED_TAR_GZ=$(mktemp -t "TAR_GZ_XXXXXXXXXX") 98 99clean_non_source_files() { 100( cd "$1" 101 find . -type f \ 102 | grep -v '\.c$' | grep -v '\.cc$' | grep -v '\.cpp$' \ 103 | grep -v '\.h$' | grep -v '\.hh$' | grep -v '\.inc$' \ 104 | grep -v '\.s$' | grep -v '\.py$' | grep -v '\.hpp$' \ 105 | grep -v '\.S$' | grep -v '\.asm$' \ 106 | while read -r file; do 107 rm -f "$file" || true 108 done 109 find . -type d -empty -delete 110) 111} 112 113tar xzf "${GRPCIO_TAR_GZ}" -C "${GRPCIO_STRIP_TEMPDIR}" 114( cd "${GRPCIO_STRIP_TEMPDIR}" 115 find . -type d -name .git -exec rm -fr {} \; || true 116 for dir in */third_party/*; do 117 clean_non_source_files "${dir}" || true 118 done 119 tar czf "${GRPCIO_STRIPPED_TAR_GZ}" -- * 120 chmod ugo+r "${GRPCIO_STRIPPED_TAR_GZ}" 121) 122mv "${GRPCIO_STRIPPED_TAR_GZ}" "${GRPCIO_TAR_GZ}" 123 124# Build gRPC tools package distribution 125"${PYTHON}" tools/distrib/python/make_grpcio_tools.py 126 127# Build gRPC tools package source distribution 128${SETARCH_CMD} "${PYTHON}" tools/distrib/python/grpcio_tools/setup.py sdist 129 130# Build gRPC tools package binary distribution 131# shellcheck disable=SC2086 132${SETARCH_CMD} "${PYTHON}" tools/distrib/python/grpcio_tools/setup.py bdist_wheel $WHEEL_PLAT_NAME_FLAG 133 134# run twine check before auditwheel, because auditwheel puts the repaired wheels into 135# the artifacts output dir. 136if [ "$GRPC_SKIP_TWINE_CHECK" == "" ] 137then 138 # Install virtualenv if it isn't already available. 139 # TODO(jtattermusch): cleanup the virtualenv version fallback logic. 140 "${PYTHON}" -m pip install virtualenv 141 "${PYTHON}" -m virtualenv venv || { "${PYTHON}" -m pip install virtualenv==20.0.23 && "${PYTHON}" -m virtualenv venv; } 142 # Ensure the generated artifacts are valid using "twine check" 143 venv/bin/python -m pip install "twine<=2.0" 144 venv/bin/python -m twine check dist/* tools/distrib/python/grpcio_tools/dist/* 145 rm -rf venv/ 146fi 147 148assert_is_universal_wheel() { 149 WHL="$1" 150 TMPDIR=$(mktemp -d) 151 unzip "$WHL" -d "$TMPDIR" 152 SO=$(find "$TMPDIR" -name '*.so' | head -n1) 153 if ! file "$SO" | grep "Mach-O universal binary with 2 architectures"; then 154 echo "$WHL is not universal2. Found the following:" >/dev/stderr 155 file "$SO" >/dev/stderr 156 exit 1 157 fi 158} 159 160fix_faulty_universal2_wheel() { 161 WHL="$1" 162 assert_is_universal_wheel "$WHL" 163 if echo "$WHL" | grep "x86_64"; then 164 UPDATED_NAME="${WHL//x86_64/universal2}" 165 mv "$WHL" "$UPDATED_NAME" 166 fi 167} 168 169# This is necessary due to https://github.com/pypa/wheel/issues/406. 170# distutils incorrectly generates a universal2 artifact that only contains 171# x86_64 libraries. 172if [ "$GRPC_UNIVERSAL2_REPAIR" != "" ]; then 173 for WHEEL in dist/*.whl tools/distrib/python/grpcio_tools/dist/*.whl; do 174 fix_faulty_universal2_wheel "$WHEEL" 175 done 176fi 177 178 179if [ "$GRPC_RUN_AUDITWHEEL_REPAIR" != "" ] 180then 181 for wheel in dist/*.whl; do 182 "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep -E -w "$AUDITWHEEL_PLAT" 183 "${AUDITWHEEL}" repair "$wheel" --strip --wheel-dir "$ARTIFACT_DIR" 184 rm "$wheel" 185 done 186 for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do 187 "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr | grep -E -w "$AUDITWHEEL_PLAT" 188 "${AUDITWHEEL}" repair "$wheel" --strip --wheel-dir "$ARTIFACT_DIR" 189 rm "$wheel" 190 done 191else 192 cp -r dist/*.whl "$ARTIFACT_DIR" 193 cp -r tools/distrib/python/grpcio_tools/dist/*.whl "$ARTIFACT_DIR" 194fi 195 196# grpcio and grpcio-tools wheels have already been copied to artifact_dir 197# by "auditwheel repair", now copy the .tar.gz source archives as well. 198cp -r dist/*.tar.gz "$ARTIFACT_DIR" 199cp -r tools/distrib/python/grpcio_tools/dist/*.tar.gz "$ARTIFACT_DIR" 200 201# We need to use the built grpcio-tools/grpcio to compile the health proto 202# Wheels are not supported by setup_requires/dependency_links, so we 203# manually install the dependency. Note we should only do this if we 204# are in a docker image or in a virtualenv. 205if [ "$GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS" != "" ] 206then 207 "${PYTHON}" -m pip install -rrequirements.txt 208 209 if [ "$("$PYTHON" -c "import sys; print(sys.version_info[0])")" == "2" ] 210 then 211 "${PYTHON}" -m pip install futures>=2.2.0 enum34>=1.0.4 212 fi 213 214 "${PYTHON}" -m pip install grpcio --no-index --find-links "file://$ARTIFACT_DIR/" 215 "${PYTHON}" -m pip install grpcio-tools --no-index --find-links "file://$ARTIFACT_DIR/" 216 217 # Note(lidiz) setuptools's "sdist" command creates a source tarball, which 218 # demands an extra step of building the wheel. The building step is merely ran 219 # through setup.py, but we can optimize it with "bdist_wheel" command, which 220 # skips the wheel building step. 221 222 # Build grpcio_testing source distribution 223 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_testing/setup.py preprocess \ 224 sdist bdist_wheel 225 cp -r src/python/grpcio_testing/dist/* "$ARTIFACT_DIR" 226 227 # Build grpcio_channelz source distribution 228 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_channelz/setup.py \ 229 preprocess build_package_protos sdist bdist_wheel 230 cp -r src/python/grpcio_channelz/dist/* "$ARTIFACT_DIR" 231 232 # Build grpcio_health_checking source distribution 233 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_health_checking/setup.py \ 234 preprocess build_package_protos sdist bdist_wheel 235 cp -r src/python/grpcio_health_checking/dist/* "$ARTIFACT_DIR" 236 237 # Build grpcio_reflection source distribution 238 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_reflection/setup.py \ 239 preprocess build_package_protos sdist bdist_wheel 240 cp -r src/python/grpcio_reflection/dist/* "$ARTIFACT_DIR" 241 242 # Build grpcio_status source distribution 243 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_status/setup.py \ 244 preprocess sdist bdist_wheel 245 cp -r src/python/grpcio_status/dist/* "$ARTIFACT_DIR" 246 247 # Build grpcio_csds source distribution 248 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_csds/setup.py \ 249 sdist bdist_wheel 250 cp -r src/python/grpcio_csds/dist/* "$ARTIFACT_DIR" 251 252 # Build grpcio_admin source distribution and it needs the cutting-edge version 253 # of Channelz and CSDS to be installed. 254 "${PYTHON}" -m pip install --upgrade xds-protos==0.0.8 255 "${PYTHON}" -m pip install grpcio-channelz --no-index --find-links "file://$ARTIFACT_DIR/" 256 "${PYTHON}" -m pip install grpcio-csds --no-index --find-links "file://$ARTIFACT_DIR/" 257 ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_admin/setup.py \ 258 sdist bdist_wheel 259 cp -r src/python/grpcio_admin/dist/* "$ARTIFACT_DIR" 260fi 261