xref: /aosp_15_r20/external/grpc-grpc/tools/run_tests/artifacts/build_artifact_python.sh (revision cc02d7e222339f7a4f6ba5f422e6413f4bd931f2)
1#!/bin/bash
2# Copyright 2016 gRPC authors.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16set -ex
17
18cd "$(dirname "$0")/../../.."
19
20export GRPC_PYTHON_BUILD_WITH_CYTHON=1
21export PYTHON=${PYTHON:-python}
22export AUDITWHEEL=${AUDITWHEEL:-auditwheel}
23
24# activate ccache if desired
25# shellcheck disable=SC1091
26source tools/internal_ci/helper_scripts/prepare_ccache_symlinks_rc
27
28# Needed for building binary distribution wheels -- bdist_wheel
29"${PYTHON}" -m pip install --upgrade pip wheel setuptools
30
31if [ "$GRPC_SKIP_PIP_CYTHON_UPGRADE" == "" ]
32then
33  # Install Cython to avoid source wheel build failure.
34  # This only needs to be done when not running under docker (=on MacOS)
35  # since the docker images used for building python wheels
36  # already have a new-enough version of cython pre-installed.
37  # Any installation step is a potential source of breakages,
38  # so we are trying to perform as few download-and-install operations
39  # as possible.
40  "${PYTHON}" -m pip install --upgrade 'cython<3.0.0rc1'
41fi
42
43# Allow build_ext to build C/C++ files in parallel
44# by enabling a monkeypatch. It speeds up the build a lot.
45# Use externally provided GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS value if set.
46export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=${GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS:-2}
47
48mkdir -p "${ARTIFACTS_OUT}"
49ARTIFACT_DIR="$PWD/${ARTIFACTS_OUT}"
50
51# check whether we are crosscompiling. AUDITWHEEL_ARCH is set by the dockcross docker image.
52if [ "$AUDITWHEEL_ARCH" == "aarch64" ]
53then
54  # when crosscompiling for aarch64, --plat-name needs to be set explicitly
55  # to end up with correctly named wheel file
56  # the value should be manylinuxABC_ARCH and dockcross docker image
57  # conveniently provides the value in the AUDITWHEEL_PLAT env
58  WHEEL_PLAT_NAME_FLAG="--plat-name=$AUDITWHEEL_PLAT"
59
60  # override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix
61  GRPC_PYTHON_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-aarch64-linux-gnu.so"))')"
62  export GRPC_PYTHON_OVERRIDE_EXT_SUFFIX
63
64  # since we're crosscompiling, we need to explicitly choose the right platform for boringssl assembly optimizations
65  export GRPC_BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM="linux-aarch64"
66fi
67
68# check whether we are crosscompiling. AUDITWHEEL_ARCH is set by the dockcross docker image.
69if [ "$AUDITWHEEL_ARCH" == "armv7l" ]
70then
71  # when crosscompiling for arm, --plat-name needs to be set explicitly
72  # to end up with correctly named wheel file
73  # our dockcross-based docker image onveniently provides the value in the AUDITWHEEL_PLAT env
74  WHEEL_PLAT_NAME_FLAG="--plat-name=$AUDITWHEEL_PLAT"
75
76  # override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix
77  GRPC_PYTHON_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-arm-linux-gnueabihf.so"))')"
78  export GRPC_PYTHON_OVERRIDE_EXT_SUFFIX
79
80  # since we're crosscompiling, we need to explicitly choose the right platform for boringssl assembly optimizations
81  export GRPC_BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM="linux-arm"
82fi
83
84ancillary_package_dir=(
85  "src/python/grpcio_admin/"
86  "src/python/grpcio_channelz/"
87  "src/python/grpcio_csds/"
88  "src/python/grpcio_health_checking/"
89  "src/python/grpcio_reflection/"
90  "src/python/grpcio_status/"
91  "src/python/grpcio_testing/"
92  "src/python/grpcio_observability/"
93)
94
95# Copy license to ancillary package directories so it will be distributed.
96for directory in "${ancillary_package_dir[@]}"; do
97  cp "LICENSE" "${directory}"
98done
99
100# Build the source distribution first because MANIFEST.in cannot override
101# exclusion of built shared objects among package resources (for some
102# inexplicable reason).
103${SETARCH_CMD} "${PYTHON}" setup.py sdist
104
105# Wheel has a bug where directories don't get excluded.
106# https://bitbucket.org/pypa/wheel/issues/99/cannot-exclude-directory
107# shellcheck disable=SC2086
108${SETARCH_CMD} "${PYTHON}" setup.py bdist_wheel $WHEEL_PLAT_NAME_FLAG
109
110GRPCIO_STRIP_TEMPDIR=$(mktemp -d)
111GRPCIO_TAR_GZ_LIST=( dist/grpcio-*.tar.gz )
112GRPCIO_TAR_GZ=${GRPCIO_TAR_GZ_LIST[0]}
113GRPCIO_STRIPPED_TAR_GZ=$(mktemp -t "TAR_GZ_XXXXXXXXXX")
114
115clean_non_source_files() {
116( cd "$1"
117  find . -type f \
118    | grep -v '\.c$' | grep -v '\.cc$' | grep -v '\.cpp$' \
119    | grep -v '\.h$' | grep -v '\.hh$' | grep -v '\.inc$' \
120    | grep -v '\.s$' | grep -v '\.py$' | grep -v '\.hpp$' \
121    | grep -v '\.S$' | grep -v '\.asm$'                   \
122    | while read -r file; do
123      rm -f "$file" || true
124    done
125  find . -type d -empty -delete
126)
127}
128
129tar xzf "${GRPCIO_TAR_GZ}" -C "${GRPCIO_STRIP_TEMPDIR}"
130( cd "${GRPCIO_STRIP_TEMPDIR}"
131  find . -type d -name .git -exec rm -fr {} \; || true
132  for dir in */third_party/*; do
133    clean_non_source_files "${dir}" || true
134  done
135  tar czf "${GRPCIO_STRIPPED_TAR_GZ}" -- *
136  chmod ugo+r "${GRPCIO_STRIPPED_TAR_GZ}"
137)
138mv "${GRPCIO_STRIPPED_TAR_GZ}" "${GRPCIO_TAR_GZ}"
139
140# Build gRPC tools package distribution
141"${PYTHON}" tools/distrib/python/make_grpcio_tools.py
142
143# Build gRPC tools package source distribution
144${SETARCH_CMD} "${PYTHON}" tools/distrib/python/grpcio_tools/setup.py sdist
145
146# Build gRPC tools package binary distribution
147# shellcheck disable=SC2086
148${SETARCH_CMD} "${PYTHON}" tools/distrib/python/grpcio_tools/setup.py bdist_wheel $WHEEL_PLAT_NAME_FLAG
149
150if [ "$GRPC_BUILD_MAC" == "" ]; then
151  "${PYTHON}" src/python/grpcio_observability/make_grpcio_observability.py
152  ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_observability/setup.py sdist
153  # shellcheck disable=SC2086
154  ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_observability/setup.py bdist_wheel $WHEEL_PLAT_NAME_FLAG
155fi
156
157
158# run twine check before auditwheel, because auditwheel puts the repaired wheels into
159# the artifacts output dir.
160if [ "$GRPC_SKIP_TWINE_CHECK" == "" ]
161then
162  # Install virtualenv if it isn't already available.
163  # TODO(jtattermusch): cleanup the virtualenv version fallback logic.
164  "${PYTHON}" -m pip install virtualenv
165  "${PYTHON}" -m virtualenv venv || { "${PYTHON}" -m pip install virtualenv==20.0.23 && "${PYTHON}" -m virtualenv venv; }
166  # Ensure the generated artifacts are valid using "twine check"
167  venv/bin/python -m pip install "twine<=2.0" "readme_renderer<40.0"
168  venv/bin/python -m twine check dist/* tools/distrib/python/grpcio_tools/dist/*
169  if [ "$GRPC_BUILD_MAC" == "" ]; then
170    venv/bin/python -m twine check src/python/grpcio_observability/dist/*
171  fi
172  rm -rf venv/
173fi
174
175assert_is_universal_wheel()  {
176  WHL="$1"
177  TMPDIR=$(mktemp -d)
178  unzip "$WHL" -d "$TMPDIR"
179  SO=$(find "$TMPDIR" -name '*.so' | head -n1)
180  if ! file "$SO" | grep "Mach-O universal binary with 2 architectures"; then
181    echo "$WHL is not universal2. Found the following:" >/dev/stderr
182    file "$SO" >/dev/stderr
183    exit 1
184  fi
185}
186
187fix_faulty_universal2_wheel() {
188  WHL="$1"
189  assert_is_universal_wheel "$WHL"
190  if echo "$WHL" | grep "x86_64"; then
191    UPDATED_NAME="${WHL//x86_64/universal2}"
192    mv "$WHL" "$UPDATED_NAME"
193  fi
194}
195
196# This is necessary due to https://github.com/pypa/wheel/issues/406.
197# wheel incorrectly generates a universal2 artifact that only contains
198# x86_64 libraries.
199if [ "$GRPC_BUILD_MAC" != "" ]; then
200  for WHEEL in dist/*.whl tools/distrib/python/grpcio_tools/dist/*.whl; do
201    fix_faulty_universal2_wheel "$WHEEL"
202  done
203fi
204
205
206if [ "$GRPC_RUN_AUDITWHEEL_REPAIR" != "" ]
207then
208  for wheel in dist/*.whl; do
209    "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr |  grep -E -w "$AUDITWHEEL_PLAT"
210    "${AUDITWHEEL}" repair "$wheel" --strip --wheel-dir "$ARTIFACT_DIR"
211    rm "$wheel"
212  done
213  for wheel in tools/distrib/python/grpcio_tools/dist/*.whl; do
214    "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr |  grep -E -w "$AUDITWHEEL_PLAT"
215    "${AUDITWHEEL}" repair "$wheel" --strip --wheel-dir "$ARTIFACT_DIR"
216    rm "$wheel"
217  done
218else
219  cp -r dist/*.whl "$ARTIFACT_DIR"
220  cp -r tools/distrib/python/grpcio_tools/dist/*.whl "$ARTIFACT_DIR"
221fi
222
223# grpcio and grpcio-tools have already been copied to artifact_dir
224# by "auditwheel repair", now copy the .tar.gz source archives as well.
225cp -r dist/*.tar.gz "$ARTIFACT_DIR"
226cp -r tools/distrib/python/grpcio_tools/dist/*.tar.gz "$ARTIFACT_DIR"
227
228
229if [ "$GRPC_BUILD_MAC" == "" ]; then
230  if [ "$GRPC_RUN_AUDITWHEEL_REPAIR" != "" ]
231  then
232    for wheel in src/python/grpcio_observability/dist/*.whl; do
233      "${AUDITWHEEL}" show "$wheel" | tee /dev/stderr |  grep -E -w "$AUDITWHEEL_PLAT"
234      "${AUDITWHEEL}" repair "$wheel" --strip --wheel-dir "$ARTIFACT_DIR"
235      rm "$wheel"
236    done
237  else
238    cp -r src/python/grpcio_observability/dist/*.whl "$ARTIFACT_DIR"
239  fi
240  cp -r src/python/grpcio_observability/dist/*.tar.gz "$ARTIFACT_DIR"
241fi
242
243# We need to use the built grpcio-tools/grpcio to compile the health proto
244# Wheels are not supported by setup_requires/dependency_links, so we
245# manually install the dependency.  Note we should only do this if we
246# are in a docker image or in a virtualenv.
247if [ "$GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS" != "" ]
248then
249  "${PYTHON}" -m pip install -rrequirements.txt
250
251  if [ "$("$PYTHON" -c "import sys; print(sys.version_info[0])")" == "2" ]
252  then
253    "${PYTHON}" -m pip install futures>=2.2.0 enum34>=1.0.4
254  fi
255
256  "${PYTHON}" -m pip install grpcio --no-index --find-links "file://$ARTIFACT_DIR/"
257  "${PYTHON}" -m pip install grpcio-tools --no-index --find-links "file://$ARTIFACT_DIR/"
258
259  # Note(lidiz) setuptools's "sdist" command creates a source tarball, which
260  # demands an extra step of building the wheel. The building step is merely ran
261  # through setup.py, but we can optimize it with "bdist_wheel" command, which
262  # skips the wheel building step.
263
264  # Build xds_protos source distribution
265  # build.py is invoked as part of generate_projects.
266  ${SETARCH_CMD} "${PYTHON}" tools/distrib/python/xds_protos/setup.py \
267      sdist bdist_wheel install
268  cp -r tools/distrib/python/xds_protos/dist/* "$ARTIFACT_DIR"
269
270  # Build grpcio_testing source distribution
271  ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_testing/setup.py preprocess \
272      sdist bdist_wheel
273  cp -r src/python/grpcio_testing/dist/* "$ARTIFACT_DIR"
274
275  # Build grpcio_channelz source distribution
276  ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_channelz/setup.py \
277      preprocess build_package_protos sdist bdist_wheel
278  cp -r src/python/grpcio_channelz/dist/* "$ARTIFACT_DIR"
279
280  # Build grpcio_health_checking source distribution
281  ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_health_checking/setup.py \
282      preprocess build_package_protos sdist bdist_wheel
283  cp -r src/python/grpcio_health_checking/dist/* "$ARTIFACT_DIR"
284
285  # Build grpcio_reflection source distribution
286  ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_reflection/setup.py \
287      preprocess build_package_protos sdist bdist_wheel
288  cp -r src/python/grpcio_reflection/dist/* "$ARTIFACT_DIR"
289
290  # Build grpcio_status source distribution
291  ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_status/setup.py \
292      preprocess sdist bdist_wheel
293  cp -r src/python/grpcio_status/dist/* "$ARTIFACT_DIR"
294
295  # Install xds-protos as a dependency of grpcio-csds
296  "${PYTHON}" -m pip install xds-protos --no-index --find-links "file://$ARTIFACT_DIR/"
297
298  # Build grpcio_csds source distribution
299  ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_csds/setup.py \
300      sdist bdist_wheel
301  cp -r src/python/grpcio_csds/dist/* "$ARTIFACT_DIR"
302
303  # Build grpcio_admin source distribution and it needs the cutting-edge version
304  # of Channelz and CSDS to be installed.
305  "${PYTHON}" -m pip install grpcio-channelz --no-index --find-links "file://$ARTIFACT_DIR/"
306  "${PYTHON}" -m pip install grpcio-csds --no-index --find-links "file://$ARTIFACT_DIR/"
307  ${SETARCH_CMD} "${PYTHON}" src/python/grpcio_admin/setup.py \
308      sdist bdist_wheel
309  cp -r src/python/grpcio_admin/dist/* "$ARTIFACT_DIR"
310
311fi
312