1#!/usr/bin/env python3
2# Copyright 2016 gRPC authors.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""Definition of targets to build artifacts."""
16
17import os.path
18import random
19import string
20import sys
21
22sys.path.insert(0, os.path.abspath('..'))
23import python_utils.jobset as jobset
24
25_LATEST_MANYLINUX = "manylinux2014"
26
27
28def create_docker_jobspec(name,
29                          dockerfile_dir,
30                          shell_command,
31                          environ={},
32                          flake_retries=0,
33                          timeout_retries=0,
34                          timeout_seconds=30 * 60,
35                          extra_docker_args=None,
36                          verbose_success=False):
37    """Creates jobspec for a task running under docker."""
38    environ = environ.copy()
39    environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name
40
41    docker_args = []
42    for k, v in list(environ.items()):
43        docker_args += ['-e', '%s=%s' % (k, v)]
44    docker_env = {
45        'DOCKERFILE_DIR': dockerfile_dir,
46        'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh',
47        'DOCKER_RUN_SCRIPT_COMMAND': shell_command,
48        'OUTPUT_DIR': 'artifacts'
49    }
50    if extra_docker_args is not None:
51        docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args
52    jobspec = jobset.JobSpec(
53        cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] +
54        docker_args,
55        environ=docker_env,
56        shortname='build_artifact.%s' % (name),
57        timeout_seconds=timeout_seconds,
58        flake_retries=flake_retries,
59        timeout_retries=timeout_retries,
60        verbose_success=verbose_success)
61    return jobspec
62
63
64def create_jobspec(name,
65                   cmdline,
66                   environ={},
67                   shell=False,
68                   flake_retries=0,
69                   timeout_retries=0,
70                   timeout_seconds=30 * 60,
71                   use_workspace=False,
72                   cpu_cost=1.0,
73                   verbose_success=False):
74    """Creates jobspec."""
75    environ = environ.copy()
76    if use_workspace:
77        environ['WORKSPACE_NAME'] = 'workspace_%s' % name
78        environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name)
79        cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh'
80                  ] + cmdline
81    else:
82        environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name)
83
84    jobspec = jobset.JobSpec(cmdline=cmdline,
85                             environ=environ,
86                             shortname='build_artifact.%s' % (name),
87                             timeout_seconds=timeout_seconds,
88                             flake_retries=flake_retries,
89                             timeout_retries=timeout_retries,
90                             shell=shell,
91                             cpu_cost=cpu_cost,
92                             verbose_success=verbose_success)
93    return jobspec
94
95
96_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.10'
97
98_ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'}
99
100
101class PythonArtifact:
102    """Builds Python artifacts."""
103
104    def __init__(self, platform, arch, py_version, presubmit=False):
105        self.name = 'python_%s_%s_%s' % (platform, arch, py_version)
106        self.platform = platform
107        self.arch = arch
108        self.labels = ['artifact', 'python', platform, arch, py_version]
109        if presubmit:
110            self.labels.append('presubmit')
111        self.py_version = py_version
112        if platform == _LATEST_MANYLINUX:
113            self.labels.append('latest-manylinux')
114        if 'manylinux' in platform:
115            self.labels.append('linux')
116        if 'linux_extra' in platform:
117            # linux_extra wheels used to be built by a separate kokoro job.
118            # Their build is now much faster, so they can be included
119            # in the regular artifact build.
120            self.labels.append('linux')
121        if 'musllinux' in platform:
122            self.labels.append('linux')
123
124    def pre_build_jobspecs(self):
125        return []
126
127    def build_jobspec(self, inner_jobs=None):
128        environ = {}
129        if inner_jobs is not None:
130            # set number of parallel jobs when building native extension
131            # building the native extension is the most time-consuming part of the build
132            environ['GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS'] = str(inner_jobs)
133
134        if self.platform == "macos":
135            environ['ARCHFLAGS'] = "-arch arm64 -arch x86_64"
136            environ["GRPC_UNIVERSAL2_REPAIR"] = "true"
137            environ['GRPC_BUILD_WITH_BORING_SSL_ASM'] = "false"
138
139        if self.platform == 'linux_extra':
140            # Crosscompilation build for armv7 (e.g. Raspberry Pi)
141            environ['PYTHON'] = '/opt/python/{}/bin/python3'.format(
142                self.py_version)
143            environ['PIP'] = '/opt/python/{}/bin/pip3'.format(self.py_version)
144            environ['GRPC_SKIP_PIP_CYTHON_UPGRADE'] = 'TRUE'
145            environ['GRPC_SKIP_TWINE_CHECK'] = 'TRUE'
146            return create_docker_jobspec(
147                self.name,
148                'tools/dockerfile/grpc_artifact_python_linux_{}'.format(
149                    self.arch),
150                'tools/run_tests/artifacts/build_artifact_python.sh',
151                environ=environ,
152                timeout_seconds=60 * 60)
153        elif 'manylinux' in self.platform:
154            if self.arch == 'x86':
155                environ['SETARCH_CMD'] = 'linux32'
156            # Inside the manylinux container, the python installations are located in
157            # special places...
158            environ['PYTHON'] = '/opt/python/{}/bin/python'.format(
159                self.py_version)
160            environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version)
161            environ['GRPC_SKIP_PIP_CYTHON_UPGRADE'] = 'TRUE'
162            if self.arch == 'aarch64':
163                environ['GRPC_SKIP_TWINE_CHECK'] = 'TRUE'
164                # As we won't strip the binary with auditwheel (see below), strip
165                # it at link time.
166                environ['LDFLAGS'] = '-s'
167            else:
168                # only run auditwheel if we're not crosscompiling
169                environ['GRPC_RUN_AUDITWHEEL_REPAIR'] = 'TRUE'
170                # only build the packages that depend on grpcio-tools
171                # if we're not crosscompiling.
172                # - they require protoc to run on current architecture
173                # - they only have sdist packages anyway, so it's useless to build them again
174                environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE'
175            return create_docker_jobspec(
176                self.name,
177                'tools/dockerfile/grpc_artifact_python_%s_%s' %
178                (self.platform, self.arch),
179                'tools/run_tests/artifacts/build_artifact_python.sh',
180                environ=environ,
181                timeout_seconds=60 * 60 * 2)
182        elif 'musllinux' in self.platform:
183            environ['PYTHON'] = '/opt/python/{}/bin/python'.format(
184                self.py_version)
185            environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version)
186            environ['GRPC_SKIP_PIP_CYTHON_UPGRADE'] = 'TRUE'
187            environ['GRPC_RUN_AUDITWHEEL_REPAIR'] = 'TRUE'
188            environ['GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX'] = 'TRUE'
189            return create_docker_jobspec(
190                self.name,
191                'tools/dockerfile/grpc_artifact_python_%s_%s' %
192                (self.platform, self.arch),
193                'tools/run_tests/artifacts/build_artifact_python.sh',
194                environ=environ,
195                timeout_seconds=60 * 60 * 2)
196        elif self.platform == 'windows':
197            environ['EXT_COMPILER'] = 'msvc'
198            # For some reason, the batch script %random% always runs with the same
199            # seed.  We create a random temp-dir here
200            dir = ''.join(
201                random.choice(string.ascii_uppercase) for _ in range(10))
202            return create_jobspec(self.name, [
203                'tools\\run_tests\\artifacts\\build_artifact_python.bat',
204                self.py_version, '32' if self.arch == 'x86' else '64'
205            ],
206                                  environ=environ,
207                                  timeout_seconds=45 * 60,
208                                  use_workspace=True)
209        else:
210            environ['PYTHON'] = self.py_version
211            environ['SKIP_PIP_INSTALL'] = 'TRUE'
212            return create_jobspec(
213                self.name,
214                ['tools/run_tests/artifacts/build_artifact_python.sh'],
215                environ=environ,
216                timeout_seconds=60 * 60 * 2,
217                use_workspace=True)
218
219    def __str__(self):
220        return self.name
221
222
223class RubyArtifact:
224    """Builds ruby native gem."""
225
226    def __init__(self, platform, gem_platform, presubmit=False):
227        self.name = 'ruby_native_gem_%s_%s' % (platform, gem_platform)
228        self.platform = platform
229        self.gem_platform = gem_platform
230        self.labels = ['artifact', 'ruby', platform, gem_platform]
231        if presubmit:
232            self.labels.append('presubmit')
233
234    def pre_build_jobspecs(self):
235        return []
236
237    def build_jobspec(self, inner_jobs=None):
238        environ = {}
239        if inner_jobs is not None:
240            # set number of parallel jobs when building native extension
241            environ['GRPC_RUBY_BUILD_PROCS'] = str(inner_jobs)
242        # Ruby build uses docker internally and docker cannot be nested.
243        # We are using a custom workspace instead.
244        return create_jobspec(self.name, [
245            'tools/run_tests/artifacts/build_artifact_ruby.sh',
246            self.gem_platform
247        ],
248                              use_workspace=True,
249                              timeout_seconds=90 * 60,
250                              environ=environ)
251
252
253class PHPArtifact:
254    """Builds PHP PECL package"""
255
256    def __init__(self, platform, arch, presubmit=False):
257        self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch)
258        self.platform = platform
259        self.arch = arch
260        self.labels = ['artifact', 'php', platform, arch]
261        if presubmit:
262            self.labels.append('presubmit')
263
264    def pre_build_jobspecs(self):
265        return []
266
267    def build_jobspec(self, inner_jobs=None):
268        del inner_jobs  # arg unused as PHP artifact build is basically just packing an archive
269        if self.platform == 'linux':
270            return create_docker_jobspec(
271                self.name,
272                'tools/dockerfile/test/php73_zts_debian11_{}'.format(self.arch),
273                'tools/run_tests/artifacts/build_artifact_php.sh')
274        else:
275            return create_jobspec(
276                self.name, ['tools/run_tests/artifacts/build_artifact_php.sh'],
277                use_workspace=True)
278
279
280class ProtocArtifact:
281    """Builds protoc and protoc-plugin artifacts"""
282
283    def __init__(self, platform, arch, presubmit=False):
284        self.name = 'protoc_%s_%s' % (platform, arch)
285        self.platform = platform
286        self.arch = arch
287        self.labels = ['artifact', 'protoc', platform, arch]
288        if presubmit:
289            self.labels.append('presubmit')
290
291    def pre_build_jobspecs(self):
292        return []
293
294    def build_jobspec(self, inner_jobs=None):
295        environ = {}
296        if inner_jobs is not None:
297            # set number of parallel jobs when building protoc
298            environ['GRPC_PROTOC_BUILD_COMPILER_JOBS'] = str(inner_jobs)
299
300        if self.platform != 'windows':
301            environ['CXXFLAGS'] = ''
302            environ['LDFLAGS'] = ''
303            if self.platform == 'linux':
304                dockerfile_dir = 'tools/dockerfile/grpc_artifact_centos6_{}'.format(
305                    self.arch)
306                if self.arch == 'aarch64':
307                    # for aarch64, use a dockcross manylinux image that will
308                    # give us both ready to use crosscompiler and sufficient backward compatibility
309                    dockerfile_dir = 'tools/dockerfile/grpc_artifact_protoc_aarch64'
310                environ['LDFLAGS'] += ' -static-libgcc -static-libstdc++ -s'
311                return create_docker_jobspec(
312                    self.name,
313                    dockerfile_dir,
314                    'tools/run_tests/artifacts/build_artifact_protoc.sh',
315                    environ=environ)
316            else:
317                environ[
318                    'CXXFLAGS'] += ' -std=c++14 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG
319                return create_jobspec(
320                    self.name,
321                    ['tools/run_tests/artifacts/build_artifact_protoc.sh'],
322                    environ=environ,
323                    timeout_seconds=60 * 60,
324                    use_workspace=True)
325        else:
326            vs_tools_architecture = self.arch  # architecture selector passed to vcvarsall.bat
327            environ['ARCHITECTURE'] = vs_tools_architecture
328            return create_jobspec(
329                self.name,
330                ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'],
331                environ=environ,
332                use_workspace=True)
333
334    def __str__(self):
335        return self.name
336
337
338def _reorder_targets_for_build_speed(targets):
339    """Reorder targets to achieve optimal build speed"""
340    # ruby artifact build builds multiple artifacts at once, so make sure
341    # we start building ruby artifacts first, so that they don't end up
342    # being a long tail once everything else finishes.
343    return list(
344        sorted(targets,
345               key=lambda target: 0 if target.name.startswith('ruby_') else 1))
346
347
348def targets():
349    """Gets list of supported targets"""
350    return _reorder_targets_for_build_speed([
351        ProtocArtifact('linux', 'x64', presubmit=True),
352        ProtocArtifact('linux', 'x86', presubmit=True),
353        ProtocArtifact('linux', 'aarch64', presubmit=True),
354        ProtocArtifact('macos', 'x64', presubmit=True),
355        ProtocArtifact('windows', 'x64', presubmit=True),
356        ProtocArtifact('windows', 'x86', presubmit=True),
357        PythonArtifact('manylinux2014', 'x64', 'cp37-cp37m', presubmit=True),
358        PythonArtifact('manylinux2014', 'x64', 'cp38-cp38', presubmit=True),
359        PythonArtifact('manylinux2014', 'x64', 'cp39-cp39'),
360        PythonArtifact('manylinux2014', 'x64', 'cp310-cp310'),
361        PythonArtifact('manylinux2014', 'x64', 'cp311-cp311', presubmit=True),
362        PythonArtifact('manylinux2014', 'x86', 'cp37-cp37m', presubmit=True),
363        PythonArtifact('manylinux2014', 'x86', 'cp38-cp38', presubmit=True),
364        PythonArtifact('manylinux2014', 'x86', 'cp39-cp39'),
365        PythonArtifact('manylinux2014', 'x86', 'cp310-cp310'),
366        PythonArtifact('manylinux2014', 'x86', 'cp311-cp311', presubmit=True),
367        PythonArtifact('manylinux2014', 'aarch64', 'cp37-cp37m',
368                       presubmit=True),
369        PythonArtifact('manylinux2014', 'aarch64', 'cp38-cp38', presubmit=True),
370        PythonArtifact('manylinux2014', 'aarch64', 'cp39-cp39'),
371        PythonArtifact('manylinux2014', 'aarch64', 'cp310-cp310'),
372        PythonArtifact('manylinux2014', 'aarch64', 'cp311-cp311'),
373        PythonArtifact('linux_extra', 'armv7', 'cp37-cp37m', presubmit=True),
374        PythonArtifact('linux_extra', 'armv7', 'cp38-cp38'),
375        PythonArtifact('linux_extra', 'armv7', 'cp39-cp39'),
376        PythonArtifact('linux_extra', 'armv7', 'cp310-cp310'),
377        PythonArtifact('linux_extra', 'armv7', 'cp311-cp311', presubmit=True),
378        PythonArtifact('musllinux_1_1', 'x64', 'cp310-cp310'),
379        PythonArtifact('musllinux_1_1', 'x64', 'cp311-cp311', presubmit=True),
380        PythonArtifact('musllinux_1_1', 'x64', 'cp37-cp37m', presubmit=True),
381        PythonArtifact('musllinux_1_1', 'x64', 'cp38-cp38'),
382        PythonArtifact('musllinux_1_1', 'x64', 'cp39-cp39'),
383        PythonArtifact('musllinux_1_1', 'x86', 'cp310-cp310'),
384        PythonArtifact('musllinux_1_1', 'x86', 'cp311-cp311', presubmit=True),
385        PythonArtifact('musllinux_1_1', 'x86', 'cp37-cp37m', presubmit=True),
386        PythonArtifact('musllinux_1_1', 'x86', 'cp38-cp38'),
387        PythonArtifact('musllinux_1_1', 'x86', 'cp39-cp39'),
388        PythonArtifact('macos', 'x64', 'python3.7', presubmit=True),
389        PythonArtifact('macos', 'x64', 'python3.8'),
390        PythonArtifact('macos', 'x64', 'python3.9'),
391        PythonArtifact('macos', 'x64', 'python3.10', presubmit=True),
392        PythonArtifact('macos', 'x64', 'python3.11', presubmit=True),
393        PythonArtifact('windows', 'x86', 'Python37_32bit', presubmit=True),
394        PythonArtifact('windows', 'x86', 'Python38_32bit'),
395        PythonArtifact('windows', 'x86', 'Python39_32bit'),
396        PythonArtifact('windows', 'x86', 'Python310_32bit'),
397        PythonArtifact('windows', 'x86', 'Python311_32bit', presubmit=True),
398        PythonArtifact('windows', 'x64', 'Python37', presubmit=True),
399        PythonArtifact('windows', 'x64', 'Python38'),
400        PythonArtifact('windows', 'x64', 'Python39'),
401        PythonArtifact('windows', 'x64', 'Python310'),
402        PythonArtifact('windows', 'x64', 'Python311', presubmit=True),
403        RubyArtifact('linux', 'x86-mingw32', presubmit=True),
404        RubyArtifact('linux', 'x64-mingw32', presubmit=True),
405        RubyArtifact('linux', 'x64-mingw-ucrt', presubmit=True),
406        RubyArtifact('linux', 'x86_64-linux', presubmit=True),
407        RubyArtifact('linux', 'x86-linux', presubmit=True),
408        RubyArtifact('linux', 'x86_64-darwin', presubmit=True),
409        RubyArtifact('linux', 'arm64-darwin', presubmit=True),
410        PHPArtifact('linux', 'x64', presubmit=True),
411        PHPArtifact('macos', 'x64', presubmit=True),
412    ])
413