1#!/usr/bin/env python3
2#
3# Copyright 2021 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17from pathlib import Path
18import re
19import shutil
20import subprocess
21import sys
22from typing import List, Union
23from xml.dom import minidom
24
25
26# Use resolve to ensure consistent capitalization between runs of this script, which is important
27# for patching functions that insert a path only if it doesn't already exist.
28PYTHON_SRC = Path(__file__).parent.parent.resolve()
29TOP = PYTHON_SRC.parent.parent.parent
30
31CMAKE_BIN = TOP / 'prebuilts/cmake/windows-x86/bin/cmake.exe'
32VS_GENERATOR = 'Visual Studio 17 2022'
33
34
35def create_new_dir(path: Path) -> None:
36    if path.exists():
37        shutil.rmtree(path)
38    path.mkdir(parents=True)
39
40
41def run_cmd(args: List[Union[str, Path]], cwd: Path) -> None:
42    print(f'cd {cwd}')
43    str_args = [str(arg) for arg in args]
44    print(subprocess.list2cmdline(str_args))
45    subprocess.run(str_args, cwd=cwd, check=True)
46
47
48def read_xml_file(path: Path) -> minidom.Element:
49    doc = minidom.parse(str(path))
50    return doc.documentElement
51
52
53def write_xml_file(root: minidom.Element, path: Path) -> None:
54    with open(path, 'w', encoding='utf8') as out:
55        out.write('<?xml version="1.0" encoding="utf-8"?>\n')
56        root.writexml(out)
57
58
59def get_text_element(root: minidom.Element, tag: str) -> str:
60    (node,) = root.getElementsByTagName(tag)
61    (node,) = node.childNodes
62    assert node.nodeType == root.TEXT_NODE
63    assert isinstance(node.data, str)
64    return node.data
65
66
67def set_text_element(root: minidom.Element, tag: str, new_text: str) -> None:
68    (node,) = root.getElementsByTagName(tag)
69    (node,) = node.childNodes
70    assert node.nodeType == root.TEXT_NODE
71    node.data = new_text
72
73
74def patch_python_for_licenses():
75    # Python already handles bzip2 and libffi itself.
76    notice_files = [
77        TOP / 'external/zlib/LICENSE',
78        TOP / 'toolchain/xz/COPYING',
79    ]
80
81    xml_path = PYTHON_SRC / 'PCbuild/regen.targets'
82    proj = read_xml_file(xml_path)
83
84    # Pick the unconditional <_LicenseSources> element and add extra notices to the end.
85    elements = proj.getElementsByTagName('_LicenseSources')
86    (element,) = [e for e in elements if not e.hasAttribute('Condition')]
87    includes = element.getAttribute('Include').split(';')
88    for notice in notice_files:
89        if str(notice) not in includes:
90            includes.append(str(notice))
91    element.setAttribute('Include', ';'.join(includes))
92
93    write_xml_file(proj, xml_path)
94
95
96def remove_modules_from_pcbuild_proj():
97    modules_to_remove = [
98        '_sqlite3',
99    ]
100
101    xml_path = PYTHON_SRC / 'PCbuild/pcbuild.proj'
102    proj = read_xml_file(xml_path)
103    for tag in ('ExternalModules', 'ExtensionModules'):
104        for element in proj.getElementsByTagName(tag):
105            deps = element.getAttribute('Include').split(';')
106            for unwanted in modules_to_remove:
107                if unwanted in deps:
108                    deps.remove(unwanted)
109                    element.setAttribute('Include', ';'.join(deps))
110    write_xml_file(proj, xml_path)
111
112
113def build_using_cmake(out: Path, src: Path) -> None:
114    create_new_dir(out)
115    run_cmd([CMAKE_BIN, src, '-G', VS_GENERATOR, '-A', 'x64'], cwd=out)
116    run_cmd([CMAKE_BIN, '--build', '.', '--config', 'Release'], cwd=out)
117
118
119def build_using_cmake_and_install(install_dir: Path, build_dir: Path, src: Path) -> None:
120    create_new_dir(build_dir)
121    run_cmd([CMAKE_BIN, src, '-G', VS_GENERATOR, '-A', 'x64', f'-DCMAKE_INSTALL_PREFIX={install_dir}'], cwd=build_dir)
122    run_cmd([CMAKE_BIN, '--build', '.', '--config', 'Release', '--target', 'install'], cwd=build_dir)
123
124
125def patch_libffi_props() -> None:
126    """The libffi.props file uses libffi-N.lib and libffi-N.dll. (At time of writing, N is 7, but
127    upstream Python uses 8 instead.) The CMake-based build of libffi produces unsuffixed files, so
128    fix libffi.props to match.
129    """
130    path = PYTHON_SRC / 'PCbuild/libffi.props'
131    content = path.read_text(encoding='utf8')
132    content = re.sub(r'libffi-\d+\.lib', 'libffi.lib', content)
133    content = re.sub(r'libffi-\d+\.dll', 'libffi.dll', content)
134    path.write_text(content, encoding='utf8')
135
136
137def patch_pythoncore_for_zlib() -> None:
138    """pythoncore.vcxproj builds zlib into itself by listing individual zlib C files. AOSP uses
139    Chromium's zlib fork, which has a different set of C files and defines. Switch to AOSP zlib:
140     - Build a static library using CMake: zlibstatic.lib, zconf.h, zlib.h
141     - Strip ClCompile/ClInclude elements from the project file that point to $(zlibDir).
142     - Add a dependency on the static library.
143    """
144
145    xml_path = PYTHON_SRC / 'PCbuild/pythoncore.vcxproj'
146    proj = read_xml_file(xml_path)
147
148    # Strip ClCompile/ClInclude that point into the zlib directory.
149    for tag in ('ClCompile', 'ClInclude'):
150        for element in proj.getElementsByTagName(tag):
151            if element.getAttribute('Include').startswith('$(zlibDir)'):
152                element.parentNode.removeChild(element)
153
154    # Add a dependency on the static zlib archive.
155    deps = get_text_element(proj, 'AdditionalDependencies').split(';')
156    libz_path = str(TOP / 'out/zlib-install/lib/zlibstatic.lib')
157    if libz_path not in deps:
158        deps.insert(0, libz_path)
159        set_text_element(proj, 'AdditionalDependencies', ';'.join(deps))
160
161    write_xml_file(proj, xml_path)
162
163
164def main() -> None:
165    # Point the Python MSBuild project at the paths where repo/Kokoro would put the various
166    # dependencies. The existing python.props uses trailing slashes in the path, and some (but not
167    # all) uses of these variables expect the trailing slash.
168    xml_path = PYTHON_SRC / 'PCbuild/python.props'
169    root = read_xml_file(xml_path)
170    set_text_element(root, 'bz2Dir', str(TOP / 'external/bzip2') + '\\')
171    set_text_element(root, 'libffiDir', str(TOP / 'external/libffi') + '\\') # Provides LICENSE
172    set_text_element(root, 'libffiIncludeDir', str(TOP / 'out/libffi/dist/include') + '\\') # headers
173    set_text_element(root, 'libffiOutDir', str(TOP / 'out/libffi/Release') + '\\') # dll+lib
174    set_text_element(root, 'lzmaDir', str(TOP / 'toolchain/xz') + '\\')
175    set_text_element(root, 'zlibDir', str(TOP / 'out/zlib-install/include') + '\\')
176    write_xml_file(root, xml_path)
177
178    # liblzma.vcxproj adds $(lzmaDir)windows to the include path for config.h, but AOSP has a newer
179    # version of xz that moves config.h into a subdir like vs2017 or vs2019. See this upstream
180    # commit [1]. Copy the file into the place Python currently expects. (This can go away if Python
181    # updates its xz dependency.)
182    #
183    # [1] https://git.tukaani.org/?p=xz.git;a=commit;h=82388980187b0e3794d187762054200bbdcc9a53
184    #
185    # Use vs2019 because there is no vs2022 header currently. (The existing vs2013, vs2017, and
186    # vs2019 headers are all the same anyway.)
187    xz = TOP / 'toolchain/xz'
188    shutil.copy2(xz / 'windows/vs2019/config.h',
189                 xz / 'windows/config.h')
190
191    patch_python_for_licenses()
192    remove_modules_from_pcbuild_proj()
193    build_using_cmake(TOP / 'out/libffi', TOP / 'external/libffi')
194    build_using_cmake_and_install(TOP / 'out/zlib-install', TOP / 'out/zlib-build', TOP / 'external/zlib')
195    patch_libffi_props()
196    patch_pythoncore_for_zlib()
197
198
199if __name__ == '__main__':
200    main()
201