1#!/usr/bin/env python 2# 3# Copyright (C) 2016 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17import argparse 18import logging 19import os 20from pathlib import Path 21import shutil 22import subprocess 23from tempfile import TemporaryDirectory 24import textwrap 25 26 27THIS_DIR = Path(__file__).resolve().parent 28 29 30def logger(): 31 return logging.getLogger(__name__) 32 33 34def check_call(cmd): 35 logger().debug("Running `%s`", " ".join(cmd)) 36 subprocess.check_call(cmd) 37 38 39def remove(path): 40 logger().debug("remove `%s`", path) 41 os.remove(path) 42 43 44def fetch_artifact(branch: str, build: str, pattern: str) -> None: 45 """Fetches an artifact from the build server. 46 47 Use OAuth2 authentication and the gLinux android-fetch-artifact package, 48 which work with both on-corp and off-corp workstations.""" 49 fetch_artifact_path = shutil.which("fetch_artifact") 50 if fetch_artifact_path is None: 51 raise RuntimeError( 52 "error: cannot find fetch_artifact in PATH. Install it using:\n" 53 " sudo glinux-add-repo android\n" 54 " sudo apt update\n" 55 " sudo apt install android-fetch-artifact\n" 56 ) 57 cmd = [ 58 fetch_artifact_path, 59 "--use_oauth2", 60 "--branch", 61 branch, 62 "--target=linux", 63 "--bid", 64 build, 65 pattern, 66 ] 67 check_call(cmd) 68 69 70def api_str(api_level): 71 return f"android-{api_level}" 72 73 74def start_branch(build): 75 branch_name = "update-" + (build or "latest") 76 logger().info("Creating branch %s", branch_name) 77 check_call(["repo", "start", branch_name, "."]) 78 79 80def remove_old_release(install_dir: Path) -> None: 81 if (install_dir / ".git").exists(): 82 logger().info('Removing old install directory "%s"', install_dir) 83 check_call(["git", "rm", "-rf", install_dir]) 84 85 # Need to check again because git won't remove directories if they have 86 # non-git files in them. 87 if install_dir.exists(): 88 shutil.rmtree(install_dir) 89 90 91LIBUNWIND_GLOB = "toolchains/llvm/prebuilt/*/lib64/clang/*/lib/linux/*/libunwind.a" 92LIBCXX_SHARED_GLOB = ( 93 "toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/*/libc++_shared.so" 94) 95LIBCXX_STATIC_GLOB = ( 96 "toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/*/libc++_static.a" 97) 98LIBCXXABI_GLOB = "toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/*/libc++abi.a" 99LIBANDROID_SUPPORT_GLOB = ( 100 "toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/*/libandroid_support.a" 101) 102 103 104def unzip_single_directory(artifact: Path, destination: Path) -> None: 105 # Use cwd so that we can use rename without having to worry about crossing 106 # file systems. 107 with TemporaryDirectory(dir=os.getcwd()) as temp_dir: 108 cmd = [ 109 "unzip", 110 str(artifact), 111 "-d", 112 temp_dir, 113 "*/sources/android/cpufeatures/*", 114 "*/sources/android/native_app_glue/*", 115 "*/sources/android/support/*", 116 "*/sources/cxx-stl/*", 117 "*/source.properties", 118 os.path.join("*", LIBUNWIND_GLOB), 119 os.path.join("*", LIBCXX_SHARED_GLOB), 120 os.path.join("*", LIBCXX_STATIC_GLOB), 121 os.path.join("*", LIBCXXABI_GLOB), 122 os.path.join("*", LIBANDROID_SUPPORT_GLOB), 123 ] 124 check_call(cmd) 125 126 dirs = os.listdir(temp_dir) 127 assert len(dirs) == 1 128 ndk_dir = Path(temp_dir) / dirs[0] 129 for child in ndk_dir.iterdir(): 130 child.rename(destination / child.name) 131 132 133def relocate_libcxx(install_dir: Path) -> None: 134 """Copies the libc++ libraries from the toolchain to sources. 135 136 New versions of the NDK have removed the libraries in the sources directory because 137 they are duplicates and they aren't needed in typical builds. Soong still expects to 138 find them in that directory though. We could fix Soong, but since this whole 139 directory should be dead soon we'll just fix-up the install for now. 140 """ 141 dest_base = install_dir / "sources/cxx-stl/llvm-libc++/libs" 142 for glob in { 143 LIBCXX_SHARED_GLOB, 144 LIBCXX_STATIC_GLOB, 145 LIBCXXABI_GLOB, 146 LIBANDROID_SUPPORT_GLOB, 147 }: 148 file_name = Path(glob).name 149 for file_path in install_dir.glob(glob): 150 triple = file_path.parent.name 151 abi = { 152 "arm-linux-androideabi": "armeabi-v7a", 153 "aarch64-linux-android": "arm64-v8a", 154 "i686-linux-android": "x86", 155 "x86_64-linux-android": "x86_64", 156 }[triple] 157 dest_dir = dest_base / abi 158 dest_dir.mkdir(parents=True, exist_ok=True) 159 dest = dest_dir / file_name 160 logger().info("Relocating %s to %s", file_path, dest) 161 file_path.rename(dest) 162 163 164def relocate_libunwind(install_dir: Path) -> None: 165 dest_base = install_dir / "sources/cxx-stl/llvm-libc++/libs" 166 for libunwind in install_dir.glob(LIBUNWIND_GLOB): 167 arch = libunwind.parent.name 168 abi = { 169 "arm": "armeabi-v7a", 170 "aarch64": "arm64-v8a", 171 "i386": "x86", 172 "x86_64": "x86_64", 173 }[arch] 174 dest_dir = dest_base / abi 175 dest = dest_dir / "libunwind.a" 176 logger().info("Relocating %s to %s", libunwind, dest) 177 libunwind.rename(dest) 178 179 180def delete_android_mks(install_dir: Path) -> None: 181 for android_mk in install_dir.glob("**/Android.mk"): 182 android_mk.unlink() 183 184 185def install_new_release(branch: str, build: str, install_dir: Path) -> None: 186 install_dir.mkdir() 187 188 artifact_pattern = "android-ndk-*.zip" 189 logger().info( 190 "Fetching %s from %s (artifacts matching %s)", build, branch, artifact_pattern 191 ) 192 fetch_artifact(branch, build, artifact_pattern) 193 artifacts = list(Path().glob("android-ndk-*.zip")) 194 try: 195 assert len(artifacts) == 1 196 artifact = artifacts[0] 197 198 logger().info("Extracting release") 199 unzip_single_directory(artifact, install_dir) 200 relocate_libcxx(install_dir) 201 relocate_libunwind(install_dir) 202 delete_android_mks(install_dir) 203 finally: 204 for artifact in artifacts: 205 artifact.unlink() 206 207 208def commit(branch: str, build: str, install_dir: Path) -> None: 209 logger().info("Making commit") 210 check_call(["git", "add", str(install_dir)]) 211 message = textwrap.dedent( 212 f"""\ 213 Update NDK prebuilts to build {build}. 214 215 Taken from branch {branch}. 216 217 Bug: None 218 Test: treehugger 219 """ 220 ) 221 check_call(["git", "commit", "-m", message]) 222 223 224def get_args(): 225 parser = argparse.ArgumentParser() 226 parser.add_argument( 227 "-b", "--branch", default="master-ndk", help="Branch to pull build from." 228 ) 229 parser.add_argument("--build", required=True, help="Build number to pull.") 230 parser.add_argument( 231 "--use-current-branch", 232 action="store_true", 233 help="Perform the update in the current branch. Do not repo start.", 234 ) 235 parser.add_argument( 236 "-v", "--verbose", action="count", default=0, help="Increase output verbosity." 237 ) 238 return parser.parse_args() 239 240 241def main() -> None: 242 os.chdir(THIS_DIR) 243 244 args = get_args() 245 verbose_map = (logging.WARNING, logging.INFO, logging.DEBUG) 246 verbosity = min(args.verbose, 2) 247 logging.basicConfig(level=verbose_map[verbosity]) 248 249 install_dir = THIS_DIR / "current" 250 251 if not args.use_current_branch: 252 start_branch(args.build) 253 remove_old_release(install_dir) 254 install_new_release(args.branch, args.build, install_dir) 255 commit(args.branch, args.build, install_dir) 256 257 258if __name__ == "__main__": 259 main() 260