1#!/usr/bin/env python3 2 3# Copyright 2018 The SwiftShader Authors. All Rights Reserved. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import argparse 18import contextlib 19import multiprocessing 20import os 21import platform 22import re 23import shutil 24import subprocess 25import sys 26import tempfile 27from os import path 28 29# LLVM_BRANCH must match the value of the same variable in third_party/update-llvm-16.sh 30LLVM_BRANCH = "release/16.x" 31 32# LLVM_COMMIT must be set to the commit hash that we last updated to when running third_party/update-llvm-16.sh. 33# Run 'git show -s origin/llvm16-clean' and look for 'llvm-16-update: <hash>' to retrieve it. 34LLVM_COMMIT = "fce3e75e01babe38576b1519dab5f752955525f9" 35 36SCRIPT_DIR = path.dirname(path.realpath(sys.argv[0])) 37LLVM_STAGING_DIR = path.abspath(path.join(tempfile.gettempdir(), "llvm-16")) 38LLVM_DIR = path.abspath(path.join(LLVM_STAGING_DIR, "llvm")) 39LLVM_OBJS = path.join(LLVM_STAGING_DIR, "build") 40LLVM_CONFIGS = path.abspath(path.join(SCRIPT_DIR, '..', 'configs')) 41 42# List of all arches SwiftShader supports 43LLVM_TARGETS = [ 44 ('AArch64', ('__aarch64__',)), 45 ('ARM', ('__arm__',)), 46 ('X86', ('__i386__', '__x86_64__')), 47 ('LoongArch', ('__loongarch__',)), 48 ('Mips', ('__mips__',)), 49 ('PowerPC', ('__powerpc64__',)), 50 ('RISCV', ('__riscv',)), 51] 52 53# Per-platform arches 54LLVM_TRIPLES = { 55 'android': [ 56 ('__x86_64__', 'x86_64-linux-android'), 57 ('__i386__', 'i686-linux-android'), 58 ('__arm__', 'armv7-linux-androideabi'), 59 ('__aarch64__', 'aarch64-linux-android'), 60 ('__riscv', 'riscv64-linux-android'), 61 ], 62 'linux': [ 63 ('__x86_64__', 'x86_64-unknown-linux-gnu'), 64 ('__i386__', 'i686-pc-linux-gnu'), 65 ('__arm__', 'armv7-linux-gnueabihf'), 66 ('__aarch64__', 'aarch64-linux-gnu'), 67 ('__loongarch__', 'loongarch64-unknown-linux-gnu'), 68 ('__mips__', 'mipsel-linux-gnu'), 69 ('__mips64', 'mips64el-linux-gnuabi64'), 70 ('__powerpc64__', 'powerpc64le-unknown-linux-gnu'), 71 ('__riscv', 'riscv64-unknown-linux-gnu'), 72 ], 73 'darwin': [ 74 ('__x86_64__', 'x86_64-apple-darwin'), 75 ('__aarch64__', 'arm64-apple-darwin'), 76 ], 77 'windows': [ 78 ('__x86_64__', 'x86_64-pc-win32'), 79 ('__i386__', 'i686-pc-win32'), 80 ('__arm__', 'armv7-pc-win32'), 81 ('__aarch64__', 'aarch64-pc-win32'), 82 ('__mips__', 'mipsel-pc-win32'), 83 ('__mips64', 'mips64el-pc-win32'), 84 ], 85 'fuchsia': [ 86 ('__x86_64__', 'x86_64-unknown-fuchsia'), 87 ('__aarch64__', 'aarch64-unknown-fuchsia'), 88 ] 89} 90 91# Mapping of target platform to the host it must be built on 92LLVM_PLATFORM_TO_HOST_SYSTEM = { 93 'android': 'Linux', 94 'darwin': 'Darwin', 95 'linux': 'Linux', 96 'windows': 'Windows', 97 'fuchsia': 'Linux' 98} 99 100# LLVM configurations to be undefined. 101LLVM_UNDEF_MACROS = [ 102 'BACKTRACE_HEADER', 103 'ENABLE_BACKTRACES', 104 'ENABLE_CRASH_OVERRIDES', 105 'HAVE_BACKTRACE', 106 'HAVE_POSIX_SPAWN', 107 'HAVE_PTHREAD_GETNAME_NP', 108 'HAVE_PTHREAD_SETNAME_NP', 109 'HAVE_TERMIOS_H', 110 'HAVE_ZLIB_H', 111 'HAVE__UNWIND_BACKTRACE', 112] 113 114# General CMake options for building LLVM 115LLVM_CMAKE_OPTIONS = [ 116 '-DCMAKE_BUILD_TYPE=Release', 117 '-DLLVM_ENABLE_THREADS=ON', 118 '-DLLVM_ENABLE_TERMINFO=OFF', 119 '-DLLVM_ENABLE_LIBXML2=OFF', 120 '-DLLVM_ENABLE_LIBEDIT=OFF', 121 '-DLLVM_ENABLE_LIBPFM=OFF', 122 '-DLLVM_ENABLE_ZLIB=OFF', 123 '-DLLVM_INCLUDE_BENCHMARKS=OFF', 124 '-DLLVM_INCLUDE_TESTS=OFF', 125 '-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON' 126] 127 128# Used when building LLVM for darwin. Affects values set in the generated config files. 129MIN_MACOS_VERSION = '10.10' 130 131@contextlib.contextmanager 132def pushd(new_dir): 133 previous_dir = os.getcwd() 134 os.chdir(new_dir) 135 try: 136 yield 137 finally: 138 os.chdir(previous_dir) 139 140 141def log(message, level=1): 142 print(' ' * level + '> ' + message) 143 144 145def run_command(command, log_level=1): 146 log(command, log_level) 147 os.system(command) 148 149 150def run_subprocess(*popenargs, log_level=1, cwd=None): 151 log([' '.join(t) for t in popenargs][0], log_level) 152 return subprocess.run(*popenargs, cwd=cwd) 153 154 155def _parse_args(): 156 parser = argparse.ArgumentParser() 157 parser.add_argument('name', help='destination name', 158 choices=['android', 'linux', 'darwin', 'windows', 'fuchsia']) 159 parser.add_argument('-j', '--jobs', help='parallel compilation', type=int) 160 return parser.parse_args() 161 162 163def validate_args(args): 164 host = platform.system() 165 if host not in LLVM_PLATFORM_TO_HOST_SYSTEM.values(): 166 raise Exception(f"Host system not supported: '{host}'") 167 168 if args.name not in LLVM_PLATFORM_TO_HOST_SYSTEM.keys(): 169 raise Exception(f"Unknown target platform: '{args.name}'") 170 171 expected_host = LLVM_PLATFORM_TO_HOST_SYSTEM[args.name] 172 if LLVM_PLATFORM_TO_HOST_SYSTEM[args.name] != host: 173 raise Exception( 174 f"Target platform '{args.name}' must be built on '{expected_host}', not on '{host}'") 175 176 177def get_cmake_targets_to_build(platform): 178 """Returns list of LLVM targets to build for the input platform""" 179 targets = set() 180 for arch_def, triplet in LLVM_TRIPLES[platform]: 181 for arch, defs in LLVM_TARGETS: 182 if arch_def in defs: 183 targets.add(arch) 184 185 # Maintain the sort order of LLVM_TARGETS as this affects how config 186 # headers are generated 187 return [t[0] for t in LLVM_TARGETS if t[0] in targets] 188 189 190def clone_llvm(): 191 log("Cloning/Updating LLVM", 1) 192 # Clone or update staging directory 193 if not path.exists(LLVM_STAGING_DIR): 194 os.mkdir(LLVM_STAGING_DIR) 195 with pushd(LLVM_STAGING_DIR): 196 run_command('git init', 2) 197 run_command( 198 'git remote add origin https://github.com/llvm/llvm-project.git', 2) 199 run_command('git config core.sparsecheckout true', 2) 200 run_command('echo /llvm > .git/info/sparse-checkout', 2) 201 run_command('echo /cmake >> .git/info/sparse-checkout', 2) 202 203 with pushd(LLVM_STAGING_DIR): 204 run_command('echo /llvm > .git/info/sparse-checkout', 2) 205 run_command('echo /cmake >> .git/info/sparse-checkout', 2) 206 run_command('git fetch origin {}'.format(LLVM_BRANCH), 2) 207 run_command('git checkout {}'.format(LLVM_COMMIT), 2) 208 return 209 210 211def build_llvm(name, num_jobs): 212 """Build LLVM and get all generated files.""" 213 log("Building LLVM", 1) 214 if num_jobs is None: 215 num_jobs = multiprocessing.cpu_count() 216 217 """On Windows we need to have CMake generate build files for the 64-bit 218 Visual Studio host toolchain.""" 219 host = '-Thost=x64' if name == 'windows' else '' 220 221 cmake_options = LLVM_CMAKE_OPTIONS + ['-DLLVM_TARGETS_TO_BUILD=' + 222 ';'.join(t for t in get_cmake_targets_to_build(name))] 223 224 if name == 'darwin': 225 cmake_options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET={}'.format(MIN_MACOS_VERSION)) 226 227 os.makedirs(LLVM_OBJS, exist_ok=True) 228 run_subprocess(['cmake', host, LLVM_DIR] + 229 cmake_options, log_level=2, cwd=LLVM_OBJS) 230 run_subprocess(['cmake', '--build', '.', '-j', 231 str(num_jobs)], log_level=2, cwd=LLVM_OBJS) 232 233 234def list_files(src_base, src, dst_base, suffixes): 235 """Enumerate the files that are under `src` and end with one of the 236 `suffixes` and yield the source path and the destination path.""" 237 src_base = path.abspath(src_base) 238 src = path.join(src_base, src) 239 for base_dir, dirnames, filenames in os.walk(src): 240 for filename in filenames: 241 if path.splitext(filename)[1] in suffixes: 242 relative = path.relpath(base_dir, src_base) 243 yield (path.join(base_dir, filename), 244 path.join(dst_base, relative, filename)) 245 246 247def copy_common_generated_files(dst_base): 248 """Copy platform-independent generated files.""" 249 log("Copying platform-independent generated files", 1) 250 suffixes = {'.inc', '.h', '.def'} 251 subdirs = [ 252 path.join('include', 'llvm', 'IR'), 253 path.join('include', 'llvm', 'Support'), 254 path.join('include', 'llvm', 'TargetParser'), 255 path.join('include', 'llvm', 'Frontend'), 256 path.join('lib', 'IR'), 257 path.join('lib', 'ExecutionEngine'), 258 path.join('lib', 'Transforms', 'InstCombine'), 259 ] + [path.join('lib', 'Target', arch) for arch, defs in LLVM_TARGETS] 260 for subdir in subdirs: 261 for src, dst in list_files(LLVM_OBJS, subdir, dst_base, suffixes): 262 log('{} -> {}'.format(src, dst), 2) 263 os.makedirs(path.dirname(dst), exist_ok=True) 264 shutil.copyfile(src, dst) 265 266 267def copy_platform_file(platform, src, dst): 268 """Copy platform-dependent generated files and add platform-specific 269 modifications.""" 270 271 # LLVM configuration patterns to be post-processed. 272 llvm_target_pattern = re.compile('^LLVM_[A-Z_]+\\(([A-Za-z0-9_]+)\\)$') 273 llvm_native_pattern = re.compile( 274 '^#define LLVM_NATIVE_([A-Z]+) (LLVMInitialize)?(.*)$') 275 llvm_triple_pattern = re.compile('^#define (LLVM_[A-Z_]+_TRIPLE) "(.*)"$') 276 llvm_define_pattern = re.compile('^#define ([A-Za-z0-9_]+) (.*)$') 277 278 # Build architecture-specific conditions. 279 conds = {} 280 for arch, defs in LLVM_TARGETS: 281 conds[arch] = ' || '.join('defined(' + v + ')' for v in defs) 282 283 # Get a set of platform-specific triples. 284 triples = LLVM_TRIPLES[platform] 285 286 with open(src, 'r') as src_file: 287 os.makedirs(path.dirname(dst), exist_ok=True) 288 with open(dst, 'w') as dst_file: 289 for line in src_file: 290 if line == '#define LLVM_CONFIG_H\n': 291 print(line, file=dst_file, end='') 292 print('', file=dst_file) 293 print('#if !defined(__i386__) && defined(_M_IX86)', 294 file=dst_file) 295 print('#define __i386__ 1', file=dst_file) 296 print('#endif', file=dst_file) 297 print('', file=dst_file) 298 print( 299 '#if !defined(__x86_64__) && (defined(_M_AMD64) || defined (_M_X64))', file=dst_file) 300 print('#define __x86_64__ 1', file=dst_file) 301 print('#endif', file=dst_file) 302 print('', file=dst_file) 303 304 match = llvm_target_pattern.match(line) 305 if match: 306 arch = match.group(1) 307 print('#if ' + conds[arch], file=dst_file) 308 print(line, file=dst_file, end='') 309 print('#endif', file=dst_file) 310 continue 311 312 match = llvm_native_pattern.match(line) 313 if match: 314 name = match.group(1) 315 init = match.group(2) or '' 316 arch = match.group(3) 317 end = '' 318 if arch.lower().endswith(name.lower()): 319 end = arch[-len(name):] 320 directive = '#if ' 321 for arch, defs in LLVM_TARGETS: 322 print(directive + conds[arch], file=dst_file) 323 print('#define LLVM_NATIVE_' + name + ' ' + 324 init + arch + end, file=dst_file) 325 directive = '#elif ' 326 print('#else', file=dst_file) 327 print('#error "unknown architecture"', file=dst_file) 328 print('#endif', file=dst_file) 329 continue 330 331 match = llvm_triple_pattern.match(line) 332 if match: 333 name = match.group(1) 334 directive = '#if' 335 for defs, triple in triples: 336 print(directive + ' defined(' + defs + ')', 337 file=dst_file) 338 print('#define ' + name + ' "' + triple + '"', 339 file=dst_file) 340 directive = '#elif' 341 print('#else', file=dst_file) 342 print('#error "unknown architecture"', file=dst_file) 343 print('#endif', file=dst_file) 344 continue 345 346 match = llvm_define_pattern.match(line) 347 if match and match.group(1) in LLVM_UNDEF_MACROS: 348 print('/* #undef ' + match.group(1) + ' */', file=dst_file) 349 continue 350 351 print(line, file=dst_file, end='') 352 353 354def copy_platform_generated_files(platform, dst_base): 355 """Copy platform-specific generated files.""" 356 log("Copying platform-specific generated files", 1) 357 suffixes = {'.inc', '.h', '.def'} 358 src_dir = path.join('include', 'llvm', 'Config') 359 for src, dst in list_files(LLVM_OBJS, src_dir, dst_base, suffixes): 360 log('{}, {} -> {}'.format(platform, src, dst), 2) 361 copy_platform_file(platform, src, dst) 362 363 364def main(): 365 args = _parse_args() 366 validate_args(args) 367 clone_llvm() 368 build_llvm(args.name, args.jobs) 369 copy_common_generated_files(path.join(LLVM_CONFIGS, 'common')) 370 copy_platform_generated_files( 371 args.name, path.join(LLVM_CONFIGS, args.name)) 372 373 374if __name__ == '__main__': 375 main() 376