1#!/usr/bin/env python3 2 3# Copyright 2021 Google, Inc. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at: 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16""" Build BT targets on the host system. 17 18For building, you will first have to stage a platform directory that has the 19following structure: 20|-common-mk 21|-bt 22|-external 23|-|-rust 24|-|-|-vendor 25 26The simplest way to do this is to check out platform2 to another directory (that 27is not a subdir of this bt directory), symlink bt there and symlink the rust 28vendor repository as well. 29""" 30import argparse 31import multiprocessing 32import os 33import platform 34import shutil 35import six 36import subprocess 37import sys 38import tarfile 39import time 40 41# Use flags required by common-mk (find -type f | grep -nE 'use[.]' {}) 42COMMON_MK_USES = [ 43 'asan', 44 'coverage', 45 'cros_host', 46 'cros_debug', 47 'floss_rootcanal', 48 'function_elimination_experiment', 49 'fuzzer', 50 'fuzzer', 51 'lto_experiment', 52 'msan', 53 'profiling', 54 'proto_force_optimize_speed', 55 'tcmalloc', 56 'test', 57 'ubsan', 58] 59 60# Use a specific commit version for common-mk to avoid build surprises. 61COMMON_MK_COMMIT = "d014d561eaf5ece08166edd98b10c145ef81312d" 62 63# Default use flags. 64USE_DEFAULTS = { 65 'android': False, 66 'bt_nonstandard_codecs': False, 67 'test': False, 68} 69 70VALID_TARGETS = [ 71 'all', # All targets except test and clean 72 'bloat', # Check bloat of crates 73 'clean', # Clean up output directory 74 'docs', # Build Rust docs 75 'hosttools', # Build the host tools (i.e. packetgen) 76 'main', # Build the main C++ codebase 77 'prepare', # Prepare the output directory (gn gen + rust setup) 78 'rust', # Build only the rust components + copy artifacts to output dir 79 'test', # Run the unit tests 80 'clippy', # Run cargo clippy 81 'utils', # Build Floss utils 82] 83 84# TODO(b/190750167) - Host tests are disabled until we are full bazel build 85HOST_TESTS = [ 86 # 'bluetooth_test_common', 87 # 'bluetoothtbd_test', 88 # 'net_test_avrcp', 89 # 'net_test_btcore', 90 # 'net_test_types', 91 # 'net_test_btm_iso', 92 # 'net_test_btpackets', 93] 94 95# Map of git repos to bootstrap and what commit to check them out at. None 96# values will just checkout to HEAD. 97BOOTSTRAP_GIT_REPOS = { 98 'platform2': ('https://chromium.googlesource.com/chromiumos/platform2', COMMON_MK_COMMIT), 99 'rust_crates': ('https://chromium.googlesource.com/chromiumos/third_party/rust_crates', None), 100 'proto_logging': ('https://android.googlesource.com/platform/frameworks/proto_logging', None), 101} 102 103# List of packages required for linux build 104REQUIRED_APT_PACKAGES = [ 105 'bison', 106 'build-essential', 107 'curl', 108 'debmake', 109 'flatbuffers-compiler', 110 'flex', 111 'g++-multilib', 112 'gcc-multilib', 113 'generate-ninja', 114 'gnupg', 115 'gperf', 116 'libabsl-dev', 117 'libc++abi-dev', 118 'libc++-dev', 119 'libdbus-1-dev', 120 'libdouble-conversion-dev', 121 'libevent-dev', 122 'libevent-dev', 123 'libflatbuffers-dev', 124 'libfmt-dev', 125 'libgl1-mesa-dev', 126 'libglib2.0-dev', 127 'libgtest-dev', 128 'libgmock-dev', 129 'liblc3-dev', 130 'liblz4-tool', 131 'libncurses5', 132 'libnss3-dev', 133 'libfmt-dev', 134 'libprotobuf-dev', 135 'libre2-9', 136 'libre2-dev', 137 'libssl-dev', 138 'libtinyxml2-dev', 139 'libx11-dev', 140 'libxml2-utils', 141 'ninja-build', 142 'openssl', 143 'protobuf-compiler', 144 'unzip', 145 'x11proto-core-dev', 146 'xsltproc', 147 'zip', 148 'zlib1g-dev', 149] 150 151# List of cargo packages required for linux build 152REQUIRED_CARGO_PACKAGES = ['cxxbridge-cmd', 'pdl-compiler', 'grpcio-compiler', 'cargo-bloat'] 153 154APT_PKG_LIST = ['apt', '-qq', 'list'] 155CARGO_PKG_LIST = ['cargo', 'install', '--list'] 156 157 158class UseFlags(): 159 160 def __init__(self, use_flags): 161 """ Construct the use flags. 162 163 Args: 164 use_flags: List of use flags parsed from the command. 165 """ 166 self.flags = {} 167 168 # Import use flags required by common-mk 169 for use in COMMON_MK_USES: 170 self.set_flag(use, False) 171 172 # Set our defaults 173 for use, value in USE_DEFAULTS.items(): 174 self.set_flag(use, value) 175 176 # Set use flags - value is set to True unless the use starts with - 177 # All given use flags always override the defaults 178 for use in use_flags: 179 value = not use.startswith('-') 180 self.set_flag(use, value) 181 182 def set_flag(self, key, value=True): 183 setattr(self, key, value) 184 self.flags[key] = value 185 186 187class HostBuild(): 188 189 def __init__(self, args): 190 """ Construct the builder. 191 192 Args: 193 args: Parsed arguments from ArgumentParser 194 """ 195 self.args = args 196 197 # Set jobs to number of cpus unless explicitly set 198 self.jobs = self.args.jobs 199 if not self.jobs: 200 self.jobs = multiprocessing.cpu_count() 201 sys.stderr.write("Number of jobs = {}\n".format(self.jobs)) 202 203 # Normalize bootstrap dir and make sure it exists 204 self.bootstrap_dir = os.path.abspath(self.args.bootstrap_dir) 205 os.makedirs(self.bootstrap_dir, exist_ok=True) 206 207 # Output and platform directories are based on bootstrap 208 self.output_dir = os.path.join(self.bootstrap_dir, 'output') 209 self.platform_dir = os.path.join(self.bootstrap_dir, 'staging') 210 self.bt_dir = os.path.join(self.platform_dir, 'bt') 211 self.sysroot = self.args.sysroot 212 self.libdir = self.args.libdir 213 self.install_dir = os.path.join(self.output_dir, 'install') 214 215 assert os.path.samefile(self.bt_dir, 216 os.path.dirname(__file__)), "Please rerun bootstrap for the current project!" 217 218 # If default target isn't set, build everything 219 self.target = 'all' 220 if hasattr(self.args, 'target') and self.args.target: 221 self.target = self.args.target 222 223 target_use = self.args.use if self.args.use else [] 224 225 # Unless set, always build test code 226 if not self.args.notest: 227 target_use.append('test') 228 229 self.use = UseFlags(target_use) 230 231 # Validate platform directory 232 assert os.path.isdir(self.platform_dir), 'Platform dir does not exist' 233 assert os.path.isfile(os.path.join(self.platform_dir, '.gn')), 'Platform dir does not have .gn at root' 234 235 # Make sure output directory exists (or create it) 236 os.makedirs(self.output_dir, exist_ok=True) 237 238 # Set some default attributes 239 self.libbase_ver = None 240 241 self.configure_environ() 242 243 def _generate_rustflags(self): 244 """ Rustflags to include for the build. 245 """ 246 rust_flags = [ 247 '-L', 248 '{}/out/Default'.format(self.output_dir), 249 '-C', 250 'link-arg=-Wl,--allow-multiple-definition', 251 # exclude uninteresting warnings 252 '-A improper_ctypes_definitions -A improper_ctypes -A unknown_lints', 253 '-Cstrip=debuginfo', 254 '-Copt-level=z', 255 ] 256 257 return ' '.join(rust_flags) 258 259 def configure_environ(self): 260 """ Configure environment variables for GN and Cargo. 261 """ 262 self.env = os.environ.copy() 263 264 # Make sure cargo home dir exists and has a bin directory 265 cargo_home = os.path.join(self.output_dir, 'cargo_home') 266 os.makedirs(cargo_home, exist_ok=True) 267 os.makedirs(os.path.join(cargo_home, 'bin'), exist_ok=True) 268 269 # Configure Rust env variables 270 self.custom_env = {} 271 self.custom_env['CARGO_TARGET_DIR'] = self.output_dir 272 self.custom_env['CARGO_HOME'] = os.path.join(self.output_dir, 'cargo_home') 273 self.custom_env['RUSTFLAGS'] = self._generate_rustflags() 274 self.custom_env['CXX_ROOT_PATH'] = os.path.join(self.platform_dir, 'bt') 275 self.custom_env['CROS_SYSTEM_API_ROOT'] = os.path.join(self.platform_dir, 'system_api') 276 self.custom_env['CXX_OUTDIR'] = self._gn_default_output() 277 278 # On ChromeOS, this is /usr/bin/grpc_rust_plugin 279 # In the container, this is /root/.cargo/bin/grpc_rust_plugin 280 self.custom_env['GRPC_RUST_PLUGIN_PATH'] = shutil.which('grpc_rust_plugin') 281 self.env.update(self.custom_env) 282 283 def print_env(self): 284 """ Print the custom environment variables that are used in build. 285 286 Useful so that external tools can mimic the environment to be the same 287 as build.py, e.g. rust-analyzer. 288 """ 289 for k, v in self.custom_env.items(): 290 print("export {}='{}'".format(k, v)) 291 292 def run_command(self, target, args, cwd=None, env=None): 293 """ Run command and stream the output. 294 """ 295 # Set some defaults 296 if not cwd: 297 cwd = self.platform_dir 298 if not env: 299 env = self.env 300 301 for k, v in env.items(): 302 if env[k] is None: 303 env[k] = "" 304 305 log_file = os.path.join(self.output_dir, '{}.log'.format(target)) 306 with open(log_file, 'wb') as lf: 307 rc = 0 308 process = subprocess.Popen(args, cwd=cwd, env=env, stdout=subprocess.PIPE) 309 while True: 310 line = process.stdout.readline() 311 print(line.decode('utf-8'), end="") 312 lf.write(line) 313 if not line: 314 rc = process.poll() 315 if rc is not None: 316 break 317 318 time.sleep(0.1) 319 320 if rc != 0: 321 raise Exception("Return code is {}".format(rc)) 322 323 def _get_basever(self): 324 if self.libbase_ver: 325 return self.libbase_ver 326 327 self.libbase_ver = os.environ.get('BASE_VER', '') 328 if not self.libbase_ver: 329 base_file = os.path.join(self.sysroot, 'usr/share/libchrome/BASE_VER') 330 try: 331 with open(base_file, 'r') as f: 332 self.libbase_ver = f.read().strip('\n') 333 except: 334 self.libbase_ver = 'NOT-INSTALLED' 335 336 return self.libbase_ver 337 338 def _gn_default_output(self): 339 return os.path.join(self.output_dir, 'out/Default') 340 341 def _gn_configure(self): 342 """ Configure all required parameters for platform2. 343 344 Mostly copied from //common-mk/platform2.py 345 """ 346 clang = not self.args.no_clang 347 348 def to_gn_string(s): 349 return '"%s"' % s.replace('"', '\\"') 350 351 def to_gn_list(strs): 352 return '[%s]' % ','.join([to_gn_string(s) for s in strs]) 353 354 def to_gn_args_args(gn_args): 355 for k, v in gn_args.items(): 356 if isinstance(v, bool): 357 v = str(v).lower() 358 elif isinstance(v, list): 359 v = to_gn_list(v) 360 elif isinstance(v, six.string_types): 361 v = to_gn_string(v) 362 else: 363 raise AssertionError('Unexpected %s, %r=%r' % (type(v), k, v)) 364 yield '%s=%s' % (k.replace('-', '_'), v) 365 366 gn_args = { 367 'platform_subdir': 'bt', 368 'cc': 'clang' if clang else 'gcc', 369 'cxx': 'clang++' if clang else 'g++', 370 'ar': 'llvm-ar' if clang else 'ar', 371 'pkg-config': 'pkg-config', 372 'clang_cc': clang, 373 'clang_cxx': clang, 374 'OS': 'linux', 375 'sysroot': self.sysroot, 376 'libdir': os.path.join(self.sysroot, self.libdir), 377 'build_root': self.output_dir, 378 'platform2_root': self.platform_dir, 379 'libbase_ver': self._get_basever(), 380 'enable_exceptions': os.environ.get('CXXEXCEPTIONS', 0) == '1', 381 'external_cflags': [], 382 'external_cxxflags': ["-DNDEBUG"], 383 'enable_werror': True, 384 } 385 386 if clang: 387 # Make sure to mark the clang use flag as true 388 self.use.set_flag('clang', True) 389 gn_args['external_cxxflags'] += ['-I/usr/include/'] 390 391 gn_args_args = list(to_gn_args_args(gn_args)) 392 use_args = ['%s=%s' % (k, str(v).lower()) for k, v in self.use.flags.items()] 393 gn_args_args += ['use={%s}' % (' '.join(use_args))] 394 395 gn_args = [ 396 'gn', 397 'gen', 398 ] 399 400 if self.args.verbose: 401 gn_args.append('-v') 402 403 gn_args += [ 404 '--root=%s' % self.platform_dir, 405 '--args=%s' % ' '.join(gn_args_args), 406 self._gn_default_output(), 407 ] 408 409 if 'PKG_CONFIG_PATH' in self.env: 410 print('DEBUG: PKG_CONFIG_PATH is', self.env['PKG_CONFIG_PATH']) 411 412 self.run_command('configure', gn_args) 413 414 def _gn_build(self, target): 415 """ Generate the ninja command for the target and run it. 416 """ 417 args = ['%s:%s' % ('bt', target)] 418 ninja_args = ['ninja', '-C', self._gn_default_output()] 419 if self.jobs: 420 ninja_args += ['-j', str(self.jobs)] 421 ninja_args += args 422 423 if self.args.verbose: 424 ninja_args.append('-v') 425 426 self.run_command('build', ninja_args) 427 428 def _rust_configure(self): 429 """ Generate config file at cargo_home so we use vendored crates. 430 """ 431 template = """ 432 [source.systembt] 433 directory = "{}/external/rust/vendor" 434 435 [source.crates-io] 436 replace-with = "systembt" 437 local-registry = "/nonexistent" 438 """ 439 440 if not self.args.no_vendored_rust: 441 contents = template.format(self.platform_dir) 442 with open(os.path.join(self.env['CARGO_HOME'], 'config'), 'w') as f: 443 f.write(contents) 444 445 def _rust_build(self): 446 """ Run `cargo build` from platform2/bt directory. 447 """ 448 cmd = ['cargo', 'build'] 449 if not self.args.rust_debug: 450 cmd.append('--release') 451 452 self.run_command('rust', cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) 453 454 def _target_prepare(self): 455 """ Target to prepare the output directory for building. 456 457 This runs gn gen to generate all rquired files and set up the Rust 458 config properly. This will be run 459 """ 460 self._gn_configure() 461 self._rust_configure() 462 463 def _target_hosttools(self): 464 """ Build the tools target in an already prepared environment. 465 """ 466 self._gn_build('tools') 467 468 # Also copy bluetooth_packetgen to CARGO_HOME so it's available 469 shutil.copy(os.path.join(self._gn_default_output(), 'bluetooth_packetgen'), 470 os.path.join(self.env['CARGO_HOME'], 'bin')) 471 472 def _target_docs(self): 473 """Build the Rust docs.""" 474 self.run_command('docs', ['cargo', 'doc'], cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) 475 476 def _target_rust(self): 477 """ Build rust artifacts in an already prepared environment. 478 """ 479 self._rust_build() 480 481 def _target_main(self): 482 """ Build the main GN artifacts in an already prepared environment. 483 """ 484 self._gn_build('all') 485 486 def _target_test(self): 487 """ Runs the host tests. 488 """ 489 # Rust tests first 490 rust_test_cmd = ['cargo', 'test'] 491 if not self.args.rust_debug: 492 rust_test_cmd.append('--release') 493 494 if self.args.test_name: 495 rust_test_cmd = rust_test_cmd + [self.args.test_name, "--", "--test-threads=1", "--nocapture"] 496 497 self.run_command('test', rust_test_cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) 498 499 # Host tests second based on host test list 500 for t in HOST_TESTS: 501 self.run_command('test', [os.path.join(self.output_dir, 'out/Default', t)], 502 cwd=os.path.join(self.output_dir), 503 env=self.env) 504 505 def _target_clippy(self): 506 """ Runs cargo clippy, a collection of lints to catch common mistakes. 507 """ 508 cmd = ['cargo', 'clippy'] 509 self.run_command('rust', cmd, cwd=os.path.join(self.platform_dir, 'bt'), env=self.env) 510 511 def _target_utils(self): 512 """ Builds the utility applications. 513 """ 514 rust_targets = ['hcidoc'] 515 516 # Build targets 517 for target in rust_targets: 518 self.run_command('utils', ['cargo', 'build', '-p', target], 519 cwd=os.path.join(self.platform_dir, 'bt'), 520 env=self.env) 521 522 def _target_install(self): 523 """ Installs files required to run Floss to install directory. 524 """ 525 # First make the install directory 526 prefix = self.install_dir 527 os.makedirs(prefix, exist_ok=True) 528 529 # Next save the cwd and change to install directory 530 last_cwd = os.getcwd() 531 os.chdir(prefix) 532 533 bindir = os.path.join(self.output_dir, 'debug') 534 srcdir = os.path.dirname(__file__) 535 536 install_map = [ 537 { 538 'src': os.path.join(bindir, 'btadapterd'), 539 'dst': 'usr/libexec/bluetooth/btadapterd', 540 'strip': True 541 }, 542 { 543 'src': os.path.join(bindir, 'btmanagerd'), 544 'dst': 'usr/libexec/bluetooth/btmanagerd', 545 'strip': True 546 }, 547 { 548 'src': os.path.join(bindir, 'btclient'), 549 'dst': 'usr/local/bin/btclient', 550 'strip': True 551 }, 552 ] 553 554 for v in install_map: 555 src, partial_dst, strip = (v['src'], v['dst'], v['strip']) 556 dst = os.path.join(prefix, partial_dst) 557 558 # Create dst directory first and copy file there 559 os.makedirs(os.path.dirname(dst), exist_ok=True) 560 print('Installing {}'.format(dst)) 561 shutil.copy(src, dst) 562 563 # Binary should be marked for strip and no-strip option shouldn't be 564 # set. No-strip is useful while debugging. 565 if strip and not self.args.no_strip: 566 self.run_command('install', ['llvm-strip', dst]) 567 568 # Put all files into a tar.gz for easier installation 569 tar_location = os.path.join(prefix, 'floss.tar.gz') 570 with tarfile.open(tar_location, 'w:gz') as tar: 571 for v in install_map: 572 tar.add(v['dst']) 573 574 print('Tarball created at {}'.format(tar_location)) 575 576 def _target_bloat(self): 577 """Run cargo bloat on workspace. 578 """ 579 crate_paths = [ 580 os.path.join(self.platform_dir, 'bt', 'system', 'gd', 'rust', 'linux', 'mgmt'), 581 os.path.join(self.platform_dir, 'bt', 'system', 'gd', 'rust', 'linux', 'service'), 582 os.path.join(self.platform_dir, 'bt', 'system', 'gd', 'rust', 'linux', 'client') 583 ] 584 for crate in crate_paths: 585 self.run_command('bloat', ['cargo', 'bloat', '--release', '--crates', '--wide'], cwd=crate, env=self.env) 586 587 def _target_clean(self): 588 """ Delete the output directory entirely. 589 """ 590 shutil.rmtree(self.output_dir) 591 592 # Remove Cargo.lock that may have become generated 593 cargo_lock_files = [ 594 os.path.join(self.platform_dir, 'bt', 'Cargo.lock'), 595 ] 596 for lock_file in cargo_lock_files: 597 try: 598 os.remove(lock_file) 599 print('Removed {}'.format(lock_file)) 600 except FileNotFoundError: 601 pass 602 603 def _target_all(self): 604 """ Build all common targets (skipping doc, test, and clean). 605 """ 606 self._target_prepare() 607 self._target_hosttools() 608 self._target_main() 609 self._target_rust() 610 611 def build(self): 612 """ Builds according to self.target 613 """ 614 print('Building target ', self.target) 615 616 # Validate that the target is valid 617 if self.target not in VALID_TARGETS: 618 print('Target {} is not valid. Must be in {}'.format(self.target, VALID_TARGETS)) 619 return 620 621 if self.target == 'prepare': 622 self._target_prepare() 623 elif self.target == 'hosttools': 624 self._target_hosttools() 625 elif self.target == 'rust': 626 self._target_rust() 627 elif self.target == 'docs': 628 self._target_docs() 629 elif self.target == 'main': 630 self._target_main() 631 elif self.target == 'test': 632 self._target_test() 633 elif self.target == 'clippy': 634 self._target_clippy() 635 elif self.target == 'clean': 636 self._target_clean() 637 elif self.target == 'install': 638 self._target_install() 639 elif self.target == 'utils': 640 self._target_utils() 641 elif self.target == 'bloat': 642 self._target_bloat() 643 elif self.target == 'all': 644 self._target_all() 645 646 647# Default to 10 min timeouts on all git operations. 648GIT_TIMEOUT_SEC = 600 649 650 651class Bootstrap(): 652 653 def __init__(self, base_dir, bt_dir, partial_staging, clone_timeout): 654 """ Construct bootstrapper. 655 656 Args: 657 base_dir: Where to stage everything. 658 bt_dir: Where bluetooth source is kept (will be symlinked) 659 partial_staging: Whether to do a partial clone for staging. 660 clone_timeout: Timeout for clone operations. 661 """ 662 self.base_dir = os.path.abspath(base_dir) 663 self.bt_dir = os.path.abspath(bt_dir) 664 self.partial_staging = partial_staging 665 self.clone_timeout = clone_timeout 666 667 # Create base directory if it doesn't already exist 668 os.makedirs(self.base_dir, exist_ok=True) 669 670 if not os.path.isdir(self.bt_dir): 671 raise Exception('{} is not a valid directory'.format(self.bt_dir)) 672 673 self.git_dir = os.path.join(self.base_dir, 'repos') 674 self.staging_dir = os.path.join(self.base_dir, 'staging') 675 self.output_dir = os.path.join(self.base_dir, 'output') 676 self.external_dir = os.path.join(self.base_dir, 'staging', 'external') 677 678 self.dir_setup_complete = os.path.join(self.base_dir, '.setup-complete') 679 680 def _run_with_timeout(self, cmd, cwd, timeout=None): 681 """Runs a command using subprocess.check_output. """ 682 print('Running command: {} [at cwd={}]'.format(' '.join(cmd), cwd)) 683 with subprocess.Popen(cmd, cwd=cwd) as proc: 684 try: 685 outs, errs = proc.communicate(timeout=timeout) 686 except subprocess.TimeoutExpired: 687 proc.kill() 688 outs, errs = proc.communicate() 689 print('Timeout on {}'.format(' '.join(cmd)), file=sys.stderr) 690 raise 691 692 if proc.returncode != 0: 693 raise Exception('Cmd {} had return code {}'.format(' '.join(cmd), proc.returncode)) 694 695 def _update_platform2(self): 696 """Updates repositories used for build.""" 697 for project in BOOTSTRAP_GIT_REPOS.keys(): 698 cwd = os.path.join(self.git_dir, project) 699 (repo, commit) = BOOTSTRAP_GIT_REPOS[project] 700 701 # Update to required commit when necessary or pull the latest code. 702 if commit is not None: 703 head = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=cwd).strip() 704 if head != commit: 705 subprocess.check_call(['git', 'fetch'], cwd=cwd) 706 subprocess.check_call(['git', 'checkout', commit], cwd=cwd) 707 else: 708 subprocess.check_call(['git', 'pull'], cwd=cwd) 709 710 def _setup_platform2(self): 711 """ Set up platform2. 712 713 This will check out all the git repos and symlink everything correctly. 714 """ 715 716 # Create all directories we will need to use 717 for dirpath in [self.git_dir, self.staging_dir, self.output_dir, self.external_dir]: 718 os.makedirs(dirpath, exist_ok=True) 719 720 # If already set up, only update platform2 721 if os.path.isfile(self.dir_setup_complete): 722 print('{} already set-up. Updating instead.'.format(self.base_dir)) 723 self._update_platform2() 724 else: 725 clone_options = [] 726 # When doing a partial staging, we use a treeless clone which allows 727 # us to access all commits but downloads things on demand. This 728 # helps speed up the initial git clone during builds but isn't good 729 # for long-term development. 730 if self.partial_staging: 731 clone_options = ['--filter=tree:0'] 732 # Check out all repos in git directory 733 for project in BOOTSTRAP_GIT_REPOS.keys(): 734 (repo, commit) = BOOTSTRAP_GIT_REPOS[project] 735 736 # Try repo clone several times. 737 # Currently, we set timeout on this operation after 738 # |self.clone_timeout|. If it fails, try to recover. 739 tries = 2 740 for x in range(tries): 741 try: 742 self._run_with_timeout(['git', 'clone', repo, project] + clone_options, 743 cwd=self.git_dir, 744 timeout=self.clone_timeout) 745 except subprocess.TimeoutExpired: 746 shutil.rmtree(os.path.join(self.git_dir, project)) 747 if x == tries - 1: 748 raise 749 # All other exceptions should raise 750 except: 751 raise 752 # No exceptions/problems should not retry. 753 else: 754 break 755 756 # Pin to commit. 757 if commit is not None: 758 subprocess.check_call(['git', 'checkout', commit], cwd=os.path.join(self.git_dir, project)) 759 760 # Symlink things 761 symlinks = [ 762 (os.path.join(self.git_dir, 'platform2', 'common-mk'), os.path.join(self.staging_dir, 'common-mk')), 763 (os.path.join(self.git_dir, 'platform2', 'system_api'), os.path.join(self.staging_dir, 'system_api')), 764 (os.path.join(self.git_dir, 'platform2', '.gn'), os.path.join(self.staging_dir, '.gn')), 765 (os.path.join(self.bt_dir), os.path.join(self.staging_dir, 'bt')), 766 (os.path.join(self.git_dir, 'rust_crates'), os.path.join(self.external_dir, 'rust')), 767 (os.path.join(self.git_dir, 'proto_logging'), os.path.join(self.external_dir, 'proto_logging')), 768 ] 769 770 # Create symlinks 771 for pairs in symlinks: 772 (src, dst) = pairs 773 try: 774 os.unlink(dst) 775 except Exception as e: 776 print(e) 777 os.symlink(src, dst) 778 779 # Write to setup complete file so we don't repeat this step 780 with open(self.dir_setup_complete, 'w') as f: 781 f.write('Setup complete.') 782 783 def _pretty_print_install(self, install_cmd, packages, line_limit=80): 784 """ Pretty print an install command. 785 786 Args: 787 install_cmd: Prefixed install command. 788 packages: Enumerate packages and append them to install command. 789 line_limit: Number of characters per line. 790 791 Return: 792 Array of lines to join and print. 793 """ 794 install = [install_cmd] 795 line = ' ' 796 # Remainder needed = space + len(pkg) + space + \ 797 # Assuming 80 character lines, that's 80 - 3 = 77 798 line_limit = line_limit - 3 799 for pkg in packages: 800 if len(line) + len(pkg) < line_limit: 801 line = '{}{} '.format(line, pkg) 802 else: 803 install.append(line) 804 line = ' {} '.format(pkg) 805 806 if len(line) > 0: 807 install.append(line) 808 809 return install 810 811 def _check_package_installed(self, package, cmd, predicate): 812 """Check that the given package is installed. 813 814 Args: 815 package: Check that this package is installed. 816 cmd: Command prefix to check if installed (package appended to end) 817 predicate: Function/lambda to check if package is installed based 818 on output. Takes string output and returns boolean. 819 820 Return: 821 True if package is installed. 822 """ 823 try: 824 output = subprocess.check_output(cmd + [package], stderr=subprocess.STDOUT) 825 is_installed = predicate(output.decode('utf-8')) 826 print(' {} is {}'.format(package, 'installed' if is_installed else 'missing')) 827 828 return is_installed 829 except Exception as e: 830 print(e) 831 return False 832 833 def _get_command_output(self, cmd): 834 """Runs the command and gets the output. 835 836 Args: 837 cmd: Command to run. 838 839 Return: 840 Tuple (Success, Output). Success represents if the command ran ok. 841 """ 842 try: 843 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) 844 return (True, output.decode('utf-8').split('\n')) 845 except Exception as e: 846 print(e) 847 return (False, "") 848 849 def _print_missing_packages(self): 850 """Print any missing packages found via apt. 851 852 This will find any missing packages necessary for build using apt and 853 print it out as an apt-get install printf. 854 """ 855 print('Checking for any missing packages...') 856 857 (success, output) = self._get_command_output(APT_PKG_LIST) 858 if not success: 859 raise Exception("Could not query apt for packages.") 860 861 packages_installed = {} 862 for line in output: 863 if 'installed' in line: 864 split = line.split('/', 2) 865 packages_installed[split[0]] = True 866 867 need_packages = [] 868 for pkg in REQUIRED_APT_PACKAGES: 869 if pkg not in packages_installed: 870 need_packages.append(pkg) 871 872 # No packages need to be installed 873 if len(need_packages) == 0: 874 print('+ All required packages are installed') 875 return 876 877 install = self._pretty_print_install('sudo apt-get install', need_packages) 878 879 # Print all lines so they can be run in cmdline 880 print('Missing system packages. Run the following command: ') 881 print(' \\\n'.join(install)) 882 883 def _print_missing_rust_packages(self): 884 """Print any missing packages found via cargo. 885 886 This will find any missing packages necessary for build using cargo and 887 print it out as a cargo-install printf. 888 """ 889 print('Checking for any missing cargo packages...') 890 891 (success, output) = self._get_command_output(CARGO_PKG_LIST) 892 if not success: 893 raise Exception("Could not query cargo for packages.") 894 895 packages_installed = {} 896 for line in output: 897 # Cargo installed packages have this format 898 # <crate name> <version>: 899 # <binary name> 900 # We only care about the crates themselves 901 if ':' not in line: 902 continue 903 904 split = line.split(' ', 2) 905 packages_installed[split[0]] = True 906 907 need_packages = [] 908 for pkg in REQUIRED_CARGO_PACKAGES: 909 if pkg not in packages_installed: 910 need_packages.append(pkg) 911 912 # No packages to be installed 913 if len(need_packages) == 0: 914 print('+ All required cargo packages are installed') 915 return 916 917 install = self._pretty_print_install('cargo install', need_packages) 918 print('Missing cargo packages. Run the following command: ') 919 print(' \\\n'.join(install)) 920 921 def bootstrap(self): 922 """ Bootstrap the Linux build.""" 923 self._setup_platform2() 924 self._print_missing_packages() 925 self._print_missing_rust_packages() 926 927 928if __name__ == '__main__': 929 parser = argparse.ArgumentParser(description='Simple build for host.') 930 parser.add_argument('--bootstrap-dir', 931 help='Directory to run bootstrap on (or was previously run on).', 932 default="~/.floss") 933 parser.add_argument('--run-bootstrap', 934 help='Run bootstrap code to verify build env is ok to build.', 935 default=False, 936 action='store_true') 937 parser.add_argument('--print-env', 938 help='Print environment variables used for build.', 939 default=False, 940 action='store_true') 941 parser.add_argument('--no-clang', help='Don\'t use clang compiler.', default=False, action='store_true') 942 parser.add_argument('--no-strip', 943 help='Skip stripping binaries during install.', 944 default=False, 945 action='store_true') 946 parser.add_argument('--use', help='Set a specific use flag.') 947 parser.add_argument('--notest', help='Don\'t compile test code.', default=False, action='store_true') 948 parser.add_argument('--test-name', help='Run test with this string in the name.', default=None) 949 parser.add_argument('--target', help='Run specific build target') 950 parser.add_argument('--sysroot', help='Set a specific sysroot path', default='/') 951 parser.add_argument('--libdir', help='Libdir - default = usr/lib', default='usr/lib') 952 parser.add_argument('--jobs', help='Number of jobs to run', default=0, type=int) 953 parser.add_argument('--no-vendored-rust', 954 help='Do not use vendored rust crates', 955 default=False, 956 action='store_true') 957 parser.add_argument('--verbose', help='Verbose logs for build.') 958 parser.add_argument('--rust-debug', help='Build Rust code as debug.', default=False, action='store_true') 959 parser.add_argument( 960 '--partial-staging', 961 help='Bootstrap git repositories with partial clones. Use to speed up initial git clone for automated builds.', 962 default=False, 963 action='store_true') 964 parser.add_argument('--clone-timeout', 965 help='Timeout for repository cloning during bootstrap.', 966 default=GIT_TIMEOUT_SEC, 967 type=int) 968 args = parser.parse_args() 969 970 # Make sure we get absolute path + expanded path for bootstrap directory 971 args.bootstrap_dir = os.path.abspath(os.path.expanduser(args.bootstrap_dir)) 972 973 # Possible values for machine() come from 'uname -m' 974 # Since this script only runs on Linux, x86_64 machines must have this value 975 if platform.machine() != 'x86_64': 976 raise Exception("Only x86_64 machines are currently supported by this build script.") 977 978 if args.run_bootstrap: 979 bootstrap = Bootstrap(args.bootstrap_dir, os.path.dirname(__file__), args.partial_staging, args.clone_timeout) 980 bootstrap.bootstrap() 981 elif args.print_env: 982 build = HostBuild(args) 983 build.print_env() 984 else: 985 build = HostBuild(args) 986 build.build() 987