1#!/usr/bin/env python3 2# Copyright 2014 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6# This script computs the number of concurrent links we want to run in the build 7# as a function of machine spec. It's based on GetDefaultConcurrentLinks in GYP. 8 9import argparse 10import multiprocessing 11import os 12import re 13import subprocess 14import sys 15 16sys.path.insert(1, os.path.join(os.path.dirname(__file__), '..')) 17import gn_helpers 18 19 20def _GetTotalMemoryInBytes(): 21 if sys.platform in ('win32', 'cygwin'): 22 import ctypes 23 24 class MEMORYSTATUSEX(ctypes.Structure): 25 _fields_ = [ 26 ("dwLength", ctypes.c_ulong), 27 ("dwMemoryLoad", ctypes.c_ulong), 28 ("ullTotalPhys", ctypes.c_ulonglong), 29 ("ullAvailPhys", ctypes.c_ulonglong), 30 ("ullTotalPageFile", ctypes.c_ulonglong), 31 ("ullAvailPageFile", ctypes.c_ulonglong), 32 ("ullTotalVirtual", ctypes.c_ulonglong), 33 ("ullAvailVirtual", ctypes.c_ulonglong), 34 ("sullAvailExtendedVirtual", ctypes.c_ulonglong), 35 ] 36 37 stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX)) 38 ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) 39 return stat.ullTotalPhys 40 elif sys.platform.startswith('linux'): 41 if os.path.exists("/proc/meminfo"): 42 with open("/proc/meminfo") as meminfo: 43 memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') 44 for line in meminfo: 45 match = memtotal_re.match(line) 46 if not match: 47 continue 48 return float(match.group(1)) * 2**10 49 elif sys.platform == 'darwin': 50 try: 51 return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize'])) 52 except Exception: 53 return 0 54 # TODO(scottmg): Implement this for other platforms. 55 return 0 56 57 58def _GetDefaultConcurrentLinks(per_link_gb, reserve_gb, thin_lto_type, 59 secondary_per_link_gb, override_ram_in_gb): 60 explanation = [] 61 explanation.append( 62 'per_link_gb={} reserve_gb={} secondary_per_link_gb={}'.format( 63 per_link_gb, reserve_gb, secondary_per_link_gb)) 64 if override_ram_in_gb: 65 mem_total_gb = override_ram_in_gb 66 else: 67 mem_total_gb = float(_GetTotalMemoryInBytes()) / 2**30 68 adjusted_mem_total_gb = max(0, mem_total_gb - reserve_gb) 69 70 # Ensure that there is at least as many links allocated for the secondary as 71 # there is for the primary. The secondary link usually uses fewer gbs. 72 mem_cap = int( 73 max(1, adjusted_mem_total_gb / (per_link_gb + secondary_per_link_gb))) 74 75 try: 76 cpu_count = multiprocessing.cpu_count() 77 except: 78 cpu_count = 1 79 80 # A local LTO links saturate all cores, but only for some amount of the link. 81 cpu_cap = cpu_count 82 if thin_lto_type is not None: 83 assert thin_lto_type == 'local' 84 cpu_cap = min(cpu_count, 6) 85 86 explanation.append( 87 'cpu_count={} cpu_cap={} mem_total_gb={:.1f}GiB adjusted_mem_total_gb={:.1f}GiB' 88 .format(cpu_count, cpu_cap, mem_total_gb, adjusted_mem_total_gb)) 89 90 num_links = min(mem_cap, cpu_cap) 91 if num_links == cpu_cap: 92 if cpu_cap == cpu_count: 93 reason = 'cpu_count' 94 else: 95 reason = 'cpu_cap (thinlto)' 96 else: 97 reason = 'RAM' 98 99 # static link see too many open files if we have many concurrent links. 100 # ref: http://b/233068481 101 if num_links > 30: 102 num_links = 30 103 reason = 'nofile' 104 105 explanation.append('concurrent_links={} (reason: {})'.format( 106 num_links, reason)) 107 108 # Use remaining RAM for a secondary pool if needed. 109 if secondary_per_link_gb: 110 mem_remaining = adjusted_mem_total_gb - num_links * per_link_gb 111 secondary_size = int(max(0, mem_remaining / secondary_per_link_gb)) 112 if secondary_size > cpu_count: 113 secondary_size = cpu_count 114 reason = 'cpu_count' 115 else: 116 reason = 'mem_remaining={:.1f}GiB'.format(mem_remaining) 117 explanation.append('secondary_size={} (reason: {})'.format( 118 secondary_size, reason)) 119 else: 120 secondary_size = 0 121 122 return num_links, secondary_size, explanation 123 124 125def main(): 126 parser = argparse.ArgumentParser() 127 parser.add_argument('--mem_per_link_gb', type=int, default=8) 128 parser.add_argument('--reserve_mem_gb', type=int, default=0) 129 parser.add_argument('--secondary_mem_per_link', type=int, default=0) 130 parser.add_argument('--override-ram-in-gb-for-testing', type=float, default=0) 131 parser.add_argument('--thin-lto') 132 options = parser.parse_args() 133 134 primary_pool_size, secondary_pool_size, explanation = ( 135 _GetDefaultConcurrentLinks(options.mem_per_link_gb, 136 options.reserve_mem_gb, options.thin_lto, 137 options.secondary_mem_per_link, 138 options.override_ram_in_gb_for_testing)) 139 if options.override_ram_in_gb_for_testing: 140 print('primary={} secondary={} explanation={}'.format( 141 primary_pool_size, secondary_pool_size, explanation)) 142 else: 143 sys.stdout.write( 144 gn_helpers.ToGNString({ 145 'primary_pool_size': primary_pool_size, 146 'secondary_pool_size': secondary_pool_size, 147 'explanation': explanation, 148 })) 149 return 0 150 151 152if __name__ == '__main__': 153 sys.exit(main()) 154