1#!prebuilts/build-tools/linux-x86/bin/py3-cmd -B 2 3# Copyright 2024, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Script to collect all of the make variables from all product config combos. 18 19This script must be run from the root of the source tree. 20 21See GetArgs() below or run dump_product_config for more information. 22""" 23 24import argparse 25import asyncio 26import contextlib 27import csv 28import dataclasses 29import json 30import multiprocessing 31import os 32import subprocess 33import sys 34import time 35from typing import List, Dict, Tuple, Optional 36 37import buildbot 38 39# We have some BIG variables 40csv.field_size_limit(sys.maxsize) 41 42 43class DataclassJSONEncoder(json.JSONEncoder): 44 """JSONEncoder for our custom types.""" 45 def default(self, o): 46 if dataclasses.is_dataclass(o): 47 return dataclasses.asdict(o) 48 return super().default(o) 49 50 51def GetProducts(): 52 """Get the all of the available TARGET_PRODUCT values.""" 53 try: 54 stdout = subprocess.check_output(["build/soong/bin/list_products"], text=True) 55 except subprocess.CalledProcessError: 56 sys.exit(1) 57 return [s.strip() for s in stdout.splitlines() if s.strip()] 58 59 60def GetReleases(product): 61 """For a given product, get the release configs available to it.""" 62 if True: 63 # Hard code the list 64 mainline_products = [ 65 "module_arm", 66 "module_x86", 67 "module_arm64", 68 "module_riscv64", 69 "module_x86_64", 70 "module_arm64only", 71 "module_x86_64only", 72 ] 73 if product in mainline_products: 74 return ["trunk_staging", "trunk", "mainline"] 75 else: 76 return ["trunk_staging", "trunk", "next"] 77 else: 78 # Get it from the build system 79 try: 80 stdout = subprocess.check_output(["build/soong/bin/list_releases", product], text=True) 81 except subprocess.CalledProcessError: 82 sys.exit(1) 83 return [s.strip() for s in stdout.splitlines() if s.strip()] 84 85 86def GenerateAllLunchTargets(): 87 """Generate the full list of lunch targets.""" 88 for product in GetProducts(): 89 for release in GetReleases(product): 90 for variant in ["user", "userdebug", "eng"]: 91 yield (product, release, variant) 92 93 94async def ParallelExec(parallelism, tasks): 95 ''' 96 ParallelExec takes a parallelism number, and an iterator of tasks to run. 97 Then it will run all the tasks, but a maximum of parallelism will be run at 98 any given time. The tasks must be async functions that accept one argument, 99 which will be an integer id of the worker that they're running on. 100 ''' 101 tasks = iter(tasks) 102 103 overall_start = time.monotonic() 104 # lists so they can be modified from the inner function 105 total_duration = [0] 106 count = [0] 107 async def dispatch(worker): 108 while True: 109 try: 110 task = next(tasks) 111 item_start = time.monotonic() 112 await task(worker) 113 now = time.monotonic() 114 item_duration = now - item_start 115 count[0] += 1 116 total_duration[0] += item_duration 117 sys.stderr.write(f"Timing: Items processed: {count[0]}, Wall time: {now-overall_start:0.1f} sec, Throughput: {(now-overall_start)/count[0]:0.3f} sec per item, Average duration: {total_duration[0]/count[0]:0.1f} sec\n") 118 except StopIteration: 119 return 120 121 await asyncio.gather(*[dispatch(worker) for worker in range(parallelism)]) 122 123 124async def DumpProductConfigs(out, generator, out_dir): 125 """Collects all of the product config data and store it in file.""" 126 # Write the outer json list by hand so we can stream it 127 out.write("[") 128 try: 129 first_result = [True] # a list so it can be modified from the inner function 130 def run(lunch): 131 async def curried(worker): 132 sys.stderr.write(f"running: {'-'.join(lunch)}\n") 133 result = await DumpOneProductConfig(lunch, os.path.join(out_dir, f"lunchable_{worker}")) 134 if first_result[0]: 135 out.write("\n") 136 first_result[0] = False 137 else: 138 out.write(",\n") 139 result.dumpToFile(out) 140 sys.stderr.write(f"finished: {'-'.join(lunch)}\n") 141 return curried 142 143 await ParallelExec(multiprocessing.cpu_count(), (run(lunch) for lunch in generator)) 144 finally: 145 # Close the json regardless of how we exit 146 out.write("\n]\n") 147 148 149@dataclasses.dataclass(frozen=True) 150class Variable: 151 """A variable name, value and where it was set.""" 152 name: str 153 value: str 154 location: str 155 156 157@dataclasses.dataclass(frozen=True) 158class ProductResult: 159 product: str 160 release: str 161 variant: str 162 board_includes: List[str] 163 product_includes: Dict[str, List[str]] 164 product_graph: List[Tuple[str, str]] 165 board_vars: List[Variable] 166 product_vars: List[Variable] 167 168 def dumpToFile(self, f): 169 json.dump(self, f, sort_keys=True, indent=2, cls=DataclassJSONEncoder) 170 171 172@dataclasses.dataclass(frozen=True) 173class ProductError: 174 product: str 175 release: str 176 variant: str 177 error: str 178 179 def dumpToFile(self, f): 180 json.dump(self, f, sort_keys=True, indent=2, cls=DataclassJSONEncoder) 181 182 183def NormalizeInheritGraph(lists): 184 """Flatten the inheritance graph to a simple list for easier querying.""" 185 result = set() 186 for item in lists: 187 for i in range(len(item)): 188 result.add((item[i+1] if i < len(item)-1 else "", item[i])) 189 return sorted(list(result)) 190 191 192def ParseDump(lunch, filename) -> ProductResult: 193 """Parses the csv and returns a tuple of the data.""" 194 def diff(initial, final): 195 return [after for after in final.values() if 196 initial.get(after.name, Variable(after.name, "", "<unset>")).value != after.value] 197 product_initial = {} 198 product_final = {} 199 board_initial = {} 200 board_final = {} 201 inherit_product = [] # The stack of inherit-product calls 202 product_includes = {} # Other files included by each of the properly imported files 203 board_includes = [] # Files included by boardconfig 204 with open(filename) as f: 205 phase = "" 206 for line in csv.reader(f): 207 if line[0] == "phase": 208 phase = line[1] 209 elif line[0] == "val": 210 # TOOD: We should skip these somewhere else. 211 if line[3].startswith("_ALL_RELEASE_FLAGS"): 212 continue 213 if line[3].startswith("PRODUCTS."): 214 continue 215 if phase == "PRODUCTS": 216 if line[2] == "initial": 217 product_initial[line[3]] = Variable(line[3], line[4], line[5]) 218 if phase == "PRODUCT-EXPAND": 219 if line[2] == "final": 220 product_final[line[3]] = Variable(line[3], line[4], line[5]) 221 if phase == "BOARD": 222 if line[2] == "initial": 223 board_initial[line[3]] = Variable(line[3], line[4], line[5]) 224 if line[2] == "final": 225 board_final[line[3]] = Variable(line[3], line[4], line[5]) 226 elif line[0] == "imported": 227 imports = [s.strip() for s in line[1].split()] 228 if imports: 229 inherit_product.append(imports) 230 inc = [s.strip() for s in line[2].split()] 231 for f in inc: 232 product_includes.setdefault(imports[0], []).append(f) 233 elif line[0] == "board_config_files": 234 board_includes += [s.strip() for s in line[1].split()] 235 return ProductResult( 236 product = lunch[0], 237 release = lunch[1], 238 variant = lunch[2], 239 product_vars = diff(product_initial, product_final), 240 board_vars = diff(board_initial, board_final), 241 product_graph = NormalizeInheritGraph(inherit_product), 242 product_includes = product_includes, 243 board_includes = board_includes 244 ) 245 246 247async def DumpOneProductConfig(lunch, out_dir) -> ProductResult | ProductError: 248 """Print a single config's lunch info to stdout.""" 249 product, release, variant = lunch 250 251 dumpconfig_file = os.path.join(out_dir, f"{product}-{release}-{variant}.csv") 252 253 # Run get_build_var to bootstrap soong_ui for this target 254 env = dict(os.environ) 255 env["TARGET_PRODUCT"] = product 256 env["TARGET_RELEASE"] = release 257 env["TARGET_BUILD_VARIANT"] = variant 258 env["OUT_DIR"] = out_dir 259 process = await asyncio.create_subprocess_exec( 260 "build/soong/bin/get_build_var", 261 "TARGET_PRODUCT", 262 stdout=subprocess.PIPE, 263 stderr=subprocess.STDOUT, 264 env=env 265 ) 266 stdout, _ = await process.communicate() 267 stdout = stdout.decode() 268 269 if process.returncode != 0: 270 return ProductError( 271 product = product, 272 release = release, 273 variant = variant, 274 error = stdout 275 ) 276 else: 277 # Run kati to extract the data 278 process = await asyncio.create_subprocess_exec( 279 "prebuilts/build-tools/linux-x86/bin/ckati", 280 "-f", 281 "build/make/core/dumpconfig.mk", 282 f"TARGET_PRODUCT={product}", 283 f"TARGET_RELEASE={release}", 284 f"TARGET_BUILD_VARIANT={variant}", 285 f"DUMPCONFIG_FILE={dumpconfig_file}", 286 stdout=subprocess.PIPE, 287 stderr=subprocess.STDOUT, 288 env=env 289 ) 290 stdout, _ = await process.communicate() 291 if process.returncode != 0: 292 stdout = stdout.decode() 293 return ProductError( 294 product = product, 295 release = release, 296 variant = variant, 297 error = stdout 298 ) 299 else: 300 # Parse and record the output 301 return ParseDump(lunch, dumpconfig_file) 302 303 304def GetArgs(): 305 """Parse command line arguments.""" 306 parser = argparse.ArgumentParser( 307 description="Collect all of the make variables from product config.", 308 epilog="NOTE: This script must be run from the root of the source tree.") 309 parser.add_argument("--lunch", nargs="*") 310 parser.add_argument("--dist", action="store_true") 311 312 return parser.parse_args() 313 314 315async def main(): 316 args = GetArgs() 317 318 out_dir = buildbot.OutDir() 319 320 if args.dist: 321 cm = open(os.path.join(buildbot.DistDir(), "all_product_config.json"), "w") 322 else: 323 cm = contextlib.nullcontext(sys.stdout) 324 325 326 with cm as out: 327 if args.lunch: 328 lunches = [lunch.split("-") for lunch in args.lunch] 329 fail = False 330 for i in range(len(lunches)): 331 if len(lunches[i]) != 3: 332 sys.stderr.write(f"Malformed lunch targets: {args.lunch[i]}\n") 333 fail = True 334 if fail: 335 sys.exit(1) 336 if len(lunches) == 1: 337 result = await DumpOneProductConfig(lunches[0], out_dir) 338 result.dumpToFile(out) 339 out.write("\n") 340 else: 341 await DumpProductConfigs(out, lunches, out_dir) 342 else: 343 # All configs mode. This will exec single config mode in parallel 344 # for each lunch combo. Write output to $DIST_DIR. 345 await DumpProductConfigs(out, GenerateAllLunchTargets(), out_dir) 346 347 348if __name__ == "__main__": 349 asyncio.run(main()) 350 351 352# vim: set syntax=python ts=4 sw=4 sts=4: 353 354