1# -*- bazel-starlark -*- 2# Copyright 2023 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Siso configuration for rewriting remote calls into reproxy config.""" 6 7load("@builtin//encoding.star", "json") 8load("@builtin//lib/gn.star", "gn") 9load("@builtin//path.star", "path") 10load("@builtin//struct.star", "module") 11load("./clang_code_coverage_wrapper.star", "clang_code_coverage_wrapper") 12load("./config.star", "config") 13load("./platform.star", "platform") 14load("./rewrapper_cfg.star", "rewrapper_cfg") 15 16def __filegroups(ctx): 17 return {} 18 19def __parse_rewrapper_cmdline(ctx, cmd): 20 if not "rewrapper" in cmd.args[0]: 21 return [], "", False 22 23 # Example command: 24 # ../../buildtools/reclient/rewrapper 25 # -cfg=../../buildtools/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg 26 # -inputs=build/config/unsafe_buffers_paths.txt 27 # -exec_root=/path/to/your/chromium/src/ 28 # ../../third_party/llvm-build/Release+Asserts/bin/clang++ 29 # [rest of clang args] 30 # We don't need to care about: 31 # -exec_root: Siso already knows this. 32 wrapped_command_pos = -1 33 cfg_file = None 34 skip = "" 35 rw_ops = {} 36 for i, arg in enumerate(cmd.args): 37 if i == 0: 38 continue 39 if arg.startswith("-cfg="): 40 cfg_file = ctx.fs.canonpath(arg.removeprefix("-cfg=")) 41 continue 42 if arg.startswith("-inputs=") or skip == "-inputs": 43 rw_ops["inputs"] = arg.removeprefix("-inputs=").split(",") 44 skip = "" 45 continue 46 if arg == "-inputs": 47 skip = arg 48 continue 49 if not arg.startswith("-"): 50 wrapped_command_pos = i 51 break 52 if wrapped_command_pos < 1: 53 fail("couldn't find first non-arg passed to rewrapper for %s" % str(cmd.args)) 54 if not cfg_file: 55 return cmd.args[wrapped_command_pos:], rw_ops, True 56 rw_cfg_opts = rewrapper_cfg.parse(ctx, cfg_file) 57 58 # Command line options have higher priority than the ones in the cfg file. 59 rw_cfg_opts.update(rw_ops) 60 return cmd.args[wrapped_command_pos:], rw_cfg_opts, True 61 62def __parse_cros_rewrapper_cmdline(ctx, cmd): 63 # fix cros sdk clang command line and extract rewrapper cfg. 64 # Example command: 65 # ../../build/cros_cache/chrome-sdk/symlinks/amd64-generic+15629.0.0+target_toolchain/bin/x86_64-cros-linux-gnu-clang++ 66 # -MMD -MF obj/third_party/abseil-cpp/absl/base/base/spinlock.o.d 67 # ... 68 # --rewrapper-path /usr/local/google/home/ukai/src/chromium/src/build/args/chromeos/rewrapper_amd64-generic 69 # --rewrapper-cfg ../../buildtools/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg 70 # -pipe -march=x86-64 -msse3 ... 71 cfg_file = None 72 skip = "" 73 args = [] 74 toolchainpath = None 75 for i, arg in enumerate(cmd.args): 76 if i == 0: 77 toolchainpath = path.dir(path.dir(ctx.fs.canonpath(arg))) 78 args.append(arg) 79 continue 80 if skip: 81 if skip == "--rewrapper-cfg": 82 cfg_file = ctx.fs.canonpath(arg) 83 skip = "" 84 continue 85 if arg in ("--rewrapper-path", "--rewrapper-cfg"): 86 skip = arg 87 continue 88 args.append(arg) 89 if not cfg_file: 90 fail("couldn't find rewrapper cfg file in %s" % str(cmd.args)) 91 rwcfg = rewrapper_cfg.parse(ctx, cfg_file) 92 inputs = rwcfg.get("inputs", []) 93 inputs.extend([ 94 path.join(toolchainpath, "bin"), 95 path.join(toolchainpath, "lib"), 96 path.join(toolchainpath, "usr/bin"), 97 path.join(toolchainpath, "usr/lib64/clang"), 98 # TODO: b/320189180 - Simple Chrome builds should use libraries under usr/lib64. 99 # But, Ninja/Reclient also don't use them unexpectedly. 100 ]) 101 rwcfg["inputs"] = inputs 102 rwcfg["preserve_symlinks"] = True 103 return args, rwcfg 104 105# TODO(b/278225415): change gn so this wrapper (and by extension this handler) becomes unnecessary. 106def __parse_clang_code_coverage_wrapper_cmdline(ctx, cmd): 107 # Example command: 108 # python3 109 # ../../build/toolchain/clang_code_coverage_wrapper.py 110 # --target-os=... 111 # --files_to_instrument=... 112 # ../../buildtools/reclient/rewrapper 113 # -cfg=../../buildtools/reclient_cfgs/chromium-browser-clang/rewrapper_linux.cfg 114 # -inputs=build/config/unsafe_buffers_paths.txt 115 # -exec_root=/path/to/your/chromium/src/ 116 # ../../third_party/llvm-build/Release+Asserts/bin/clang++ 117 # [rest of clang args] 118 # We don't need to care about: 119 # most args to clang_code_coverage_wrapper (need --files_to_instrument as tool_input) 120 # -exec_root: Siso already knows this. 121 rewrapper_pos = -1 122 wrapped_command_pos = -1 123 cfg_file = None 124 skip = None 125 rw_ops = {} 126 for i, arg in enumerate(cmd.args): 127 if i < 2: 128 continue 129 if rewrapper_pos == -1 and not arg.startswith("-"): 130 rewrapper_pos = i 131 continue 132 if rewrapper_pos > 0 and arg.startswith("-cfg="): 133 cfg_file = ctx.fs.canonpath(arg.removeprefix("-cfg=")) 134 continue 135 if arg.startswith("-inputs=") or skip == "-inputs": 136 rw_ops["inputs"] = arg.removeprefix("-inputs=").split(",") 137 skip = "" 138 continue 139 if arg == "-inputs": 140 skip = arg 141 continue 142 if rewrapper_pos > 0 and not arg.startswith("-"): 143 wrapped_command_pos = i 144 break 145 if rewrapper_pos < 1: 146 fail("couldn't find rewrapper in %s" % str(cmd.args)) 147 if wrapped_command_pos < 1: 148 fail("couldn't find first non-arg passed to rewrapper for %s" % str(cmd.args)) 149 if not cfg_file: 150 fail("couldn't find rewrapper cfg file in %s" % str(cmd.args)) 151 coverage_wrapper_command = cmd.args[:rewrapper_pos] + cmd.args[wrapped_command_pos:] 152 clang_command = clang_code_coverage_wrapper.run(ctx, list(coverage_wrapper_command)) 153 if len(clang_command) > 1 and "/chrome-sdk/" in clang_command[0]: 154 # TODO: implement cros sdk support under code coverage wrapper 155 fail("need to fix handler for cros sdk under code coverage wrapper") 156 rw_cfg_opts = rewrapper_cfg.parse(ctx, cfg_file) 157 158 # Command line options have higher priority than the ones in the cfg file. 159 rw_cfg_opts.update(rw_ops) 160 return clang_command, rw_cfg_opts 161 162def __rewrite_rewrapper(ctx, cmd, use_large = False): 163 # If clang-coverage, needs different handling. 164 if len(cmd.args) > 2 and "clang_code_coverage_wrapper.py" in cmd.args[1]: 165 args, rwcfg = __parse_clang_code_coverage_wrapper_cmdline(ctx, cmd) 166 elif len(cmd.args) > 1 and "/chrome-sdk/" in cmd.args[0]: 167 args, rwcfg = __parse_cros_rewrapper_cmdline(ctx, cmd) 168 else: 169 # handling for generic rewrapper. 170 args, rwcfg, wrapped = __parse_rewrapper_cmdline(ctx, cmd) 171 if not wrapped: 172 print("command doesn't have rewrapper. %s" % str(cmd.args)) 173 return 174 if not rwcfg: 175 fail("couldn't find rewrapper cfg file in %s" % str(cmd.args)) 176 if use_large: 177 platform = rwcfg.get("platform", {}) 178 if platform.get("OSFamily") == "Windows": 179 # Since there is no large Windows workers, it needs to run locally. 180 ctx.actions.fix(args = args) 181 return 182 if platform: 183 action_key = None 184 for key in rwcfg["platform"]: 185 if key.startswith("label:action_"): 186 action_key = key 187 break 188 if action_key: 189 rwcfg["platform"].pop(action_key) 190 else: 191 rwcfg["platform"] = {} 192 rwcfg["platform"].update({ 193 "label:action_large": "1", 194 }) 195 196 # Some large compiles take longer than the default timeout 2m. 197 rwcfg["exec_timeout"] = "4m" 198 rwcfg["reclient_timeout"] = "4m" 199 ctx.actions.fix( 200 args = args, 201 reproxy_config = json.encode(rwcfg), 202 ) 203 204def __rewrite_rewrapper_large(ctx, cmd): 205 return __rewrite_rewrapper(ctx, cmd, use_large = True) 206 207def __strip_rewrapper(ctx, cmd): 208 # If clang-coverage, needs different handling. 209 if len(cmd.args) > 2 and "clang_code_coverage_wrapper.py" in cmd.args[1]: 210 args, _ = __parse_clang_code_coverage_wrapper_cmdline(ctx, cmd) 211 else: 212 args, _, wrapped = __parse_rewrapper_cmdline(ctx, cmd) 213 if not wrapped: 214 print("command doesn't have rewrapper. %s" % str(cmd.args)) 215 return 216 ctx.actions.fix(args = args) 217 218__handlers = { 219 "rewrite_rewrapper": __rewrite_rewrapper, 220 "rewrite_rewrapper_large": __rewrite_rewrapper_large, 221 "strip_rewrapper": __strip_rewrapper, 222} 223 224def __use_remoteexec(ctx): 225 if "args.gn" in ctx.metadata: 226 gn_args = gn.args(ctx) 227 if gn_args.get("use_remoteexec") == "true": 228 return True 229 return False 230 231def __step_config(ctx, step_config): 232 # New rules to convert commands calling rewrapper to use reproxy instead. 233 new_rules = [] 234 235 # Disable racing on builders since bots don't have many CPU cores. 236 # TODO: b/297807325 - Siso wants to handle local execution. 237 # However, Reclient's alerts require racing and local fallback to be 238 # done on Reproxy side. 239 exec_strategy = "racing" 240 if config.get(ctx, "builder"): 241 exec_strategy = "remote_local_fallback" 242 243 for rule in step_config["rules"]: 244 # Replace nacl-clang/clang++ rules without command_prefix, because they will incorrectly match rewrapper. 245 # Replace the original step rule with one that only rewrites rewrapper and convert its rewrapper config to reproxy config. 246 if rule["name"].find("nacl-clang") >= 0 and not rule.get("command_prefix"): 247 new_rule = { 248 "name": rule["name"], 249 "action": rule["action"], 250 "handler": "rewrite_rewrapper", 251 } 252 new_rules.append(new_rule) 253 continue 254 255 # clang cxx/cc/objcxx/objc will always have rewrapper config when use_remoteexec=true. 256 # Remove the native siso handling and replace with custom rewrapper-specific handling. 257 # All other rule values are not reused, instead use rewrapper config via handler. 258 # (In particular, command_prefix should be avoided because it will be rewrapper.) 259 if (rule["name"].startswith("clang/cxx") or rule["name"].startswith("clang/cc") or 260 rule["name"].startswith("clang-cl/cxx") or rule["name"].startswith("clang-cl/cc") or 261 rule["name"].startswith("clang/objc")): 262 if not rule.get("action"): 263 fail("clang rule %s found without action" % rule["name"]) 264 265 new_rule = { 266 "name": rule["name"], 267 "action": rule["action"], 268 "exclude_input_patterns": rule.get("exclude_input_patterns"), 269 "handler": "rewrite_rewrapper", 270 "input_root_absolute_path": rule.get("input_root_absolute_path"), 271 } 272 new_rules.append(new_rule) 273 continue 274 275 # clang-coverage/ is handled by the rewrite_rewrapper handler of clang/{cxx, cc} action rules above, so ignore these rules. 276 if rule["name"].startswith("clang-coverage/"): 277 continue 278 279 # Add non-remote rules as-is. 280 if not rule.get("remote"): 281 new_rules.append(rule) 282 continue 283 284 # Finally handle remaining remote rules. It's assumed it is enough to only convert native remote config to reproxy config. 285 platform_ref = rule.get("platform_ref") 286 if platform_ref: 287 p = step_config["platforms"].get(platform_ref) 288 if not p: 289 fail("Rule %s uses undefined platform '%s'" % (rule["name"], platform_ref)) 290 else: 291 p = step_config.get("platforms", {}).get("default") 292 if not p: 293 fail("Rule %s did not set platform_ref but no default platform exists" % rule["name"]) 294 rule["reproxy_config"] = { 295 "platform": p, 296 "labels": { 297 "type": "tool", 298 "siso_rule": rule["name"], 299 }, 300 "canonicalize_working_dir": rule.get("canonicalize_dir", False), 301 "exec_strategy": exec_strategy, 302 "exec_timeout": rule.get("timeout", "10m"), 303 "reclient_timeout": rule.get("timeout", "10m"), 304 "download_outputs": True, 305 } 306 new_rules.append(rule) 307 308 step_config["rules"] = new_rules 309 return step_config 310 311reproxy = module( 312 "reproxy", 313 enabled = __use_remoteexec, 314 step_config = __step_config, 315 filegroups = __filegroups, 316 handlers = __handlers, 317) 318