1# -*- bazel-starlark -*- 2# Copyright 2023 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Siso configuration main entry.""" 6 7load("@builtin//encoding.star", "json") 8load("@builtin//runtime.star", "runtime") 9load("@builtin//struct.star", "module") 10load("./blink_all.star", "blink_all") 11load("./linux.star", chromium_linux = "chromium") 12load("./mac.star", chromium_mac = "chromium") 13load("./mojo.star", "mojo") 14load("./platform.star", "platform") 15load("./reproxy.star", "reproxy") 16load("./simple.star", "simple") 17load("./windows.star", chromium_windows = "chromium") 18 19def __use_large_b289968566(ctx, step_config): 20 # TODO(b/289968566): they often faile with exit=137 (OOM?). 21 # They need to run on a machine has more memory than the default machine type n2-custom-2-3840 22 exit137_list = [ 23 # Fallback happens with follwoing args.gn (try/linux-chromeos-rel-compilator). 24 # Fallback may happen in other build config too. 25 # also_build_lacros_chrome = true 26 # dcheck_always_on = true 27 # enable_backup_ref_ptr_feature_flag = true 28 # enable_dangling_raw_ptr_checks = true 29 # enable_dangling_raw_ptr_feature_flag = true 30 # ffmpeg_branding = "ChromeOS" 31 # is_component_build = false 32 # is_debug = false 33 # proprietary_codecs = true 34 # symbol_level = 0 35 # target_os = "chromeos" 36 # use_cups = true 37 # use_remoteexec = true 38 "./lacros_clang_x64/obj/chrome/browser/browser/chrome_browser_interface_binders.o", 39 "./lacros_clang_x64/obj/chrome/browser/browser/chrome_content_browser_client.o", 40 "./lacros_clang_x64/obj/content/browser/browser/browser_interface_binders.o", 41 "./lacros_clang_x64/obj/content/browser/browser/render_frame_host_impl.o", 42 "./lacros_clang_x64/obj/net/http/transport_security_state_generated_files/transport_security_state.o", 43 "./lacros_clang_x64/obj/third_party/blink/public/mojom/mojom_platform/speech_recognition_grammar.mojom.o", 44 "./obj/chrome/browser/ash/ash/autotest_private_api.o", 45 "./obj/chrome/browser/ash/ash/chrome_browser_main_parts_ash.o", 46 "./obj/chrome/browser/ash/ash/user_session_manager.o", 47 "./obj/chrome/browser/ash/ash/webui_login_view.o", 48 "./obj/chrome/browser/ash/system_web_apps/apps/browser_tests/media_app_integration_browsertest.o", 49 "./obj/chrome/browser/ash/system_web_apps/browser_tests/system_web_app_manager_browsertest.o", 50 "./obj/chrome/browser/ash/unit_tests/wizard_controller_unittest.o", 51 "./obj/chrome/browser/browser/browser_prefs.o", 52 "./obj/chrome/browser/browser/chrome_browser_interface_binders.o", 53 "./obj/chrome/browser/browser/chrome_content_browser_client.o", 54 "./obj/chrome/browser/browser/render_view_context_menu.o", 55 "./obj/chrome/browser/ui/ash/holding_space/browser_tests/holding_space_ui_browsertest.o", 56 "./obj/chrome/test/browser_tests/browser_non_client_frame_view_chromeos_browsertest.o", 57 "./obj/chrome/test/browser_tests/chrome_shelf_controller_browsertest.o", 58 "./obj/chrome/test/browser_tests/device_local_account_browsertest.o", 59 "./obj/chrome/test/browser_tests/file_manager_browsertest_base.o", 60 "./obj/chrome/test/browser_tests/full_restore_app_launch_handler_browsertest.o", 61 "./obj/chrome/test/browser_tests/safe_browsing_blocking_page_test.o", 62 "./obj/chrome/test/browser_tests/scalable_iph_browsertest.o", 63 "./obj/chrome/test/interactive_ui_tests/local_card_migration_uitest.o", 64 "./obj/chrome/test/interactive_ui_tests/system_web_app_interactive_uitest.o", 65 "./obj/chrome/test/interactive_ui_tests/tab_drag_controller_interactive_uitest.o", 66 "./obj/chrome/test/test_support_ui/web_app_integration_test_driver.o", 67 "./obj/chrome/test/unit_tests/chrome_browsing_data_remover_delegate_unittest.o", 68 "./obj/chrome/test/unit_tests/chrome_shelf_controller_unittest.o", 69 "./obj/chrome/test/unit_tests/render_view_context_menu_unittest.o", 70 "./obj/content/browser/browser/browser_interface_binders.o", 71 72 # Fallback happens with follwoing args.gn (try/linux-lacros-rel-compilator). 73 # Fallback may happen in other build config too. 74 # also_build_ash_chrome = true 75 # chromeos_is_browser_only = true 76 # dcheck_always_on = true 77 # is_clang = true 78 # is_component_build = false 79 # is_debug = false 80 # symbol_level = 0 81 # target_os = "chromeos" 82 # use_cups = true 83 # use_remoteexec = true 84 "./ash_clang_x64/obj/chrome/browser/ash/ash/autotest_private_api.o", 85 "./ash_clang_x64/obj/chrome/browser/ash/ash/chrome_browser_main_parts_ash.o", 86 "./ash_clang_x64/obj/chrome/browser/browser/browser_prefs.o", 87 "./ash_clang_x64/obj/chrome/browser/browser/chrome_browser_interface_binders.o", 88 "./ash_clang_x64/obj/chrome/browser/browser/chrome_content_browser_client.o", 89 "./ash_clang_x64/obj/content/browser/browser/browser_interface_binders.o", 90 91 # Fallback happens with follwoing args.gn (try/android_compile_dbg). 92 # Fallback may happen in other build config too. 93 # debuggable_apks = false 94 # ffmpeg_branding = "Chrome" 95 # is_component_build = true 96 # is_debug = true 97 # proprietary_codecs = true 98 # symbol_level = 0 99 # target_cpu = "arm64" 100 # target_os = "android" 101 # use_remoteexec = true 102 "./android_clang_arm/obj/content/browser/browser/browser_interface_binders.o", 103 "./obj/content/test/content_unittests__library/auction_runner_unittest.o", 104 105 # Fallback happens with follwoing args.gn (try/fuchsia-x64-cast-receiver-rel). 106 # Fallback may happen in other build config too. 107 # cast_streaming_enable_remoting = true 108 # chrome_pgo_phase = 0 109 # dcheck_always_on = true 110 # enable_cast_receiver = true 111 # enable_dav1d_decoder = false 112 # enable_hidpi = false 113 # enable_libaom = false 114 # enable_library_cdms = false 115 # enable_logging_override = true 116 # enable_pdf = false 117 # enable_plugins = false 118 # enable_printing = false 119 # fuchsia_code_coverage = true 120 # is_component_build = false 121 # is_debug = false 122 # optimize_for_size = true 123 # optional_trace_events_enabled = false 124 # produce_v8_compile_hints = false 125 # symbol_level = 0 126 # target_os = "fuchsia" 127 # use_remoteexec = true 128 # use_thin_lto = false 129 "./obj/fuchsia_web/runners/cast_runner_integration_tests__exec/cast_runner_integration_test.o", 130 "./obj/fuchsia_web/webengine/web_engine_core/frame_impl.o", 131 132 # Fallback happens with follwoing args.gn (try/linux_chromium_asan_rel_ng). 133 # dcheck_always_on = true 134 # fail_on_san_warnings = true 135 # is_asan = true 136 # is_component_build = false 137 # is_debug = false 138 # is_lsan = true 139 # symbol_level = 1 140 # use_remoteexec = true 141 "./obj/components/autofill/core/browser/unit_tests/browser_autofill_manager_unittest.o", 142 "./obj/content/browser/browser/render_frame_host_impl.o", 143 "./obj/content/browser/browser/web_contents_impl.o", 144 "./obj/content/test/content_browsertests/back_forward_cache_features_browsertest.o", 145 "./obj/content/test/content_browsertests/back_forward_cache_internal_browsertest.o", 146 "./obj/content/test/content_browsertests/fenced_frame_browsertest.o", 147 "./obj/content/test/content_browsertests/interest_group_browsertest.o", 148 "./obj/content/test/content_browsertests/navigation_controller_impl_browsertest.o", 149 "./obj/content/test/content_browsertests/navigation_request_browsertest.o", 150 "./obj/content/test/content_browsertests/prerender_browsertest.o", 151 "./obj/content/test/content_browsertests/render_frame_host_impl_browsertest.o", 152 "./obj/content/test/content_browsertests/shared_storage_browsertest.o", 153 "./obj/content/test/content_browsertests/site_per_process_browsertest.o", 154 "./obj/content/test/content_browsertests/web_contents_impl_browsertest.o", 155 "./obj/content/test/content_unittests/ad_auction_service_impl_unittest.o", 156 "./obj/content/test/content_unittests/auction_runner_unittest.o", 157 "./obj/content/test/content_unittests/authenticator_impl_unittest.o", 158 "./obj/content/test/content_unittests/web_usb_service_impl_unittest.o", 159 "./obj/net/third_party/quiche/quiche_tests/quic_connection_test.o", 160 "./obj/third_party/abseil-cpp/absl/functional/any_invocable_test/any_invocable_test.o", 161 "./obj/third_party/blink/renderer/core/unit_tests/web_frame_test.o", 162 "./obj/third_party/blink/renderer/core/unit_tests/web_media_player_impl_unittest.o", 163 "./obj/third_party/perfetto/protos/perfetto/trace/merged_trace_lite/perfetto_trace.pb.o", 164 "./obj/ui/gl/gl_unittest_utils/gl_bindings_autogen_mock.o", 165 166 # Fallback happens with follwoing args.gn (ci/Linux MSan Builder). 167 # dcheck_always_on = false 168 # is_component_build = false 169 # is_debug = false 170 # is_msan = true 171 # msan_track_origins = 2 172 # use_remoteexec = true 173 "./obj/chrome/test/unit_tests/site_settings_handler_unittest.o", 174 "./obj/components/policy/chrome_settings_proto_generated_compile_proto/chrome_settings.pb.o", 175 "./obj/content/test/content_browsertests/cross_origin_opener_policy_browsertest.o", 176 "./obj/content/test/content_browsertests/navigation_controller_impl_browsertest.o", 177 "./obj/content/test/content_unittests/auction_runner_unittest.o", 178 "./obj/content/test/test_support/service_worker_test_utils.o", 179 "./obj/net/dns/tests/host_resolver_manager_unittest.o", 180 "./obj/net/third_party/quiche/quiche_tests/oghttp2_adapter_test.o", 181 "./obj/net/third_party/quiche/quiche_tests/quic_connection_test.o", 182 "./obj/net/third_party/quiche/quiche_tests/structured_headers_generated_test.o", 183 "./obj/ui/accessibility/accessibility_unittests/ax_node_position_unittest.o", 184 "./obj/ui/gl/gl_unittest_utils/gl_bindings_autogen_mock.o", 185 "./obj/ui/gl/gl_unittest_utils/gl_mock.o", 186 "./obj/v8/v8_turboshaft/csa-optimize-phase.o", 187 # Fallback happens with following args.gn (try/android-arm64-rel) 188 # android_static_analysis = "off" 189 # coverage_instrumentation_input_file = "//.code-coverage/files_to_instrument.txt" 190 # dcheck_always_on = true 191 # debuggable_apks = false 192 # fail_on_android_expectations = true 193 # ffmpeg_branding = "Chrome" 194 # is_component_build = false 195 # is_debug = false 196 # proprietary_codecs = true 197 # skip_secondary_abi_for_cq = true 198 # strip_debug_info = true 199 # symbol_level = 0 200 # system_webview_package_name = "com.google.android.apps.chrome" 201 # target_cpu = "arm64" 202 # target_os = "android" 203 # use_clang_coverage = true 204 # use_remoteexec = false 205 # use_siso = true 206 "./obj/content/test/content_browsertests__library/interest_group_browsertest.o", 207 "./obj/content/test/content_browsertests__library/prerender_browsertest.o", 208 "./robolectric_x64/obj/net/http/transport_security_state_generated_files/transport_security_state.o", 209 # Fallback happens with the following args.gn 210 # (linux-build-perf-developer, win-build-perf-developer) 211 # is_component_build = true 212 # is_debug = true 213 # symbol_level = 2 214 # use_siso = true 215 "./clang_x64/obj/net/http/transport_security_state_generated_files/transport_security_state.o", 216 "./obj/content/browser/browser/render_process_host_impl.o", 217 "./obj/net/http/transport_security_state_generated_files/transport_security_state.o", 218 # Fallback happens with the following args.gn 219 # (try/mac-rel) 220 # coverage_instrumentation_input_file = "//.code-coverage/files_to_instrument.txt" 221 # dcheck_always_on = true 222 # enable_backup_ref_ptr_feature_flag = true 223 # enable_dangling_raw_ptr_checks = true 224 # enable_dangling_raw_ptr_feature_flag = true 225 # ffmpeg_branding = "Chrome" 226 # is_component_build = false 227 # is_debug = false 228 # proprietary_codecs = true 229 # symbol_level = 0 230 # target_cpu = "x64" 231 # use_clang_coverage = true 232 # use_remoteexec = false 233 # use_siso = true 234 "./arm64/obj/net/http/transport_security_state_generated_files/transport_security_state.o", 235 "./arm64_v8_x64/obj/net/http/transport_security_state_generated_files/transport_security_state.o", 236 "./clang_arm64_v8_x64/obj/net/http/transport_security_state_generated_files/transport_security_state.o", 237 # Fallback happens with the following args.gn 238 # (try/ios-simulator) 239 # coverage_instrumentation_input_file = "//.code-coverage/files_to_instrument.txt" 240 # enable_run_ios_unittests_with_xctest = true 241 # is_component_build = false 242 # is_debug = true 243 # symbol_level = 1 244 # target_cpu = "x64" 245 # target_environment = "simulator" 246 # target_os = "ios" 247 # use_clang_coverage = true 248 # use_remoteexec = false 249 # use_siso = true 250 "./clang_arm64/obj/net/http/transport_security_state_generated_files/transport_security_state.o", 251 # Fallback happens with the following args.gn 252 # (mac-build-perf-developer, win-build-perf-developer) 253 # is_component_build = true 254 # is_debug = true 255 # symbol_level = 2 256 # use_siso = true 257 "./obj/content/browser/browser/storage_partition_impl.o", 258 ] 259 if runtime.os == "windows": 260 exit137_list = [obj.removesuffix(".o") + ".obj" for obj in exit137_list if obj.startswith("./obj/")] 261 262 # Fallback happens with the following args.gn 263 # (win-build-perf-developer) 264 # is_component_build = true 265 # is_debug = true 266 # symbol_level = 2 267 # use_siso = true 268 exit137_list.extend([ 269 "./obj/third_party/blink/renderer/core/core/local_frame_view.obj", 270 "./obj/third_party/blink/renderer/core/core_hot/document.obj", 271 ]) 272 273 new_rules = [] 274 for rule in step_config["rules"]: 275 if not rule["name"].endswith("/cxx"): 276 new_rules.append(rule) 277 continue 278 if "action_outs" in rule: 279 fail("unexpeced \"action_outs\" in cxx rule %s" % rule["name"]) 280 r = {} 281 r.update(rule) 282 r["name"] += "/b289968566/exit-137" 283 r["action_outs"] = exit137_list 284 285 # Some large compile take longer than the default timeout 2m. 286 r["timeout"] = "4m" 287 288 # use `_large` variant of platform if it doesn't use default platform, 289 # i.e. mac/win case. 290 if "platform_ref" in r: 291 r["platform_ref"] = r["platform_ref"] + "_large" 292 else: 293 r["platform_ref"] = "large" 294 if r.get("handler") == "rewrite_rewrapper": 295 r["handler"] = "rewrite_rewrapper_large" 296 new_rules.append(r) 297 new_rules.append(rule) 298 step_config["rules"] = new_rules 299 return step_config 300 301def init(ctx): 302 print("runtime: os:%s arch:%s run:%d" % ( 303 runtime.os, 304 runtime.arch, 305 runtime.num_cpu, 306 )) 307 host = { 308 "linux": chromium_linux, 309 "darwin": chromium_mac, 310 "windows": chromium_windows, 311 }[runtime.os] 312 step_config = { 313 "platforms": { 314 "default": { 315 "OSFamily": "Linux", 316 "container-image": "docker://gcr.io/chops-public-images-prod/rbe/siso-chromium/linux@sha256:912808c295e578ccde53b0685bcd0d56c15d7a03e819dcce70694bfe3fdab35e", 317 "label:action_default": "1", 318 }, 319 # Large workers are usually used for Python actions like generate bindings, mojo generators etc 320 # They can run on Linux workers. 321 "large": { 322 "OSFamily": "Linux", 323 "container-image": "docker://gcr.io/chops-public-images-prod/rbe/siso-chromium/linux@sha256:912808c295e578ccde53b0685bcd0d56c15d7a03e819dcce70694bfe3fdab35e", 324 # As of Jul 2023, the action_large pool uses n2-highmem-8 with 200GB of pd-ssd. 325 # The pool is intended for the following actions. 326 # - slow actions that can benefit from multi-cores and/or faster disk I/O. e.g. link, mojo, generate bindings etc. 327 # - actions that fail for OOM. 328 "label:action_large": "1", 329 }, 330 }, 331 "input_deps": {}, 332 "rules": [], 333 } 334 step_config = blink_all.step_config(ctx, step_config) 335 step_config = host.step_config(ctx, step_config) 336 step_config = mojo.step_config(ctx, step_config) 337 step_config = simple.step_config(ctx, step_config) 338 if reproxy.enabled(ctx): 339 step_config = reproxy.step_config(ctx, step_config) 340 341 # Python actions may use an absolute path at the first argument. 342 # e.g. C:/src/depot_tools/bootstrap-2@3_8_10_chromium_26_bin/python3/bin/python3.exe 343 # It needs to set `pyhton3` or `python3.exe` to remote_command. 344 for rule in step_config["rules"]: 345 if rule["name"].startswith("clang-coverage"): 346 # clang_code_coverage_wrapper.run() strips the python wrapper. 347 # So it shouldn't set `remote_command: python3`. 348 continue 349 350 # On Linux worker, it needs to be `python3` instead of `python3.exe`. 351 arg0 = rule.get("command_prefix", "").split(" ")[0].strip("\"") 352 if arg0 != platform.python_bin: 353 continue 354 p = rule.get("reproxy_config", {}).get("platform") or step_config["platforms"].get(rule.get("platform_ref", "default")) 355 if not p: 356 continue 357 if p.get("OSFamily") == "Linux": 358 arg0 = arg0.removesuffix(".exe") 359 rule["remote_command"] = arg0 360 361 step_config = __use_large_b289968566(ctx, step_config) 362 363 filegroups = {} 364 filegroups.update(blink_all.filegroups(ctx)) 365 filegroups.update(host.filegroups(ctx)) 366 filegroups.update(simple.filegroups(ctx)) 367 368 handlers = {} 369 handlers.update(blink_all.handlers) 370 handlers.update(host.handlers) 371 handlers.update(simple.handlers) 372 handlers.update(reproxy.handlers) 373 374 return module( 375 "config", 376 step_config = json.encode(step_config), 377 filegroups = filegroups, 378 handlers = handlers, 379 ) 380