1"""Module providing migrate_legacy_fields function. 2 3migrate_legacy_fields takes parsed CROSSTOOL proto and migrates it (inplace) to 4use only the features. 5 6Tracking issue: https://github.com/bazelbuild/bazel/issues/5187 7 8Since C++ rules team is working on migrating CROSSTOOL from text proto into 9Starlark, we advise CROSSTOOL owners to wait for the CROSSTOOL -> Starlark 10migrator before they invest too much time into fixing their pipeline. Tracking 11issue for the Starlark effort is 12https://github.com/bazelbuild/bazel/issues/5380. 13""" 14 15from third_party.com.github.bazelbuild.bazel.src.main.protobuf import crosstool_config_pb2 16 17ALL_CC_COMPILE_ACTIONS = [ 18 "assemble", "preprocess-assemble", "linkstamp-compile", "c-compile", 19 "c++-compile", "c++-header-parsing", "c++-module-compile", 20 "c++-module-codegen", "lto-backend", "clif-match" 21] 22 23ALL_OBJC_COMPILE_ACTIONS = [ 24 "objc-compile", "objc++-compile" 25] 26 27ALL_CXX_COMPILE_ACTIONS = [ 28 action for action in ALL_CC_COMPILE_ACTIONS 29 if action not in ["c-compile", "preprocess-assemble", "assemble"] 30] 31 32ALL_CC_LINK_ACTIONS = [ 33 "c++-link-executable", "c++-link-dynamic-library", 34 "c++-link-nodeps-dynamic-library" 35] 36 37ALL_OBJC_LINK_ACTIONS = [ 38 "objc-executable", "objc++-executable", 39] 40 41DYNAMIC_LIBRARY_LINK_ACTIONS = [ 42 "c++-link-dynamic-library", "c++-link-nodeps-dynamic-library" 43] 44 45NODEPS_DYNAMIC_LIBRARY_LINK_ACTIONS = ["c++-link-nodeps-dynamic-library"] 46 47TRANSITIVE_DYNAMIC_LIBRARY_LINK_ACTIONS = ["c++-link-dynamic-library"] 48 49TRANSITIVE_LINK_ACTIONS = ["c++-link-executable", "c++-link-dynamic-library"] 50 51CC_LINK_EXECUTABLE = ["c++-link-executable"] 52 53 54def compile_actions(toolchain): 55 """Returns compile actions for cc or objc rules.""" 56 if _is_objc_toolchain(toolchain): 57 return ALL_CC_COMPILE_ACTIONS + ALL_OBJC_COMPILE_ACTIONS 58 else: 59 return ALL_CC_COMPILE_ACTIONS 60 61def link_actions(toolchain): 62 """Returns link actions for cc or objc rules.""" 63 if _is_objc_toolchain(toolchain): 64 return ALL_CC_LINK_ACTIONS + ALL_OBJC_LINK_ACTIONS 65 else: 66 return ALL_CC_LINK_ACTIONS 67 68 69def executable_link_actions(toolchain): 70 """Returns transitive link actions for cc or objc rules.""" 71 if _is_objc_toolchain(toolchain): 72 return CC_LINK_EXECUTABLE + ALL_OBJC_LINK_ACTIONS 73 else: 74 return CC_LINK_EXECUTABLE 75 76 77def _is_objc_toolchain(toolchain): 78 return any(ac.action_name == "objc-compile" for ac in toolchain.action_config) 79 80# Map converting from LinkingMode to corresponding feature name 81LINKING_MODE_TO_FEATURE_NAME = { 82 "FULLY_STATIC": "fully_static_link", 83 "MOSTLY_STATIC": "static_linking_mode", 84 "DYNAMIC": "dynamic_linking_mode", 85 "MOSTLY_STATIC_LIBRARIES": "static_linking_mode_nodeps_library", 86} 87 88def migrate_legacy_fields(crosstool): 89 """Migrates parsed crosstool (inplace) to not use legacy fields.""" 90 crosstool.ClearField("default_toolchain") 91 for toolchain in crosstool.toolchain: 92 _ = [_migrate_expand_if_all_available(f) for f in toolchain.feature] 93 _ = [_migrate_expand_if_all_available(ac) for ac in toolchain.action_config] 94 _ = [_migrate_repeated_expands(f) for f in toolchain.feature] 95 _ = [_migrate_repeated_expands(ac) for ac in toolchain.action_config] 96 97 if (toolchain.dynamic_library_linker_flag or 98 _contains_dynamic_flags(toolchain)) and not _get_feature( 99 toolchain, "supports_dynamic_linker"): 100 feature = toolchain.feature.add() 101 feature.name = "supports_dynamic_linker" 102 feature.enabled = True 103 104 if toolchain.supports_start_end_lib and not _get_feature( 105 toolchain, "supports_start_end_lib"): 106 feature = toolchain.feature.add() 107 feature.name = "supports_start_end_lib" 108 feature.enabled = True 109 110 if toolchain.supports_interface_shared_objects and not _get_feature( 111 toolchain, "supports_interface_shared_libraries"): 112 feature = toolchain.feature.add() 113 feature.name = "supports_interface_shared_libraries" 114 feature.enabled = True 115 116 if toolchain.supports_embedded_runtimes and not _get_feature( 117 toolchain, "static_link_cpp_runtimes"): 118 feature = toolchain.feature.add() 119 feature.name = "static_link_cpp_runtimes" 120 feature.enabled = True 121 122 if toolchain.needsPic and not _get_feature(toolchain, "supports_pic"): 123 feature = toolchain.feature.add() 124 feature.name = "supports_pic" 125 feature.enabled = True 126 127 if toolchain.supports_fission and not _get_feature( 128 toolchain, "per_object_debug_info"): 129 # feature { 130 # name: "per_object_debug_info" 131 # enabled: true 132 # flag_set { 133 # action: "assemble" 134 # action: "preprocess-assemble" 135 # action: "c-compile" 136 # action: "c++-compile" 137 # action: "c++-module-codegen" 138 # action: "lto-backend" 139 # flag_group { 140 # expand_if_all_available: 'is_using_fission'", 141 # flag: "-gsplit-dwarf" 142 # } 143 # } 144 # } 145 feature = toolchain.feature.add() 146 feature.name = "per_object_debug_info" 147 feature.enabled = True 148 flag_set = feature.flag_set.add() 149 flag_set.action[:] = [ 150 "c-compile", "c++-compile", "c++-module-codegen", "assemble", 151 "preprocess-assemble", "lto-backend" 152 ] 153 flag_group = flag_set.flag_group.add() 154 flag_group.expand_if_all_available[:] = ["is_using_fission"] 155 flag_group.flag[:] = ["-gsplit-dwarf"] 156 157 if toolchain.objcopy_embed_flag and not _get_feature( 158 toolchain, "objcopy_embed_flags"): 159 feature = toolchain.feature.add() 160 feature.name = "objcopy_embed_flags" 161 feature.enabled = True 162 flag_set = feature.flag_set.add() 163 flag_set.action[:] = ["objcopy_embed_data"] 164 flag_group = flag_set.flag_group.add() 165 flag_group.flag[:] = toolchain.objcopy_embed_flag 166 167 action_config = toolchain.action_config.add() 168 action_config.action_name = "objcopy_embed_data" 169 action_config.config_name = "objcopy_embed_data" 170 action_config.enabled = True 171 tool = action_config.tool.add() 172 tool.tool_path = _find_tool_path(toolchain, "objcopy") 173 174 if toolchain.ld_embed_flag and not _get_feature( 175 toolchain, "ld_embed_flags"): 176 feature = toolchain.feature.add() 177 feature.name = "ld_embed_flags" 178 feature.enabled = True 179 flag_set = feature.flag_set.add() 180 flag_set.action[:] = ["ld_embed_data"] 181 flag_group = flag_set.flag_group.add() 182 flag_group.flag[:] = toolchain.ld_embed_flag 183 184 action_config = toolchain.action_config.add() 185 action_config.action_name = "ld_embed_data" 186 action_config.config_name = "ld_embed_data" 187 action_config.enabled = True 188 tool = action_config.tool.add() 189 tool.tool_path = _find_tool_path(toolchain, "ld") 190 191 192 # Create default_link_flags feature for linker_flag 193 flag_sets = _extract_legacy_link_flag_sets_for(toolchain) 194 if flag_sets: 195 if _get_feature(toolchain, "default_link_flags"): 196 continue 197 if _get_feature(toolchain, "legacy_link_flags"): 198 for f in toolchain.feature: 199 if f.name == "legacy_link_flags": 200 f.ClearField("flag_set") 201 feature = f 202 _rename_feature_in_toolchain(toolchain, "legacy_link_flags", 203 "default_link_flags") 204 break 205 else: 206 feature = _prepend_feature(toolchain) 207 feature.name = "default_link_flags" 208 feature.enabled = True 209 _add_flag_sets(feature, flag_sets) 210 211 # Create default_compile_flags feature for compiler_flag, cxx_flag 212 flag_sets = _extract_legacy_compile_flag_sets_for(toolchain) 213 if flag_sets and not _get_feature(toolchain, "default_compile_flags"): 214 if _get_feature(toolchain, "legacy_compile_flags"): 215 for f in toolchain.feature: 216 if f.name == "legacy_compile_flags": 217 f.ClearField("flag_set") 218 feature = f 219 _rename_feature_in_toolchain(toolchain, "legacy_compile_flags", 220 "default_compile_flags") 221 break 222 else: 223 feature = _prepend_feature(toolchain) 224 feature.enabled = True 225 feature.name = "default_compile_flags" 226 _add_flag_sets(feature, flag_sets) 227 228 # Unfiltered cxx flags have to have their own special feature. 229 # "unfiltered_compile_flags" is a well-known (by Bazel) feature name that is 230 # excluded from nocopts filtering. 231 if toolchain.unfiltered_cxx_flag: 232 # If there already is a feature named unfiltered_compile_flags, the 233 # crosstool is already migrated for unfiltered_compile_flags 234 if _get_feature(toolchain, "unfiltered_compile_flags"): 235 for f in toolchain.feature: 236 if f.name == "unfiltered_compile_flags": 237 for flag_set in f.flag_set: 238 for flag_group in flag_set.flag_group: 239 if flag_group.iterate_over == "unfiltered_compile_flags": 240 flag_group.ClearField("iterate_over") 241 flag_group.ClearField("expand_if_all_available") 242 flag_group.ClearField("flag") 243 flag_group.flag[:] = toolchain.unfiltered_cxx_flag 244 else: 245 if not _get_feature(toolchain, "user_compile_flags"): 246 feature = toolchain.feature.add() 247 feature.name = "user_compile_flags" 248 feature.enabled = True 249 flag_set = feature.flag_set.add() 250 flag_set.action[:] = compile_actions(toolchain) 251 flag_group = flag_set.flag_group.add() 252 flag_group.expand_if_all_available[:] = ["user_compile_flags"] 253 flag_group.iterate_over = "user_compile_flags" 254 flag_group.flag[:] = ["%{user_compile_flags}"] 255 256 if not _get_feature(toolchain, "sysroot"): 257 sysroot_actions = compile_actions(toolchain) + link_actions(toolchain) 258 sysroot_actions.remove("assemble") 259 feature = toolchain.feature.add() 260 feature.name = "sysroot" 261 feature.enabled = True 262 flag_set = feature.flag_set.add() 263 flag_set.action[:] = sysroot_actions 264 flag_group = flag_set.flag_group.add() 265 flag_group.expand_if_all_available[:] = ["sysroot"] 266 flag_group.flag[:] = ["--sysroot=%{sysroot}"] 267 268 feature = toolchain.feature.add() 269 feature.name = "unfiltered_compile_flags" 270 feature.enabled = True 271 flag_set = feature.flag_set.add() 272 flag_set.action[:] = compile_actions(toolchain) 273 flag_group = flag_set.flag_group.add() 274 flag_group.flag[:] = toolchain.unfiltered_cxx_flag 275 276 # clear fields 277 toolchain.ClearField("debian_extra_requires") 278 toolchain.ClearField("gcc_plugin_compiler_flag") 279 toolchain.ClearField("ar_flag") 280 toolchain.ClearField("ar_thin_archives_flag") 281 toolchain.ClearField("gcc_plugin_header_directory") 282 toolchain.ClearField("mao_plugin_header_directory") 283 toolchain.ClearField("supports_normalizing_ar") 284 toolchain.ClearField("supports_thin_archives") 285 toolchain.ClearField("supports_incremental_linker") 286 toolchain.ClearField("supports_dsym") 287 toolchain.ClearField("supports_gold_linker") 288 toolchain.ClearField("default_python_top") 289 toolchain.ClearField("default_python_version") 290 toolchain.ClearField("python_preload_swigdeps") 291 toolchain.ClearField("needsPic") 292 toolchain.ClearField("compilation_mode_flags") 293 toolchain.ClearField("linking_mode_flags") 294 toolchain.ClearField("unfiltered_cxx_flag") 295 toolchain.ClearField("ld_embed_flag") 296 toolchain.ClearField("objcopy_embed_flag") 297 toolchain.ClearField("supports_start_end_lib") 298 toolchain.ClearField("supports_interface_shared_objects") 299 toolchain.ClearField("supports_fission") 300 toolchain.ClearField("supports_embedded_runtimes") 301 toolchain.ClearField("compiler_flag") 302 toolchain.ClearField("cxx_flag") 303 toolchain.ClearField("linker_flag") 304 toolchain.ClearField("dynamic_library_linker_flag") 305 toolchain.ClearField("static_runtimes_filegroup") 306 toolchain.ClearField("dynamic_runtimes_filegroup") 307 308 # Enable features that were previously enabled by Bazel 309 default_features = [ 310 "dependency_file", "random_seed", "module_maps", "module_map_home_cwd", 311 "header_module_compile", "include_paths", "pic", "preprocessor_define" 312 ] 313 for feature_name in default_features: 314 feature = _get_feature(toolchain, feature_name) 315 if feature: 316 feature.enabled = True 317 318 319def _find_tool_path(toolchain, tool_name): 320 """Returns the tool path of the tool with the given name.""" 321 for tool in toolchain.tool_path: 322 if tool.name == tool_name: 323 return tool.path 324 return None 325 326 327def _add_flag_sets(feature, flag_sets): 328 """Add flag sets into a feature.""" 329 for flag_set in flag_sets: 330 with_feature = flag_set[0] 331 actions = flag_set[1] 332 flags = flag_set[2] 333 expand_if_all_available = flag_set[3] 334 not_feature = None 335 if len(flag_set) >= 5: 336 not_feature = flag_set[4] 337 flag_set = feature.flag_set.add() 338 if with_feature is not None: 339 flag_set.with_feature.add().feature[:] = [with_feature] 340 if not_feature is not None: 341 flag_set.with_feature.add().not_feature[:] = [not_feature] 342 flag_set.action[:] = actions 343 flag_group = flag_set.flag_group.add() 344 flag_group.expand_if_all_available[:] = expand_if_all_available 345 flag_group.flag[:] = flags 346 return feature 347 348 349def _extract_legacy_compile_flag_sets_for(toolchain): 350 """Get flag sets for default_compile_flags feature.""" 351 result = [] 352 if toolchain.compiler_flag: 353 result.append( 354 [None, compile_actions(toolchain), toolchain.compiler_flag, []]) 355 356 # Migrate compiler_flag from compilation_mode_flags 357 for cmf in toolchain.compilation_mode_flags: 358 mode = crosstool_config_pb2.CompilationMode.Name(cmf.mode).lower() 359 # coverage mode has been a noop since a while 360 if mode == "coverage": 361 continue 362 363 if (cmf.compiler_flag or 364 cmf.cxx_flag) and not _get_feature(toolchain, mode): 365 feature = toolchain.feature.add() 366 feature.name = mode 367 368 if cmf.compiler_flag: 369 result.append([mode, compile_actions(toolchain), cmf.compiler_flag, []]) 370 371 if toolchain.cxx_flag: 372 result.append([None, ALL_CXX_COMPILE_ACTIONS, toolchain.cxx_flag, []]) 373 374 # Migrate compiler_flag/cxx_flag from compilation_mode_flags 375 for cmf in toolchain.compilation_mode_flags: 376 mode = crosstool_config_pb2.CompilationMode.Name(cmf.mode).lower() 377 # coverage mode has been a noop since a while 378 if mode == "coverage": 379 continue 380 381 if cmf.cxx_flag: 382 result.append([mode, ALL_CXX_COMPILE_ACTIONS, cmf.cxx_flag, []]) 383 384 return result 385 386 387def _extract_legacy_link_flag_sets_for(toolchain): 388 """Get flag sets for default_link_flags feature.""" 389 result = [] 390 391 # Migrate linker_flag 392 if toolchain.linker_flag: 393 result.append([None, link_actions(toolchain), toolchain.linker_flag, []]) 394 395 # Migrate linker_flags from compilation_mode_flags 396 for cmf in toolchain.compilation_mode_flags: 397 mode = crosstool_config_pb2.CompilationMode.Name(cmf.mode).lower() 398 # coverage mode has beed a noop since a while 399 if mode == "coverage": 400 continue 401 402 if cmf.linker_flag and not _get_feature(toolchain, mode): 403 feature = toolchain.feature.add() 404 feature.name = mode 405 406 if cmf.linker_flag: 407 result.append([mode, link_actions(toolchain), cmf.linker_flag, []]) 408 409 # Migrate linker_flags from linking_mode_flags 410 for lmf in toolchain.linking_mode_flags: 411 mode = crosstool_config_pb2.LinkingMode.Name(lmf.mode) 412 feature_name = LINKING_MODE_TO_FEATURE_NAME.get(mode) 413 # if the feature is already there, we don't migrate, lmf is not used 414 if _get_feature(toolchain, feature_name): 415 continue 416 417 if lmf.linker_flag: 418 feature = toolchain.feature.add() 419 feature.name = feature_name 420 if mode == "DYNAMIC": 421 result.append( 422 [None, NODEPS_DYNAMIC_LIBRARY_LINK_ACTIONS, lmf.linker_flag, []]) 423 result.append([ 424 None, 425 TRANSITIVE_DYNAMIC_LIBRARY_LINK_ACTIONS, 426 lmf.linker_flag, 427 [], 428 "static_link_cpp_runtimes", 429 ]) 430 result.append([ 431 feature_name, 432 executable_link_actions(toolchain), lmf.linker_flag, [] 433 ]) 434 elif mode == "MOSTLY_STATIC": 435 result.append( 436 [feature_name, 437 CC_LINK_EXECUTABLE, lmf.linker_flag, []]) 438 else: 439 result.append( 440 [feature_name, 441 link_actions(toolchain), lmf.linker_flag, []]) 442 443 if toolchain.dynamic_library_linker_flag: 444 result.append([ 445 None, DYNAMIC_LIBRARY_LINK_ACTIONS, 446 toolchain.dynamic_library_linker_flag, [] 447 ]) 448 449 if toolchain.test_only_linker_flag: 450 result.append([ 451 None, 452 link_actions(toolchain), toolchain.test_only_linker_flag, 453 ["is_cc_test"] 454 ]) 455 456 return result 457 458 459def _prepend_feature(toolchain): 460 """Create a new feature and make it be the first in the toolchain.""" 461 features = toolchain.feature 462 toolchain.ClearField("feature") 463 new_feature = toolchain.feature.add() 464 toolchain.feature.extend(features) 465 return new_feature 466 467 468def _get_feature(toolchain, name): 469 """Returns feature with a given name or None.""" 470 for feature in toolchain.feature: 471 if feature.name == name: 472 return feature 473 return None 474 475 476def _migrate_expand_if_all_available(message): 477 """Move expand_if_all_available field to flag_groups.""" 478 for flag_set in message.flag_set: 479 if flag_set.expand_if_all_available: 480 for flag_group in flag_set.flag_group: 481 new_vars = ( 482 flag_group.expand_if_all_available[:] + 483 flag_set.expand_if_all_available[:]) 484 flag_group.expand_if_all_available[:] = new_vars 485 flag_set.ClearField("expand_if_all_available") 486 487 488def _migrate_repeated_expands(message): 489 """Replace repeated legacy fields with nesting.""" 490 todo_queue = [] 491 for flag_set in message.flag_set: 492 todo_queue.extend(flag_set.flag_group) 493 while todo_queue: 494 flag_group = todo_queue.pop() 495 todo_queue.extend(flag_group.flag_group) 496 if len(flag_group.expand_if_all_available) <= 1 and len( 497 flag_group.expand_if_none_available) <= 1: 498 continue 499 500 current_children = flag_group.flag_group 501 current_flags = flag_group.flag 502 flag_group.ClearField("flag_group") 503 flag_group.ClearField("flag") 504 505 new_flag_group = flag_group.flag_group.add() 506 new_flag_group.flag_group.extend(current_children) 507 new_flag_group.flag.extend(current_flags) 508 509 if len(flag_group.expand_if_all_available) > 1: 510 expands_to_move = flag_group.expand_if_all_available[1:] 511 flag_group.expand_if_all_available[:] = [ 512 flag_group.expand_if_all_available[0] 513 ] 514 new_flag_group.expand_if_all_available.extend(expands_to_move) 515 516 if len(flag_group.expand_if_none_available) > 1: 517 expands_to_move = flag_group.expand_if_none_available[1:] 518 flag_group.expand_if_none_available[:] = [ 519 flag_group.expand_if_none_available[0] 520 ] 521 new_flag_group.expand_if_none_available.extend(expands_to_move) 522 523 todo_queue.append(new_flag_group) 524 todo_queue.append(flag_group) 525 526 527def _contains_dynamic_flags(toolchain): 528 for lmf in toolchain.linking_mode_flags: 529 mode = crosstool_config_pb2.LinkingMode.Name(lmf.mode) 530 if mode == "DYNAMIC": 531 return True 532 return False 533 534 535def _rename_feature_in_toolchain(toolchain, from_name, to_name): 536 for f in toolchain.feature: 537 _rename_feature_in(f, from_name, to_name) 538 for a in toolchain.action_config: 539 _rename_feature_in(a, from_name, to_name) 540 541 542def _rename_feature_in(msg, from_name, to_name): 543 if from_name in msg.implies: 544 msg.implies.remove(from_name) 545 for requires in msg.requires: 546 if from_name in requires.feature: 547 requires.feature.remove(from_name) 548 requires.feature.extend([to_name]) 549 for flag_set in msg.flag_set: 550 for with_feature in flag_set.with_feature: 551 if from_name in with_feature.feature: 552 with_feature.feature.remove(from_name) 553 with_feature.feature.extend([to_name]) 554 if from_name in with_feature.not_feature: 555 with_feature.not_feature.remove(from_name) 556 with_feature.not_feature.extend([to_name]) 557 for env_set in msg.env_set: 558 for with_feature in env_set.with_feature: 559 if from_name in with_feature.feature: 560 with_feature.feature.remove(from_name) 561 with_feature.feature.extend([to_name]) 562 if from_name in with_feature.not_feature: 563 with_feature.not_feature.remove(from_name) 564 with_feature.not_feature.extend([to_name]) 565