1#!/usr/bin/env python3 2# Copyright 2020 The Pigweed Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); you may not 5# use this file except in compliance with the License. You may obtain a copy of 6# the License at 7# 8# https://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13# License for the specific language governing permissions and limitations under 14# the License. 15"""Extracts build information from Arduino cores.""" 16 17import glob 18import logging 19import os 20import platform 21import pprint 22import re 23import sys 24import time 25from collections import OrderedDict 26from pathlib import Path 27 28try: 29 from pw_arduino_build import file_operations 30 31except ImportError: 32 # Load from this directory if pw_arduino_build is not available. 33 import file_operations # type: ignore 34 35_LOG = logging.getLogger(__name__) 36 37_pretty_print = pprint.PrettyPrinter(indent=1, width=120).pprint 38_pretty_format = pprint.PrettyPrinter(indent=1, width=120).pformat 39 40 41def arduino_runtime_os_string(): 42 arduno_platform = { 43 "Linux": "linux", 44 "Windows": "windows", 45 "Darwin": "macosx", 46 } 47 return arduno_platform[platform.system()] 48 49 50class ArduinoBuilder: 51 """Used to interpret arduino boards.txt and platform.txt files.""" 52 53 # pylint: disable=too-many-instance-attributes,too-many-public-methods 54 55 BOARD_MENU_REGEX = re.compile( 56 r"^(?P<name>menu\.[^#=]+)=(?P<description>.*)$", re.MULTILINE 57 ) 58 59 BOARD_NAME_REGEX = re.compile( 60 r"^(?P<name>[^\s#\.]+)\.name=(?P<description>.*)$", re.MULTILINE 61 ) 62 63 VARIABLE_REGEX = re.compile( 64 r"^(?P<name>[^\s#=]+)=(?P<value>.*)$", re.MULTILINE 65 ) 66 67 MENU_OPTION_REGEX = re.compile( 68 r"^menu\." # starts with "menu" 69 r"(?P<menu_option_name>[^.]+)\." # first token after . 70 r"(?P<menu_option_value>[^.]+)$" 71 ) # second (final) token after . 72 73 TOOL_NAME_REGEX = re.compile( 74 r"^tools\." r"(?P<tool_name>[^.]+)\." # starts with "tools" 75 ) # first token after . 76 77 INTERPOLATED_VARIABLE_REGEX = re.compile(r"{[^}]+}", re.MULTILINE) 78 79 OBJCOPY_STEP_NAME_REGEX = re.compile(r"^recipe.objcopy.([^.]+).pattern$") 80 81 def __init__( 82 self, 83 arduino_path, 84 package_name, 85 build_path=None, 86 project_path=None, 87 project_source_path=None, 88 library_path=None, 89 library_names=None, 90 build_project_name=None, 91 compiler_path_override=False, 92 ): 93 self.arduino_path = arduino_path 94 self.arduino_package_name = package_name 95 self.selected_board = None 96 self.build_path = build_path 97 self.project_path = project_path 98 self.project_source_path = project_source_path 99 self.build_project_name = build_project_name 100 self.compiler_path_override = compiler_path_override 101 self.variant_includes = "" 102 self.build_variant_path = False 103 self.library_names = library_names 104 self.library_path = library_path 105 106 self.compiler_path_override_binaries = [] 107 if self.compiler_path_override: 108 self.compiler_path_override_binaries = file_operations.find_files( 109 self.compiler_path_override, "*" 110 ) 111 112 # Container dicts for boards.txt and platform.txt file data. 113 self.board = OrderedDict() 114 self.platform = OrderedDict() 115 self.menu_options = OrderedDict( 116 {"global_options": {}, "default_board_values": {}, "selected": {}} 117 ) 118 self.tools_variables = {} 119 120 # Set and check for valid hardware folder. 121 self.hardware_path = os.path.join(self.arduino_path, "hardware") 122 123 if not os.path.exists(self.hardware_path): 124 raise FileNotFoundError( 125 "Arduino package path '{}' does not exist.".format( 126 self.hardware_path 127 ) 128 ) 129 130 # Set and check for valid package name 131 self.package_path = os.path.join( 132 self.arduino_path, "hardware", package_name 133 ) 134 # {build.arch} is the first folder name of the package (upcased) 135 self.build_arch = os.path.split(package_name)[0].upper() 136 137 if not os.path.exists(self.package_path): 138 _LOG.error( 139 "Error: Arduino package name '%s' does not exist.", package_name 140 ) 141 _LOG.error("Did you mean:\n") 142 # TODO(tonymd): On Windows concatenating "/" may not work 143 possible_alternatives = [ 144 d.replace(self.hardware_path + os.sep, "", 1) 145 for d in glob.glob(self.hardware_path + "/*/*") 146 ] 147 _LOG.error("\n".join(possible_alternatives)) 148 sys.exit(1) 149 150 # Populate library paths. 151 if not library_path: 152 self.library_path = [] 153 # Append core libraries directory. 154 core_lib_path = Path(self.package_path) / "libraries" 155 if core_lib_path.is_dir(): 156 self.library_path.append(Path(self.package_path) / "libraries") 157 if library_path: 158 self.library_path = [ 159 os.path.realpath(os.path.expanduser(os.path.expandvars(l_path))) 160 for l_path in library_path 161 ] 162 163 # Grab all folder names in the cores directory. These are typically 164 # sub-core source files. 165 self.sub_core_folders = os.listdir( 166 os.path.join(self.package_path, "cores") 167 ) 168 169 self._find_tools_variables() 170 171 self.boards_txt = os.path.join(self.package_path, "boards.txt") 172 self.platform_txt = os.path.join(self.package_path, "platform.txt") 173 174 def select_board(self, board_name, menu_option_overrides=False): 175 self.selected_board = board_name 176 177 # Load default menu options for a selected board. 178 if self.selected_board not in self.board.keys(): 179 _LOG.error("Error board: '%s' not supported.", self.selected_board) 180 # TODO(tonymd): Print supported boards here 181 sys.exit(1) 182 183 # Override default menu options if any are specified. 184 if menu_option_overrides: 185 for moption in menu_option_overrides: 186 if not self.set_menu_option(moption): 187 # TODO(tonymd): Print supported menu options here 188 sys.exit(1) 189 190 self._copy_default_menu_options_to_build_variables() 191 self._apply_recipe_overrides() 192 self._substitute_variables() 193 194 def set_variables(self, variable_list: list[str]): 195 # Convert the string list containing 'name=value' items into a dict 196 variable_source = {} 197 for var in variable_list: 198 var_name, value = var.split("=") 199 variable_source[var_name] = value 200 201 # Replace variables in platform 202 for var, value in self.platform.items(): 203 self.platform[var] = self._replace_variables(value, variable_source) 204 205 def _apply_recipe_overrides(self): 206 # Override link recipes with per-core exceptions 207 # Teensyduino cores 208 if self.build_arch == "TEENSY": 209 # Change {build.path}/{archive_file} 210 # To {archive_file_path} (which should contain the core.a file) 211 new_link_line = self.platform["recipe.c.combine.pattern"].replace( 212 "{object_files} \"{build.path}/{archive_file}\"", 213 "{object_files} {archive_file_path}", 214 1, 215 ) 216 # Add the teensy provided toolchain lib folder for link access to 217 # libarm_cortexM*_math.a 218 new_link_line = new_link_line.replace( 219 "\"-L{build.path}\"", 220 "\"-L{build.path}\" -L{compiler.path}/arm/arm-none-eabi/lib", 221 1, 222 ) 223 self.platform["recipe.c.combine.pattern"] = new_link_line 224 # Remove the pre-compiled header include 225 self.platform["recipe.cpp.o.pattern"] = self.platform[ 226 "recipe.cpp.o.pattern" 227 ].replace("\"-I{build.path}/pch\"", "", 1) 228 229 # Adafruit-samd core 230 # TODO(tonymd): This build_arch may clash with Arduino-SAMD core 231 elif self.build_arch == "SAMD": 232 new_link_line = self.platform["recipe.c.combine.pattern"].replace( 233 "\"{build.path}/{archive_file}\" -Wl,--end-group", 234 "{archive_file_path} -Wl,--end-group", 235 1, 236 ) 237 self.platform["recipe.c.combine.pattern"] = new_link_line 238 239 # STM32L4 Core: 240 # https://github.com/GrumpyOldPizza/arduino-STM32L4 241 elif self.build_arch == "STM32L4": 242 # TODO(tonymd): {build.path}/{archive_file} for the link step always 243 # seems to be core.a (except STM32 core) 244 line_to_delete = "-Wl,--start-group \"{build.path}/{archive_file}\"" 245 new_link_line = self.platform["recipe.c.combine.pattern"].replace( 246 line_to_delete, "-Wl,--start-group {archive_file_path}", 1 247 ) 248 self.platform["recipe.c.combine.pattern"] = new_link_line 249 250 # stm32duino core 251 elif self.build_arch == "STM32": 252 # Must link in SrcWrapper for all projects. 253 if not self.library_names: 254 self.library_names = [] 255 self.library_names.append("SrcWrapper") 256 257 # Surround args in quotes if they contain any quote characters. 258 # Example: 259 # before: -DVARIANT_H="variant_generic.h" 260 # after: "-DVARIANT_H=\"variant_generic.h\"" 261 build_info_args = [] 262 for arg in self.platform["build.info.flags"].split(): 263 if '"' not in arg: 264 build_info_args.append(arg) 265 continue 266 new_arg = arg.replace('"', '\\"') 267 build_info_args.append(f'"{new_arg}"') 268 self.platform["build.info.flags"] = ' '.join(build_info_args) 269 270 def _copy_default_menu_options_to_build_variables(self): 271 # Clear existing options 272 self.menu_options["selected"] = {} 273 # Set default menu options for selected board 274 for menu_key, menu_dict in self.menu_options["default_board_values"][ 275 self.selected_board 276 ].items(): 277 for name, var in self.board[self.selected_board].items(): 278 starting_key = "{}.{}.".format(menu_key, menu_dict["name"]) 279 if name.startswith(starting_key): 280 new_var_name = name.replace(starting_key, "", 1) 281 self.menu_options["selected"][new_var_name] = var 282 283 def set_menu_option(self, moption): 284 if moption not in self.board[self.selected_board]: 285 _LOG.error("Error: '%s' is not a valid menu option.", moption) 286 return False 287 288 # Override default menu option with new value. 289 menu_match_result = self.MENU_OPTION_REGEX.match(moption) 290 if menu_match_result: 291 menu_match = menu_match_result.groupdict() 292 menu_value = menu_match["menu_option_value"] 293 menu_key = "menu.{}".format(menu_match["menu_option_name"]) 294 self.menu_options["default_board_values"][self.selected_board][ 295 menu_key 296 ]["name"] = menu_value 297 298 # Update build variables 299 self._copy_default_menu_options_to_build_variables() 300 return True 301 302 def _set_global_arduino_variables(self): 303 """Set some global variables defined by the Arduino-IDE. 304 305 See Docs: 306 https://arduino.github.io/arduino-cli/platform-specification/#global-predefined-properties 307 """ 308 309 # TODO(tonymd): Make sure these variables are replaced in recipe lines 310 # even if they are None: build_path, project_path, project_source_path, 311 # build_project_name 312 for current_board in self.board.values(): 313 if self.build_path: 314 current_board["build.path"] = self.build_path 315 if self.build_project_name: 316 current_board["build.project_name"] = self.build_project_name 317 # {archive_file} is the final *.elf 318 archive_file = "{}.elf".format(self.build_project_name) 319 current_board["archive_file"] = archive_file 320 # {archive_file_path} is the final core.a archive 321 if self.build_path: 322 current_board["archive_file_path"] = os.path.join( 323 self.build_path, "core.a" 324 ) 325 if self.project_source_path: 326 current_board["build.source.path"] = self.project_source_path 327 328 current_board["extra.time.local"] = str(int(time.time())) 329 current_board["runtime.ide.version"] = "10812" 330 current_board["runtime.hardware.path"] = self.hardware_path 331 332 # Copy {runtime.tools.TOOL_NAME.path} vars 333 self._set_tools_variables(current_board) 334 335 current_board["runtime.platform.path"] = self.package_path 336 if self.platform["name"] == "Teensyduino": 337 # Teensyduino is installed into the arduino IDE folder 338 # rather than ~/.arduino15/packages/ 339 current_board["runtime.hardware.path"] = os.path.join( 340 self.hardware_path, "teensy" 341 ) 342 343 current_board["build.system.path"] = os.path.join( 344 self.package_path, "system" 345 ) 346 347 # Set the {build.core.path} variable that pointing to a sub-core 348 # folder. For Teensys this is: 349 # 'teensy/hardware/avr/1.58.1/cores/teensy{3,4}'. For other cores 350 # it's typically just the 'arduino' folder. For example: 351 # 'arduino-samd/hardware/samd/1.8.8/cores/arduino' 352 core_path = Path(self.package_path) / "cores" 353 core_path /= current_board.get( 354 "build.core", self.sub_core_folders[0] 355 ) 356 current_board["build.core.path"] = core_path.as_posix() 357 358 current_board["build.arch"] = self.build_arch 359 360 for name, var in current_board.items(): 361 current_board[name] = var.replace( 362 "{build.core.path}", core_path.as_posix() 363 ) 364 365 def load_board_definitions(self): 366 """Loads Arduino boards.txt and platform.txt files into dictionaries. 367 368 Populates the following dictionaries: 369 self.menu_options 370 self.boards 371 self.platform 372 """ 373 # Load platform.txt 374 with open(self.platform_txt, "r") as pfile: 375 platform_file = pfile.read() 376 platform_var_matches = self.VARIABLE_REGEX.finditer(platform_file) 377 for p_match in [m.groupdict() for m in platform_var_matches]: 378 self.platform[p_match["name"]] = p_match["value"] 379 380 # Load boards.txt 381 with open(self.boards_txt, "r") as bfile: 382 board_file = bfile.read() 383 # Get all top-level menu options, e.g. menu.usb=USB Type 384 board_menu_matches = self.BOARD_MENU_REGEX.finditer(board_file) 385 for menuitem in [m.groupdict() for m in board_menu_matches]: 386 self.menu_options["global_options"][menuitem["name"]] = { 387 "description": menuitem["description"] 388 } 389 390 # Get all board names, e.g. teensy40.name=Teensy 4.0 391 board_name_matches = self.BOARD_NAME_REGEX.finditer(board_file) 392 for b_match in [m.groupdict() for m in board_name_matches]: 393 self.board[b_match["name"]] = OrderedDict() 394 self.menu_options["default_board_values"][ 395 b_match["name"] 396 ] = OrderedDict() 397 398 # Get all board variables, e.g. teensy40.* 399 for current_board_name, current_board in self.board.items(): 400 board_line_matches = re.finditer( 401 fr"^\s*{current_board_name}\." 402 fr"(?P<key>[^#=]+)=(?P<value>.*)$", 403 board_file, 404 re.MULTILINE, 405 ) 406 for b_match in [m.groupdict() for m in board_line_matches]: 407 # Check if this line is a menu option 408 # (e.g. 'menu.usb.serial') and save as default if it's the 409 # first one seen. 410 ArduinoBuilder.save_default_menu_option( 411 current_board_name, 412 b_match["key"], 413 b_match["value"], 414 self.menu_options, 415 ) 416 current_board[b_match["key"]] = b_match["value"].strip() 417 418 self._set_global_arduino_variables() 419 420 @staticmethod 421 def save_default_menu_option(current_board_name, key, value, menu_options): 422 """Save a given menu option as the default. 423 424 Saves the key and value into menu_options["default_board_values"] 425 if it doesn't already exist. Assumes menu options are added in the order 426 specified in boards.txt. The first value for a menu key is the default. 427 """ 428 # Check if key is a menu option 429 # e.g. menu.usb.serial 430 # menu.usb.serial.build.usbtype 431 menu_match_result = re.match( 432 r'^menu\.' # starts with "menu" 433 r'(?P<menu_option_name>[^.]+)\.' # first token after . 434 r'(?P<menu_option_value>[^.]+)' # second token after . 435 r'(\.(?P<rest>.+))?', # optionally any trailing tokens after a . 436 key, 437 ) 438 if menu_match_result: 439 menu_match = menu_match_result.groupdict() 440 current_menu_key = "menu.{}".format(menu_match["menu_option_name"]) 441 # If this is the first menu option seen for current_board_name, save 442 # as the default. 443 if ( 444 current_menu_key 445 not in menu_options["default_board_values"][current_board_name] 446 ): 447 menu_options["default_board_values"][current_board_name][ 448 current_menu_key 449 ] = { 450 "name": menu_match["menu_option_value"], 451 "description": value, 452 } 453 454 def _replace_variables(self, line, variable_lookup_source): 455 """Replace {variables} from loaded boards.txt or platform.txt. 456 457 Replace interpolated variables surrounded by curly braces in line with 458 definitions from variable_lookup_source. 459 """ 460 new_line = line 461 for current_var_match in self.INTERPOLATED_VARIABLE_REGEX.findall(line): 462 # {build.flags.c} --> build.flags.c 463 current_var = current_var_match.strip("{}") 464 465 # check for matches in board definition 466 if current_var in variable_lookup_source: 467 replacement = variable_lookup_source.get(current_var, "") 468 new_line = new_line.replace(current_var_match, replacement) 469 return new_line 470 471 def _find_tools_variables(self): 472 # Gather tool directories in order of increasing precedence 473 runtime_tool_paths = [] 474 475 # Check for tools installed in ~/.arduino15/packages/arduino/tools/ 476 # TODO(tonymd): Is this Mac & Linux specific? 477 runtime_tool_paths += glob.glob( 478 os.path.join( 479 os.path.realpath(os.path.expanduser(os.path.expandvars("~"))), 480 ".arduino15", 481 "packages", 482 "arduino", 483 "tools", 484 "*", 485 ) 486 ) 487 488 # <ARDUINO_PATH>/tools/<OS_STRING>/<TOOL_NAMES> 489 runtime_tool_paths += glob.glob( 490 os.path.join( 491 self.arduino_path, "tools", arduino_runtime_os_string(), "*" 492 ) 493 ) 494 # <ARDUINO_PATH>/tools/<TOOL_NAMES> 495 # This will grab linux/windows/macosx/share as <TOOL_NAMES>. 496 runtime_tool_paths += glob.glob( 497 os.path.join(self.arduino_path, "tools", "*") 498 ) 499 500 # Process package tools after arduino tools. 501 # They should overwrite vars & take precedence. 502 503 # <PACKAGE_PATH>/tools/<OS_STRING>/<TOOL_NAMES> 504 runtime_tool_paths += glob.glob( 505 os.path.join( 506 self.package_path, "tools", arduino_runtime_os_string(), "*" 507 ) 508 ) 509 # <PACKAGE_PATH>/tools/<TOOL_NAMES> 510 # This will grab linux/windows/macosx/share as <TOOL_NAMES>. 511 runtime_tool_paths += glob.glob( 512 os.path.join(self.package_path, "tools", "*") 513 ) 514 515 for path in runtime_tool_paths: 516 # Make sure TOOL_NAME is not an OS string 517 if not ( 518 path.endswith("linux") 519 or path.endswith("windows") 520 or path.endswith("macosx") 521 or path.endswith("share") 522 ): 523 # TODO(tonymd): Check if a file & do nothing? 524 525 # Check if it's a directory with subdir(s) as a version string 526 # create all 'runtime.tools.{tool_folder}-{version.path}' 527 # (for each version) 528 # create 'runtime.tools.{tool_folder}.path' 529 # (with latest version) 530 if os.path.isdir(path): 531 # Grab the tool name (folder) by itself. 532 tool_folder = os.path.basename(path) 533 # Sort so that [-1] is the latest version. 534 version_paths = sorted(glob.glob(os.path.join(path, "*"))) 535 # Check if all sub folders start with a version string. 536 if len(version_paths) == sum( 537 bool(re.match(r"^[0-9.]+", os.path.basename(vp))) 538 for vp in version_paths 539 ): 540 for version_path in version_paths: 541 version_string = os.path.basename(version_path) 542 var_name = "runtime.tools.{}-{}.path".format( 543 tool_folder, version_string 544 ) 545 self.tools_variables[var_name] = os.path.join( 546 path, version_string 547 ) 548 var_name = "runtime.tools.{}.path".format(tool_folder) 549 self.tools_variables[var_name] = os.path.join( 550 path, os.path.basename(version_paths[-1]) 551 ) 552 # Else set toolpath to path. 553 else: 554 var_name = "runtime.tools.{}.path".format(tool_folder) 555 self.tools_variables[var_name] = path 556 557 _LOG.debug("TOOL VARIABLES: %s", _pretty_format(self.tools_variables)) 558 559 # Copy self.tools_variables into destination 560 def _set_tools_variables(self, destination): 561 for key, value in self.tools_variables.items(): 562 destination[key] = value 563 564 def _substitute_variables(self): 565 """Perform variable substitution in board and platform metadata.""" 566 567 # menu -> board 568 # Copy selected menu variables into board definiton 569 for name, value in self.menu_options["selected"].items(): 570 self.board[self.selected_board][name] = value 571 572 # board -> board 573 # Replace any {vars} in the selected board with values defined within 574 # (and from copied in menu options). 575 for var, value in self.board[self.selected_board].items(): 576 self.board[self.selected_board][var] = self._replace_variables( 577 value, self.board[self.selected_board] 578 ) 579 580 # Check for build.variant variable 581 # This will be set in selected board after menu options substitution 582 build_variant = self.board[self.selected_board].get( 583 "build.variant", None 584 ) 585 if build_variant: 586 # Set build.variant.path 587 bvp = os.path.join(self.package_path, "variants", build_variant) 588 self.build_variant_path = bvp 589 self.board[self.selected_board]["build.variant.path"] = bvp 590 # Add the variant folder as an include directory 591 # This is used in the stm32l4 and stm32duino cores. Include 592 # directories should be surrounded in quotes in case they contain 593 # spaces or parens. 594 self.variant_includes = "\"-I{}\"".format(bvp) 595 596 _LOG.debug("PLATFORM INITIAL: %s", _pretty_format(self.platform)) 597 598 # board -> platform 599 # Replace {vars} in platform from the selected board definition 600 for var, value in self.platform.items(): 601 self.platform[var] = self._replace_variables( 602 value, self.board[self.selected_board] 603 ) 604 605 # platform -> platform 606 # Replace any remaining {vars} in platform from platform 607 for var, value in self.platform.items(): 608 self.platform[var] = self._replace_variables(value, self.platform) 609 610 # Repeat platform -> platform for any lingering variables 611 # Example: {build.opt.name} in STM32 core 612 for var, value in self.platform.items(): 613 self.platform[var] = self._replace_variables(value, self.platform) 614 615 _LOG.debug("MENU_OPTIONS: %s", _pretty_format(self.menu_options)) 616 _LOG.debug( 617 "SELECTED_BOARD: %s", 618 _pretty_format(self.board[self.selected_board]), 619 ) 620 _LOG.debug("PLATFORM: %s", _pretty_format(self.platform)) 621 622 def selected_board_spec(self): 623 return self.board[self.selected_board] 624 625 def get_menu_options(self): 626 all_options = [] 627 max_string_length = [0, 0] 628 629 for key_name, description in self.board[self.selected_board].items(): 630 menu_match_result = self.MENU_OPTION_REGEX.match(key_name) 631 if menu_match_result: 632 menu_match = menu_match_result.groupdict() 633 name = "menu.{}.{}".format( 634 menu_match["menu_option_name"], 635 menu_match["menu_option_value"], 636 ) 637 if len(name) > max_string_length[0]: 638 max_string_length[0] = len(name) 639 if len(description) > max_string_length[1]: 640 max_string_length[1] = len(description) 641 all_options.append((name, description)) 642 643 return all_options, max_string_length 644 645 def get_default_menu_options(self): 646 default_options = [] 647 max_string_length = [0, 0] 648 649 for key_name, value in self.menu_options["default_board_values"][ 650 self.selected_board 651 ].items(): 652 full_key = key_name + "." + value["name"] 653 if len(full_key) > max_string_length[0]: 654 max_string_length[0] = len(full_key) 655 if len(value["description"]) > max_string_length[1]: 656 max_string_length[1] = len(value["description"]) 657 default_options.append((full_key, value["description"])) 658 659 return default_options, max_string_length 660 661 @staticmethod 662 def split_binary_from_arguments(compile_line): 663 compile_binary = None 664 rest_of_line = compile_line 665 666 compile_binary_match = re.search(r'^("[^"]+") ', compile_line) 667 if compile_binary_match: 668 compile_binary = compile_binary_match[1] 669 rest_of_line = compile_line.replace(compile_binary_match[0], "", 1) 670 671 return compile_binary, rest_of_line 672 673 def _strip_includes_source_file_object_file_vars(self, compile_line): 674 line = compile_line 675 if self.variant_includes: 676 line = compile_line.replace( 677 "{includes} \"{source_file}\" -o \"{object_file}\"", 678 self.variant_includes, 679 1, 680 ) 681 else: 682 line = compile_line.replace( 683 "{includes} \"{source_file}\" -o \"{object_file}\"", "", 1 684 ) 685 return line 686 687 def _get_tool_name(self, line): 688 tool_match_result = self.TOOL_NAME_REGEX.match(line) 689 if tool_match_result: 690 return tool_match_result[1] 691 return False 692 693 def get_upload_tool_names(self): 694 return [ 695 self._get_tool_name(t) 696 for t in self.platform.keys() 697 if self.TOOL_NAME_REGEX.match(t) and 'upload.pattern' in t 698 ] 699 700 # TODO(tonymd): Use these getters in _replace_variables() or 701 # _substitute_variables() 702 703 def _get_platform_variable(self, variable): 704 # TODO(tonymd): Check for '.macos' '.linux' '.windows' in variable name, 705 # compare with platform.system() and return that instead. 706 return self.platform.get(variable, False) 707 708 def _get_platform_variable_with_substitutions(self, variable, namespace): 709 line = self.platform.get(variable, False) 710 # Get all unique variables used in this line in line. 711 unique_vars = sorted( 712 set(self.INTERPOLATED_VARIABLE_REGEX.findall(line)) 713 ) 714 # Search for each unique_vars in namespace and global. 715 for var in unique_vars: 716 v_raw_name = var.strip("{}") 717 718 # Check for namespace.variable 719 # eg: 'tools.stm32CubeProg.cmd' 720 possible_var_name = "{}.{}".format(namespace, v_raw_name) 721 result = self._get_platform_variable(possible_var_name) 722 # Check for os overriden variable 723 # eg: 724 # ('tools.stm32CubeProg.cmd', 'stm32CubeProg.sh'), 725 # ('tools.stm32CubeProg.cmd.windows', 'stm32CubeProg.bat'), 726 possible_var_name = "{}.{}.{}".format( 727 namespace, v_raw_name, arduino_runtime_os_string() 728 ) 729 os_override_result = self._get_platform_variable(possible_var_name) 730 731 if os_override_result: 732 line = line.replace(var, os_override_result) 733 elif result: 734 line = line.replace(var, result) 735 # Check for variable at top level? 736 # elif self._get_platform_variable(v_raw_name): 737 # line = line.replace(self._get_platform_variable(v_raw_name), 738 # result) 739 return line 740 741 def get_upload_line(self, tool_name, serial_port=False): 742 """TODO(tonymd) Add docstring.""" 743 # TODO(tonymd): Error if tool_name does not exist 744 tool_namespace = "tools.{}".format(tool_name) 745 pattern = "tools.{}.upload.pattern".format(tool_name) 746 747 if not self._get_platform_variable(pattern): 748 _LOG.error("Error: upload tool '%s' does not exist.", tool_name) 749 tools = self.get_upload_tool_names() 750 _LOG.error("Valid tools: %s", ", ".join(tools)) 751 return sys.exit(1) 752 753 line = self._get_platform_variable_with_substitutions( 754 pattern, tool_namespace 755 ) 756 757 # TODO(tonymd): Teensy specific tool overrides. 758 if tool_name == "teensyloader": 759 # Remove un-necessary lines 760 # {serial.port.label} and {serial.port.protocol} are returned by 761 # the teensy_ports binary. 762 line = line.replace("\"-portlabel={serial.port.label}\"", "", 1) 763 line = line.replace( 764 "\"-portprotocol={serial.port.protocol}\"", "", 1 765 ) 766 767 if serial_port == "UNKNOWN" or not serial_port: 768 line = line.replace('"-port={serial.port}"', "", 1) 769 else: 770 line = line.replace("{serial.port}", serial_port, 1) 771 772 return line 773 774 def _get_binary_path(self, variable_pattern): 775 compile_line = self.replace_compile_binary_with_override_path( 776 self._get_platform_variable(variable_pattern) 777 ) 778 compile_binary, _ = ArduinoBuilder.split_binary_from_arguments( 779 compile_line 780 ) 781 return compile_binary 782 783 def get_cc_binary(self): 784 return self._get_binary_path("recipe.c.o.pattern") 785 786 def get_cxx_binary(self): 787 return self._get_binary_path("recipe.cpp.o.pattern") 788 789 def get_objcopy_binary(self): 790 objcopy_step_name = self.get_objcopy_step_names()[0] 791 objcopy_binary = self._get_binary_path(objcopy_step_name) 792 return objcopy_binary 793 794 def get_ar_binary(self): 795 return self._get_binary_path("recipe.ar.pattern") 796 797 def get_size_binary(self): 798 return self._get_binary_path("recipe.size.pattern") 799 800 def replace_command_args_with_compiler_override_path(self, compile_line): 801 if not self.compiler_path_override: 802 return compile_line 803 replacement_line = compile_line 804 replacement_line_args = compile_line.split() 805 for arg in replacement_line_args: 806 compile_binary_basename = os.path.basename(arg.strip("\"")) 807 if compile_binary_basename in self.compiler_path_override_binaries: 808 new_compiler = os.path.join( 809 self.compiler_path_override, compile_binary_basename 810 ) 811 replacement_line = replacement_line.replace( 812 arg, new_compiler, 1 813 ) 814 return replacement_line 815 816 def replace_compile_binary_with_override_path(self, compile_line): 817 replacement_compile_line = compile_line 818 819 # Change the compiler path if there's an override path set 820 if self.compiler_path_override: 821 compile_binary, line = ArduinoBuilder.split_binary_from_arguments( 822 compile_line 823 ) 824 compile_binary_basename = os.path.basename( 825 compile_binary.strip("\"") 826 ) 827 new_compiler = os.path.join( 828 self.compiler_path_override, compile_binary_basename 829 ) 830 if platform.system() == "Windows" and not re.match( 831 r".*\.exe$", new_compiler, flags=re.IGNORECASE 832 ): 833 new_compiler += ".exe" 834 835 if os.path.isfile(new_compiler): 836 replacement_compile_line = "\"{}\" {}".format( 837 new_compiler, line 838 ) 839 840 return replacement_compile_line 841 842 def get_c_compile_line(self): 843 _LOG.debug( 844 "ARDUINO_C_COMPILE: %s", 845 _pretty_format(self.platform["recipe.c.o.pattern"]), 846 ) 847 848 compile_line = self.platform["recipe.c.o.pattern"] 849 compile_line = self._strip_includes_source_file_object_file_vars( 850 compile_line 851 ) 852 compile_line += " -I{}".format( 853 self.board[self.selected_board]["build.core.path"] 854 ) 855 856 compile_line = self.replace_compile_binary_with_override_path( 857 compile_line 858 ) 859 return compile_line 860 861 def get_s_compile_line(self): 862 _LOG.debug( 863 "ARDUINO_S_COMPILE %s", 864 _pretty_format(self.platform["recipe.S.o.pattern"]), 865 ) 866 867 compile_line = self.platform["recipe.S.o.pattern"] 868 compile_line = self._strip_includes_source_file_object_file_vars( 869 compile_line 870 ) 871 compile_line += " -I{}".format( 872 self.board[self.selected_board]["build.core.path"] 873 ) 874 875 compile_line = self.replace_compile_binary_with_override_path( 876 compile_line 877 ) 878 return compile_line 879 880 def get_ar_compile_line(self): 881 _LOG.debug( 882 "ARDUINO_AR_COMPILE: %s", 883 _pretty_format(self.platform["recipe.ar.pattern"]), 884 ) 885 886 compile_line = self.platform["recipe.ar.pattern"].replace( 887 "\"{object_file}\"", "", 1 888 ) 889 890 compile_line = self.replace_compile_binary_with_override_path( 891 compile_line 892 ) 893 return compile_line 894 895 def get_cpp_compile_line(self): 896 _LOG.debug( 897 "ARDUINO_CPP_COMPILE: %s", 898 _pretty_format(self.platform["recipe.cpp.o.pattern"]), 899 ) 900 901 compile_line = self.platform["recipe.cpp.o.pattern"] 902 compile_line = self._strip_includes_source_file_object_file_vars( 903 compile_line 904 ) 905 compile_line += " -I{}".format( 906 self.board[self.selected_board]["build.core.path"] 907 ) 908 909 compile_line = self.replace_compile_binary_with_override_path( 910 compile_line 911 ) 912 return compile_line 913 914 def get_link_line(self): 915 _LOG.debug( 916 "ARDUINO_LINK: %s", 917 _pretty_format(self.platform["recipe.c.combine.pattern"]), 918 ) 919 920 compile_line = self.platform["recipe.c.combine.pattern"] 921 922 compile_line = self.replace_compile_binary_with_override_path( 923 compile_line 924 ) 925 return compile_line 926 927 def get_objcopy_step_names(self): 928 names = [ 929 name 930 for name, line in self.platform.items() 931 if self.OBJCOPY_STEP_NAME_REGEX.match(name) 932 ] 933 return names 934 935 def get_objcopy_steps(self) -> list[str]: 936 lines = [ 937 line 938 for name, line in self.platform.items() 939 if self.OBJCOPY_STEP_NAME_REGEX.match(name) 940 ] 941 lines = [ 942 self.replace_compile_binary_with_override_path(line) 943 for line in lines 944 ] 945 return lines 946 947 # TODO(tonymd): These recipes are probably run in sorted order 948 def get_objcopy(self, suffix): 949 # Expected vars: 950 # teensy: 951 # recipe.objcopy.eep.pattern 952 # recipe.objcopy.hex.pattern 953 954 pattern = "recipe.objcopy.{}.pattern".format(suffix) 955 objcopy_step_names = self.get_objcopy_step_names() 956 957 objcopy_suffixes = [ 958 m[1] 959 for m in [ 960 self.OBJCOPY_STEP_NAME_REGEX.match(line) 961 for line in objcopy_step_names 962 ] 963 if m 964 ] 965 if pattern not in objcopy_step_names: 966 _LOG.error("Error: objcopy suffix '%s' does not exist.", suffix) 967 _LOG.error("Valid suffixes: %s", ", ".join(objcopy_suffixes)) 968 return sys.exit(1) 969 970 line = self._get_platform_variable(pattern) 971 972 _LOG.debug("ARDUINO_OBJCOPY_%s: %s", suffix, line) 973 974 line = self.replace_compile_binary_with_override_path(line) 975 976 return line 977 978 def get_objcopy_flags(self, suffix): 979 # TODO(tonymd): Possibly teensy specific variables. 980 flags = "" 981 if suffix == "hex": 982 flags = self.platform.get("compiler.elf2hex.flags", "") 983 elif suffix == "bin": 984 flags = self.platform.get("compiler.elf2bin.flags", "") 985 elif suffix == "eep": 986 flags = self.platform.get("compiler.objcopy.eep.flags", "") 987 return flags 988 989 # TODO(tonymd): There are more recipe hooks besides postbuild. 990 # They are run in sorted order. 991 # TODO(tonymd): Rename this to get_hooks(hook_name, step). 992 # TODO(tonymd): Add a list-hooks and or run-hooks command 993 def get_postbuild_line(self, step_number): 994 line = self.platform[ 995 "recipe.hooks.postbuild.{}.pattern".format(step_number) 996 ] 997 line = self.replace_command_args_with_compiler_override_path(line) 998 return line 999 1000 def get_prebuild_steps(self) -> list[str]: 1001 # Teensy core uses recipe.hooks.sketch.prebuild.1.pattern 1002 # stm32 core uses recipe.hooks.prebuild.1.pattern 1003 # TODO(tonymd): STM32 core uses recipe.hooks.prebuild.1.pattern.windows 1004 # (should override non-windows key) 1005 lines = [ 1006 line 1007 for name, line in self.platform.items() 1008 if re.match( 1009 r"^recipe.hooks.(?:sketch.)?prebuild.[^.]+.pattern$", name 1010 ) 1011 ] 1012 # TODO(tonymd): Write a function to fetch/replace OS specific patterns 1013 # (ending in an OS string) 1014 lines = [ 1015 self.replace_compile_binary_with_override_path(line) 1016 for line in lines 1017 ] 1018 return lines 1019 1020 def get_postbuild_steps(self) -> list[str]: 1021 lines = [ 1022 line 1023 for name, line in self.platform.items() 1024 if re.match(r"^recipe.hooks.postbuild.[^.]+.pattern$", name) 1025 ] 1026 1027 lines = [ 1028 self.replace_command_args_with_compiler_override_path(line) 1029 for line in lines 1030 ] 1031 return lines 1032 1033 def get_s_flags(self): 1034 compile_line = self.get_s_compile_line() 1035 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 1036 compile_line 1037 ) 1038 compile_line = compile_line.replace("-c", "", 1) 1039 return compile_line.strip() 1040 1041 def get_c_flags(self): 1042 compile_line = self.get_c_compile_line() 1043 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 1044 compile_line 1045 ) 1046 compile_line = compile_line.replace("-c", "", 1) 1047 return compile_line.strip() 1048 1049 def get_cpp_flags(self): 1050 compile_line = self.get_cpp_compile_line() 1051 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 1052 compile_line 1053 ) 1054 compile_line = compile_line.replace("-c", "", 1) 1055 return compile_line.strip() 1056 1057 def get_ar_flags(self): 1058 compile_line = self.get_ar_compile_line() 1059 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 1060 compile_line 1061 ) 1062 return compile_line.strip() 1063 1064 def get_ld_flags(self): 1065 compile_line = self.get_link_line() 1066 _, compile_line = ArduinoBuilder.split_binary_from_arguments( 1067 compile_line 1068 ) 1069 1070 # TODO(tonymd): This is teensy specific 1071 line_to_delete = ( 1072 "-o \"{build.path}/{build.project_name}.elf\" " 1073 "{object_files} \"-L{build.path}\"" 1074 ) 1075 if self.build_path: 1076 line_to_delete = line_to_delete.replace( 1077 "{build.path}", self.build_path 1078 ) 1079 if self.build_project_name: 1080 line_to_delete = line_to_delete.replace( 1081 "{build.project_name}", self.build_project_name 1082 ) 1083 1084 compile_line = compile_line.replace(line_to_delete, "", 1) 1085 libs = re.findall(r'(-l[^ ]+ ?)', compile_line) 1086 for lib in libs: 1087 compile_line = compile_line.replace(lib, "", 1) 1088 libs = [lib.strip() for lib in libs] 1089 1090 return compile_line.strip() 1091 1092 def get_ld_libs(self, name_only=False): 1093 compile_line = self.get_link_line() 1094 libs = re.findall(r'(?P<arg>-l(?P<name>[^ ]+) ?)', compile_line) 1095 if name_only: 1096 libs = [lib_name.strip() for lib_arg, lib_name in libs] 1097 else: 1098 libs = [lib_arg.strip() for lib_arg, lib_name in libs] 1099 return " ".join(libs) 1100 1101 def library_folders(self): 1102 """TODO(tonymd) Add docstring.""" 1103 # Arduino library format documentation: 1104 # https://arduino.github.io/arduino-cli/library-specification/#layout-of-folders-and-files 1105 # - If src folder exists, 1106 # use that as the root include directory -Ilibraries/libname/src 1107 # - Else lib folder as root include -Ilibraries/libname 1108 # (exclude source files in the examples folder in this case) 1109 1110 if not self.library_names or not self.library_path: 1111 return [] 1112 1113 folder_patterns = ["*"] 1114 if self.library_names: 1115 folder_patterns = self.library_names 1116 1117 library_folders = OrderedDict() 1118 for library_dir in self.library_path: 1119 found_library_names = file_operations.find_files( 1120 library_dir, folder_patterns, directories_only=True 1121 ) 1122 _LOG.debug( 1123 "Found Libraries %s: %s", library_dir, found_library_names 1124 ) 1125 for lib_name in found_library_names: 1126 lib_dir = os.path.join(library_dir, lib_name) 1127 src_dir = os.path.join(lib_dir, "src") 1128 if os.path.exists(src_dir) and os.path.isdir(src_dir): 1129 library_folders[lib_name] = src_dir 1130 else: 1131 library_folders[lib_name] = lib_dir 1132 1133 return list(library_folders.values()) 1134 1135 def library_include_dirs(self): 1136 return [Path(lib).as_posix() for lib in self.library_folders()] 1137 1138 def library_includes(self): 1139 include_args = [] 1140 library_folders = self.library_folders() 1141 for lib_dir in library_folders: 1142 include_args.append("-I{}".format(os.path.relpath(lib_dir))) 1143 return include_args 1144 1145 def library_files(self, pattern, only_library_name=None): 1146 sources = [] 1147 library_folders = self.library_folders() 1148 if only_library_name: 1149 library_folders = [ 1150 lf for lf in self.library_folders() if only_library_name in lf 1151 ] 1152 for lib_dir in library_folders: 1153 for file_path in file_operations.find_files(lib_dir, [pattern]): 1154 if not file_path.startswith("examples"): 1155 sources.append((Path(lib_dir) / file_path).as_posix()) 1156 return sources 1157 1158 def library_c_files(self): 1159 return self.library_files("**/*.c") 1160 1161 def library_s_files(self): 1162 return self.library_files("**/*.S") 1163 1164 def library_cpp_files(self): 1165 return self.library_files("**/*.cpp") 1166 1167 def get_core_path(self): 1168 return self.board[self.selected_board]["build.core.path"] 1169 1170 def core_files(self, pattern): 1171 sources = [] 1172 for file_path in file_operations.find_files( 1173 self.get_core_path(), [pattern] 1174 ): 1175 sources.append(os.path.join(self.get_core_path(), file_path)) 1176 return sources 1177 1178 def core_c_files(self): 1179 return self.core_files("**/*.c") 1180 1181 def core_s_files(self): 1182 return self.core_files("**/*.S") 1183 1184 def core_cpp_files(self): 1185 return self.core_files("**/*.cpp") 1186 1187 def get_variant_path(self): 1188 return self.build_variant_path 1189 1190 def variant_files(self, pattern): 1191 sources = [] 1192 if self.build_variant_path: 1193 for file_path in file_operations.find_files( 1194 self.get_variant_path(), [pattern] 1195 ): 1196 sources.append(os.path.join(self.get_variant_path(), file_path)) 1197 return sources 1198 1199 def variant_c_files(self): 1200 return self.variant_files("**/*.c") 1201 1202 def variant_s_files(self): 1203 return self.variant_files("**/*.S") 1204 1205 def variant_cpp_files(self): 1206 return self.variant_files("**/*.cpp") 1207 1208 def project_files(self, pattern): 1209 sources = [] 1210 for file_path in file_operations.find_files( 1211 self.project_path, [pattern] 1212 ): 1213 if not file_path.startswith( 1214 "examples" 1215 ) and not file_path.startswith("libraries"): 1216 sources.append(file_path) 1217 return sources 1218 1219 def project_c_files(self): 1220 return self.project_files("**/*.c") 1221 1222 def project_cpp_files(self): 1223 return self.project_files("**/*.cpp") 1224 1225 def project_ino_files(self): 1226 return self.project_files("**/*.ino") 1227