xref: /aosp_15_r20/external/pigweed/pw_arduino_build/py/pw_arduino_build/unit_test_runner.py (revision 61c4878ac05f98d0ceed94b57d316916de578985)

#!/usr/bin/env python3
# Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
"""This script flashes and runs unit tests onto Arduino boards."""

import argparse
import logging
import os
import platform
import re
import subprocess
import sys
import time
from pathlib import Path

import serial
import serial.tools.list_ports
import pw_arduino_build.log
from pw_arduino_build import teensy_detector
from pw_arduino_build.file_operations import decode_file_json

_LOG = logging.getLogger('unit_test_runner')

# Verification of test pass/failure depends on these strings. If the formatting
# or output of the simple_printing_event_handler changes, this may need to be
# updated.
_TESTS_STARTING_STRING = b'[==========] Running all tests.'
_TESTS_DONE_STRING = b'[==========] Done running all tests.'
_TEST_FAILURE_STRING = b'[  FAILED  ]'

# How long to wait for the first byte of a test to be emitted. This is longer
# than the user-configurable timeout as there's a delay while the device is
# flashed.
_FLASH_TIMEOUT = 5.0


class TestingFailure(Exception):
    """A simple exception to be raised when a testing step fails."""


class DeviceNotFound(Exception):
    """A simple exception to be raised when unable to connect to a device."""


class ArduinoCoreNotSupported(Exception):
    """Exception raised when a given core does not support unit testing."""


def valid_file_name(arg):
    file_path = Path(os.path.expandvars(arg)).absolute()
    if not file_path.is_file():
        raise argparse.ArgumentTypeError(f"'{arg}' does not exist.")
    return file_path


def parse_args():
    """Parses command-line arguments."""

    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        'binary', help='The target test binary to run', type=valid_file_name
    )
    parser.add_argument(
        '--port',
        help='The name of the serial port to connect to when ' 'running tests',
    )
    parser.add_argument(
        '--baud',
        type=int,
        default=115200,
        help='Target baud rate to use for serial communication'
        ' with target device',
    )
    parser.add_argument(
        '--test-timeout',
        type=float,
        default=5.0,
        help='Maximum communication delay in seconds before a '
        'test is considered unresponsive and aborted',
    )
    parser.add_argument(
        '--verbose',
        '-v',
        dest='verbose',
        action='store_true',
        help='Output additional logs as the script runs',
    )

    parser.add_argument(
        '--flash-only',
        action='store_true',
        help="Don't check for test output after flashing.",
    )

    # arduino_builder arguments
    # TODO(tonymd): Get these args from __main__.py or elsewhere.
    parser.add_argument(
        "-c", "--config-file", required=True, help="Path to a config file."
    )
    parser.add_argument(
        "--arduino-package-path",
        help="Path to the arduino IDE install location.",
    )
    parser.add_argument(
        "--arduino-package-name",
        help="Name of the Arduino board package to use.",
    )
    parser.add_argument(
        "--compiler-path-override",
        help="Path to arm-none-eabi-gcc bin folder. "
        "Default: Arduino core specified gcc",
    )
    parser.add_argument("--board", help="Name of the Arduino board to use.")
    parser.add_argument(
        "--upload-tool",
        required=True,
        help="Name of the Arduino upload tool to use.",
    )
    parser.add_argument(
        "--set-variable",
        action="append",
        metavar='some.variable=NEW_VALUE',
        help="Override an Arduino recipe variable. May be "
        "specified multiple times. For example: "
        "--set-variable 'serial.port.label=/dev/ttyACM0' "
        "--set-variable 'serial.port.protocol=Teensy'",
    )
    return parser.parse_args()


def log_subprocess_output(level, output):
    """Logs subprocess output line-by-line."""

    lines = output.decode('utf-8', errors='replace').splitlines()
    for line in lines:
        _LOG.log(level, line)


def read_serial(port, baud_rate, test_timeout) -> bytes:
    """Reads lines from a serial port until a line read times out.

    Returns bytes object containing the read serial data.
    """

    serial_data = bytearray()
    device = serial.Serial(
        baudrate=baud_rate, port=port, timeout=_FLASH_TIMEOUT
    )
    if not device.is_open:
        raise TestingFailure('Failed to open device')

    # Flush input buffer and reset the device to begin the test.
    device.reset_input_buffer()

    # Block and wait for the first byte.
    serial_data += device.read()
    if not serial_data:
        raise TestingFailure('Device not producing output')

    device.timeout = test_timeout

    # Read with a reasonable timeout until we stop getting characters.
    while True:
        bytes_read = device.readline()
        if not bytes_read:
            break
        serial_data += bytes_read
        if serial_data.rfind(_TESTS_DONE_STRING) != -1:
            # Set to much more aggressive timeout since the last one or two
            # lines should print out immediately. (one line if all fails or all
            # passes, two lines if mixed.)
            device.timeout = 0.01

    # Remove carriage returns.
    serial_data = serial_data.replace(b'\r', b'')

    # Try to trim captured results to only contain most recent test run.
    test_start_index = serial_data.rfind(_TESTS_STARTING_STRING)
    return (
        serial_data
        if test_start_index == -1
        else serial_data[test_start_index:]
    )


def wait_for_port(port):
    """Wait for the serial port to be available."""
    while port not in [sp.device for sp in serial.tools.list_ports.comports()]:
        time.sleep(1)


def flash_device(test_runner_args, upload_tool):
    """Flash binary to a connected device using the provided configuration."""

    # TODO(tonymd): Create a library function to call rather than launching
    # the arduino_builder script.
    flash_tool = 'arduino_builder'
    cmd = (
        [flash_tool, "--quiet"]
        + test_runner_args
        + ["--run-objcopy", "--run-postbuilds", "--run-upload", upload_tool]
    )
    _LOG.info('Flashing firmware to device')
    _LOG.debug('Running: %s', " ".join(cmd))

    env = os.environ.copy()
    process = subprocess.run(
        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env
    )
    if process.returncode:
        log_subprocess_output(logging.ERROR, process.stdout)
        raise TestingFailure('Failed to flash target device')

    log_subprocess_output(logging.DEBUG, process.stdout)

    _LOG.debug('Successfully flashed firmware to device')


def handle_test_results(test_output):
    """Parses test output to determine whether tests passed or failed."""

    if test_output.find(_TESTS_STARTING_STRING) == -1:
        raise TestingFailure('Failed to find test start')

    if test_output.rfind(_TESTS_DONE_STRING) == -1:
        log_subprocess_output(logging.INFO, test_output)
        raise TestingFailure('Tests did not complete')

    if test_output.rfind(_TEST_FAILURE_STRING) != -1:
        log_subprocess_output(logging.INFO, test_output)
        raise TestingFailure('Test suite had one or more failures')

    log_subprocess_output(logging.DEBUG, test_output)

    _LOG.info('Test passed!')


def run_device_test(
    binary,
    flash_only,
    port,
    baud,
    test_timeout,
    upload_tool,
    arduino_package_path,
    test_runner_args,
) -> bool:
    """Flashes, runs, and checks an on-device test binary.

    Returns true on test pass.
    """
    if test_runner_args is None:
        test_runner_args = []

    if "teensy" not in arduino_package_path:
        raise ArduinoCoreNotSupported(arduino_package_path)

    if port is None or "--set-variable" not in test_runner_args:
        _LOG.debug('Attempting to automatically detect dev board')
        boards = teensy_detector.detect_boards(arduino_package_path)
        if not boards:
            error = 'Could not find an attached device'
            _LOG.error(error)
            raise DeviceNotFound(error)
        test_runner_args += boards[0].test_runner_args()
        upload_tool = boards[0].arduino_upload_tool_name
        if port is None:
            port = boards[0].dev_name

    # TODO(tonymd): Remove this when teensy_ports is working in teensy_detector
    if platform.system() == "Windows":
        # Delete the incorrect serial port.
        index_of_port = [
            i
            for i, l in enumerate(test_runner_args)
            if l.startswith('serial.port=')
        ]
        if index_of_port:
            # Delete the '--set-variable' arg
            del test_runner_args[index_of_port[0] - 1]
            # Delete the 'serial.port=*' arg
            del test_runner_args[index_of_port[0] - 1]

    _LOG.debug('Launching test binary %s', binary)
    try:
        result: list[bytes] = []
        _LOG.info('Running test')
        # Warning: A race condition is possible here. This assumes the host is
        # able to connect to the port and that there isn't a test running on
        # this serial port.
        flash_device(test_runner_args, upload_tool)
        wait_for_port(port)
        if flash_only:
            return True
        result.append(read_serial(port, baud, test_timeout))
        if result:
            handle_test_results(result[0])
    except TestingFailure as err:
        _LOG.error(err)
        return False

    return True


def get_option(key, config_file_values, args, required=False):
    command_line_option = getattr(args, key, None)
    final_option = config_file_values.get(key, command_line_option)
    if required and command_line_option is None and final_option is None:
        # Print a similar error message to argparse
        executable = os.path.basename(sys.argv[0])
        option = "--" + key.replace("_", "-")
        print(
            f"{executable}: error: the following arguments are required: "
            f"{option}"
        )
        sys.exit(1)
    return final_option


def main():
    """Set up runner, and then flash/run device test."""
    args = parse_args()

    json_file_options, unused_config_path = decode_file_json(args.config_file)

    log_level = logging.DEBUG if args.verbose else logging.INFO
    pw_arduino_build.log.install(log_level)

    # Construct arduino_builder flash arguments for a given .elf binary.
    arduino_package_path = get_option(
        "arduino_package_path", json_file_options, args, required=True
    )
    # Arduino core args.
    arduino_builder_args = [
        "--arduino-package-path",
        arduino_package_path,
        "--arduino-package-name",
        get_option(
            "arduino_package_name", json_file_options, args, required=True
        ),
    ]

    # Use CIPD installed compilers.
    compiler_path_override = get_option(
        "compiler_path_override", json_file_options, args
    )
    if compiler_path_override:
        arduino_builder_args += [
            "--compiler-path-override",
            compiler_path_override,
        ]

    # Run subcommand with board selection arg.
    arduino_builder_args += [
        "run",
        "--board",
        get_option("board", json_file_options, args, required=True),
    ]

    # .elf file location args.
    binary = args.binary
    build_path = binary.parent.as_posix()
    arduino_builder_args += ["--build-path", build_path]
    build_project_name = binary.name
    # Remove '.elf' extension.
    match_result = re.match(r'(.*?)\.elf$', binary.name, re.IGNORECASE)
    if match_result:
        build_project_name = match_result[1]
        arduino_builder_args += ["--build-project-name", build_project_name]

    # USB port is passed to arduino_builder_args via --set-variable args.
    if args.set_variable:
        for var in args.set_variable:
            arduino_builder_args += ["--set-variable", var]

    if run_device_test(
        binary.as_posix(),
        args.flash_only,
        args.port,
        args.baud,
        args.test_timeout,
        args.upload_tool,
        arduino_package_path,
        test_runner_args=arduino_builder_args,
    ):
        sys.exit(0)
    else:
        sys.exit(1)


if __name__ == '__main__':
    main()