xref: /aosp_15_r20/development/scripts/cargo2rulesmk.py (revision 90c8c64db3049935a07c6143d7fd006e26f8ecca)

#!/usr/bin/env python3
# Copyright (C) 2023 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Call cargo -v, parse its output, and generate a Trusty build system module.

Usage: Run this script in a crate workspace root directory. The Cargo.toml file
should work at least for the host platform.

Without other flags, "cargo2rulesmk.py --run" calls cargo clean, calls cargo
build -v, and generates makefile rules. The cargo build only generates crates
for the host without test crates.

If there are rustc warning messages, this script will add a warning comment to
the owner crate module in rules.mk.
"""

import argparse
import glob
import json
import os
import os.path
import platform
import re
import shutil
import subprocess
import sys

from typing import List


assert "/development/scripts" in os.path.dirname(__file__)
TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))

# Some Rust packages include extra unwanted crates.
# This set contains all such excluded crate names.
EXCLUDED_CRATES = {"protobuf_bin_gen_rust_do_not_use"}


CUSTOM_MODULE_CRATES = {
    # This map tracks Rust crates that have special modules that
    # were not generated automatically by this script. Examples
    # include compiler builtins and other foundational libraries.
    # It also tracks crates tht are not under external/rust/crates.
    "compiler_builtins": "trusty/user/base/lib/libcompiler_builtins-rust",
    "core": "trusty/user/base/lib/libcore-rust",
}

RENAME_STEM_MAP = {
    # This map includes all changes to the default rust module stem names,
    # which is used for output files when different from the module name.
    "protoc_gen_rust": "protoc-gen-rust",
}

# Header added to all generated rules.mk files.
RULES_MK_HEADER = (
    "# This file is generated by cargo2rulesmk.py {args}.\n"
    + "# Do not modify this file as changes will be overridden on upgrade.\n\n"
)

CARGO_OUT = "cargo.out"  # Name of file to keep cargo build -v output.

# This should be kept in sync with tools/external_updater/crates_updater.py.
ERRORS_LINE = "Errors in " + CARGO_OUT + ":"

TARGET_TMP = "target.tmp"  # Name of temporary output directory.

# Message to be displayed when this script is called without the --run flag.
DRY_RUN_NOTE = (
    "Dry-run: This script uses ./"
    + TARGET_TMP
    + " for output directory,\n"
    + "runs cargo clean, runs cargo build -v, saves output to ./cargo.out,\n"
    + "and writes to rules.mk in the current and subdirectories.\n\n"
    + "To do do all of the above, use the --run flag.\n"
    + "See --help for other flags, and more usage notes in this script.\n"
)

# Cargo -v output of a call to rustc.
RUSTC_PAT = re.compile("^ +Running `(.*\/)?rustc (.*)`$")

# Cargo -vv output of a call to rustc could be split into multiple lines.
# Assume that the first line will contain some CARGO_* env definition.
RUSTC_VV_PAT = re.compile("^ +Running `.*CARGO_.*=.*$")
# The combined -vv output rustc command line pattern.
RUSTC_VV_CMD_ARGS = re.compile("^ *Running `.*CARGO_.*=.* (.*\/)?rustc (.*)`$")

# Cargo -vv output of a "cc" or "ar" command; all in one line.
CC_AR_VV_PAT = re.compile(r'^\[([^ ]*)[^\]]*\] running:? "(cc|ar)" (.*)$')
# Some package, such as ring-0.13.5, has pattern '... running "cc"'.

# Rustc output of file location path pattern for a warning message.
WARNING_FILE_PAT = re.compile("^ *--> ([^:]*):[0-9]+")

# cargo test --list output of the start of running a binary.
CARGO_TEST_LIST_START_PAT = re.compile(r"^\s*Running (.*) \(.*\)$")

# cargo test --list output of the end of running a binary.
CARGO_TEST_LIST_END_PAT = re.compile(r"^(\d+) tests?, (\d+) benchmarks$")

CARGO2ANDROID_RUNNING_PAT = re.compile("^### Running: .*$")

# Rust package name with suffix -d1.d2.d3(+.*)?.
VERSION_SUFFIX_PAT = re.compile(
    r"^(.*)-[0-9]+\.[0-9]+\.[0-9]+(?:-(alpha|beta)\.[0-9]+)?(?:\+.*)?$"
)

# Crate types corresponding to a C ABI library
C_LIBRARY_CRATE_TYPES = ["staticlib", "cdylib"]
# Crate types corresponding to a Rust ABI library
RUST_LIBRARY_CRATE_TYPES = ["lib", "rlib", "dylib", "proc-macro"]
# Crate types corresponding to a library
LIBRARY_CRATE_TYPES = C_LIBRARY_CRATE_TYPES + RUST_LIBRARY_CRATE_TYPES


def altered_stem(name):
    return RENAME_STEM_MAP[name] if (name in RENAME_STEM_MAP) else name


def is_build_crate_name(name):
    # We added special prefix to build script crate names.
    return name.startswith("build_script_")


def is_dependent_file_path(path):
    # Absolute or dependent '.../' paths are not main files of this crate.
    return path.startswith("/") or path.startswith(".../")


def get_module_name(crate):  # to sort crates in a list
    return crate.module_name


def pkg2crate_name(s):
    return s.replace("-", "_").replace(".", "_")


def file_base_name(path):
    return os.path.splitext(os.path.basename(path))[0]


def test_base_name(path):
    return pkg2crate_name(file_base_name(path))


def unquote(s):  # remove quotes around str
    if s and len(s) > 1 and s[0] == s[-1] and s[0] in ('"', "'"):
        return s[1:-1]
    return s


def remove_version_suffix(s):  # remove -d1.d2.d3 suffix
    if match := VERSION_SUFFIX_PAT.match(s):
        return match.group(1)
    return s


def short_out_name(pkg, s):  # replace /.../pkg-*/out/* with .../out/*
    return re.sub("^/.*/" + pkg + "-[0-9a-f]*/out/", ".../out/", s)


class Crate(object):
    """Information of a Rust crate to collect/emit for a rules.mk module."""

    def __init__(self, runner, outf_name):
        # Remembered global runner and its members.
        self.runner = runner
        self.debug = runner.args.debug
        self.cargo_dir = ""  # directory of my Cargo.toml
        self.outf_name = outf_name  # path to rules.mk
        self.outf = None  # open file handle of outf_name during dump*
        self.has_warning = False
        # Trusty module properties derived from rustc parameters.
        self.module_name = ""
        self.defaults = ""  # rust_defaults used by rust_test* modules
        self.default_srcs = False  # use 'srcs' defined in self.defaults
        self.root_pkg = ""  # parent package name of a sub/test packge, from -L
        self.srcs = []  # main_src or merged multiple source files
        self.stem = ""  # real base name of output file
        # Kept parsed status
        self.errors = ""  # all errors found during parsing
        self.line_num = 1  # runner told input source line number
        self.line = ""  # original rustc command line parameters
        # Parameters collected from rustc command line.
        self.crate_name = ""  # follows --crate-name
        self.main_src = ""  # follows crate_name parameter, shortened
        self.crate_types = []  # follows --crate-type
        self.cfgs = []  # follows --cfg, without feature= prefix
        self.features = []  # follows --cfg, name in 'feature="..."'
        self.codegens = []  # follows -C, some ignored
        self.static_libs = []  # e.g.  -l static=host_cpuid
        self.shared_libs = []  # e.g.  -l dylib=wayland-client, -l z
        self.cap_lints = ""  # follows --cap-lints
        self.emit_list = ""  # e.g., --emit=dep-info,metadata,link
        self.edition = "2015"  # rustc default, e.g., --edition=2018
        self.target = ""  # follows --target
        self.cargo_env_compat = True
        # Parameters collected from cargo metadata output
        self.dependencies = []  # crate dependencies output by `cargo metadata`
        self.feature_dependencies: dict[str, List[str]] = {}  # maps features to
        # optional dependencies

    def write(self, s):
        """convenient way to output one line at a time with EOL."""
        assert self.outf
        self.outf.write(s + "\n")

    def find_cargo_dir(self):
        """Deepest directory with Cargo.toml and contains the main_src."""
        if not is_dependent_file_path(self.main_src):
            dir_name = os.path.dirname(self.main_src)
            while dir_name:
                if os.path.exists(dir_name + "/Cargo.toml"):
                    self.cargo_dir = dir_name
                    return
                dir_name = os.path.dirname(dir_name)

    def add_codegens_flag(self, flag):
        """Ignore options not used by Trusty build system"""
        # 'prefer-dynamic' may be set by library.mk
        # 'embed-bitcode' is ignored; we might control LTO with other flags
        # 'codegen-units' is set globally in engine.mk
        # 'relocation-model' and 'target-feature=+reserve-x18' may be set by
        # common_flags.mk
        if not (
            flag.startswith("codegen-units=")
            or flag.startswith("debuginfo=")
            or flag.startswith("embed-bitcode=")
            or flag.startswith("extra-filename=")
            or flag.startswith("incremental=")
            or flag.startswith("metadata=")
            or flag.startswith("relocation-model=")
            or flag == "prefer-dynamic"
            or flag == "target-feature=+reserve-x18"
        ):
            self.codegens.append(flag)

    def get_dependencies(self):
        """Use output from cargo metadata to determine crate dependencies"""
        cargo_metadata = subprocess.run(
            [
                self.runner.cargo_path,
                "metadata",
                "--no-deps",
                "--format-version",
                "1",
            ],
            cwd=os.path.abspath(self.cargo_dir),
            stdout=subprocess.PIPE,
            check=False,
        )
        if cargo_metadata.returncode:
            self.errors += (
                "ERROR: unable to get cargo metadata to determine "
                f"dependencies; return code {cargo_metadata.returncode}\n"
            )
        else:
            metadata_json = json.loads(cargo_metadata.stdout)

            for package in metadata_json["packages"]:
                # package names containing '-' are changed to '_' in crate_name
                if package["name"].replace("-", "_") == self.crate_name:
                    self.dependencies = package["dependencies"]
                    for feat, props in package["features"].items():
                        feat_deps = [
                            d[4:] for d in props if d.startswith("dep:")
                        ]
                        if feat_deps and feat in self.feature_dependencies:
                            self.feature_dependencies[feat].extend(feat_deps)
                        else:
                            self.feature_dependencies[feat] = feat_deps
                    break
            else:  # package name not found in metadata
                if is_build_crate_name(self.crate_name):
                    print(
                        "### WARNING: unable to determine dependencies for "
                        + f"{self.crate_name} from cargo metadata"
                    )

    def parse(self, line_num, line):
        """Find important rustc arguments to convert to makefile rules."""
        self.line_num = line_num
        self.line = line
        args = [unquote(l) for l in line.split()]
        i = 0
        # Loop through every argument of rustc.
        while i < len(args):
            arg = args[i]
            if arg == "--crate-name":
                i += 1
                self.crate_name = args[i]
            elif arg == "--crate-type":
                i += 1
                # cargo calls rustc with multiple --crate-type flags.
                # rustc can accept:
                #  --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|proc-macro]
                self.crate_types.append(args[i])
            elif arg == "--test":
                self.crate_types.append("test")
            elif arg == "--target":
                i += 1
                self.target = args[i]
            elif arg == "--cfg":
                i += 1
                if args[i].startswith("feature="):
                    self.features.append(
                        unquote(args[i].replace("feature=", ""))
                    )
                else:
                    self.cfgs.append(args[i])
            elif arg == "--extern":
                i += 1
                pass  # ignored; get all dependencies from cargo metadata
            elif arg == "-C":  # codegen options
                i += 1
                self.add_codegens_flag(args[i])
            elif arg.startswith("-C"):
                # cargo has been passing "-C <xyz>" flag to rustc,
                # but newer cargo could pass '-Cembed-bitcode=no' to rustc.
                self.add_codegens_flag(arg[2:])
            elif arg == "--cap-lints":
                i += 1
                self.cap_lints = args[i]
            elif arg == "-L":
                i += 1
                if args[i].startswith("dependency=") and args[i].endswith(
                    "/deps"
                ):
                    if "/" + TARGET_TMP + "/" in args[i]:
                        self.root_pkg = re.sub(
                            "^.*/",
                            "",
                            re.sub("/" + TARGET_TMP + "/.*/deps$", "", args[i]),
                        )
                    else:
                        self.root_pkg = re.sub(
                            "^.*/",
                            "",
                            re.sub("/[^/]+/[^/]+/deps$", "", args[i]),
                        )
                    self.root_pkg = remove_version_suffix(self.root_pkg)
            elif arg == "-l":
                i += 1
                if args[i].startswith("static="):
                    self.static_libs.append(re.sub("static=", "", args[i]))
                elif args[i].startswith("dylib="):
                    self.shared_libs.append(re.sub("dylib=", "", args[i]))
                else:
                    self.shared_libs.append(args[i])
            elif arg in ("--out-dir", "--color"):  # ignored
                i += 1
            elif arg.startswith("--error-format=") or arg.startswith("--json="):
                pass  # ignored
            elif arg.startswith("--emit="):
                self.emit_list = arg.replace("--emit=", "")
            elif arg.startswith("--edition="):
                self.edition = arg.replace("--edition=", "")
            elif arg.startswith("-Aclippy") or arg.startswith("-Wclippy"):
                pass  # TODO: emit these flags in rules.mk
            elif arg.startswith("-W"):
                pass  # ignored
            elif arg.startswith("-Z"):
                pass  # ignore unstable flags
            elif arg.startswith("-D"):
                pass  # TODO: emit these flags in rules.mk
            elif not arg.startswith("-"):
                # shorten imported crate main source paths like $HOME/.cargo/
                # registry/src/github.com-1ecc6299db9ec823/memchr-2.3.3/src/
                # lib.rs
                self.main_src = re.sub(
                    r"^/[^ ]*/registry/src/", ".../", args[i]
                )
                self.main_src = re.sub(
                    r"^\.\.\./github.com-[0-9a-f]*/", ".../", self.main_src
                )
                self.find_cargo_dir()
                if self.cargo_dir:  # for a subdirectory
                    if (
                        self.runner.args.no_subdir
                    ):  # all .mk content to /dev/null
                        self.outf_name = "/dev/null"
                    elif not self.runner.args.onefile:
                        # Write to rules.mk in the subdirectory with Cargo.toml.
                        self.outf_name = self.cargo_dir + "/rules.mk"
                        self.main_src = self.main_src[len(self.cargo_dir) + 1 :]

            else:
                self.errors += "ERROR: unknown " + arg + "\n"
            i += 1
        if not self.crate_name:
            self.errors += "ERROR: missing --crate-name\n"
        if not self.main_src:
            self.errors += "ERROR: missing main source file\n"
        else:
            self.srcs.append(self.main_src)
        if not self.crate_types:
            # Treat "--cfg test" as "--test"
            if "test" in self.cfgs:
                self.crate_types.append("test")
            else:
                self.errors += "ERROR: missing --crate-type or --test\n"
        elif len(self.crate_types) > 1:
            if "test" in self.crate_types:
                self.errors += (
                    "ERROR: cannot handle both --crate-type and --test\n"
                )
            if "lib" in self.crate_types and "rlib" in self.crate_types:
                self.errors += (
                    "ERROR: cannot generate both lib and rlib crate types\n"
                )
        if not self.root_pkg:
            self.root_pkg = self.crate_name

        # get the package dependencies by running cargo metadata
        if not self.skip_crate():
            self.get_dependencies()
        self.cfgs = sorted(set(self.cfgs))
        self.features = sorted(set(self.features))
        self.codegens = sorted(set(self.codegens))
        self.static_libs = sorted(set(self.static_libs))
        self.shared_libs = sorted(set(self.shared_libs))
        self.crate_types = sorted(set(self.crate_types))
        self.module_name = self.stem
        return self

    def dump_line(self):
        self.write("\n// Line " + str(self.line_num) + " " + self.line)

    def feature_list(self):
        """Return a string of main_src + "feature_list"."""
        pkg = self.main_src
        if pkg.startswith(".../"):  # keep only the main package name
            pkg = re.sub("/.*", "", pkg[4:])
        elif pkg.startswith("/"):  # use relative path for a local package
            pkg = os.path.relpath(pkg)
        if not self.features:
            return pkg
        return pkg + ' "' + ",".join(self.features) + '"'

    def dump_skip_crate(self, kind):
        if self.debug:
            self.write("\n// IGNORED: " + kind + " " + self.main_src)
        return self

    def skip_crate(self):
        """Return crate_name or a message if this crate should be skipped."""
        if (
            is_build_crate_name(self.crate_name)
            or self.crate_name in EXCLUDED_CRATES
        ):
            return self.crate_name
        if is_dependent_file_path(self.main_src):
            return "dependent crate"
        return ""

    def dump(self):
        """Dump all error/debug/module code to the output rules.mk file."""
        self.runner.init_rules_file(self.outf_name)
        with open(self.outf_name, "a", encoding="utf-8") as outf:
            self.outf = outf
            if self.errors:
                self.dump_line()
                self.write(self.errors)
            elif self.skip_crate():
                self.dump_skip_crate(self.skip_crate())
            else:
                if self.debug:
                    self.dump_debug_info()
                self.dump_trusty_module()
            self.outf = None

    def dump_debug_info(self):
        """Dump parsed data, when cargo2rulesmk is called with --debug."""

        def dump(name, value):
            self.write(f"//{name:>12} = {value}")

        def opt_dump(name, value):
            if value:
                dump(name, value)

        def dump_list(fmt, values):
            for v in values:
                self.write(fmt % v)

        self.dump_line()
        dump("module_name", self.module_name)
        dump("crate_name", self.crate_name)
        dump("crate_types", self.crate_types)
        dump("main_src", self.main_src)
        dump("has_warning", self.has_warning)
        opt_dump("target", self.target)
        opt_dump("edition", self.edition)
        opt_dump("emit_list", self.emit_list)
        opt_dump("cap_lints", self.cap_lints)
        dump_list("//         cfg = %s", self.cfgs)
        dump_list("//         cfg = 'feature \"%s\"'", self.features)
        # TODO(chh): escape quotes in self.features, but not in other dump_list
        dump_list("//     codegen = %s", self.codegens)
        dump_list("//   -l static = %s", self.static_libs)
        dump_list("//  -l (dylib) = %s", self.shared_libs)

    def dump_trusty_module(self):
        """Dump one or more module definitions, depending on crate_types."""
        if len(self.crate_types) > 1:
            if "test" in self.crate_types:
                self.write("\nERROR: multiple crate types cannot include test type")
                return

            if "lib" in self.crate_types:
                print(f"### WARNING: crate {self.crate_name} has multiple "
                      f"crate types ({str(self.crate_types)}). Treating as 'lib'")
                self.crate_types = ["lib"]
            else:
                self.write("\nERROR: don't know how to handle crate types of "
                           f"crate {self.crate_name}: {str(self.crate_types)}")
                return

        self.dump_single_type_trusty_module()

    def dump_srcs_list(self):
        """Dump the srcs list, for defaults or regular modules."""
        if len(self.srcs) > 1:
            srcs = sorted(set(self.srcs))  # make a copy and dedup
        else:
            srcs = [self.main_src]
        self.write("MODULE_SRCS := \\")
        for src in srcs:
            self.write(f"\t$(LOCAL_DIR)/{src} \\")
        self.write("")

        # add rust file generated by build.rs to MODULE_SRCDEPS, if any
        # TODO(perlarsen): is there a need to support more than one output file?
        if srcdeps := [
            f for f in self.runner.build_out_files if f.endswith(".rs")
        ]:
            assert len(srcdeps) == 1
            outfile = srcdeps.pop()
            lines = [
                f"OUT_FILE := $(call TOBUILDDIR,$(MODULE))/{outfile}",
                f"$(OUT_FILE): $(MODULE)/out/{outfile}",
                "\t@echo copying $< to $@",
                "\t@$(MKDIR)",
                "\tcp $< $@",
                "",
                "MODULE_RUST_ENV += OUT_DIR=$(dir $(OUT_FILE))",
                "",
                "MODULE_SRCDEPS := $(OUT_FILE)",
            ]
            self.write("\n".join(lines))

    def dump_single_type_trusty_module(self):
        """Dump one simple Trusty module, which has only one crate_type."""
        crate_type = self.crate_types[0]
        assert crate_type != "test"
        self.dump_one_trusty_module(crate_type)

    def dump_one_trusty_module(self, crate_type):
        """Dump one Trusty module definition."""
        if crate_type in ["test", "bin"]:  # TODO: support test crates
            print(
                f"### WARNING: ignoring {crate_type} crate: {self.crate_name}")
            return
        if self.codegens:  # TODO: support crates that require codegen flags
            print(
                f"ERROR: {self.crate_name} uses unexpected codegen flags: " +
                str(self.codegens)
            )
            return

        self.dump_core_properties()
        if not self.defaults:
            self.dump_edition_flags_libs()

        # NOTE: a crate may list the same dependency as required and optional
        library_deps = set()
        for dependency in self.dependencies:
            if dependency["kind"] in ["dev", "build"]:
                continue
            name = (
                rename
                if (rename := dependency["rename"])
                else dependency["name"]
            )
            if dependency["target"]:
                print(
                    f"### WARNING: ignoring target-specific dependency: {name}")
                continue
            path = CUSTOM_MODULE_CRATES.get(
                name, f"external/rust/crates/{name}"
            )
            if dependency["optional"]:
                if not any(
                    name in self.feature_dependencies.get(f, [])
                    for f in self.features
                ):
                    continue
            library_deps.add(path)
        if library_deps:
            self.write("MODULE_LIBRARY_DEPS := \\")
            for path in sorted(library_deps):
                self.write(f"\t{path} \\")
            self.write("")
        if crate_type == "test" and not self.default_srcs:
            raise NotImplementedError("Crates with test data are not supported")

        assert crate_type in LIBRARY_CRATE_TYPES
        self.write("include make/library.mk")

    def dump_edition_flags_libs(self):
        if self.edition:
            self.write(f"MODULE_RUST_EDITION := {self.edition}")
        if self.features or self.cfgs:
            self.write("MODULE_RUSTFLAGS += \\")
            for feature in self.features:
                self.write(f"\t--cfg 'feature=\"{feature}\"' \\")
            for cfg in self.cfgs:
                self.write(f"\t--cfg '{cfg}' \\")
            self.write("")

        if self.static_libs or self.shared_libs:
            print("### WARNING: Crates with depend on static or shared "
                  "libraries are not supported")

    def main_src_basename_path(self):
        return re.sub("/", "_", re.sub(".rs$", "", self.main_src))

    def test_module_name(self):
        """Return a unique name for a test module."""
        # root_pkg+(_host|_device) + '_test_'+source_file_name
        suffix = self.main_src_basename_path()
        return self.root_pkg + "_test_" + suffix

    def dump_core_properties(self):
        """Dump the module header, name, stem, etc."""
        self.write("LOCAL_DIR := $(GET_LOCAL_DIR)")
        self.write("MODULE := $(LOCAL_DIR)")
        self.write(f"MODULE_CRATE_NAME := {self.crate_name}")

        # Trusty's module system only supports bin, rlib, and proc-macro so map
        # lib->rlib
        if self.crate_types != ["lib"]:
            crate_types = set(
                "rlib" if ct == "lib" else ct for ct in self.crate_types
            )
            self.write(f'MODULE_RUST_CRATE_TYPES := {" ".join(crate_types)}')

        if not self.default_srcs:
            self.dump_srcs_list()

        if hasattr(self.runner.args, "module_add_implicit_deps"):
            if hasattr(self.runner.args, "module_add_implicit_deps_reason"):
                self.write(self.runner.args.module_add_implicit_deps_reason)

            if self.runner.args.module_add_implicit_deps in [True, "yes"]:
                self.write("MODULE_ADD_IMPLICIT_DEPS := true")
            elif self.runner.args.module_add_implicit_deps in [False, "no"]:
                self.write("MODULE_ADD_IMPLICIT_DEPS := false")
            else:
                sys.exit(
                    "ERROR: invalid value for module_add_implicit_deps: " +
                    str(self.runner.args.module_add_implicit_deps)
                )


class Runner(object):
    """Main class to parse cargo -v output and print Trusty makefile modules."""

    def __init__(self, args):
        self.mk_files = set()  # Remember all Trusty module files.
        self.root_pkg = ""  # name of package in ./Cargo.toml
        # Saved flags, modes, and data.
        self.args = args
        self.dry_run = not args.run
        self.skip_cargo = args.skipcargo
        self.cargo_path = "./cargo"  # path to cargo, will be set later
        self.checked_out_files = False  # to check only once
        self.build_out_files = []  # output files generated by build.rs
        self.crates: List[Crate] = []
        self.warning_files = set()
        # Keep a unique mapping from (module name) to crate
        self.name_owners = {}
        # Save and dump all errors from cargo to rules.mk.
        self.errors = ""
        self.test_errors = ""
        self.setup_cargo_path()
        # Default action is cargo clean, followed by build or user given actions
        if args.cargo:
            self.cargo = ["clean"] + args.cargo
        else:
            default_target = "--target x86_64-unknown-linux-gnu"
            # Use the same target for both host and default device builds.
            # Same target is used as default in host x86_64 Android compilation.
            # Note: b/169872957, prebuilt cargo failed to build vsock
            # on x86_64-unknown-linux-musl systems.
            self.cargo = ["clean", "build " + default_target]
            if args.tests:
                self.cargo.append("build --tests " + default_target)
        self.empty_tests = set()
        self.empty_unittests = False

    def setup_cargo_path(self):
        """Find cargo in the --cargo_bin or prebuilt rust bin directory."""
        if self.args.cargo_bin:
            self.cargo_path = os.path.join(self.args.cargo_bin, "cargo")
            if not os.path.isfile(self.cargo_path):
                sys.exit("ERROR: cannot find cargo in " + self.args.cargo_bin)
            print("INFO: using cargo in " + self.args.cargo_bin)
            return

        # We have only tested this on Linux.
        if platform.system() != "Linux":
            sys.exit(
                "ERROR: this script has only been tested on Linux with cargo."
            )

        # Assuming that this script is in development/scripts
        env_setup_sh = os.path.join(
            TOP_DIR, "trusty/vendor/google/aosp/scripts/envsetup.sh"
        )
        if not os.path.exists(env_setup_sh):
            sys.exit("ERROR: missing " + env_setup_sh)
        rust_version = self.find_rust_version(env_setup_sh)
        self.cargo_path =  os.path.join(
            TOP_DIR, f"prebuilts/rust/linux-x86/{rust_version}/bin/cargo"
        )

        if not os.path.isfile(self.cargo_path):
            sys.exit(
                "ERROR: no cargo at "
                + self.cargo_path
                + "; consider using the --cargo_bin= flag."
            )

        if self.args.verbose:
            print(f"### INFO: using cargo from {self.cargo_path}")

    def find_rust_version(self, env_setup_sh):
        """find the Rust version used by Trusty from envsetup.sh"""

        version_pat = re.compile(r"prebuilts/rust/linux-x86/([0-9]+\.[0-9]+\..+)/bin")

        with open(env_setup_sh) as fh:
            for line in fh.readlines():
                if line.lstrip().startswith("export RUST_BINDIR"):
                    if not (result := version_pat.search(line)):
                        sys.exit("ERROR: failed to parse rust version "
                                 + "from RUST_BINDIR in envsetup.sh: "
                                 + line
                        )
                    version = result.group(1)

                    if self.args.verbose:
                        print(f"### INFO: using rust version {version}")

                    return version

        sys.exit("ERROR: failed to parse {env_setup_sh}; is RUST_BINDIR exported?")

    def find_out_files(self):
        # list1 has build.rs output for normal crates
        list1 = glob.glob(
            TARGET_TMP + "/*/*/build/" + self.root_pkg + "-*/out/*"
        )
        # list2 has build.rs output for proc-macro crates
        list2 = glob.glob(TARGET_TMP + "/*/build/" + self.root_pkg + "-*/out/*")
        return list1 + list2

    def copy_out_files(self):
        """Copy build.rs output files to ./out and set up build_out_files."""
        if self.checked_out_files:
            return
        self.checked_out_files = True
        cargo_out_files = self.find_out_files()
        out_files = set()
        if cargo_out_files:
            os.makedirs("out", exist_ok=True)
        for path in cargo_out_files:
            file_name = path.split("/")[-1]
            out_files.add(file_name)
            shutil.copy(path, "out/" + file_name)
        self.build_out_files = sorted(out_files)

    def has_used_out_dir(self):
        """Returns true if env!("OUT_DIR") is found."""
        return 0 == os.system(
            "grep -rl --exclude build.rs --include \\*.rs"
            + " 'env!(\"OUT_DIR\")' * > /dev/null"
        )

    def init_rules_file(self, name):
        # name could be rules.mk or sub_dir_path/rules.mk
        if name not in self.mk_files:
            self.mk_files.add(name)
            with open(name, "w", encoding="utf-8") as outf:
                print_args = sys.argv[1:].copy()
                if "--cargo_bin" in print_args:
                    index = print_args.index("--cargo_bin")
                    del print_args[index : index + 2]
                outf.write(RULES_MK_HEADER.format(args=" ".join(print_args)))

    def find_root_pkg(self):
        """Read name of [package] in ./Cargo.toml."""
        if not os.path.exists("./Cargo.toml"):
            return
        with open("./Cargo.toml", "r", encoding="utf-8") as inf:
            pkg_section = re.compile(r"^ *\[package\]")
            name = re.compile('^ *name *= * "([^"]*)"')
            in_pkg = False
            for line in inf:
                if in_pkg:
                    if match := name.match(line):
                        self.root_pkg = match.group(1)
                        break
                else:
                    in_pkg = pkg_section.match(line) is not None

    def run_cargo(self):
        """Calls cargo -v and save its output to ./cargo.out."""
        if self.skip_cargo:
            return self
        cargo_toml = "./Cargo.toml"
        cargo_out = "./cargo.out"

        # Do not use Cargo.lock, because Trusty makefile rules are designed
        # to run with the latest available vendored crates in Trusty.
        cargo_lock = "./Cargo.lock"
        cargo_lock_saved = "./cargo.lock.saved"
        had_cargo_lock = os.path.exists(cargo_lock)
        if not os.access(cargo_toml, os.R_OK):
            print("ERROR: Cannot find or read", cargo_toml)
            return self
        if not self.dry_run:
            if os.path.exists(cargo_out):
                os.remove(cargo_out)
            if not self.args.use_cargo_lock and had_cargo_lock:  # save it
                os.rename(cargo_lock, cargo_lock_saved)
        cmd_tail_target = " --target-dir " + TARGET_TMP
        cmd_tail_redir = " >> " + cargo_out + " 2>&1"
        # set up search PATH for cargo to find the correct rustc
        saved_path = os.environ["PATH"]
        os.environ["PATH"] = os.path.dirname(self.cargo_path) + ":" + saved_path
        # Add [workspace] to Cargo.toml if it is not there.
        added_workspace = False
        cargo_toml_lines = None
        if self.args.add_workspace:
            with open(cargo_toml, "r", encoding="utf-8") as in_file:
                cargo_toml_lines = in_file.readlines()
            found_workspace = "[workspace]\n" in cargo_toml_lines
            if found_workspace:
                print("### WARNING: found [workspace] in Cargo.toml")
            else:
                with open(cargo_toml, "a", encoding="utf-8") as out_file:
                    out_file.write("\n\n[workspace]\n")
                    added_workspace = True
                    if self.args.verbose:
                        print("### INFO: added [workspace] to Cargo.toml")
        features = ""
        for c in self.cargo:
            features = ""
            if c != "clean":
                if self.args.features is not None:
                    features = " --no-default-features"
                if self.args.features:
                    features += " --features " + self.args.features
            cmd_v_flag = " -vv " if self.args.vv else " -v "
            cmd = self.cargo_path + cmd_v_flag
            cmd += c + features + cmd_tail_target + cmd_tail_redir
            if c != "clean":
                rustflags = self.args.rustflags if self.args.rustflags else ""
                # linting issues shouldn't prevent us from generating rules.mk
                rustflags = f'RUSTFLAGS="{rustflags} --cap-lints allow" '
                cmd = rustflags + cmd
            self.run_cmd(cmd, cargo_out)
        if self.args.tests:
            cmd = (
                self.cargo_path
                + " test"
                + features
                + cmd_tail_target
                + " -- --list"
                + cmd_tail_redir
            )
            self.run_cmd(cmd, cargo_out)
        if added_workspace:  # restore original Cargo.toml
            with open(cargo_toml, "w", encoding="utf-8") as out_file:
                assert cargo_toml_lines
                out_file.writelines(cargo_toml_lines)
            if self.args.verbose:
                print("### INFO: restored original Cargo.toml")
        os.environ["PATH"] = saved_path
        if not self.dry_run:
            if not had_cargo_lock:  # restore to no Cargo.lock state
                if os.path.exists(cargo_lock):
                    os.remove(cargo_lock)
            elif not self.args.use_cargo_lock:  # restore saved Cargo.lock
                os.rename(cargo_lock_saved, cargo_lock)
        return self

    def run_cmd(self, cmd, cargo_out):
        if self.dry_run:
            print("Dry-run skip:", cmd)
        else:
            if self.args.verbose:
                print("Running:", cmd)
            with open(cargo_out, "a+", encoding="utf-8") as out_file:
                out_file.write("### Running: " + cmd + "\n")
            ret = os.system(cmd)
            if ret != 0:
                print(
                    "*** There was an error while running cargo.  "
                    + f"See the {cargo_out} file for details."
                )

    def apply_patch(self):
        """Apply local patch file if it is given."""
        if self.args.patch:
            if self.dry_run:
                print("Dry-run skip patch file:", self.args.patch)
            else:
                if not os.path.exists(self.args.patch):
                    self.append_to_rules(
                        "ERROR cannot find patch file: " + self.args.patch
                    )
                    return self
                if self.args.verbose:
                    print(
                        "### INFO: applying local patch file:", self.args.patch
                    )
                subprocess.run(
                    [
                        "patch",
                        "-s",
                        "--no-backup-if-mismatch",
                        "./rules.mk",
                        self.args.patch,
                    ],
                    check=True,
                )
        return self

    def gen_rules(self):
        """Parse cargo.out and generate Trusty makefile rules"""
        if self.dry_run:
            print("Dry-run skip: read", CARGO_OUT, "write rules.mk")
        elif os.path.exists(CARGO_OUT):
            self.find_root_pkg()
            if self.args.copy_out:
                self.copy_out_files()
            elif self.find_out_files() and self.has_used_out_dir():
                print(
                    "WARNING: "
                    + self.root_pkg
                    + " has cargo output files; "
                    + "please rerun with the --copy-out flag."
                )
            with open(CARGO_OUT, "r", encoding="utf-8") as cargo_out:
                self.parse(cargo_out, "rules.mk")
            self.crates.sort(key=get_module_name)
            for crate in self.crates:
                crate.dump()
            if self.errors:
                self.append_to_rules("\n" + ERRORS_LINE + "\n" + self.errors)
            if self.test_errors:
                self.append_to_rules(
                    "\n// Errors when listing tests:\n" + self.test_errors
                )
        return self

    def add_crate(self, crate: Crate):
        """Append crate to list unless it meets criteria for being skipped."""
        if crate.skip_crate():
            if self.args.debug:  # include debug info of all crates
                self.crates.append(crate)
        elif crate.crate_types == set(["bin"]):
            print("WARNING: skipping binary crate: " + crate.crate_name)
        else:
            self.crates.append(crate)

    def find_warning_owners(self):
        """For each warning file, find its owner crate."""
        missing_owner = False
        for f in self.warning_files:
            cargo_dir = ""  # find lowest crate, with longest path
            owner = None  # owner crate of this warning
            for c in self.crates:
                if f.startswith(c.cargo_dir + "/") and len(cargo_dir) < len(
                    c.cargo_dir
                ):
                    cargo_dir = c.cargo_dir
                    owner = c
            if owner:
                owner.has_warning = True
            else:
                missing_owner = True
        if missing_owner and os.path.exists("Cargo.toml"):
            # owner is the root cargo, with empty cargo_dir
            for c in self.crates:
                if not c.cargo_dir:
                    c.has_warning = True

    def rustc_command(self, n, rustc_line, line, outf_name):
        """Process a rustc command line from cargo -vv output."""
        # cargo build -vv output can have multiple lines for a rustc command
        # due to '\n' in strings for environment variables.
        # strip removes leading spaces and '\n' at the end
        new_rustc = (rustc_line.strip() + line) if rustc_line else line
        # Use an heuristic to detect the completions of a multi-line command.
        # This might fail for some very rare case, but easy to fix manually.
        if not line.endswith("`\n") or (new_rustc.count("`") % 2) != 0:
            return new_rustc
        if match := RUSTC_VV_CMD_ARGS.match(new_rustc):
            args = match.group(2)
            self.add_crate(Crate(self, outf_name).parse(n, args))
        else:
            self.assert_empty_vv_line(new_rustc)
        return ""

    def append_to_rules(self, line):
        self.init_rules_file("rules.mk")
        with open("rules.mk", "a", encoding="utf-8") as outf:
            outf.write(line)

    def assert_empty_vv_line(self, line):
        if line:  # report error if line is not empty
            self.append_to_rules("ERROR -vv line: " + line)
        return ""

    def add_empty_test(self, name):
        if name.startswith("unittests"):
            self.empty_unittests = True
        else:
            self.empty_tests.add(name)

    def should_ignore_test(self, src):
        # cargo test outputs the source file for integration tests but
        # "unittests" for unit tests. To figure out to which crate this
        # corresponds, we check if the current source file is the main source of
        # a non-test crate, e.g., a library or a binary.
        return (
            src in self.args.test_blocklist
            or src in self.empty_tests
            or (
                self.empty_unittests
                and src
                in [
                    c.main_src for c in self.crates if c.crate_types != ["test"]
                ]
            )
        )

    def parse(self, inf, outf_name):
        """Parse rustc, test, and warning messages in input file."""
        n = 0  # line number
        # We read the file in two passes, where the first simply checks for
        # empty tests. Otherwise we would add and merge tests before seeing
        # they're empty.
        cur_test_name = None
        for line in inf:
            if match := CARGO_TEST_LIST_START_PAT.match(line):
                cur_test_name = match.group(1)
            elif cur_test_name and (
                match := CARGO_TEST_LIST_END_PAT.match(line)
            ):
                if int(match.group(1)) + int(match.group(2)) == 0:
                    self.add_empty_test(cur_test_name)
                cur_test_name = None
        inf.seek(0)
        prev_warning = False  # true if the previous line was warning: ...
        rustc_line = ""  # previous line(s) matching RUSTC_VV_PAT
        in_tests = False
        for line in inf:
            n += 1
            if line.startswith("warning: "):
                prev_warning = True
                rustc_line = self.assert_empty_vv_line(rustc_line)
                continue
            new_rustc = ""
            if match := RUSTC_PAT.match(line):
                args_line = match.group(2)
                self.add_crate(Crate(self, outf_name).parse(n, args_line))
                self.assert_empty_vv_line(rustc_line)
            elif rustc_line or RUSTC_VV_PAT.match(line):
                new_rustc = self.rustc_command(n, rustc_line, line, outf_name)
            elif CC_AR_VV_PAT.match(line):
                raise NotImplementedError("$CC or $AR commands not supported")
            elif prev_warning and (match := WARNING_FILE_PAT.match(line)):
                self.assert_empty_vv_line(rustc_line)
                fpath = match.group(1)
                if fpath[0] != "/":  # ignore absolute path
                    self.warning_files.add(fpath)
            elif line.startswith("error: ") or line.startswith("error[E"):
                if not self.args.ignore_cargo_errors:
                    if in_tests:
                        self.test_errors += "// " + line
                    else:
                        self.errors += line
            elif CARGO2ANDROID_RUNNING_PAT.match(line):
                in_tests = "cargo test" in line and "--list" in line
            prev_warning = False
            rustc_line = new_rustc
        self.find_warning_owners()


def get_parser():
    """Parse main arguments."""
    parser = argparse.ArgumentParser("cargo2rulesmk")
    parser.add_argument(
        "--add_workspace",
        action="store_true",
        default=False,
        help=(
            "append [workspace] to Cargo.toml before calling cargo,"
            + " to treat current directory as root of package source;"
            + " otherwise the relative source file path in generated"
            + " rules.mk file will be from the parent directory."
        ),
    )
    parser.add_argument(
        "--cargo",
        action="append",
        metavar="args_string",
        help=(
            "extra cargo build -v args in a string, "
            + "each --cargo flag calls cargo build -v once"
        ),
    )
    parser.add_argument(
        "--cargo_bin",
        type=str,
        help="use cargo in the cargo_bin directory instead of the prebuilt one",
    )
    parser.add_argument(
        "--copy-out",
        action="store_true",
        default=False,
        help=(
            "only for root directory, "
            + "copy build.rs output to ./out/* and declare source deps "
            + "for ./out/*.rs; for crates with code pattern: "
            + 'include!(concat!(env!("OUT_DIR"), "/<some_file>.rs"))'
        ),
    )
    parser.add_argument(
        "--debug",
        action="store_true",
        default=False,
        help="dump debug info into rules.mk",
    )
    parser.add_argument(
        "--features",
        type=str,
        help=(
            "pass features to cargo build, "
            + "empty string means no default features"
        ),
    )
    parser.add_argument(
        "--ignore-cargo-errors",
        action="store_true",
        default=False,
        help="do not append cargo/rustc error messages to rules.mk",
    )
    parser.add_argument(
        "--no-subdir",
        action="store_true",
        default=False,
        help="do not output anything for sub-directories",
    )
    parser.add_argument(
        "--onefile",
        action="store_true",
        default=False,
        help=(
            "output all into one ./rules.mk, default will generate "
            + "one rules.mk per Cargo.toml in subdirectories"
        ),
    )
    parser.add_argument(
        "--patch",
        type=str,
        help="apply the given patch file to generated ./rules.mk",
    )
    parser.add_argument(
        "--run",
        action="store_true",
        default=False,
        help="run it, default is dry-run",
    )
    parser.add_argument("--rustflags", type=str, help="passing flags to rustc")
    parser.add_argument(
        "--skipcargo",
        action="store_true",
        default=False,
        help="skip cargo command, parse cargo.out, and generate ./rules.mk",
    )
    parser.add_argument(
        "--tests",
        action="store_true",
        default=False,
        help="run cargo build --tests after normal build",
    )
    parser.add_argument(
        "--use-cargo-lock",
        action="store_true",
        default=False,
        help=(
            "run cargo build with existing Cargo.lock "
            + "(used when some latest dependent crates failed)"
        ),
    )
    parser.add_argument(
        "--test-data",
        nargs="*",
        default=[],
        help=(
            "Add the given file to the given test's data property. "
            + "Usage: test-path=data-path"
        ),
    )
    parser.add_argument(
        "--dependency-blocklist",
        nargs="*",
        default=[],
        help="Do not emit the given dependencies (without lib prefixes).",
    )
    parser.add_argument(
        "--test-blocklist",
        nargs="*",
        default=[],
        help=(
            "Do not emit the given tests. "
            + "Pass the path to the test file to exclude."
        ),
    )
    parser.add_argument(
        "--cfg-blocklist",
        nargs="*",
        default=[],
        help="Do not emit the given cfg.",
    )
    parser.add_argument(
        "--verbose",
        action="store_true",
        default=False,
        help="echo executed commands",
    )
    parser.add_argument(
        "--vv",
        action="store_true",
        default=False,
        help="run cargo with -vv instead of default -v",
    )
    parser.add_argument(
        "--dump-config-and-exit",
        type=str,
        help=(
            "Dump command-line arguments (minus this flag) to a config file and"
            " exit. This is intended to help migrate from command line options "
            "to config files."
        ),
    )
    parser.add_argument(
        "--config",
        type=str,
        help=(
            "Load command-line options from the given config file. Options in "
            "this file will override those passed on the command line."
        ),
    )
    return parser


def parse_args(parser):
    """Parses command-line options."""
    args = parser.parse_args()
    # Use the values specified in a config file if one was found.
    if args.config:
        with open(args.config, "r", encoding="utf-8") as f:
            config = json.load(f)
            args_dict = vars(args)
            for arg in config:
                args_dict[arg.replace("-", "_")] = config[arg]
    return args


def dump_config(parser, args):
    """Writes the non-default command-line options to the specified file."""
    args_dict = vars(args)
    # Filter out the arguments that have their default value.
    # Also filter certain "temporary" arguments.
    non_default_args = {}
    for arg in args_dict:
        if (
            args_dict[arg] != parser.get_default(arg)
            and arg != "dump_config_and_exit"
            and arg != "config"
            and arg != "cargo_bin"
        ):
            non_default_args[arg.replace("_", "-")] = args_dict[arg]
    # Write to the specified file.
    with open(args.dump_config_and_exit, "w", encoding="utf-8") as f:
        json.dump(non_default_args, f, indent=2, sort_keys=True)


def main():
    parser = get_parser()
    args = parse_args(parser)
    if not args.run:  # default is dry-run
        print(DRY_RUN_NOTE)
    if args.dump_config_and_exit:
        dump_config(parser, args)
    else:
        Runner(args).run_cargo().gen_rules().apply_patch()


if __name__ == "__main__":
    main()