xref: /aosp_15_r20/tools/asuite/aidegen/lib/project_info.py (revision c2e18aaa1096c836b086f94603d04f4eb9cf37f5)

#!/usr/bin/env python3
#
# Copyright 2018 - The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Project information."""

from __future__ import absolute_import

import logging
import os
import time

from aidegen import constant
from aidegen.lib import aidegen_metrics
from aidegen.lib import common_util
from aidegen.lib import errors
from aidegen.lib import module_info
from aidegen.lib import project_config
from aidegen.lib import source_locator
from aidegen.idea import iml

from atest import atest_utils

_CONVERT_MK_URL = ('https://android.googlesource.com/platform/build/soong/'
                   '#convert-android_mk-files')
_ROBOLECTRIC_MODULE = 'Robolectric_all'
_NOT_TARGET = ('The module %s does not contain any Java or Kotlin file, '
               'therefore we skip this module in the project.')
# The module fake-framework have the same package name with framework but empty
# content. It will impact the dependency for framework when referencing the
# package from fake-framework in IntelliJ.
_EXCLUDE_MODULES = ['fake-framework']
# When we use atest_utils.build(), there is a command length limit on
# soong_ui.bash. We reserve 5000 characters for rewriting the command line
# in soong_ui.bash.
_CMD_LENGTH_BUFFER = 5000
# For each argument, it needs a space to separate following argument.
_BLANK_SIZE = 1
_CORE_MODULES = [constant.FRAMEWORK_ALL, constant.CORE_ALL,
                 'org.apache.http.legacy.stubs.system']
_BUILD_BP_JSON_ENV_ON = {
    constant.GEN_JAVA_DEPS: 'true',
    constant.GEN_CC_DEPS: 'true',
    constant.GEN_COMPDB: 'true',
    constant.GEN_RUST: 'true'
}


class ProjectInfo:
    """Project information.

    Users should call config_project first before starting using ProjectInfo.

    Class attributes:
        modules_info: An AidegenModuleInfo instance whose name_to_module_info is
                      combining module-info.json with module_bp_java_deps.json.
        projects: A list of instances of ProjectInfo that are generated in an
                  AIDEGen command.

    Attributes:
        project_absolute_path: The absolute path of the project.
        project_relative_path: The relative path of the project to
                               common_util.get_android_root_dir().
        project_module_names: A set of module names under project_absolute_path
                              directory or it's subdirectories.
        dep_modules: A dict has recursively dependent modules of
                     project_module_names.
        iml_path: The project's iml file path.
        source_path: A dictionary to keep following data:
                     source_folder_path: A set contains the source folder
                                         relative paths.
                     test_folder_path: A set contains the test folder relative
                                       paths.
                     jar_path: A set contains the jar file paths.
                     jar_module_path: A dictionary contains the jar file and
                                      the module's path mapping, only used in
                                      Eclipse.
                     r_java_path: A set contains the relative path to the
                                  R.java files, only used in Eclipse.
                     srcjar_path: A source content descriptor only used in
                                  IntelliJ.
                                  e.g. out/.../aapt2.srcjar!/
                                  The "!/" is a content descriptor for
                                  compressed files in IntelliJ.
        is_main_project: A boolean to verify the project is main project.
        dependencies: A list of dependency projects' iml file names, e.g. base,
                      framework-all.
        iml_name: The iml project file name of this project.
        rel_out_soong_jar_path: A string of relative project path in the
                                'out/soong/.intermediates' directory, e.g., if
                                self.project_relative_path = 'frameworks/base'
                                the rel_out_soong_jar_path should be
                                'out/soong/.intermediates/frameworks/base/'.
    """

    modules_info = None
    projects = []

    def __init__(self, target=None, is_main_project=False):
        """ProjectInfo initialize.

        Args:
            target: Includes target module or project path from user input, when
                    locating the target, project with matching module name of
                    the given target has a higher priority than project path.
            is_main_project: A boolean, default is False. True if the target is
                             the main project, otherwise False.
        """
        rel_path, abs_path = common_util.get_related_paths(
            self.modules_info, target)
        self.module_name = self.get_target_name(target, abs_path)
        self.is_main_project = is_main_project
        self.project_module_names = set(
            self.modules_info.get_module_names(rel_path))
        self.project_relative_path = rel_path
        self.project_absolute_path = abs_path
        self.iml_path = ''
        self._set_default_modules()
        self._init_source_path()
        if target == constant.FRAMEWORK_ALL:
            self.dep_modules = self.get_dep_modules([target])
        else:
            self.dep_modules = self.get_dep_modules()
        self._filter_out_modules()
        self.dependencies = []
        self.iml_name = iml.IMLGenerator.get_unique_iml_name(abs_path)
        self.rel_out_soong_jar_path = self._get_rel_project_out_soong_jar_path()

    def _set_default_modules(self):
        """Append default hard-code modules, source paths and jar files.

        1. framework: Framework module is always needed for dependencies, but it
            might not always be located by module dependency.
        2. org.apache.http.legacy.stubs.system: The module can't be located
            through module dependency. Without it, a lot of java files will have
            error of "cannot resolve symbol" in IntelliJ since they import
            packages android.Manifest and com.android.internal.R.
        """
        # Set the default modules framework-all and core-all as the core
        # dependency modules.
        self.project_module_names.update(_CORE_MODULES)

    def _init_source_path(self):
        """Initialize source_path dictionary."""
        self.source_path = {
            'source_folder_path': set(),
            'test_folder_path': set(),
            'jar_path': set(),
            'jar_module_path': {},
            'r_java_path': set(),
            'srcjar_path': set()
        }

    def _search_android_make_files(self):
        """Search project and dependency modules contain Android.mk files.

        If there is only Android.mk but no Android.bp, we'll show the warning
        message, otherwise we won't.

        Yields:
            A string: the relative path of Android.mk.
        """
        if (common_util.exist_android_mk(self.project_absolute_path) and
                not common_util.exist_android_bp(self.project_absolute_path)):
            yield '\t' + os.path.join(self.project_relative_path,
                                      constant.ANDROID_MK)
        for mod_name in self.dep_modules:
            rel_path, abs_path = common_util.get_related_paths(
                self.modules_info, mod_name)
            if rel_path and abs_path:
                if (common_util.exist_android_mk(abs_path)
                        and not common_util.exist_android_bp(abs_path)):
                    yield '\t' + os.path.join(rel_path, constant.ANDROID_MK)

    def _get_modules_under_project_path(self, rel_path):
        """Find qualified modules under the rel_path.

        Find modules which contain any Java or Kotlin file as a target module.
        If it's the whole source tree project, add all modules into it.

        Args:
            rel_path: A string, the project's relative path.

        Returns:
            A set of module names.
        """
        logging.info('Find modules contain any Java or Kotlin file under %s.',
                     rel_path)
        if rel_path == '':
            return self.modules_info.name_to_module_info.keys()
        modules = set()
        root_dir = common_util.get_android_root_dir()
        for name, data in self.modules_info.name_to_module_info.items():
            if module_info.AidegenModuleInfo.is_project_path_relative_module(
                    data, rel_path):
                if common_util.check_java_or_kotlin_file_exists(
                        os.path.join(root_dir, data[constant.KEY_PATH][0])):
                    modules.add(name)
                else:
                    logging.debug(_NOT_TARGET, name)
        return modules

    def _get_robolectric_dep_module(self, modules):
        """Return the robolectric module set as dependency if any module is a
           robolectric test.

        Args:
            modules: A set of modules.

        Returns:
            A set with a robolectric_all module name if one of the modules
            needs the robolectric test module. Otherwise return empty list.
        """
        for module in modules:
            if self.modules_info.is_robolectric_test(module):
                return {_ROBOLECTRIC_MODULE}
        return set()

    def _filter_out_modules(self):
        """Filter out unnecessary modules."""
        for module in _EXCLUDE_MODULES:
            self.dep_modules.pop(module, None)

    def get_dep_modules(self, module_names=None, depth=0):
        """Recursively find dependent modules of the project.

        Find dependent modules by dependencies parameter of each module.
        For example:
            The module_names is ['m1'].
            The modules_info is
            {
                'm1': {'dependencies': ['m2'], 'path': ['path_to_m1']},
                'm2': {'path': ['path_to_m4']},
                'm3': {'path': ['path_to_m1']}
                'm4': {'path': []}
            }
            The result dependent modules are:
            {
                'm1': {'dependencies': ['m2'], 'path': ['path_to_m1']
                       'depth': 0},
                'm2': {'path': ['path_to_m4'], 'depth': 1},
                'm3': {'path': ['path_to_m1'], 'depth': 0}
            }
            Note that:
                1. m4 is not in the result as it's not among dependent modules.
                2. m3 is in the result as it has the same path to m1.

        Args:
            module_names: A set of module names.
            depth: An integer shows the depth of module dependency referenced by
                   source. Zero means the max module depth.

        Returns:
            deps: A dict contains all dependent modules data of given modules.
        """
        dep = {}
        children = set()
        if not module_names:
            module_names = self.project_module_names
            module_names.update(
                self._get_modules_under_project_path(
                    self.project_relative_path))
            module_names.update(self._get_robolectric_dep_module(module_names))
            self.project_module_names = set()
        for name in module_names:
            if (name in self.modules_info.name_to_module_info
                    and name not in self.project_module_names):
                dep[name] = self.modules_info.name_to_module_info[name]
                dep[name][constant.KEY_DEPTH] = depth
                self.project_module_names.add(name)
                if (constant.KEY_DEPENDENCIES in dep[name]
                        and dep[name][constant.KEY_DEPENDENCIES]):
                    children.update(dep[name][constant.KEY_DEPENDENCIES])
        if children:
            dep.update(self.get_dep_modules(children, depth + 1))
        return dep

    @staticmethod
    def generate_projects(targets):
        """Generate a list of projects in one time by a list of module names.

        Args:
            targets: A list of target modules or project paths from user input,
                     when locating the target, project with matched module name
                     of the target has a higher priority than project path.

        Returns:
            List: A list of ProjectInfo instances.
        """
        return [ProjectInfo(target, i == 0) for i, target in enumerate(targets)]

    @staticmethod
    def get_target_name(target, abs_path):
        """Get target name from target's absolute path.

        If the project is for entire Android source tree, change the target to
        source tree's root folder name. In this way, we give IDE project file
        a more specific name. e.g, master.iml.

        Args:
            target: Includes target module or project path from user input, when
                    locating the target, project with matching module name of
                    the given target has a higher priority than project path.
            abs_path: A string, target's absolute path.

        Returns:
            A string, the target name.
        """
        if abs_path == common_util.get_android_root_dir():
            return os.path.basename(abs_path)
        return target

    def locate_source(self, build=True):
        """Locate the paths of dependent source folders and jar files.

        Try to reference source folder path as dependent module unless the
        dependent module should be referenced to a jar file, such as modules
        have jars and jarjar_rules parameter.
        For example:
            Module: asm-6.0
                java_import {
                    name: 'asm-6.0',
                    host_supported: true,
                    jars: ['asm-6.0.jar'],
                }
            Module: bouncycastle
                java_library {
                    name: 'bouncycastle',
                    ...
                    target: {
                        android: {
                            jarjar_rules: 'jarjar-rules.txt',
                        },
                    },
                }

        Args:
            build: A boolean default to true. If false, skip building jar and
                   srcjar files, otherwise build them.

        Example usage:
            project.source_path = project.locate_source()
            E.g.
                project.source_path = {
                    'source_folder_path': ['path/to/source/folder1',
                                           'path/to/source/folder2', ...],
                    'test_folder_path': ['path/to/test/folder', ...],
                    'jar_path': ['path/to/jar/file1', 'path/to/jar/file2', ...]
                }
        """
        if not hasattr(self, 'dep_modules') or not self.dep_modules:
            raise errors.EmptyModuleDependencyError(
                'Dependent modules dictionary is empty.')
        rebuild_targets = set()
        for module_name, module_data in self.dep_modules.items():
            module = self._generate_moduledata(module_name, module_data)
            module.locate_sources_path()
            self.source_path['source_folder_path'].update(set(module.src_dirs))
            self.source_path['test_folder_path'].update(set(module.test_dirs))
            self.source_path['r_java_path'].update(set(module.r_java_paths))
            self.source_path['srcjar_path'].update(set(module.srcjar_paths))
            self._append_jars_as_dependencies(module)
            rebuild_targets.update(module.build_targets)
        config = project_config.ProjectConfig.get_instance()
        if config.is_skip_build:
            return
        if rebuild_targets:
            if build:
                logging.info('\nThe batch_build_dependencies function is '
                             'called by ProjectInfo\'s locate_source method.')
                batch_build_dependencies(rebuild_targets)
                self.locate_source(build=False)
            else:
                logging.warning('Jar or srcjar files build skipped:\n\t%s.',
                                '\n\t'.join(rebuild_targets))

    def _generate_moduledata(self, module_name, module_data):
        """Generate a module class to collect dependencies in IDE.

        The rules of initialize a module data instance: if ide_object isn't None
        and its ide_name is 'eclipse', we'll create an EclipseModuleData
        instance otherwise create a ModuleData instance.

        Args:
            module_name: Name of the module.
            module_data: A dictionary holding a module information.

        Returns:
            A ModuleData class.
        """
        ide_name = project_config.ProjectConfig.get_instance().ide_name
        if ide_name == constant.IDE_ECLIPSE:
            return source_locator.EclipseModuleData(
                module_name, module_data, self.project_relative_path)
        depth = project_config.ProjectConfig.get_instance().depth
        return source_locator.ModuleData(module_name, module_data, depth)

    def _append_jars_as_dependencies(self, module):
        """Add given module's jar files into dependent_data as dependencies.

        Args:
            module: A ModuleData instance.
        """
        if module.jar_files:
            self.source_path['jar_path'].update(module.jar_files)
            for jar in list(module.jar_files):
                self.source_path['jar_module_path'].update({
                    jar:
                    module.module_path
                })
        # Collecting the jar files of default core modules as dependencies.
        if constant.KEY_DEPENDENCIES in module.module_data:
            self.source_path['jar_path'].update([
                x for x in module.module_data[constant.KEY_DEPENDENCIES]
                if common_util.is_target(x, constant.TARGET_LIBS)
            ])

    def _get_rel_project_out_soong_jar_path(self):
        """Gets the projects' jar path in 'out/soong/.intermediates' folder.

        Gets the relative project's jar path in the 'out/soong/.intermediates'
        directory. For example, if the self.project_relative_path is
        'frameworks/base', the returned value should be
        'out/soong/.intermediates/frameworks/base/'.

        Returns:
            A string of relative project path in out/soong/.intermediates/
            directory, e.g. 'out/soong/.intermediates/frameworks/base/'.
        """
        rdir = os.path.relpath(common_util.get_soong_out_path(),
                               common_util.get_android_root_dir())
        return os.sep.join(
            [rdir, constant.INTERMEDIATES, self.project_relative_path]) + os.sep

    @classmethod
    def multi_projects_locate_source(cls, projects):
        """Locate the paths of dependent source folders and jar files.

        Args:
            projects: A list of ProjectInfo instances. Information of a project
                      such as project relative path, project real path, project
                      dependencies.
        """
        cls.projects = projects
        for project in projects:
            project.locate_source()
            _update_iml_dep_modules(project)


class MultiProjectsInfo(ProjectInfo):
    """Multiple projects info.

    Usage example:
        if folder_base:
            project = MultiProjectsInfo(['module_name'])
            project.collect_all_dep_modules()
            project.gen_folder_base_dependencies()
        else:
            ProjectInfo.generate_projects(['module_name'])

    Attributes:
        _targets: A list of module names or project paths.
        path_to_sources: A dictionary of modules' sources, the module's path
                         as key and the sources as value.
                         e.g.
                         {
                             'frameworks/base': {
                                 'src_dirs': [],
                                 'test_dirs': [],
                                 'r_java_paths': [],
                                 'srcjar_paths': [],
                                 'jar_files': [],
                                 'dep_paths': [],
                             }
                         }
    """

    def __init__(self, targets=None):
        """MultiProjectsInfo initialize.

        Args:
            targets: A list of module names or project paths from user's input.
        """
        super().__init__(targets[0], True)
        self._targets = targets
        self.path_to_sources = {}

    @staticmethod
    def _clear_srcjar_paths(module):
        """Clears the srcjar_paths.

        Args:
            module: A ModuleData instance.
        """
        module.srcjar_paths = []

    def _collect_framework_srcjar_info(self, module):
        """Clears the framework's srcjars.

        Args:
            module: A ModuleData instance.
        """
        if module.module_path == constant.FRAMEWORK_PATH:
            framework_srcjar_path = os.path.join(constant.FRAMEWORK_PATH,
                                                 constant.FRAMEWORK_SRCJARS)
            if module.module_name == constant.FRAMEWORK_ALL:
                self.path_to_sources[framework_srcjar_path] = {
                    'src_dirs': [],
                    'test_dirs': [],
                    'r_java_paths': [],
                    'srcjar_paths': module.srcjar_paths,
                    'jar_files': [],
                    'dep_paths': [constant.FRAMEWORK_PATH],
                }
            # In the folder base case, AIDEGen has to ignore all module's srcjar
            # files under the frameworks/base except the framework-all. Because
            # there are too many duplicate srcjars of modules under the
            # frameworks/base. So that AIDEGen keeps the srcjar files only from
            # the framework-all module. Other modules' srcjar files will be
            # removed. However, when users choose the module base case, srcjar
            # files will be collected by the ProjectInfo class, so that the
            # removing srcjar_paths in this class does not impact the
            # srcjar_paths collection of modules in the ProjectInfo class.
            self._clear_srcjar_paths(module)

    def collect_all_dep_modules(self):
        """Collects all dependency modules for the projects."""
        self.project_module_names.clear()
        module_names = set(_CORE_MODULES)
        for target in self._targets:
            relpath, _ = common_util.get_related_paths(self.modules_info,
                                                       target)
            module_names.update(self._get_modules_under_project_path(relpath))
        module_names.update(self._get_robolectric_dep_module(module_names))
        self.dep_modules = self.get_dep_modules(module_names)

    def gen_folder_base_dependencies(self, module):
        """Generates the folder base dependencies dictionary.

        Args:
            module: A ModuleData instance.
        """
        mod_path = module.module_path
        if not mod_path:
            logging.debug('The %s\'s path is empty.', module.module_name)
            return
        self._collect_framework_srcjar_info(module)
        if mod_path not in self.path_to_sources:
            self.path_to_sources[mod_path] = {
                'src_dirs': module.src_dirs,
                'test_dirs': module.test_dirs,
                'r_java_paths': module.r_java_paths,
                'srcjar_paths': module.srcjar_paths,
                'jar_files': module.jar_files,
                'dep_paths': module.dep_paths,
            }
        else:
            for key, val in self.path_to_sources[mod_path].items():
                val.extend([v for v in getattr(module, key) if v not in val])


def batch_build_dependencies(rebuild_targets):
    """Batch build the jar or srcjar files of the modules if they don't exist.

    Command line has the max length limit, MAX_ARG_STRLEN, and
    MAX_ARG_STRLEN = (PAGE_SIZE * 32).
    If the build command is longer than MAX_ARG_STRLEN, this function will
    separate the rebuild_targets into chunks with size less or equal to
    MAX_ARG_STRLEN to make sure it can be built successfully.

    Args:
        rebuild_targets: A set of jar or srcjar files which do not exist.
    """
    start_time = time.time()
    logging.info('Ready to build the jar or srcjar files. Files count = %s',
                 str(len(rebuild_targets)))
    arg_max = os.sysconf('SC_PAGE_SIZE') * 32 - _CMD_LENGTH_BUFFER
    rebuild_targets = list(rebuild_targets)
    for start, end in iter(_separate_build_targets(rebuild_targets, arg_max)):
        _build_target(rebuild_targets[start:end])
    duration = time.time() - start_time
    logging.debug('Build Time,  duration = %s', str(duration))
    aidegen_metrics.performance_metrics(constant.TYPE_AIDEGEN_BUILD_TIME,
                                        duration)


def _build_target(targets):
    """Build the jar or srcjar files.

    Use -k to keep going when some targets can't be built or build failed.
    Use -j to speed up building.

    Args:
        targets: A list of jar or srcjar files which need to build.
    """
    build_cmd = ['-k', '-j']
    build_cmd.extend(list(targets))
    atest_utils.update_build_env(_BUILD_BP_JSON_ENV_ON)
    if not atest_utils.build(build_cmd):
        message = ('Build failed!\n{}\nAIDEGen will proceed but dependency '
                   'correctness is not guaranteed if not all targets being '
                   'built successfully.'.format('\n'.join(targets)))
        print('\n{} {}\n'.format(common_util.COLORED_INFO('Warning:'), message))


def _separate_build_targets(build_targets, max_length):
    """Separate the build_targets by limit the command size to max command
    length.

    Args:
        build_targets: A list to be separated.
        max_length: The max number of each build command length.

    Yields:
        The start index and end index of build_targets.
    """
    arg_len = 0
    first_item_index = 0
    for i, item in enumerate(build_targets):
        arg_len = arg_len + len(item) + _BLANK_SIZE
        if arg_len > max_length:
            yield first_item_index, i
            first_item_index = i
            arg_len = len(item) + _BLANK_SIZE
    if first_item_index < len(build_targets):
        yield first_item_index, len(build_targets)


def _update_iml_dep_modules(project):
    """Gets the dependent modules in the project's iml file.

    The jar files which have the same source codes as cls.projects' source files
    should be removed from the dependencies.iml file's jar paths. The codes are
    written in aidegen.project.project_splitter.py.
    We should also add the jar project's unique iml name into self.dependencies
    which later will be written into its own iml project file. If we don't
    remove these files in dependencies.iml, it will cause the duplicated codes
    in IDE and raise issues. For example, when users do 'refactor' and rename a
    class in the IDE, it will search all sources and dependencies' jar paths and
    lead to the error.
    """
    keys = ('source_folder_path', 'test_folder_path', 'r_java_path',
            'srcjar_path', 'jar_path')
    for key in keys:
        for jar in project.source_path[key]:
            for prj in ProjectInfo.projects:
                if prj is project:
                    continue
                if (prj.rel_out_soong_jar_path in jar and
                        jar.endswith(constant.JAR_EXT)):
                    if prj.iml_name not in project.dependencies:
                        project.dependencies.append(prj.iml_name)
                    break