xref: /aosp_15_r20/external/ltp/android/tools/compare_ltp_projects.py (revision 49cdfc7efb34551c7342be41a7384b9c40d7cab7)

#! /usr/bin/env python3
#
# Copyright 2018 - The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

"""Tool for comparing 2 LTP projects to find added / removed tests & testsuites.

The tool can be run in two separate steps by dumping the list of tests as a json
file, i.e.:

$ git checkout master
$ compare_ltp_projects.py --ltp-old . --ltp-old-json ./ltp_old.json
$ git checkout mymergebranch
$ compare_ltp_projects.py --ltp-old-json ./ltp_old.json --ltp-new .
"""

import os
import argparse
import os.path
import sys
import json

def scan_tests(ltp_root, suite):
    ''' Find all tests that are run as part of given test suite in LTP.

    Args:
        ltp_root: Path ot the LTP project.
        suite: Name of testsuite.

    Returns:
        List of tests names that are run as part of the given test suite.
    '''

    tests = []
    if not suite:
        return tests

    runtest_dir = ltp_root + os.path.sep + 'runtest'
    test_suiteFile = runtest_dir + os.path.sep + suite
    if not os.path.isfile(test_suiteFile):
        print('No tests defined for suite {}',format(suite))
        return tests

    with open(test_suiteFile) as f:
        for line in f:
            line = line.strip()
            if line and not line.startswith('#'):
                tests.append(line.split()[0])
    f.close()
    return tests

def scan_test_suites(ltp_root, scenario):
    ''' Find all testuites and tests run as part of given LTP scenario

    Args:
        ltp_root: Path to the LTP project.
        scenario: name of the scenario (found in ltp_root/scenario_groups). E.g. "vts"

    Returns:
        List of testsuite names that are run as part of given scenario (e.g. vts).
        If scenario is not specified, return all testsuites.
    '''

    runtest_dir = ltp_root + os.path.sep + 'runtest'
    if not os.path.isdir(runtest_dir):
        print('Invalid ltp_root {}, runtest directory doesnt exist'.format(ltp_root))
        sys.exit(2)

    test_suites = []
    if scenario:
        scenarioFile = ltp_root + os.path.sep + 'scenario_groups' + os.path.sep + scenario
        if not os.path.isfile(scenarioFile):
            return test_suites
        with open(scenarioFile) as f:
            for line in f:
                line = line.strip()
                if not line.startswith('#'):
                    test_suites.append(line)
        return test_suites

    runtest_dir = ltp_root + os.path.sep + 'runtest'
    test_suites = [f for f in os.listdir(runtest_dir) if os.path.isfile(runtest_dir + os.path.sep + f)]

    return test_suites

def scan_ltp(ltp_root, scenario):
    ''' scan LTP project and return all tests and testuites present.

    Args:
        ltp_root: Path to the LTP project.
        scenario: specific scenario we want to scan. e.g. vts

    Returns:
        Dictionary of all LTP test names keyed by testsuite they are run as part of.
        E.g.
             {
                 'mem': ['oom01', 'mem02',...],
                 'syscalls': ['open01', 'read02',...],
                 ...
             }
    '''

    if not os.path.isdir(ltp_root):
        print('ltp_root {} does not exist'.format(ltp_root))
        sys.exit(1)

    test_suites = scan_test_suites(ltp_root, scenario)
    if not test_suites:
        print('No Testsuites found for scenario {}'.format(scenario))
        sys.exit(3)

    ltp_tests = {}
    for suite in test_suites:
        ltp_tests[suite] = scan_tests(ltp_root, suite)
    return ltp_tests

def show_diff(ltp_tests_1, ltp_tests_2):
    ''' Find and print diff between testcases in 2 LTP project checkouts.

    Args:
        ltp_tests_1: dictionary of tests keyed by test suite names
        ltp_tests_2: dictionary of tests keyed by test suite names
    '''
    test_suites1 = set(ltp_tests_1.keys())
    test_suites2 = set(ltp_tests_2.keys())

    # Generate lists of deleted, added and common test suites between
    # LTP1 & LTP2
    deleted_test_suites = sorted(test_suites1.difference(test_suites2))
    added_test_suites = sorted(test_suites2.difference(test_suites1))
    common_test_suites = test_suites1.intersection(test_suites2)

    deleted_tests = []
    added_tests = []
    for suite in common_test_suites:
        tests1 = set(ltp_tests_1[suite])
        tests2 = set(ltp_tests_2[suite])

        exclusive_test1 = tests1.difference(tests2)
        exclusive_test2 = tests2.difference(tests1)
        for e in exclusive_test1:
            deleted_tests.append(suite + '.' + e)
        for e in exclusive_test2:
            added_tests.append(suite + '.' + e)
    deleted_tests = sorted(deleted_tests)
    added_tests = sorted(added_tests)
    print_columns(added_test_suites, deleted_test_suites, added_tests, deleted_tests)


def print_columns(added_test_suites, deleted_test_suites, added_tests, deleted_tests):
    DEFAULT_WIDTH = 8
    # find the maximum width of a test suite name or test name
    # we have to print to decide the alignment.
    if not deleted_test_suites:
        width_suites = DEFAULT_WIDTH
    else:
        width_suites = max([len(x) for x in deleted_test_suites])

    if not deleted_tests:
        width_tests = DEFAULT_WIDTH
    else:
        width_tests = max([len(x) for x in deleted_tests])
    width = max(width_suites, width_tests);

    # total rows we have to print
    total_rows = max(len(deleted_test_suites), len(added_test_suites))
    added_text = 'Added ({})'.format(len(added_test_suites))
    deleted_text = 'Deleted ({})'.format(len(deleted_test_suites))
    if total_rows > 0:
        print('{:*^{len}}'.format(' Tests Suites ', len=width*2+2))
        print('{:<{len}} {:<{len}}'.format(added_text, deleted_text, len=width))
        for i in range(total_rows):
            print('{:<{len}} {:<{len}}'.format('' if i >= len(added_test_suites) else str(added_test_suites[i]),
                                 '' if i >= len(deleted_test_suites) else str(deleted_test_suites[i]), len=width))

    print('')
    # total rows we have to print
    total_rows = max(len(deleted_tests), len(added_tests))
    added_text = 'Added ({})'.format(len(added_tests))
    deleted_text = 'Deleted ({})'.format(len(deleted_tests))
    if total_rows:
        print('{:*^{len}}'.format(' Tests ', len=width*2+2))
        print('{:^{len}} {:^{len}}'.format(added_text, deleted_text, len=width))
        for i in range(total_rows):
            print('{:<{len}} {:<{len}}'.format('' if i >= len(added_tests) else str(added_tests[i]),
                                 '' if i >= len(deleted_tests) else str(deleted_tests[i]), len=width))
def main():
    arg_parser = argparse.ArgumentParser(
        description='Diff 2 LTP projects for supported test cases')
    arg_parser.add_argument('-o', '--ltp-old',
                            dest='ltp_old',
                            help="LTP Root Directory before merge")
    arg_parser.add_argument('-oj', '--ltp-old-json',
                            dest='ltp_old_json',
                            default='ltp_old.json',
                            help="Old LTP parsed directory in json format")
    arg_parser.add_argument('-n', '--ltp-new',
                            dest='ltp_new',
                            help="LTP Root Directory after merge")
    arg_parser.add_argument('-nj', '--ltp-new-json',
                            dest='ltp_new_json',
                            default='ltp_new.json',
                            help="New LTP parsed directory in json format")
    arg_parser.add_argument('--scenario', default=None,
                            dest='scenario',
                            help="LTP scenario to list tests for")
    args = arg_parser.parse_args()

    # Find tests in the "old" directory or read the json output from a prior run
    if args.ltp_old:
        ltp_tests1 = scan_ltp(args.ltp_old, args.scenario)
    elif args.ltp_old_json and os.path.isfile(args.ltp_old_json):
        with open(args.ltp_old_json) as f:
            ltp_tests1 = json.load(f)
    else:
        ltp_tests1 = None

    # Do the same for the "new" directory
    if args.ltp_new:
        ltp_tests2 = scan_ltp(args.ltp_new, args.scenario)
    elif args.ltp_new_json and os.path.isfile(args.ltp_new_json):
        with open(args.ltp_new_json) as f:
            ltp_tests2 = json.load(f)
    else:
        ltp_tests2 = None

    if ltp_tests1 and ltp_tests2:
        # Show the difference of the two directories if both are present
        show_diff(ltp_tests1, ltp_tests2)
    elif ltp_tests1:
        # If just the old directory was read, dump it as json
        with open(args.ltp_old_json, 'w') as f:
            json.dump(ltp_tests1, f)
    elif ltp_tests2:
        # If just the new directory was read, dump it as json
        with open(args.ltp_new_json, 'w') as f:
            json.dump(ltp_tests2, f)

if __name__ == '__main__':
    main()