deqp/scripts/testset.py

# -*- coding: utf-8 -*-

#-------------------------------------------------------------------------
# drawElements Quality Program utilities
# --------------------------------------
#
# Copyright 2015 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#-------------------------------------------------------------------------

import sys
import random
import string
import subprocess
from optparse import OptionParser

def all (results, predicate):
    for result in results:
        if not predicate(result):
            return False
    return True

def any (results, predicate):
    for result in results:
        if predicate(result):
            return True
    return False

class FilterRule:
    def __init__ (self, name, description, filters):
        self.name = name
        self.description = description
        self.filters = filters

class TestCaseResult:
    def __init__ (self, name, results):
        self.name = name
        self.results = results

class Group:
    def __init__ (self, name):
        self.name = name
        self.cases = []

def readCaseList (filename):
    f = open(filename, 'rb')
    cases = []
    for line in f:
        if line[:6] == "TEST: ":
            case = line[6:].strip()
            if len(case) > 0:
                cases.append(case)
    return cases

def toResultList (caselist):
    results = []
    for case in caselist:
        results.append(TestCaseResult(case, []))
    return results

def addResultsToCaseList (caselist, results):
    resultMap = {}
    caseListRes = toResultList(caselist)

    for res in caseListRes:
        resultMap[res.name] = res

    for result in results:
        if result.name in resultMap:
            resultMap[result.name].results += result.results

    return caseListRes

def readTestResults (filename):
    f = open(filename, 'rb')
    csvData = f.read()
    csvLines = csvData.splitlines()
    results = []

    f.close()

    for line in csvLines[1:]:
        args = line.split(',')
        if len(args) == 1:
            continue # Ignore

        results.append(TestCaseResult(args[0], args[1:]))

    if len(results) == 0:
        raise Exception("Empty result list")

    # Quick check for results
    numResultItems = len(results[0].results)
    seenResults = set()
    for result in results:
        if result.name in seenResults:
            raise Exception("Duplicate result row for test case '%s'" % result.name)
        if len(result.results) != numResultItems:
            raise Exception("Found %d results for test case '%s', expected %d" % (len(result.results), result.name, numResultItems))
        seenResults.add(result.name)

    return results

def readGroupList (filename):
    f = open(filename, 'rb')
    groups = []
    for line in f:
        group = line.strip()
        if group != "":
            groups.append(group)
    return groups

def createGroups (results, groupNames):
    groups = []
    matched = set()

    for groupName in groupNames:
        group = Group(groupName)
        groups.append(group)

        prefix = groupName + "."
        prefixLen = len(prefix)
        for case in results:
            if case.name[:prefixLen] == prefix:
                if case in matched:
                    die("Case '%s' matched by multiple groups (when processing '%s')" % (case.name, group.name))
                group.cases.append(case)
                matched.add(case)

    return groups

def createLeafGroups (results):
    groups = []
    groupMap = {}

    for case in results:
        parts = case.name.split('.')
        groupName = string.join(parts[:-1], ".")

        if not groupName in groupMap:
            group = Group(groupName)
            groups.append(group)
            groupMap[groupName] = group
        else:
            group = groupMap[groupName]

        group.cases.append(case)

    return groups

def filterList (results, condition):
    filtered = []
    for case in results:
        if condition(case.results):
            filtered.append(case)
    return filtered

def getFilter (list, name):
    for filter in list:
        if filter.name == name:
            return filter
    return None

def getNumCasesInGroups (groups):
    numCases = 0
    for group in groups:
        numCases += len(group.cases)
    return numCases

def getCasesInSet (results, caseSet):
    filtered = []
    for case in results:
        if case in caseSet:
            filtered.append(case)
    return filtered

def selectCasesInGroups (results, groups):
    casesInGroups = set()
    for group in groups:
        for case in group.cases:
            casesInGroups.add(case)
    return getCasesInSet(results, casesInGroups)

def selectRandomSubset (results, groups, limit, seed):
    selectedCases = set()
    numSelect = min(limit, getNumCasesInGroups(groups))

    random.seed(seed)
    random.shuffle(groups)

    groupNdx = 0
    while len(selectedCases) < numSelect:
        group = groups[groupNdx]
        if len(group.cases) == 0:
            del groups[groupNdx]
            if groupNdx == len(groups):
                groupNdx -= 1
            continue # Try next

        selected = random.choice(group.cases)
        selectedCases.add(selected)
        group.cases.remove(selected)

        groupNdx = (groupNdx + 1) % len(groups)

    return getCasesInSet(results, selectedCases)

def die (msg):
    print(msg)
    sys.exit(-1)

# Named filter lists
FILTER_RULES = [
    FilterRule("all", "No filtering", []),
    FilterRule("all-pass", "All results must be 'Pass'", [lambda l: all(l, lambda r: r == 'Pass')]),
    FilterRule("any-pass", "Any of results is 'Pass'", [lambda l: any(l, lambda r: r == 'Pass')]),
    FilterRule("any-fail", "Any of results is not 'Pass' or 'NotSupported'", [lambda l: not all(l, lambda r: r == 'Pass' or r == 'NotSupported')]),
    FilterRule("prev-failing", "Any except last result is failure", [lambda l: l[-1] == 'Pass' and not all(l[:-1], lambda r: r == 'Pass')]),
    FilterRule("prev-passing", "Any except last result is 'Pass'", [lambda l: l[-1] != 'Pass' and any(l[:-1], lambda r: r == 'Pass')])
]

if __name__ == "__main__":
    parser = OptionParser(usage = "usage: %prog [options] [caselist] [result csv file]")
    parser.add_option("-f", "--filter", dest="filter", default="all", help="filter rule name")
    parser.add_option("-l", "--list", action="store_true", dest="list", default=False, help="list available rules")
    parser.add_option("-n", "--num", dest="limit", default=0, help="limit number of cases")
    parser.add_option("-s", "--seed", dest="seed", default=0, help="use selected seed for random selection")
    parser.add_option("-g", "--groups", dest="groups_file", default=None, help="select cases based on group list file")

    (options, args) = parser.parse_args()

    if options.list:
        print("Available filter rules:")
        for filter in FILTER_RULES:
            print("  %s: %s" % (filter.name, filter.description))
        sys.exit(0)

    if len(args) == 0:
        die("No input files specified")
    elif len(args) > 2:
        die("Too many arguments")

    # Fetch filter
    filter = getFilter(FILTER_RULES, options.filter)
    if filter == None:
        die("Unknown filter '%s'" % options.filter)

    # Read case list
    caselist = readCaseList(args[0])
    if len(args) > 1:
        results = readTestResults(args[1])
        results = addResultsToCaseList(caselist, results)
    else:
        results = toResultList(caselist)

    # Execute filters for results
    for rule in filter.filters:
        results = filterList(results, rule)

    if options.limit != 0:
        if options.groups_file != None:
            groups = createGroups(results, readGroupList(options.groups_file))
        else:
            groups = createLeafGroups(results)
        results = selectRandomSubset(results, groups, int(options.limit), int(options.seed))
    elif options.groups_file != None:
        groups = createGroups(results, readGroupList(options.groups_file))
        results = selectCasesInGroups(results, groups)

    # Print test set
    for result in results:
        print(result.name)