1# -*- coding: utf-8 -*- 2#------------------------------------------------------------------------- 3# drawElements Quality Program utilities 4# -------------------------------------- 5# 6# Copyright 2016 The Android Open Source Project 7# 8# Licensed under the Apache License, Version 2.0 (the "License"); 9# you may not use this file except in compliance with the License. 10# You may obtain a copy of the License at 11# 12# http://www.apache.org/licenses/LICENSE-2.0 13# 14# Unless required by applicable law or agreed to in writing, software 15# distributed under the License is distributed on an "AS IS" BASIS, 16# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17# See the License for the specific language governing permissions and 18# limitations under the License. 19# 20#------------------------------------------------------------------------- 21 22from ctsbuild.common import * 23from ctsbuild.build import build 24from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET 25from fnmatch import fnmatch 26from copy import copy 27from collections import defaultdict 28import logging 29import argparse 30import re 31import xml.etree.cElementTree as ElementTree 32import xml.dom.minidom as minidom 33 34GENERATED_FILE_WARNING = """ 35 This file has been automatically generated. Edit with caution. 36 """ 37 38class Project: 39 def __init__ (self, path, copyright = None): 40 self.path = path 41 self.copyright = copyright 42 43class Configuration: 44 def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None, runByDefault = True, listOfGroupsToSplit = []): 45 self.name = name 46 self.glconfig = glconfig 47 self.rotation = rotation 48 self.surfacetype = surfacetype 49 self.required = required 50 self.filters = filters 51 self.expectedRuntime = runtime 52 self.runByDefault = runByDefault 53 self.listOfGroupsToSplit = listOfGroupsToSplit 54 55class Package: 56 def __init__ (self, module, configurations): 57 self.module = module 58 self.configurations = configurations 59 60class Mustpass: 61 def __init__ (self, project, version, packages): 62 self.project = project 63 self.version = version 64 self.packages = packages 65 66class Filter: 67 TYPE_INCLUDE = 0 68 TYPE_EXCLUDE = 1 69 70 def __init__ (self, type, filenames): 71 self.type = type 72 self.filenames = filenames 73 self.key = ",".join(filenames) 74 75class TestRoot: 76 def __init__ (self): 77 self.children = [] 78 79class TestGroup: 80 def __init__ (self, name): 81 self.name = name 82 self.children = [] 83 84class TestCase: 85 def __init__ (self, name): 86 self.name = name 87 self.configurations = [] 88 89def getSrcDir (mustpass): 90 return os.path.join(mustpass.project.path, mustpass.version, "src") 91 92def getModuleShorthand (module): 93 assert module.name[:5] == "dEQP-" 94 return module.name[5:].lower() 95 96def getCaseListFileName (package, configuration): 97 return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name) 98 99def getDstCaseListPath (mustpass): 100 return os.path.join(mustpass.project.path, mustpass.version) 101 102def getCommandLine (config): 103 cmdLine = "" 104 105 if config.glconfig != None: 106 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig 107 108 if config.rotation != None: 109 cmdLine += "--deqp-screen-rotation=%s " % config.rotation 110 111 if config.surfacetype != None: 112 cmdLine += "--deqp-surface-type=%s " % config.surfacetype 113 114 cmdLine += "--deqp-watchdog=enable" 115 116 return cmdLine 117 118class CaseList: 119 def __init__(self, filePath, sortedLines): 120 self.filePath = filePath 121 self.sortedLines = sortedLines 122 123def readAndSortCaseList (buildCfg, generator, module): 124 build(buildCfg, generator, [module.binName]) 125 genCaseList(buildCfg, generator, module, "txt") 126 filePath = getCaseListPath(buildCfg, module, "txt") 127 with open(filePath, 'r') as first_file: 128 lines = first_file.readlines() 129 lines.sort() 130 caseList = CaseList(filePath, lines) 131 return caseList 132 133def readPatternList (filename, patternList): 134 with open(filename, 'rt') as f: 135 for line in f: 136 line = line.strip() 137 if len(line) > 0 and line[0] != '#': 138 patternList.append(line) 139 140def include (*filenames): 141 return Filter(Filter.TYPE_INCLUDE, filenames) 142 143def exclude (*filenames): 144 return Filter(Filter.TYPE_EXCLUDE, filenames) 145 146def insertXMLHeaders (mustpass, doc): 147 if mustpass.project.copyright != None: 148 doc.insert(0, ElementTree.Comment(mustpass.project.copyright)) 149 doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING)) 150 151def prettifyXML (doc): 152 uglyString = ElementTree.tostring(doc, 'utf-8') 153 reparsed = minidom.parseString(uglyString) 154 return reparsed.toprettyxml(indent='\t', encoding='utf-8') 155 156def genSpecXML (mustpass): 157 mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version) 158 insertXMLHeaders(mustpass, mustpassElem) 159 160 for package in mustpass.packages: 161 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name) 162 163 for config in package.configurations: 164 configElem = ElementTree.SubElement(packageElem, "Configuration", 165 caseListFile = getCaseListFileName(package, config), 166 commandLine = getCommandLine(config), 167 name = config.name) 168 169 return mustpassElem 170 171def addOptionElement (parent, optionName, optionValue): 172 ElementTree.SubElement(parent, "option", name=optionName, value=optionValue) 173 174def genAndroidTestXml (mustpass): 175 RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner" 176 configElement = ElementTree.Element("configuration") 177 178 # have the deqp package installed on the device for us 179 preparerElement = ElementTree.SubElement(configElement, "target_preparer") 180 preparerElement.set("class", "com.android.tradefed.targetprep.suite.SuiteApkInstaller") 181 addOptionElement(preparerElement, "cleanup-apks", "true") 182 addOptionElement(preparerElement, "test-file-name", "com.drawelements.deqp.apk") 183 184 # Target preparer for incremental dEQP 185 preparerElement = ElementTree.SubElement(configElement, "target_preparer") 186 preparerElement.set("class", "com.android.compatibility.common.tradefed.targetprep.FilePusher") 187 addOptionElement(preparerElement, "cleanup", "true") 188 addOptionElement(preparerElement, "disable", "true") 189 addOptionElement(preparerElement, "push", "deqp-binary32->/data/local/tmp/deqp-binary32") 190 addOptionElement(preparerElement, "push", "deqp-binary64->/data/local/tmp/deqp-binary64") 191 addOptionElement(preparerElement, "push", "gles2->/data/local/tmp/gles2") 192 addOptionElement(preparerElement, "push", "gles2-incremental-deqp-baseline.txt->/data/local/tmp/gles2-incremental-deqp-baseline.txt") 193 addOptionElement(preparerElement, "push", "gles3->/data/local/tmp/gles3") 194 addOptionElement(preparerElement, "push", "gles3-incremental-deqp-baseline.txt->/data/local/tmp/gles3-incremental-deqp-baseline.txt") 195 addOptionElement(preparerElement, "push", "gles3-incremental-deqp.txt->/data/local/tmp/gles3-incremental-deqp.txt") 196 addOptionElement(preparerElement, "push", "gles31->/data/local/tmp/gles31") 197 addOptionElement(preparerElement, "push", "gles31-incremental-deqp-baseline.txt->/data/local/tmp/gles31-incremental-deqp-baseline.txt") 198 addOptionElement(preparerElement, "push", "internal->/data/local/tmp/internal") 199 addOptionElement(preparerElement, "push", "vk-incremental-deqp-baseline.txt->/data/local/tmp/vk-incremental-deqp-baseline.txt") 200 addOptionElement(preparerElement, "push", "vk-incremental-deqp.txt->/data/local/tmp/vk-incremental-deqp.txt") 201 addOptionElement(preparerElement, "push", "vulkan->/data/local/tmp/vulkan") 202 preparerElement = ElementTree.SubElement(configElement, "target_preparer") 203 preparerElement.set("class", "com.android.compatibility.common.tradefed.targetprep.IncrementalDeqpPreparer") 204 addOptionElement(preparerElement, "disable", "true") 205 206 # add in metadata option for component name 207 ElementTree.SubElement(configElement, "option", name="test-suite-tag", value="cts") 208 ElementTree.SubElement(configElement, "option", key="component", name="config-descriptor:metadata", value="deqp") 209 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="not_instant_app") 210 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="multi_abi") 211 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="secondary_user") 212 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="no_foldable_states") 213 controllerElement = ElementTree.SubElement(configElement, "object") 214 controllerElement.set("class", "com.android.tradefed.testtype.suite.module.TestFailureModuleController") 215 controllerElement.set("type", "module_controller") 216 addOptionElement(controllerElement, "screenshot-on-failure", "false") 217 218 for package in mustpass.packages: 219 for config in package.configurations: 220 if not config.runByDefault: 221 continue 222 223 testElement = ElementTree.SubElement(configElement, "test") 224 testElement.set("class", RUNNER_CLASS) 225 addOptionElement(testElement, "deqp-package", package.module.name) 226 caseListFile = getCaseListFileName(package,config) 227 addOptionElement(testElement, "deqp-caselist-file", caseListFile) 228 if caseListFile.startswith("gles3"): 229 addOptionElement(testElement, "incremental-deqp-include-file", "gles3-incremental-deqp.txt") 230 elif caseListFile.startswith("vk"): 231 addOptionElement(testElement, "incremental-deqp-include-file", "vk-incremental-deqp.txt") 232 # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well. 233 if config.glconfig != None: 234 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig) 235 236 if config.surfacetype != None: 237 addOptionElement(testElement, "deqp-surface-type", config.surfacetype) 238 239 if config.rotation != None: 240 addOptionElement(testElement, "deqp-screen-rotation", config.rotation) 241 242 if config.expectedRuntime != None: 243 addOptionElement(testElement, "runtime-hint", config.expectedRuntime) 244 245 if config.required: 246 addOptionElement(testElement, "deqp-config-required", "true") 247 248 insertXMLHeaders(mustpass, configElement) 249 250 return configElement 251 252class PatternSet: 253 def __init__(self): 254 self.namedPatternsTree = {} 255 self.namedPatternsDict = {} 256 self.wildcardPatternsDict = {} 257 258def readPatternSets (mustpass): 259 patternSets = {} 260 for package in mustpass.packages: 261 for cfg in package.configurations: 262 for filter in cfg.filters: 263 if not filter.key in patternSets: 264 patternList = [] 265 for filename in filter.filenames: 266 readPatternList(os.path.join(getSrcDir(mustpass), filename), patternList) 267 patternSet = PatternSet() 268 for pattern in patternList: 269 if pattern.find('*') == -1: 270 patternSet.namedPatternsDict[pattern] = 0 271 t = patternSet.namedPatternsTree 272 parts = pattern.split('.') 273 for part in parts: 274 t = t.setdefault(part, {}) 275 else: 276 # We use regex instead of fnmatch because it's faster 277 patternSet.wildcardPatternsDict[re.compile("^" + pattern.replace(".", r"\.").replace("*", ".*?") + "$")] = 0 278 patternSets[filter.key] = patternSet 279 return patternSets 280 281def genMustpassFromLists (mustpass, moduleCaseLists): 282 print("Generating mustpass '%s'" % mustpass.version) 283 patternSets = readPatternSets(mustpass) 284 285 for package in mustpass.packages: 286 currentCaseList = moduleCaseLists[package.module] 287 logging.debug("Reading " + currentCaseList.filePath) 288 289 for config in package.configurations: 290 # construct components of path to main destination file 291 mainDstFileDir = getDstCaseListPath(mustpass) 292 mainDstFileName = getCaseListFileName(package, config) 293 mainDstFilePath = os.path.join(mainDstFileDir, mainDstFileName) 294 mainGroupSubDir = mainDstFileName[:-4] 295 296 if not os.path.exists(mainDstFileDir): 297 os.makedirs(mainDstFileDir) 298 mainDstFile = open(mainDstFilePath, 'w') 299 print(mainDstFilePath) 300 output_files = {} 301 def openAndStoreFile(filePath, testFilePath, parentFile): 302 if filePath not in output_files: 303 try: 304 print(" " + filePath) 305 parentFile.write(mainGroupSubDir + "/" + testFilePath + "\n") 306 currentDir = os.path.dirname(filePath) 307 if not os.path.exists(currentDir): 308 os.makedirs(currentDir) 309 output_files[filePath] = open(filePath, 'w') 310 311 except FileNotFoundError: 312 print(f"File not found: {filePath}") 313 return output_files[filePath] 314 315 lastOutputFile = "" 316 currentOutputFile = None 317 for line in currentCaseList.sortedLines: 318 if not line.startswith("TEST: "): 319 continue 320 caseName = line.replace("TEST: ", "").strip("\n") 321 caseParts = caseName.split(".") 322 keep = True 323 # Do the includes with the complex patterns first 324 for filter in config.filters: 325 if filter.type == Filter.TYPE_INCLUDE: 326 keep = False 327 patterns = patternSets[filter.key].wildcardPatternsDict 328 for pattern in patterns.keys(): 329 keep = pattern.match(caseName) 330 if keep: 331 patterns[pattern] += 1 332 break 333 334 if not keep: 335 t = patternSets[filter.key].namedPatternsTree 336 if len(t.keys()) == 0: 337 continue 338 for part in caseParts: 339 if part in t: 340 t = t[part] 341 else: 342 t = None # Not found 343 break 344 keep = t == {} 345 if keep: 346 patternSets[filter.key].namedPatternsDict[caseName] += 1 347 348 # Do the excludes 349 if filter.type == Filter.TYPE_EXCLUDE: 350 patterns = patternSets[filter.key].wildcardPatternsDict 351 for pattern in patterns.keys(): 352 discard = pattern.match(caseName) 353 if discard: 354 patterns[pattern] += 1 355 keep = False 356 break 357 if keep: 358 t = patternSets[filter.key].namedPatternsTree 359 if len(t.keys()) == 0: 360 continue 361 for part in caseParts: 362 if part in t: 363 t = t[part] 364 else: 365 t = None # Not found 366 break 367 if t == {}: 368 patternSets[filter.key].namedPatternsDict[caseName] += 1 369 keep = False 370 if not keep: 371 break 372 if not keep: 373 continue 374 375 parts = caseName.split('.') 376 if len(config.listOfGroupsToSplit) > 0: 377 if len(parts) > 2: 378 groupName = parts[1].replace("_", "-") 379 for splitPattern in config.listOfGroupsToSplit: 380 splitParts = splitPattern.split(".") 381 if len(splitParts) > 1 and caseName.startswith(splitPattern + "."): 382 groupName = groupName + "/" + parts[2].replace("_", "-") 383 filePath = os.path.join(mainDstFileDir, mainGroupSubDir, groupName + ".txt") 384 if lastOutputFile != filePath: 385 currentOutputFile = openAndStoreFile(filePath, groupName + ".txt", mainDstFile) 386 lastOutputFile = filePath 387 currentOutputFile.write(caseName + "\n") 388 else: 389 mainDstFile.write(caseName + "\n") 390 391 # Check that all patterns have been used in the filters 392 # This check will help identifying typos and patterns becoming stale 393 for filter in config.filters: 394 if filter.type == Filter.TYPE_INCLUDE: 395 patternSet = patternSets[filter.key] 396 for pattern, usage in patternSet.namedPatternsDict.items(): 397 if usage == 0: 398 logging.debug("Case %s in file %s for module %s was never used!" % (pattern, filter.key, config.name)) 399 for pattern, usage in patternSet.wildcardPatternsDict.items(): 400 if usage == 0: 401 logging.debug("Pattern %s in file %s for module %s was never used!" % (pattern, filter.key, config.name)) 402 403 # Generate XML 404 specXML = genSpecXML(mustpass) 405 specFilename = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml") 406 407 print(" Writing spec: " + specFilename) 408 writeFile(specFilename, prettifyXML(specXML).decode()) 409 410 # TODO: Which is the best selector mechanism? 411 if (mustpass.version == "main"): 412 androidTestXML = genAndroidTestXml(mustpass) 413 androidTestFilename = os.path.join(mustpass.project.path, "AndroidTest.xml") 414 415 print(" Writing AndroidTest.xml: " + androidTestFilename) 416 writeFile(androidTestFilename, prettifyXML(androidTestXML).decode()) 417 418 print("Done!") 419 420 421def genMustpassLists (mustpassLists, generator, buildCfg): 422 moduleCaseLists = {} 423 424 # Getting case lists involves invoking build, so we want to cache the results 425 for mustpass in mustpassLists: 426 for package in mustpass.packages: 427 if not package.module in moduleCaseLists: 428 moduleCaseLists[package.module] = readAndSortCaseList(buildCfg, generator, package.module) 429 430 for mustpass in mustpassLists: 431 genMustpassFromLists(mustpass, moduleCaseLists) 432 433def parseCmdLineArgs (): 434 parser = argparse.ArgumentParser(description = "Build Android CTS mustpass", 435 formatter_class=argparse.ArgumentDefaultsHelpFormatter) 436 parser.add_argument("-b", 437 "--build-dir", 438 dest="buildDir", 439 default=DEFAULT_BUILD_DIR, 440 help="Temporary build directory") 441 parser.add_argument("-t", 442 "--build-type", 443 dest="buildType", 444 default="Debug", 445 help="Build type") 446 parser.add_argument("-c", 447 "--deqp-target", 448 dest="targetName", 449 default=DEFAULT_TARGET, 450 help="dEQP build target") 451 parser.add_argument("-v", "--verbose", 452 dest="verbose", 453 action="store_true", 454 help="Enable verbose logging") 455 return parser.parse_args() 456 457def parseBuildConfigFromCmdLineArgs (): 458 args = parseCmdLineArgs() 459 initializeLogger(args.verbose) 460 return getBuildConfig(args.buildDir, args.targetName, args.buildType) 461