1# -*- coding: utf-8 -*- 2import logging 3 4#------------------------------------------------------------------------- 5# drawElements Quality Program utilities 6# -------------------------------------- 7# 8# Copyright 2016 The Android Open Source Project 9# 10# Licensed under the Apache License, Version 2.0 (the "License"); 11# you may not use this file except in compliance with the License. 12# You may obtain a copy of the License at 13# 14# http://www.apache.org/licenses/LICENSE-2.0 15# 16# Unless required by applicable law or agreed to in writing, software 17# distributed under the License is distributed on an "AS IS" BASIS, 18# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19# See the License for the specific language governing permissions and 20# limitations under the License. 21# 22#------------------------------------------------------------------------- 23 24from ctsbuild.common import * 25from ctsbuild.build import build 26from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET 27from fnmatch import fnmatch 28from copy import copy 29from collections import defaultdict 30 31import argparse 32import re 33import xml.etree.cElementTree as ElementTree 34import xml.dom.minidom as minidom 35 36GENERATED_FILE_WARNING = """ 37 This file has been automatically generated. Edit with caution. 38 """ 39 40class Project: 41 def __init__ (self, path, copyright = None): 42 self.path = path 43 self.copyright = copyright 44 45class Configuration: 46 def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None, runByDefault = True, listOfGroupsToSplit = []): 47 self.name = name 48 self.glconfig = glconfig 49 self.rotation = rotation 50 self.surfacetype = surfacetype 51 self.required = required 52 self.filters = filters 53 self.expectedRuntime = runtime 54 self.runByDefault = runByDefault 55 self.listOfGroupsToSplit = listOfGroupsToSplit 56 57class Package: 58 def __init__ (self, module, configurations): 59 self.module = module 60 self.configurations = configurations 61 62class Mustpass: 63 def __init__ (self, project, version, packages): 64 self.project = project 65 self.version = version 66 self.packages = packages 67 68class Filter: 69 TYPE_INCLUDE = 0 70 TYPE_EXCLUDE = 1 71 72 def __init__ (self, type, filenames): 73 self.type = type 74 self.filenames = filenames 75 self.key = ",".join(filenames) 76 77class TestRoot: 78 def __init__ (self): 79 self.children = [] 80 81class TestGroup: 82 def __init__ (self, name): 83 self.name = name 84 self.children = [] 85 86class TestCase: 87 def __init__ (self, name): 88 self.name = name 89 self.configurations = [] 90 91def getSrcDir (mustpass): 92 return os.path.join(mustpass.project.path, mustpass.version, "src") 93 94def getModuleShorthand (module): 95 assert module.name[:5] == "dEQP-" 96 return module.name[5:].lower() 97 98def getCaseListFileName (package, configuration): 99 return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name) 100 101def getDstCaseListPath (mustpass): 102 return os.path.join(mustpass.project.path, mustpass.version) 103 104def getCommandLine (config): 105 cmdLine = "" 106 107 if config.glconfig != None: 108 cmdLine += "--deqp-gl-config-name=%s " % config.glconfig 109 110 if config.rotation != None: 111 cmdLine += "--deqp-screen-rotation=%s " % config.rotation 112 113 if config.surfacetype != None: 114 cmdLine += "--deqp-surface-type=%s " % config.surfacetype 115 116 cmdLine += "--deqp-watchdog=enable" 117 118 return cmdLine 119 120class CaseList: 121 def __init__(self, filePath, sortedLines): 122 self.filePath = filePath 123 self.sortedLines = sortedLines 124 125def readAndSortCaseList (buildCfg, generator, module): 126 build(buildCfg, generator, [module.binName]) 127 genCaseList(buildCfg, generator, module, "txt") 128 filePath = getCaseListPath(buildCfg, module, "txt") 129 with open(filePath, 'r') as first_file: 130 lines = first_file.readlines() 131 lines.sort() 132 caseList = CaseList(filePath, lines) 133 return caseList 134 135def readPatternList (filename, patternList): 136 with open(filename, 'rt') as f: 137 for line in f: 138 line = line.strip() 139 if len(line) > 0 and line[0] != '#': 140 patternList.append(line) 141 142def include (*filenames): 143 return Filter(Filter.TYPE_INCLUDE, filenames) 144 145def exclude (*filenames): 146 return Filter(Filter.TYPE_EXCLUDE, filenames) 147 148def insertXMLHeaders (mustpass, doc): 149 if mustpass.project.copyright != None: 150 doc.insert(0, ElementTree.Comment(mustpass.project.copyright)) 151 doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING)) 152 153def prettifyXML (doc): 154 uglyString = ElementTree.tostring(doc, 'utf-8') 155 reparsed = minidom.parseString(uglyString) 156 return reparsed.toprettyxml(indent='\t', encoding='utf-8') 157 158def genSpecXML (mustpass): 159 mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version) 160 insertXMLHeaders(mustpass, mustpassElem) 161 162 for package in mustpass.packages: 163 packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name) 164 165 for config in package.configurations: 166 configElem = ElementTree.SubElement(packageElem, "Configuration", 167 caseListFile = getCaseListFileName(package, config), 168 commandLine = getCommandLine(config), 169 name = config.name) 170 171 return mustpassElem 172 173def addOptionElement (parent, optionName, optionValue): 174 ElementTree.SubElement(parent, "option", name=optionName, value=optionValue) 175 176def genAndroidTestXml (mustpass): 177 RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner" 178 configElement = ElementTree.Element("configuration") 179 180 # have the deqp package installed on the device for us 181 preparerElement = ElementTree.SubElement(configElement, "target_preparer") 182 preparerElement.set("class", "com.android.tradefed.targetprep.suite.SuiteApkInstaller") 183 addOptionElement(preparerElement, "cleanup-apks", "true") 184 addOptionElement(preparerElement, "test-file-name", "com.drawelements.deqp.apk") 185 186 # Target preparer for incremental dEQP 187 preparerElement = ElementTree.SubElement(configElement, "target_preparer") 188 preparerElement.set("class", "com.android.compatibility.common.tradefed.targetprep.FilePusher") 189 addOptionElement(preparerElement, "cleanup", "true") 190 addOptionElement(preparerElement, "disable", "true") 191 addOptionElement(preparerElement, "push", "deqp-binary32->/data/local/tmp/deqp-binary32") 192 addOptionElement(preparerElement, "push", "deqp-binary64->/data/local/tmp/deqp-binary64") 193 addOptionElement(preparerElement, "push", "gles2->/data/local/tmp/gles2") 194 addOptionElement(preparerElement, "push", "gles3->/data/local/tmp/gles3") 195 addOptionElement(preparerElement, "push", "gles3-incremental-deqp.txt->/data/local/tmp/gles3-incremental-deqp.txt") 196 addOptionElement(preparerElement, "push", "gles31->/data/local/tmp/gles31") 197 addOptionElement(preparerElement, "push", "internal->/data/local/tmp/internal") 198 addOptionElement(preparerElement, "push", "vk-incremental-deqp.txt->/data/local/tmp/vk-incremental-deqp.txt") 199 addOptionElement(preparerElement, "push", "vulkan->/data/local/tmp/vulkan") 200 preparerElement = ElementTree.SubElement(configElement, "target_preparer") 201 preparerElement.set("class", "com.android.compatibility.common.tradefed.targetprep.IncrementalDeqpPreparer") 202 addOptionElement(preparerElement, "disable", "true") 203 204 # add in metadata option for component name 205 ElementTree.SubElement(configElement, "option", name="test-suite-tag", value="cts") 206 ElementTree.SubElement(configElement, "option", key="component", name="config-descriptor:metadata", value="deqp") 207 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="not_instant_app") 208 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="multi_abi") 209 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="secondary_user") 210 ElementTree.SubElement(configElement, "option", key="parameter", name="config-descriptor:metadata", value="no_foldable_states") 211 controllerElement = ElementTree.SubElement(configElement, "object") 212 controllerElement.set("class", "com.android.tradefed.testtype.suite.module.TestFailureModuleController") 213 controllerElement.set("type", "module_controller") 214 addOptionElement(controllerElement, "screenshot-on-failure", "false") 215 216 for package in mustpass.packages: 217 for config in package.configurations: 218 if not config.runByDefault: 219 continue 220 221 testElement = ElementTree.SubElement(configElement, "test") 222 testElement.set("class", RUNNER_CLASS) 223 addOptionElement(testElement, "deqp-package", package.module.name) 224 caseListFile = getCaseListFileName(package,config) 225 addOptionElement(testElement, "deqp-caselist-file", caseListFile) 226 if caseListFile.startswith("gles3"): 227 addOptionElement(testElement, "incremental-deqp-include-file", "gles3-incremental-deqp.txt") 228 elif caseListFile.startswith("vk"): 229 addOptionElement(testElement, "incremental-deqp-include-file", "vk-incremental-deqp.txt") 230 # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well. 231 if config.glconfig != None: 232 addOptionElement(testElement, "deqp-gl-config-name", config.glconfig) 233 234 if config.surfacetype != None: 235 addOptionElement(testElement, "deqp-surface-type", config.surfacetype) 236 237 if config.rotation != None: 238 addOptionElement(testElement, "deqp-screen-rotation", config.rotation) 239 240 if config.expectedRuntime != None: 241 addOptionElement(testElement, "runtime-hint", config.expectedRuntime) 242 243 if config.required: 244 addOptionElement(testElement, "deqp-config-required", "true") 245 246 insertXMLHeaders(mustpass, configElement) 247 248 return configElement 249 250class PatternSet: 251 def __init__(self): 252 self.namedPatternsTree = {} 253 self.namedPatternsDict = {} 254 self.wildcardPatternsDict = {} 255 256def readPatternSets (mustpass): 257 patternSets = {} 258 for package in mustpass.packages: 259 for cfg in package.configurations: 260 for filter in cfg.filters: 261 if not filter.key in patternSets: 262 patternList = [] 263 for filename in filter.filenames: 264 readPatternList(os.path.join(getSrcDir(mustpass), filename), patternList) 265 patternSet = PatternSet() 266 for pattern in patternList: 267 if pattern.find('*') == -1: 268 patternSet.namedPatternsDict[pattern] = 0 269 t = patternSet.namedPatternsTree 270 parts = pattern.split('.') 271 for part in parts: 272 t = t.setdefault(part, {}) 273 else: 274 # We use regex instead of fnmatch because it's faster 275 patternSet.wildcardPatternsDict[re.compile("^" + pattern.replace(".", r"\.").replace("*", ".*?") + "$")] = 0 276 patternSets[filter.key] = patternSet 277 return patternSets 278 279def genMustpassFromLists (mustpass, moduleCaseLists): 280 print("Generating mustpass '%s'" % mustpass.version) 281 patternSets = readPatternSets(mustpass) 282 283 for package in mustpass.packages: 284 currentCaseList = moduleCaseLists[package.module] 285 logging.debug("Reading " + currentCaseList.filePath) 286 287 for config in package.configurations: 288 # construct components of path to main destination file 289 mainDstFileDir = getDstCaseListPath(mustpass) 290 mainDstFileName = getCaseListFileName(package, config) 291 mainDstFilePath = os.path.join(mainDstFileDir, mainDstFileName) 292 mainGroupSubDir = mainDstFileName[:-4] 293 294 if not os.path.exists(mainDstFileDir): 295 os.makedirs(mainDstFileDir) 296 mainDstFile = open(mainDstFilePath, 'w') 297 print(mainDstFilePath) 298 output_files = {} 299 def openAndStoreFile(filePath, testFilePath, parentFile): 300 if filePath not in output_files: 301 try: 302 print(" " + filePath) 303 parentFile.write(mainGroupSubDir + "/" + testFilePath + "\n") 304 currentDir = os.path.dirname(filePath) 305 if not os.path.exists(currentDir): 306 os.makedirs(currentDir) 307 output_files[filePath] = open(filePath, 'w') 308 309 except FileNotFoundError: 310 print(f"File not found: {filePath}") 311 return output_files[filePath] 312 313 lastOutputFile = "" 314 currentOutputFile = None 315 for line in currentCaseList.sortedLines: 316 if not line.startswith("TEST: "): 317 continue 318 caseName = line.replace("TEST: ", "").strip("\n") 319 caseParts = caseName.split(".") 320 keep = True 321 # Do the includes with the complex patterns first 322 for filter in config.filters: 323 if filter.type == Filter.TYPE_INCLUDE: 324 keep = False 325 patterns = patternSets[filter.key].wildcardPatternsDict 326 for pattern in patterns.keys(): 327 keep = pattern.match(caseName) 328 if keep: 329 patterns[pattern] += 1 330 break 331 332 if not keep: 333 t = patternSets[filter.key].namedPatternsTree 334 if len(t.keys()) == 0: 335 continue 336 for part in caseParts: 337 if part in t: 338 t = t[part] 339 else: 340 t = None # Not found 341 break 342 keep = t == {} 343 if keep: 344 patternSets[filter.key].namedPatternsDict[caseName] += 1 345 346 # Do the excludes 347 if filter.type == Filter.TYPE_EXCLUDE: 348 patterns = patternSets[filter.key].wildcardPatternsDict 349 for pattern in patterns.keys(): 350 discard = pattern.match(caseName) 351 if discard: 352 patterns[pattern] += 1 353 keep = False 354 break 355 if keep: 356 t = patternSets[filter.key].namedPatternsTree 357 if len(t.keys()) == 0: 358 continue 359 for part in caseParts: 360 if part in t: 361 t = t[part] 362 else: 363 t = None # Not found 364 break 365 if t == {}: 366 patternSets[filter.key].namedPatternsDict[caseName] += 1 367 keep = False 368 if not keep: 369 break 370 if not keep: 371 continue 372 373 parts = caseName.split('.') 374 if len(config.listOfGroupsToSplit) > 0: 375 if len(parts) > 2: 376 groupName = parts[1].replace("_", "-") 377 for splitPattern in config.listOfGroupsToSplit: 378 splitParts = splitPattern.split(".") 379 if len(splitParts) > 1 and caseName.startswith(splitPattern + "."): 380 groupName = groupName + "/" + parts[2].replace("_", "-") 381 filePath = os.path.join(mainDstFileDir, mainGroupSubDir, groupName + ".txt") 382 if lastOutputFile != filePath: 383 currentOutputFile = openAndStoreFile(filePath, groupName + ".txt", mainDstFile) 384 lastOutputFile = filePath 385 currentOutputFile.write(caseName + "\n") 386 else: 387 mainDstFile.write(caseName + "\n") 388 389 # Check that all patterns have been used in the filters 390 # This check will help identifying typos and patterns becoming stale 391 for filter in config.filters: 392 if filter.type == Filter.TYPE_INCLUDE: 393 patternSet = patternSets[filter.key] 394 for pattern, usage in patternSet.namedPatternsDict.items(): 395 if usage == 0: 396 logging.warning("Case %s in file %s for module %s was never used!" % (pattern, filter.key, config.name)) 397 for pattern, usage in patternSet.wildcardPatternsDict.items(): 398 if usage == 0: 399 logging.warning("Pattern %s in file %s for module %s was never used!" % (pattern, filter.key, config.name)) 400 401 # Generate XML 402 specXML = genSpecXML(mustpass) 403 specFilename = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml") 404 405 print(" Writing spec: " + specFilename) 406 writeFile(specFilename, prettifyXML(specXML).decode()) 407 408 # TODO: Which is the best selector mechanism? 409 if (mustpass.version == "master"): 410 androidTestXML = genAndroidTestXml(mustpass) 411 androidTestFilename = os.path.join(mustpass.project.path, "AndroidTest.xml") 412 413 print(" Writing AndroidTest.xml: " + androidTestFilename) 414 writeFile(androidTestFilename, prettifyXML(androidTestXML).decode()) 415 416 print("Done!") 417 418 419def genMustpassLists (mustpassLists, generator, buildCfg): 420 moduleCaseLists = {} 421 422 # Getting case lists involves invoking build, so we want to cache the results 423 for mustpass in mustpassLists: 424 for package in mustpass.packages: 425 if not package.module in moduleCaseLists: 426 moduleCaseLists[package.module] = readAndSortCaseList(buildCfg, generator, package.module) 427 428 for mustpass in mustpassLists: 429 genMustpassFromLists(mustpass, moduleCaseLists) 430 431def parseCmdLineArgs (): 432 parser = argparse.ArgumentParser(description = "Build Android CTS mustpass", 433 formatter_class=argparse.ArgumentDefaultsHelpFormatter) 434 parser.add_argument("-b", 435 "--build-dir", 436 dest="buildDir", 437 default=DEFAULT_BUILD_DIR, 438 help="Temporary build directory") 439 parser.add_argument("-t", 440 "--build-type", 441 dest="buildType", 442 default="Debug", 443 help="Build type") 444 parser.add_argument("-c", 445 "--deqp-target", 446 dest="targetName", 447 default=DEFAULT_TARGET, 448 help="dEQP build target") 449 return parser.parse_args() 450 451def parseBuildConfigFromCmdLineArgs (): 452 args = parseCmdLineArgs() 453 return getBuildConfig(args.buildDir, args.targetName, args.buildType) 454