1e5c31af7Sopenharmony_ci# -*- coding: utf-8 -*- 2e5c31af7Sopenharmony_ci 3e5c31af7Sopenharmony_ci#------------------------------------------------------------------------- 4e5c31af7Sopenharmony_ci# drawElements Quality Program utilities 5e5c31af7Sopenharmony_ci# -------------------------------------- 6e5c31af7Sopenharmony_ci# 7e5c31af7Sopenharmony_ci# Copyright 2015 The Android Open Source Project 8e5c31af7Sopenharmony_ci# 9e5c31af7Sopenharmony_ci# Licensed under the Apache License, Version 2.0 (the "License"); 10e5c31af7Sopenharmony_ci# you may not use this file except in compliance with the License. 11e5c31af7Sopenharmony_ci# You may obtain a copy of the License at 12e5c31af7Sopenharmony_ci# 13e5c31af7Sopenharmony_ci# http://www.apache.org/licenses/LICENSE-2.0 14e5c31af7Sopenharmony_ci# 15e5c31af7Sopenharmony_ci# Unless required by applicable law or agreed to in writing, software 16e5c31af7Sopenharmony_ci# distributed under the License is distributed on an "AS IS" BASIS, 17e5c31af7Sopenharmony_ci# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18e5c31af7Sopenharmony_ci# See the License for the specific language governing permissions and 19e5c31af7Sopenharmony_ci# limitations under the License. 20e5c31af7Sopenharmony_ci# 21e5c31af7Sopenharmony_ci#------------------------------------------------------------------------- 22e5c31af7Sopenharmony_ci 23e5c31af7Sopenharmony_cifrom ctsbuild.common import * 24e5c31af7Sopenharmony_cifrom ctsbuild.config import * 25e5c31af7Sopenharmony_cifrom ctsbuild.build import * 26e5c31af7Sopenharmony_ci 27e5c31af7Sopenharmony_ciimport os 28e5c31af7Sopenharmony_ciimport sys 29e5c31af7Sopenharmony_ciimport string 30e5c31af7Sopenharmony_ciimport socket 31e5c31af7Sopenharmony_ciimport fnmatch 32e5c31af7Sopenharmony_cifrom datetime import datetime 33e5c31af7Sopenharmony_ci 34e5c31af7Sopenharmony_ciBASE_NIGHTLY_DIR = os.path.normpath(os.path.join(DEQP_DIR, "..", "deqp-nightly")) 35e5c31af7Sopenharmony_ciBASE_BUILD_DIR = os.path.join(BASE_NIGHTLY_DIR, "build") 36e5c31af7Sopenharmony_ciBASE_LOGS_DIR = os.path.join(BASE_NIGHTLY_DIR, "logs") 37e5c31af7Sopenharmony_ciBASE_REFS_DIR = os.path.join(BASE_NIGHTLY_DIR, "refs") 38e5c31af7Sopenharmony_ci 39e5c31af7Sopenharmony_ciEXECUTOR_PATH = "executor/executor" 40e5c31af7Sopenharmony_ciLOG_TO_CSV_PATH = "executor/testlog-to-csv" 41e5c31af7Sopenharmony_ciEXECSERVER_PATH = "execserver/execserver" 42e5c31af7Sopenharmony_ci 43e5c31af7Sopenharmony_ciCASELIST_PATH = os.path.join(DEQP_DIR, "Candy", "Data") 44e5c31af7Sopenharmony_ci 45e5c31af7Sopenharmony_ciCOMPARE_NUM_RESULTS = 4 46e5c31af7Sopenharmony_ciCOMPARE_REPORT_NAME = "nightly-report.html" 47e5c31af7Sopenharmony_ci 48e5c31af7Sopenharmony_ciCOMPARE_REPORT_TMPL = ''' 49e5c31af7Sopenharmony_ci<html> 50e5c31af7Sopenharmony_ci<head> 51e5c31af7Sopenharmony_ci<title>${TITLE}</title> 52e5c31af7Sopenharmony_ci<style type="text/css"> 53e5c31af7Sopenharmony_ci<!-- 54e5c31af7Sopenharmony_cibody { font: serif; font-size: 1em; } 55e5c31af7Sopenharmony_citable { border-spacing: 0; border-collapse: collapse; } 56e5c31af7Sopenharmony_citd { border-width: 1px; border-style: solid; border-color: #808080; } 57e5c31af7Sopenharmony_ci.Header { font-weight: bold; font-size: 1em; border-style: none; } 58e5c31af7Sopenharmony_ci.CasePath { } 59e5c31af7Sopenharmony_ci.Pass { background: #80ff80; } 60e5c31af7Sopenharmony_ci.Fail { background: #ff4040; } 61e5c31af7Sopenharmony_ci.QualityWarning { background: #ffff00; } 62e5c31af7Sopenharmony_ci.CompabilityWarning { background: #ffff00; } 63e5c31af7Sopenharmony_ci.Pending { background: #808080; } 64e5c31af7Sopenharmony_ci.Running { background: #d3d3d3; } 65e5c31af7Sopenharmony_ci.NotSupported { background: #ff69b4; } 66e5c31af7Sopenharmony_ci.ResourceError { background: #ff4040; } 67e5c31af7Sopenharmony_ci.InternalError { background: #ff1493; } 68e5c31af7Sopenharmony_ci.Canceled { background: #808080; } 69e5c31af7Sopenharmony_ci.Crash { background: #ffa500; } 70e5c31af7Sopenharmony_ci.Timeout { background: #ffa500; } 71e5c31af7Sopenharmony_ci.Disabled { background: #808080; } 72e5c31af7Sopenharmony_ci.Missing { background: #808080; } 73e5c31af7Sopenharmony_ci.Ignored { opacity: 0.5; } 74e5c31af7Sopenharmony_ci--> 75e5c31af7Sopenharmony_ci</style> 76e5c31af7Sopenharmony_ci</head> 77e5c31af7Sopenharmony_ci<body> 78e5c31af7Sopenharmony_ci<h1>${TITLE}</h1> 79e5c31af7Sopenharmony_ci<table> 80e5c31af7Sopenharmony_ci${RESULTS} 81e5c31af7Sopenharmony_ci</table> 82e5c31af7Sopenharmony_ci</body> 83e5c31af7Sopenharmony_ci</html> 84e5c31af7Sopenharmony_ci''' 85e5c31af7Sopenharmony_ci 86e5c31af7Sopenharmony_ciclass NightlyRunConfig: 87e5c31af7Sopenharmony_ci def __init__(self, name, buildConfig, generator, binaryName, testset, args = [], exclude = [], ignore = []): 88e5c31af7Sopenharmony_ci self.name = name 89e5c31af7Sopenharmony_ci self.buildConfig = buildConfig 90e5c31af7Sopenharmony_ci self.generator = generator 91e5c31af7Sopenharmony_ci self.binaryName = binaryName 92e5c31af7Sopenharmony_ci self.testset = testset 93e5c31af7Sopenharmony_ci self.args = args 94e5c31af7Sopenharmony_ci self.exclude = exclude 95e5c31af7Sopenharmony_ci self.ignore = ignore 96e5c31af7Sopenharmony_ci 97e5c31af7Sopenharmony_ci def getBinaryPath(self, basePath): 98e5c31af7Sopenharmony_ci return os.path.join(self.buildConfig.getBuildDir(), self.generator.getBinaryPath(self.buildConfig.getBuildType(), basePath)) 99e5c31af7Sopenharmony_ci 100e5c31af7Sopenharmony_ciclass NightlyBuildConfig(BuildConfig): 101e5c31af7Sopenharmony_ci def __init__(self, name, buildType, args): 102e5c31af7Sopenharmony_ci BuildConfig.__init__(self, os.path.join(BASE_BUILD_DIR, name), buildType, args) 103e5c31af7Sopenharmony_ci 104e5c31af7Sopenharmony_ciclass TestCaseResult: 105e5c31af7Sopenharmony_ci def __init__ (self, name, statusCode): 106e5c31af7Sopenharmony_ci self.name = name 107e5c31af7Sopenharmony_ci self.statusCode = statusCode 108e5c31af7Sopenharmony_ci 109e5c31af7Sopenharmony_ciclass MultiResult: 110e5c31af7Sopenharmony_ci def __init__ (self, name, statusCodes): 111e5c31af7Sopenharmony_ci self.name = name 112e5c31af7Sopenharmony_ci self.statusCodes = statusCodes 113e5c31af7Sopenharmony_ci 114e5c31af7Sopenharmony_ciclass BatchResult: 115e5c31af7Sopenharmony_ci def __init__ (self, name): 116e5c31af7Sopenharmony_ci self.name = name 117e5c31af7Sopenharmony_ci self.results = [] 118e5c31af7Sopenharmony_ci 119e5c31af7Sopenharmony_cidef parseResultCsv (data): 120e5c31af7Sopenharmony_ci lines = data.splitlines()[1:] 121e5c31af7Sopenharmony_ci results = [] 122e5c31af7Sopenharmony_ci 123e5c31af7Sopenharmony_ci for line in lines: 124e5c31af7Sopenharmony_ci items = line.split(",") 125e5c31af7Sopenharmony_ci results.append(TestCaseResult(items[0], items[1])) 126e5c31af7Sopenharmony_ci 127e5c31af7Sopenharmony_ci return results 128e5c31af7Sopenharmony_ci 129e5c31af7Sopenharmony_cidef readTestCaseResultsFromCSV (filename): 130e5c31af7Sopenharmony_ci return parseResultCsv(readFile(filename)) 131e5c31af7Sopenharmony_ci 132e5c31af7Sopenharmony_cidef readBatchResultFromCSV (filename, batchResultName = None): 133e5c31af7Sopenharmony_ci batchResult = BatchResult(batchResultName if batchResultName != None else os.path.basename(filename)) 134e5c31af7Sopenharmony_ci batchResult.results = readTestCaseResultsFromCSV(filename) 135e5c31af7Sopenharmony_ci return batchResult 136e5c31af7Sopenharmony_ci 137e5c31af7Sopenharmony_cidef getResultTimestamp (): 138e5c31af7Sopenharmony_ci return datetime.now().strftime("%Y-%m-%d-%H-%M") 139e5c31af7Sopenharmony_ci 140e5c31af7Sopenharmony_cidef getCompareFilenames (logsDir): 141e5c31af7Sopenharmony_ci files = [] 142e5c31af7Sopenharmony_ci for file in os.listdir(logsDir): 143e5c31af7Sopenharmony_ci fullPath = os.path.join(logsDir, file) 144e5c31af7Sopenharmony_ci if os.path.isfile(fullPath) and fnmatch.fnmatch(file, "*.csv"): 145e5c31af7Sopenharmony_ci files.append(fullPath) 146e5c31af7Sopenharmony_ci files.sort() 147e5c31af7Sopenharmony_ci 148e5c31af7Sopenharmony_ci return files[-COMPARE_NUM_RESULTS:] 149e5c31af7Sopenharmony_ci 150e5c31af7Sopenharmony_cidef parseAsCSV (logPath, config): 151e5c31af7Sopenharmony_ci args = [config.getBinaryPath(LOG_TO_CSV_PATH), "--mode=all", "--format=csv", logPath] 152e5c31af7Sopenharmony_ci proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 153e5c31af7Sopenharmony_ci out, err = proc.communicate() 154e5c31af7Sopenharmony_ci return out 155e5c31af7Sopenharmony_ci 156e5c31af7Sopenharmony_cidef computeUnifiedTestCaseList (batchResults): 157e5c31af7Sopenharmony_ci caseList = [] 158e5c31af7Sopenharmony_ci caseSet = set() 159e5c31af7Sopenharmony_ci 160e5c31af7Sopenharmony_ci for batchResult in batchResults: 161e5c31af7Sopenharmony_ci for result in batchResult.results: 162e5c31af7Sopenharmony_ci if not result.name in caseSet: 163e5c31af7Sopenharmony_ci caseList.append(result.name) 164e5c31af7Sopenharmony_ci caseSet.add(result.name) 165e5c31af7Sopenharmony_ci 166e5c31af7Sopenharmony_ci return caseList 167e5c31af7Sopenharmony_ci 168e5c31af7Sopenharmony_cidef computeUnifiedResults (batchResults): 169e5c31af7Sopenharmony_ci 170e5c31af7Sopenharmony_ci def genResultMap (batchResult): 171e5c31af7Sopenharmony_ci resMap = {} 172e5c31af7Sopenharmony_ci for result in batchResult.results: 173e5c31af7Sopenharmony_ci resMap[result.name] = result 174e5c31af7Sopenharmony_ci return resMap 175e5c31af7Sopenharmony_ci 176e5c31af7Sopenharmony_ci resultMap = [genResultMap(r) for r in batchResults] 177e5c31af7Sopenharmony_ci caseList = computeUnifiedTestCaseList(batchResults) 178e5c31af7Sopenharmony_ci results = [] 179e5c31af7Sopenharmony_ci 180e5c31af7Sopenharmony_ci for caseName in caseList: 181e5c31af7Sopenharmony_ci statusCodes = [] 182e5c31af7Sopenharmony_ci 183e5c31af7Sopenharmony_ci for i in range(0, len(batchResults)): 184e5c31af7Sopenharmony_ci result = resultMap[i][caseName] if caseName in resultMap[i] else None 185e5c31af7Sopenharmony_ci statusCode = result.statusCode if result != None else 'Missing' 186e5c31af7Sopenharmony_ci statusCodes.append(statusCode) 187e5c31af7Sopenharmony_ci 188e5c31af7Sopenharmony_ci results.append(MultiResult(caseName, statusCodes)) 189e5c31af7Sopenharmony_ci 190e5c31af7Sopenharmony_ci return results 191e5c31af7Sopenharmony_ci 192e5c31af7Sopenharmony_cidef allStatusCodesEqual (result): 193e5c31af7Sopenharmony_ci firstCode = result.statusCodes[0] 194e5c31af7Sopenharmony_ci for i in range(1, len(result.statusCodes)): 195e5c31af7Sopenharmony_ci if result.statusCodes[i] != firstCode: 196e5c31af7Sopenharmony_ci return False 197e5c31af7Sopenharmony_ci return True 198e5c31af7Sopenharmony_ci 199e5c31af7Sopenharmony_cidef computeDiffResults (unifiedResults): 200e5c31af7Sopenharmony_ci diff = [] 201e5c31af7Sopenharmony_ci for result in unifiedResults: 202e5c31af7Sopenharmony_ci if not allStatusCodesEqual(result): 203e5c31af7Sopenharmony_ci diff.append(result) 204e5c31af7Sopenharmony_ci return diff 205e5c31af7Sopenharmony_ci 206e5c31af7Sopenharmony_cidef genCompareReport (batchResults, title, ignoreCases): 207e5c31af7Sopenharmony_ci class TableRow: 208e5c31af7Sopenharmony_ci def __init__ (self, testCaseName, innerHTML): 209e5c31af7Sopenharmony_ci self.testCaseName = testCaseName 210e5c31af7Sopenharmony_ci self.innerHTML = innerHTML 211e5c31af7Sopenharmony_ci 212e5c31af7Sopenharmony_ci unifiedResults = computeUnifiedResults(batchResults) 213e5c31af7Sopenharmony_ci diffResults = computeDiffResults(unifiedResults) 214e5c31af7Sopenharmony_ci rows = [] 215e5c31af7Sopenharmony_ci 216e5c31af7Sopenharmony_ci # header 217e5c31af7Sopenharmony_ci headerCol = '<td class="Header">Test case</td>\n' 218e5c31af7Sopenharmony_ci for batchResult in batchResults: 219e5c31af7Sopenharmony_ci headerCol += '<td class="Header">%s</td>\n' % batchResult.name 220e5c31af7Sopenharmony_ci rows.append(TableRow(None, headerCol)) 221e5c31af7Sopenharmony_ci 222e5c31af7Sopenharmony_ci # results 223e5c31af7Sopenharmony_ci for result in diffResults: 224e5c31af7Sopenharmony_ci col = '<td class="CasePath">%s</td>\n' % result.name 225e5c31af7Sopenharmony_ci for statusCode in result.statusCodes: 226e5c31af7Sopenharmony_ci col += '<td class="%s">%s</td>\n' % (statusCode, statusCode) 227e5c31af7Sopenharmony_ci 228e5c31af7Sopenharmony_ci rows.append(TableRow(result.name, col)) 229e5c31af7Sopenharmony_ci 230e5c31af7Sopenharmony_ci tableStr = "" 231e5c31af7Sopenharmony_ci for row in rows: 232e5c31af7Sopenharmony_ci if row.testCaseName is not None and matchesAnyPattern(row.testCaseName, ignoreCases): 233e5c31af7Sopenharmony_ci tableStr += '<tr class="Ignored">\n%s</tr>\n' % row.innerHTML 234e5c31af7Sopenharmony_ci else: 235e5c31af7Sopenharmony_ci tableStr += '<tr>\n%s</tr>\n' % row.innerHTML 236e5c31af7Sopenharmony_ci 237e5c31af7Sopenharmony_ci html = COMPARE_REPORT_TMPL 238e5c31af7Sopenharmony_ci html = html.replace("${TITLE}", title) 239e5c31af7Sopenharmony_ci html = html.replace("${RESULTS}", tableStr) 240e5c31af7Sopenharmony_ci 241e5c31af7Sopenharmony_ci return html 242e5c31af7Sopenharmony_ci 243e5c31af7Sopenharmony_cidef matchesAnyPattern (name, patterns): 244e5c31af7Sopenharmony_ci for pattern in patterns: 245e5c31af7Sopenharmony_ci if fnmatch.fnmatch(name, pattern): 246e5c31af7Sopenharmony_ci return True 247e5c31af7Sopenharmony_ci return False 248e5c31af7Sopenharmony_ci 249e5c31af7Sopenharmony_cidef statusCodesMatch (refResult, resResult): 250e5c31af7Sopenharmony_ci return refResult == 'Missing' or resResult == 'Missing' or refResult == resResult 251e5c31af7Sopenharmony_ci 252e5c31af7Sopenharmony_cidef compareBatchResults (referenceBatch, resultBatch, ignoreCases): 253e5c31af7Sopenharmony_ci unifiedResults = computeUnifiedResults([referenceBatch, resultBatch]) 254e5c31af7Sopenharmony_ci failedCases = [] 255e5c31af7Sopenharmony_ci 256e5c31af7Sopenharmony_ci for result in unifiedResults: 257e5c31af7Sopenharmony_ci if not matchesAnyPattern(result.name, ignoreCases): 258e5c31af7Sopenharmony_ci refResult = result.statusCodes[0] 259e5c31af7Sopenharmony_ci resResult = result.statusCodes[1] 260e5c31af7Sopenharmony_ci 261e5c31af7Sopenharmony_ci if not statusCodesMatch(refResult, resResult): 262e5c31af7Sopenharmony_ci failedCases.append(result) 263e5c31af7Sopenharmony_ci 264e5c31af7Sopenharmony_ci return failedCases 265e5c31af7Sopenharmony_ci 266e5c31af7Sopenharmony_cidef getUnusedPort (): 267e5c31af7Sopenharmony_ci # \note Not 100%-proof method as other apps may grab this port before we launch execserver 268e5c31af7Sopenharmony_ci s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 269e5c31af7Sopenharmony_ci s.bind(('localhost', 0)) 270e5c31af7Sopenharmony_ci addr, port = s.getsockname() 271e5c31af7Sopenharmony_ci s.close() 272e5c31af7Sopenharmony_ci return port 273e5c31af7Sopenharmony_ci 274e5c31af7Sopenharmony_cidef runNightly (config): 275e5c31af7Sopenharmony_ci build(config.buildConfig, config.generator) 276e5c31af7Sopenharmony_ci 277e5c31af7Sopenharmony_ci # Run parameters 278e5c31af7Sopenharmony_ci timestamp = getResultTimestamp() 279e5c31af7Sopenharmony_ci logDir = os.path.join(BASE_LOGS_DIR, config.name) 280e5c31af7Sopenharmony_ci testLogPath = os.path.join(logDir, timestamp + ".qpa") 281e5c31af7Sopenharmony_ci infoLogPath = os.path.join(logDir, timestamp + ".txt") 282e5c31af7Sopenharmony_ci csvLogPath = os.path.join(logDir, timestamp + ".csv") 283e5c31af7Sopenharmony_ci compareLogPath = os.path.join(BASE_REFS_DIR, config.name + ".csv") 284e5c31af7Sopenharmony_ci port = getUnusedPort() 285e5c31af7Sopenharmony_ci 286e5c31af7Sopenharmony_ci if not os.path.exists(logDir): 287e5c31af7Sopenharmony_ci os.makedirs(logDir) 288e5c31af7Sopenharmony_ci 289e5c31af7Sopenharmony_ci if os.path.exists(testLogPath) or os.path.exists(infoLogPath): 290e5c31af7Sopenharmony_ci raise Exception("Result '%s' already exists", timestamp) 291e5c31af7Sopenharmony_ci 292e5c31af7Sopenharmony_ci # Paths, etc. 293e5c31af7Sopenharmony_ci binaryName = config.generator.getBinaryPath(config.buildConfig.getBuildType(), os.path.basename(config.binaryName)) 294e5c31af7Sopenharmony_ci workingDir = os.path.join(config.buildConfig.getBuildDir(), os.path.dirname(config.binaryName)) 295e5c31af7Sopenharmony_ci 296e5c31af7Sopenharmony_ci execArgs = [ 297e5c31af7Sopenharmony_ci config.getBinaryPath(EXECUTOR_PATH), 298e5c31af7Sopenharmony_ci '--start-server=%s' % config.getBinaryPath(EXECSERVER_PATH), 299e5c31af7Sopenharmony_ci '--port=%d' % port, 300e5c31af7Sopenharmony_ci '--binaryname=%s' % binaryName, 301e5c31af7Sopenharmony_ci '--cmdline=%s' % string.join([shellquote(arg) for arg in config.args], " "), 302e5c31af7Sopenharmony_ci '--workdir=%s' % workingDir, 303e5c31af7Sopenharmony_ci '--caselistdir=%s' % CASELIST_PATH, 304e5c31af7Sopenharmony_ci '--testset=%s' % string.join(config.testset, ","), 305e5c31af7Sopenharmony_ci '--out=%s' % testLogPath, 306e5c31af7Sopenharmony_ci '--info=%s' % infoLogPath, 307e5c31af7Sopenharmony_ci '--summary=no' 308e5c31af7Sopenharmony_ci ] 309e5c31af7Sopenharmony_ci 310e5c31af7Sopenharmony_ci if len(config.exclude) > 0: 311e5c31af7Sopenharmony_ci execArgs += ['--exclude=%s' % string.join(config.exclude, ",")] 312e5c31af7Sopenharmony_ci 313e5c31af7Sopenharmony_ci execute(execArgs) 314e5c31af7Sopenharmony_ci 315e5c31af7Sopenharmony_ci # Translate to CSV for comparison purposes 316e5c31af7Sopenharmony_ci lastResultCsv = parseAsCSV(testLogPath, config) 317e5c31af7Sopenharmony_ci writeFile(csvLogPath, lastResultCsv) 318e5c31af7Sopenharmony_ci 319e5c31af7Sopenharmony_ci if os.path.exists(compareLogPath): 320e5c31af7Sopenharmony_ci refBatchResult = readBatchResultFromCSV(compareLogPath, "reference") 321e5c31af7Sopenharmony_ci else: 322e5c31af7Sopenharmony_ci refBatchResult = None 323e5c31af7Sopenharmony_ci 324e5c31af7Sopenharmony_ci # Generate comparison report 325e5c31af7Sopenharmony_ci compareFilenames = getCompareFilenames(logDir) 326e5c31af7Sopenharmony_ci batchResults = [readBatchResultFromCSV(filename) for filename in compareFilenames] 327e5c31af7Sopenharmony_ci 328e5c31af7Sopenharmony_ci if refBatchResult != None: 329e5c31af7Sopenharmony_ci batchResults = [refBatchResult] + batchResults 330e5c31af7Sopenharmony_ci 331e5c31af7Sopenharmony_ci writeFile(COMPARE_REPORT_NAME, genCompareReport(batchResults, config.name, config.ignore)) 332e5c31af7Sopenharmony_ci print("Comparison report written to %s" % COMPARE_REPORT_NAME) 333e5c31af7Sopenharmony_ci 334e5c31af7Sopenharmony_ci # Compare to reference 335e5c31af7Sopenharmony_ci if refBatchResult != None: 336e5c31af7Sopenharmony_ci curBatchResult = BatchResult("current") 337e5c31af7Sopenharmony_ci curBatchResult.results = parseResultCsv(lastResultCsv) 338e5c31af7Sopenharmony_ci failedCases = compareBatchResults(refBatchResult, curBatchResult, config.ignore) 339e5c31af7Sopenharmony_ci 340e5c31af7Sopenharmony_ci print("") 341e5c31af7Sopenharmony_ci for result in failedCases: 342e5c31af7Sopenharmony_ci print("MISMATCH: %s: expected %s, got %s" % (result.name, result.statusCodes[0], result.statusCodes[1])) 343e5c31af7Sopenharmony_ci 344e5c31af7Sopenharmony_ci print("") 345e5c31af7Sopenharmony_ci print("%d / %d cases passed, run %s" % (len(curBatchResult.results)-len(failedCases), len(curBatchResult.results), "FAILED" if len(failedCases) > 0 else "passed")) 346e5c31af7Sopenharmony_ci 347e5c31af7Sopenharmony_ci if len(failedCases) > 0: 348e5c31af7Sopenharmony_ci return False 349e5c31af7Sopenharmony_ci 350e5c31af7Sopenharmony_ci return True 351e5c31af7Sopenharmony_ci 352e5c31af7Sopenharmony_ci# Configurations 353e5c31af7Sopenharmony_ci 354e5c31af7Sopenharmony_ciDEFAULT_WIN32_GENERATOR = ANY_VS_X32_GENERATOR 355e5c31af7Sopenharmony_ciDEFAULT_WIN64_GENERATOR = ANY_VS_X64_GENERATOR 356e5c31af7Sopenharmony_ci 357e5c31af7Sopenharmony_ciWGL_X64_RELEASE_BUILD_CFG = NightlyBuildConfig("wgl_x64_release", "Release", ['-DDEQP_TARGET=win32_wgl']) 358e5c31af7Sopenharmony_ciARM_GLES3_EMU_X32_RELEASE_BUILD_CFG = NightlyBuildConfig("arm_gles3_emu_release", "Release", ['-DDEQP_TARGET=arm_gles3_emu']) 359e5c31af7Sopenharmony_ci 360e5c31af7Sopenharmony_ciBASE_ARGS = ['--deqp-visibility=hidden', '--deqp-watchdog=enable', '--deqp-crashhandler=enable'] 361e5c31af7Sopenharmony_ci 362e5c31af7Sopenharmony_ciCONFIGS = [ 363e5c31af7Sopenharmony_ci NightlyRunConfig( 364e5c31af7Sopenharmony_ci name = "wgl_x64_release_gles2", 365e5c31af7Sopenharmony_ci buildConfig = WGL_X64_RELEASE_BUILD_CFG, 366e5c31af7Sopenharmony_ci generator = DEFAULT_WIN64_GENERATOR, 367e5c31af7Sopenharmony_ci binaryName = "modules/gles2/deqp-gles2", 368e5c31af7Sopenharmony_ci args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 369e5c31af7Sopenharmony_ci testset = ["dEQP-GLES2.info.*", "dEQP-GLES2.functional.*", "dEQP-GLES2.usecases.*"], 370e5c31af7Sopenharmony_ci exclude = [ 371e5c31af7Sopenharmony_ci "dEQP-GLES2.functional.shaders.loops.*while*unconditional_continue*", 372e5c31af7Sopenharmony_ci "dEQP-GLES2.functional.shaders.loops.*while*only_continue*", 373e5c31af7Sopenharmony_ci "dEQP-GLES2.functional.shaders.loops.*while*double_continue*", 374e5c31af7Sopenharmony_ci ], 375e5c31af7Sopenharmony_ci ignore = [] 376e5c31af7Sopenharmony_ci ), 377e5c31af7Sopenharmony_ci NightlyRunConfig( 378e5c31af7Sopenharmony_ci name = "wgl_x64_release_gles3", 379e5c31af7Sopenharmony_ci buildConfig = WGL_X64_RELEASE_BUILD_CFG, 380e5c31af7Sopenharmony_ci generator = DEFAULT_WIN64_GENERATOR, 381e5c31af7Sopenharmony_ci binaryName = "modules/gles3/deqp-gles3", 382e5c31af7Sopenharmony_ci args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 383e5c31af7Sopenharmony_ci testset = ["dEQP-GLES3.info.*", "dEQP-GLES3.functional.*", "dEQP-GLES3.usecases.*"], 384e5c31af7Sopenharmony_ci exclude = [ 385e5c31af7Sopenharmony_ci "dEQP-GLES3.functional.shaders.loops.*while*unconditional_continue*", 386e5c31af7Sopenharmony_ci "dEQP-GLES3.functional.shaders.loops.*while*only_continue*", 387e5c31af7Sopenharmony_ci "dEQP-GLES3.functional.shaders.loops.*while*double_continue*", 388e5c31af7Sopenharmony_ci ], 389e5c31af7Sopenharmony_ci ignore = [ 390e5c31af7Sopenharmony_ci "dEQP-GLES3.functional.transform_feedback.*", 391e5c31af7Sopenharmony_ci "dEQP-GLES3.functional.occlusion_query.*", 392e5c31af7Sopenharmony_ci "dEQP-GLES3.functional.lifetime.*", 393e5c31af7Sopenharmony_ci "dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_ops", 394e5c31af7Sopenharmony_ci ] 395e5c31af7Sopenharmony_ci ), 396e5c31af7Sopenharmony_ci NightlyRunConfig( 397e5c31af7Sopenharmony_ci name = "wgl_x64_release_gles31", 398e5c31af7Sopenharmony_ci buildConfig = WGL_X64_RELEASE_BUILD_CFG, 399e5c31af7Sopenharmony_ci generator = DEFAULT_WIN64_GENERATOR, 400e5c31af7Sopenharmony_ci binaryName = "modules/gles31/deqp-gles31", 401e5c31af7Sopenharmony_ci args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 402e5c31af7Sopenharmony_ci testset = ["dEQP-GLES31.*"], 403e5c31af7Sopenharmony_ci exclude = [], 404e5c31af7Sopenharmony_ci ignore = [ 405e5c31af7Sopenharmony_ci "dEQP-GLES31.functional.draw_indirect.negative.command_bad_alignment_3", 406e5c31af7Sopenharmony_ci "dEQP-GLES31.functional.draw_indirect.negative.command_offset_not_in_buffer", 407e5c31af7Sopenharmony_ci "dEQP-GLES31.functional.vertex_attribute_binding.negative.bind_vertex_buffer_negative_offset", 408e5c31af7Sopenharmony_ci "dEQP-GLES31.functional.ssbo.layout.single_basic_type.packed.mediump_uint", 409e5c31af7Sopenharmony_ci "dEQP-GLES31.functional.blend_equation_advanced.basic.*", 410e5c31af7Sopenharmony_ci "dEQP-GLES31.functional.blend_equation_advanced.srgb.*", 411e5c31af7Sopenharmony_ci "dEQP-GLES31.functional.blend_equation_advanced.barrier.*", 412e5c31af7Sopenharmony_ci "dEQP-GLES31.functional.uniform_location.*", 413e5c31af7Sopenharmony_ci "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_framebuffer_attachment_parameteriv", 414e5c31af7Sopenharmony_ci "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_renderbuffer_parameteriv", 415e5c31af7Sopenharmony_ci "dEQP-GLES31.functional.debug.error_filters.case_0", 416e5c31af7Sopenharmony_ci "dEQP-GLES31.functional.debug.error_filters.case_2", 417e5c31af7Sopenharmony_ci ] 418e5c31af7Sopenharmony_ci ), 419e5c31af7Sopenharmony_ci NightlyRunConfig( 420e5c31af7Sopenharmony_ci name = "wgl_x64_release_gl3", 421e5c31af7Sopenharmony_ci buildConfig = WGL_X64_RELEASE_BUILD_CFG, 422e5c31af7Sopenharmony_ci generator = DEFAULT_WIN64_GENERATOR, 423e5c31af7Sopenharmony_ci binaryName = "modules/gl3/deqp-gl3", 424e5c31af7Sopenharmony_ci args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, 425e5c31af7Sopenharmony_ci testset = ["dEQP-GL3.info.*", "dEQP-GL3.functional.*"], 426e5c31af7Sopenharmony_ci exclude = [ 427e5c31af7Sopenharmony_ci "dEQP-GL3.functional.shaders.loops.*while*unconditional_continue*", 428e5c31af7Sopenharmony_ci "dEQP-GL3.functional.shaders.loops.*while*only_continue*", 429e5c31af7Sopenharmony_ci "dEQP-GL3.functional.shaders.loops.*while*double_continue*", 430e5c31af7Sopenharmony_ci ], 431e5c31af7Sopenharmony_ci ignore = [ 432e5c31af7Sopenharmony_ci "dEQP-GL3.functional.transform_feedback.*" 433e5c31af7Sopenharmony_ci ] 434e5c31af7Sopenharmony_ci ), 435e5c31af7Sopenharmony_ci NightlyRunConfig( 436e5c31af7Sopenharmony_ci name = "arm_gles3_emu_x32_egl", 437e5c31af7Sopenharmony_ci buildConfig = ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG, 438e5c31af7Sopenharmony_ci generator = DEFAULT_WIN32_GENERATOR, 439e5c31af7Sopenharmony_ci binaryName = "modules/egl/deqp-egl", 440e5c31af7Sopenharmony_ci args = BASE_ARGS, 441e5c31af7Sopenharmony_ci testset = ["dEQP-EGL.info.*", "dEQP-EGL.functional.*"], 442e5c31af7Sopenharmony_ci exclude = [ 443e5c31af7Sopenharmony_ci "dEQP-EGL.functional.sharing.gles2.multithread.*", 444e5c31af7Sopenharmony_ci "dEQP-EGL.functional.multithread.*", 445e5c31af7Sopenharmony_ci ], 446e5c31af7Sopenharmony_ci ignore = [] 447e5c31af7Sopenharmony_ci ), 448e5c31af7Sopenharmony_ci NightlyRunConfig( 449e5c31af7Sopenharmony_ci name = "opencl_x64_release", 450e5c31af7Sopenharmony_ci buildConfig = NightlyBuildConfig("opencl_x64_release", "Release", ['-DDEQP_TARGET=opencl_icd']), 451e5c31af7Sopenharmony_ci generator = DEFAULT_WIN64_GENERATOR, 452e5c31af7Sopenharmony_ci binaryName = "modules/opencl/deqp-opencl", 453e5c31af7Sopenharmony_ci args = ['--deqp-cl-platform-id=2 --deqp-cl-device-ids=1'] + BASE_ARGS, 454e5c31af7Sopenharmony_ci testset = ["dEQP-CL.*"], 455e5c31af7Sopenharmony_ci exclude = ["dEQP-CL.performance.*", "dEQP-CL.robustness.*", "dEQP-CL.stress.memory.*"], 456e5c31af7Sopenharmony_ci ignore = [ 457e5c31af7Sopenharmony_ci "dEQP-CL.scheduler.random.*", 458e5c31af7Sopenharmony_ci "dEQP-CL.language.set_kernel_arg.random_structs.*", 459e5c31af7Sopenharmony_ci "dEQP-CL.language.builtin_function.work_item.invalid_get_global_offset", 460e5c31af7Sopenharmony_ci "dEQP-CL.language.call_function.arguments.random_structs.*", 461e5c31af7Sopenharmony_ci "dEQP-CL.language.call_kernel.random_structs.*", 462e5c31af7Sopenharmony_ci "dEQP-CL.language.inf_nan.nan.frexp.float", 463e5c31af7Sopenharmony_ci "dEQP-CL.language.inf_nan.nan.lgamma_r.float", 464e5c31af7Sopenharmony_ci "dEQP-CL.language.inf_nan.nan.modf.float", 465e5c31af7Sopenharmony_ci "dEQP-CL.language.inf_nan.nan.sqrt.float", 466e5c31af7Sopenharmony_ci "dEQP-CL.api.multithread.*", 467e5c31af7Sopenharmony_ci "dEQP-CL.api.callback.random.nested.*", 468e5c31af7Sopenharmony_ci "dEQP-CL.api.memory_migration.out_of_order_host.image2d.single_device_kernel_migrate_validate_abb", 469e5c31af7Sopenharmony_ci "dEQP-CL.api.memory_migration.out_of_order.image2d.single_device_kernel_migrate_kernel_validate_abbb", 470e5c31af7Sopenharmony_ci "dEQP-CL.image.addressing_filtering12.1d_array.*", 471e5c31af7Sopenharmony_ci "dEQP-CL.image.addressing_filtering12.2d_array.*" 472e5c31af7Sopenharmony_ci ] 473e5c31af7Sopenharmony_ci ) 474e5c31af7Sopenharmony_ci] 475e5c31af7Sopenharmony_ci 476e5c31af7Sopenharmony_ciif __name__ == "__main__": 477e5c31af7Sopenharmony_ci config = None 478e5c31af7Sopenharmony_ci 479e5c31af7Sopenharmony_ci if len(sys.argv) == 2: 480e5c31af7Sopenharmony_ci cfgName = sys.argv[1] 481e5c31af7Sopenharmony_ci for curCfg in CONFIGS: 482e5c31af7Sopenharmony_ci if curCfg.name == cfgName: 483e5c31af7Sopenharmony_ci config = curCfg 484e5c31af7Sopenharmony_ci break 485e5c31af7Sopenharmony_ci 486e5c31af7Sopenharmony_ci if config != None: 487e5c31af7Sopenharmony_ci isOk = runNightly(config) 488e5c31af7Sopenharmony_ci if not isOk: 489e5c31af7Sopenharmony_ci sys.exit(-1) 490e5c31af7Sopenharmony_ci else: 491e5c31af7Sopenharmony_ci print("%s: [config]" % sys.argv[0]) 492e5c31af7Sopenharmony_ci print("") 493e5c31af7Sopenharmony_ci print(" Available configs:") 494e5c31af7Sopenharmony_ci for config in CONFIGS: 495e5c31af7Sopenharmony_ci print(" %s" % config.name) 496e5c31af7Sopenharmony_ci sys.exit(-1) 497