xref: /third_party/vk-gl-cts/scripts/testset.py

# -*- coding: utf-8 -*-

#-------------------------------------------------------------------------
# drawElements Quality Program utilities
# --------------------------------------
#
# Copyright 2015 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#-------------------------------------------------------------------------

import sys
import random
import string
import subprocess
from optparse import OptionParser

def all (results, predicate):
	for result in results:
		if not predicate(result):
			return False
	return True

def any (results, predicate):
	for result in results:
		if predicate(result):
			return True
	return False

class FilterRule:
	def __init__ (self, name, description, filters):
		self.name			= name
		self.description	= description
		self.filters		= filters

class TestCaseResult:
	def __init__ (self, name, results):
		self.name		= name
		self.results	= results

class Group:
	def __init__ (self, name):
		self.name		= name
		self.cases		= []

def readCaseList (filename):
	f = open(filename, 'rb')
	cases = []
	for line in f:
		if line[:6] == "TEST: ":
			case = line[6:].strip()
			if len(case) > 0:
				cases.append(case)
	return cases

def toResultList (caselist):
	results = []
	for case in caselist:
		results.append(TestCaseResult(case, []))
	return results

def addResultsToCaseList (caselist, results):
	resultMap	= {}
	caseListRes	= toResultList(caselist)

	for res in caseListRes:
		resultMap[res.name] = res

	for result in results:
		if result.name in resultMap:
			resultMap[result.name].results += result.results

	return caseListRes

def readTestResults (filename):
	f			= open(filename, 'rb')
	csvData		= f.read()
	csvLines	= csvData.splitlines()
	results		= []

	f.close()

	for line in csvLines[1:]:
		args = line.split(',')
		if len(args) == 1:
			continue # Ignore

		results.append(TestCaseResult(args[0], args[1:]))

	if len(results) == 0:
		raise Exception("Empty result list")

	# Quick check for results
	numResultItems	= len(results[0].results)
	seenResults		= set()
	for result in results:
		if result.name in seenResults:
			raise Exception("Duplicate result row for test case '%s'" % result.name)
		if len(result.results) != numResultItems:
			raise Exception("Found %d results for test case '%s', expected %d" % (len(result.results), result.name, numResultItems))
		seenResults.add(result.name)

	return results

def readGroupList (filename):
	f = open(filename, 'rb')
	groups = []
	for line in f:
		group = line.strip()
		if group != "":
			groups.append(group)
	return groups

def createGroups (results, groupNames):
	groups	= []
	matched	= set()

	for groupName in groupNames:
		group = Group(groupName)
		groups.append(group)

		prefix		= groupName + "."
		prefixLen	= len(prefix)
		for case in results:
			if case.name[:prefixLen] == prefix:
				if case in matched:
					die("Case '%s' matched by multiple groups (when processing '%s')" % (case.name, group.name))
				group.cases.append(case)
				matched.add(case)

	return groups

def createLeafGroups (results):
	groups = []
	groupMap = {}

	for case in results:
		parts		= case.name.split('.')
		groupName	= string.join(parts[:-1], ".")

		if not groupName in groupMap:
			group = Group(groupName)
			groups.append(group)
			groupMap[groupName] = group
		else:
			group = groupMap[groupName]

		group.cases.append(case)

	return groups

def filterList (results, condition):
	filtered = []
	for case in results:
		if condition(case.results):
			filtered.append(case)
	return filtered

def getFilter (list, name):
	for filter in list:
		if filter.name == name:
			return filter
	return None

def getNumCasesInGroups (groups):
	numCases = 0
	for group in groups:
		numCases += len(group.cases)
	return numCases

def getCasesInSet (results, caseSet):
	filtered = []
	for case in results:
		if case in caseSet:
			filtered.append(case)
	return filtered

def selectCasesInGroups (results, groups):
	casesInGroups = set()
	for group in groups:
		for case in group.cases:
			casesInGroups.add(case)
	return getCasesInSet(results, casesInGroups)

def selectRandomSubset (results, groups, limit, seed):
	selectedCases	= set()
	numSelect		= min(limit, getNumCasesInGroups(groups))

	random.seed(seed)
	random.shuffle(groups)

	groupNdx = 0
	while len(selectedCases) < numSelect:
		group = groups[groupNdx]
		if len(group.cases) == 0:
			del groups[groupNdx]
			if groupNdx == len(groups):
				groupNdx -= 1
			continue # Try next

		selected = random.choice(group.cases)
		selectedCases.add(selected)
		group.cases.remove(selected)

		groupNdx = (groupNdx + 1) % len(groups)

	return getCasesInSet(results, selectedCases)

def die (msg):
	print(msg)
	sys.exit(-1)

# Named filter lists
FILTER_RULES = [
	FilterRule("all",			"No filtering",											[]),
	FilterRule("all-pass",		"All results must be 'Pass'",							[lambda l: all(l, lambda r: r == 'Pass')]),
	FilterRule("any-pass",		"Any of results is 'Pass'",								[lambda l: any(l, lambda r: r == 'Pass')]),
	FilterRule("any-fail",		"Any of results is not 'Pass' or 'NotSupported'",		[lambda l: not all(l, lambda r: r == 'Pass' or r == 'NotSupported')]),
	FilterRule("prev-failing",	"Any except last result is failure",					[lambda l: l[-1] == 'Pass' and not all(l[:-1], lambda r: r == 'Pass')]),
	FilterRule("prev-passing",	"Any except last result is 'Pass'",						[lambda l: l[-1] != 'Pass' and any(l[:-1], lambda r: r == 'Pass')])
]

if __name__ == "__main__":
	parser = OptionParser(usage = "usage: %prog [options] [caselist] [result csv file]")
	parser.add_option("-f", "--filter", dest="filter", default="all", help="filter rule name")
	parser.add_option("-l", "--list", action="store_true", dest="list", default=False, help="list available rules")
	parser.add_option("-n", "--num", dest="limit", default=0, help="limit number of cases")
	parser.add_option("-s", "--seed", dest="seed", default=0, help="use selected seed for random selection")
	parser.add_option("-g", "--groups", dest="groups_file", default=None, help="select cases based on group list file")

	(options, args)	= parser.parse_args()

	if options.list:
		print("Available filter rules:")
		for filter in FILTER_RULES:
			print("  %s: %s" % (filter.name, filter.description))
		sys.exit(0)

	if len(args) == 0:
		die("No input files specified")
	elif len(args) > 2:
		die("Too many arguments")

	# Fetch filter
	filter = getFilter(FILTER_RULES, options.filter)
	if filter == None:
		die("Unknown filter '%s'" % options.filter)

	# Read case list
	caselist = readCaseList(args[0])
	if len(args) > 1:
		results = readTestResults(args[1])
		results = addResultsToCaseList(caselist, results)
	else:
		results = toResultList(caselist)

	# Execute filters for results
	for rule in filter.filters:
		results = filterList(results, rule)

	if options.limit != 0:
		if options.groups_file != None:
			groups = createGroups(results, readGroupList(options.groups_file))
		else:
			groups = createLeafGroups(results)
		results = selectRandomSubset(results, groups, int(options.limit), int(options.seed))
	elif options.groups_file != None:
		groups = createGroups(results, readGroupList(options.groups_file))
		results = selectCasesInGroups(results, groups)

	# Print test set
	for result in results:
		print(result.name)