1/*-------------------------------------------------------------------------
2 * drawElements Quality Program Tester Core
3 * ----------------------------------------
4 *
5 * Copyright 2014 The Android Open Source Project
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 *      http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *//*!
20 * \file
21 * \brief Test executor.
22 *//*--------------------------------------------------------------------*/
23
24#include "tcuTestSessionExecutor.hpp"
25#include "tcuCommandLine.hpp"
26#include "tcuTestLog.hpp"
27
28#include "deClock.h"
29
30namespace tcu
31{
32
33using std::vector;
34
35static qpTestCaseType nodeTypeToTestCaseType (TestNodeType nodeType)
36{
37	switch (nodeType)
38	{
39		case NODETYPE_SELF_VALIDATE:	return QP_TEST_CASE_TYPE_SELF_VALIDATE;
40		case NODETYPE_PERFORMANCE:		return QP_TEST_CASE_TYPE_PERFORMANCE;
41		case NODETYPE_CAPABILITY:		return QP_TEST_CASE_TYPE_CAPABILITY;
42		case NODETYPE_ACCURACY:			return QP_TEST_CASE_TYPE_ACCURACY;
43		default:
44			DE_ASSERT(false);
45			return QP_TEST_CASE_TYPE_LAST;
46	}
47}
48
49TestSessionExecutor::TestSessionExecutor (TestPackageRoot& root, TestContext& testCtx)
50	: m_testCtx				(testCtx)
51	, m_inflater			(testCtx)
52	, m_caseListFilter		(testCtx.getCommandLine().createCaseListFilter(testCtx.getArchive()))
53	, m_iterator			(root, m_inflater, *m_caseListFilter)
54	, m_state				(STATE_TRAVERSE_HIERARCHY)
55	, m_abortSession		(false)
56	, m_isInTestCase		(false)
57	, m_testStartTime		(0)
58	, m_packageStartTime	(0)
59{
60}
61
62TestSessionExecutor::~TestSessionExecutor (void)
63{
64}
65
66bool TestSessionExecutor::iterate (void)
67{
68	while (!m_abortSession)
69	{
70		switch (m_state)
71		{
72			case STATE_TRAVERSE_HIERARCHY:
73			{
74				const TestHierarchyIterator::State	hierIterState	= m_iterator.getState();
75
76				if (hierIterState == TestHierarchyIterator::STATE_ENTER_NODE ||
77					hierIterState == TestHierarchyIterator::STATE_LEAVE_NODE)
78				{
79					TestNode* const		curNode		= m_iterator.getNode();
80					const TestNodeType	nodeType	= curNode->getNodeType();
81					const bool			isEnter		= hierIterState == TestHierarchyIterator::STATE_ENTER_NODE;
82
83					switch (nodeType)
84					{
85						case NODETYPE_PACKAGE:
86						{
87							TestPackage* const testPackage = static_cast<TestPackage*>(curNode);
88							isEnter ? enterTestPackage(testPackage) : leaveTestPackage(testPackage);
89							break;
90						}
91
92						case NODETYPE_GROUP:
93						{
94							isEnter ? enterTestGroup(m_iterator.getNodePath()) : leaveTestGroup(m_iterator.getNodePath());
95							break; // nada
96						}
97
98						case NODETYPE_SELF_VALIDATE:
99						case NODETYPE_PERFORMANCE:
100						case NODETYPE_CAPABILITY:
101						case NODETYPE_ACCURACY:
102						{
103							TestCase* const testCase = static_cast<TestCase*>(curNode);
104
105							if (isEnter)
106							{
107								if (enterTestCase(testCase, m_iterator.getNodePath()))
108									m_state = STATE_EXECUTE_TEST_CASE;
109								// else remain in TRAVERSING_HIERARCHY => node will be exited from in the next iteration
110							}
111							else
112								leaveTestCase(testCase);
113
114							break;
115						}
116
117						default:
118							DE_ASSERT(false);
119							break;
120					}
121
122					m_iterator.next();
123					break;
124				}
125				else
126				{
127					DE_ASSERT(hierIterState == TestHierarchyIterator::STATE_FINISHED);
128					m_status.isComplete = true;
129					return false;
130				}
131			}
132
133			case STATE_EXECUTE_TEST_CASE:
134			{
135				DE_ASSERT(m_iterator.getState() == TestHierarchyIterator::STATE_LEAVE_NODE &&
136						  isTestNodeTypeExecutable(m_iterator.getNode()->getNodeType()));
137
138				TestCase* const					testCase	= static_cast<TestCase*>(m_iterator.getNode());
139				const TestCase::IterateResult	iterResult	= iterateTestCase(testCase);
140
141				if (iterResult == TestCase::STOP)
142					m_state = STATE_TRAVERSE_HIERARCHY;
143
144				return true;
145			}
146
147			default:
148				DE_ASSERT(false);
149				break;
150		}
151	}
152
153	return false;
154}
155
156void TestSessionExecutor::enterTestPackage (TestPackage* testPackage)
157{
158	// Create test case wrapper
159	DE_ASSERT(!m_caseExecutor);
160	m_caseExecutor = de::MovePtr<TestCaseExecutor>(testPackage->createExecutor());
161	testPackage->setCaseListFilter(m_caseListFilter.get());
162	m_packageStartTime	= deGetMicroseconds();
163}
164
165void TestSessionExecutor::leaveTestPackage (TestPackage* testPackage)
166{
167	DE_UNREF(testPackage);
168	m_caseExecutor->deinitTestPackage(m_testCtx);
169	// If m_caseExecutor uses local status then it may perform some tests in deinitTestPackage(). We have to update TestSessionExecutor::m_status
170	if (m_caseExecutor->usesLocalStatus())
171		m_caseExecutor->updateGlobalStatus(m_status);
172
173	const deInt64 duration	= deGetMicroseconds() - m_packageStartTime;
174	m_packageStartTime		= 0;
175
176	if (!std::string(m_testCtx.getCommandLine().getServerAddress()).empty())
177		m_caseExecutor->reportDurations(m_testCtx, std::string(testPackage->getName()), duration, m_groupsDurationTime);
178
179	m_caseExecutor.clear();
180
181	if (!std::string(m_testCtx.getCommandLine().getServerAddress()).empty())
182	{
183		m_testCtx.getLog().startTestsCasesTime();
184
185		m_testCtx.getLog() << TestLog::Integer(testPackage->getName(), "Total tests case duration in microseconds", "us", QP_KEY_TAG_TIME, duration);
186
187		for (std::map<std::string, deUint64>::iterator it = m_groupsDurationTime.begin(); it != m_groupsDurationTime.end(); ++it)
188			m_testCtx.getLog() << TestLog::Integer(it->first, "The test group case duration in microseconds", "us", QP_KEY_TAG_TIME, it->second);
189
190		m_testCtx.getLog().endTestsCasesTime();
191	}
192}
193
194void TestSessionExecutor::enterTestGroup (const std::string& casePath)
195{
196	m_groupsDurationTime[casePath] = deGetMicroseconds();
197}
198
199void TestSessionExecutor::leaveTestGroup (const std::string& casePath)
200{
201	m_groupsDurationTime[casePath] = deGetMicroseconds() - m_groupsDurationTime[casePath];
202}
203
204bool TestSessionExecutor::enterTestCase (TestCase* testCase, const std::string& casePath)
205{
206	TestLog&				log			= m_testCtx.getLog();
207	const qpTestCaseType	caseType	= nodeTypeToTestCaseType(testCase->getNodeType());
208	bool					initOk		= false;
209
210	print("\nTest case '%s'..\n", casePath.c_str());
211
212#if (DE_OS == DE_OS_WIN32)
213	fflush(stdout);
214#endif
215
216	m_testCtx.setTestResult(QP_TEST_RESULT_LAST, "");
217	m_testCtx.setTerminateAfter(false);
218	log.startCase(casePath.c_str(), caseType);
219
220	m_isInTestCase	= true;
221	m_testStartTime	= deGetMicroseconds();
222
223	try
224	{
225		m_caseExecutor->init(testCase, casePath);
226		initOk = true;
227	}
228	catch (const std::bad_alloc&)
229	{
230		DE_ASSERT(!initOk);
231		m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory in test case init");
232		m_testCtx.setTerminateAfter(true);
233	}
234	catch (const tcu::TestException& e)
235	{
236		DE_ASSERT(!initOk);
237		DE_ASSERT(e.getTestResult() != QP_TEST_RESULT_LAST);
238		m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
239		m_testCtx.setTerminateAfter(e.isFatal());
240		log << e;
241	}
242	catch (const tcu::Exception& e)
243	{
244		DE_ASSERT(!initOk);
245		m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
246		log << e;
247	}
248
249	DE_ASSERT(initOk || m_testCtx.getTestResult() != QP_TEST_RESULT_LAST);
250
251	return initOk;
252}
253
254void TestSessionExecutor::leaveTestCase (TestCase* testCase)
255{
256	TestLog&	log		= m_testCtx.getLog();
257
258	// De-init case.
259	try
260	{
261		m_caseExecutor->deinit(testCase);
262	}
263	catch (const tcu::Exception& e)
264	{
265		const bool suppressLogging = m_testCtx.getLog().isSupressLogging();
266
267		if (suppressLogging)
268			m_testCtx.getLog().supressLogging(false);
269
270		log << e << TestLog::Message << "Error in test case deinit, test program will terminate." << TestLog::EndMessage;
271		m_testCtx.setTerminateAfter(true);
272
273		m_testCtx.getLog().supressLogging(suppressLogging);
274	}
275
276	{
277		const deInt64 duration = deGetMicroseconds()-m_testStartTime;
278		m_testStartTime = 0;
279		m_testCtx.getLog() << TestLog::Integer("TestDuration", "Test case duration in microseconds", "us", QP_KEY_TAG_TIME, duration);
280	}
281
282	{
283		const qpTestResult	testResult		= m_testCtx.getTestResult();
284		const char* const	testResultDesc	= m_testCtx.getTestResultDesc();
285		const bool			terminateAfter	= m_testCtx.getTerminateAfter();
286		DE_ASSERT(testResult != QP_TEST_RESULT_LAST);
287
288		m_isInTestCase = false;
289		m_testCtx.getLog().endCase(testResult, testResultDesc);
290
291		// Update statistics.
292		print("  %s (%s)\n", qpGetTestResultName(testResult), testResultDesc);
293
294#if (DE_OS == DE_OS_WIN32)
295		fflush(stdout);
296#endif
297		if(!m_caseExecutor->usesLocalStatus())
298		{
299			m_status.numExecuted += 1;
300			switch (testResult)
301			{
302				case QP_TEST_RESULT_PASS:					m_status.numPassed			+= 1;	break;
303				case QP_TEST_RESULT_NOT_SUPPORTED:			m_status.numNotSupported	+= 1;	break;
304				case QP_TEST_RESULT_QUALITY_WARNING:		m_status.numWarnings		+= 1;	break;
305				case QP_TEST_RESULT_COMPATIBILITY_WARNING:	m_status.numWarnings		+= 1;	break;
306				case QP_TEST_RESULT_WAIVER:					m_status.numWaived			+= 1;	break;
307				default:									m_status.numFailed			+= 1;	break;
308			}
309		}
310		else
311		{
312			m_caseExecutor->updateGlobalStatus(m_status);
313		}
314
315		// terminateAfter, Resource error or any error in deinit means that execution should end
316		if (terminateAfter || testResult == QP_TEST_RESULT_RESOURCE_ERROR ||
317			(m_status.numFailed > 0 && m_testCtx.getCommandLine().isTerminateOnFailEnabled()))
318
319			m_abortSession = true;
320	}
321
322	if (m_testCtx.getWatchDog())
323		qpWatchDog_reset(m_testCtx.getWatchDog());
324}
325
326TestCase::IterateResult TestSessionExecutor::iterateTestCase (TestCase* testCase)
327{
328	TestLog&				log				= m_testCtx.getLog();
329	TestCase::IterateResult	iterateResult	= TestCase::STOP;
330
331	m_testCtx.touchWatchdog();
332
333	try
334	{
335		iterateResult = m_caseExecutor->iterate(testCase);
336	}
337	catch (const std::bad_alloc&)
338	{
339		m_testCtx.setTestResult(QP_TEST_RESULT_RESOURCE_ERROR, "Failed to allocate memory during test execution");
340		m_testCtx.setTerminateAfter(true);
341	}
342	catch (const tcu::TestException& e)
343	{
344		log << e;
345		m_testCtx.setTestResult(e.getTestResult(), e.getMessage());
346		m_testCtx.setTerminateAfter(e.isFatal());
347	}
348	catch (const tcu::Exception& e)
349	{
350		log << e;
351		m_testCtx.setTestResult(QP_TEST_RESULT_FAIL, e.getMessage());
352	}
353
354	return iterateResult;
355}
356
357} // tcu
358