1/*-------------------------------------------------------------------------
2 * drawElements Quality Program Test Executor
3 * ------------------------------------------
4 *
5 * Copyright 2014 The Android Open Source Project
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 *
11 *      http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *
19 *//*!
20 * \file
21 * \brief Command line test executor.
22 *//*--------------------------------------------------------------------*/
23
24#include "xeBatchExecutor.hpp"
25#include "xeLocalTcpIpLink.hpp"
26#include "xeTcpIpLink.hpp"
27#include "xeTestCaseListParser.hpp"
28#include "xeTestLogWriter.hpp"
29#include "xeTestResultParser.hpp"
30
31#include "deCommandLine.hpp"
32#include "deDirectoryIterator.hpp"
33#include "deStringUtil.hpp"
34#include "deUniquePtr.hpp"
35
36#include "deString.h"
37
38#include <algorithm>
39#include <cstdio>
40#include <cstdlib>
41#include <fstream>
42#include <iostream>
43#include <memory>
44#include <sstream>
45#include <string>
46#include <vector>
47
48#if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID) || (DE_OS == DE_OS_WIN32)
49#	include <signal.h>
50#endif
51
52using std::vector;
53using std::string;
54
55namespace
56{
57
58// Command line arguments.
59namespace opt
60{
61
62DE_DECLARE_COMMAND_LINE_OPT(StartServer,	string);
63DE_DECLARE_COMMAND_LINE_OPT(Host,			string);
64DE_DECLARE_COMMAND_LINE_OPT(Port,			int);
65DE_DECLARE_COMMAND_LINE_OPT(CaseListDir,	string);
66DE_DECLARE_COMMAND_LINE_OPT(TestSet,		vector<string>);
67DE_DECLARE_COMMAND_LINE_OPT(ExcludeSet,		vector<string>);
68DE_DECLARE_COMMAND_LINE_OPT(ContinueFile,	string);
69DE_DECLARE_COMMAND_LINE_OPT(TestLogFile,	string);
70DE_DECLARE_COMMAND_LINE_OPT(InfoLogFile,	string);
71DE_DECLARE_COMMAND_LINE_OPT(Summary,		bool);
72
73// TargetConfiguration
74DE_DECLARE_COMMAND_LINE_OPT(BinaryName,		string);
75DE_DECLARE_COMMAND_LINE_OPT(WorkingDir,		string);
76DE_DECLARE_COMMAND_LINE_OPT(CmdLineArgs,	string);
77
78void parseCommaSeparatedList (const char* src, vector<string>* dst)
79{
80	std::istringstream	inStr	(src);
81	string			comp;
82
83	while (std::getline(inStr, comp, ','))
84		dst->push_back(comp);
85}
86
87void registerOptions (de::cmdline::Parser& parser)
88{
89	using de::cmdline::Option;
90	using de::cmdline::NamedValue;
91
92	static const NamedValue<bool> s_yesNo[] =
93	{
94		{ "yes",	true	},
95		{ "no",		false	}
96	};
97
98	parser << Option<StartServer>	("s",		"start-server",	"Start local execserver. Path to the execserver binary.")
99		   << Option<Host>			("c",		"connect",		"Connect to host. Address of the execserver.")
100		   << Option<Port>			("p",		"port",			"TCP port of the execserver.",											"50016")
101		   << Option<CaseListDir>	("cd",		"caselistdir",	"Path to the directory containing test case XML files.",				".")
102		   << Option<TestSet>		("t",		"testset",		"Comma-separated list of include filters.",								parseCommaSeparatedList)
103		   << Option<ExcludeSet>	("e",		"exclude",		"Comma-separated list of exclude filters.",								parseCommaSeparatedList, "")
104		   << Option<ContinueFile>	(DE_NULL,	"continue",		"Continue execution by initializing results from existing test log.")
105		   << Option<TestLogFile>	("o",		"out",			"Output test log filename.",											"TestLog.qpa")
106		   << Option<InfoLogFile>	("i",		"info",			"Output info log filename.",											"InfoLog.txt")
107		   << Option<Summary>		(DE_NULL,	"summary",		"Print summary after running tests.",									s_yesNo, "yes")
108		   << Option<BinaryName>	("b",		"binaryname",	"Test binary path. Relative to working directory.",						"<Unused>")
109		   << Option<WorkingDir>	("wd",		"workdir",		"Working directory for the test execution.",							".")
110		   << Option<CmdLineArgs>	(DE_NULL,	"cmdline",		"Additional command line arguments for the test binary.",				"");
111}
112
113} // opt
114
115enum RunMode
116{
117	RUNMODE_CONNECT,
118	RUNMODE_START_SERVER
119};
120
121struct CommandLine
122{
123	CommandLine (void)
124		: port		(0)
125		, summary	(false)
126	{
127	}
128
129	xe::TargetConfiguration	targetCfg;
130	RunMode					runMode;
131	string					serverBinOrAddress;
132	int						port;
133	string					caseListDir;
134	vector<string>			testset;
135	vector<string>			exclude;
136	string					inFile;
137	string					outFile;
138	string					infoFile;
139	bool					summary;
140};
141
142bool parseCommandLine (CommandLine& cmdLine, int argc, const char* const* argv)
143{
144	de::cmdline::Parser			parser;
145	de::cmdline::CommandLine	opts;
146
147	XE_CHECK(argc >= 1);
148
149	opt::registerOptions(parser);
150
151	if (!parser.parse(argc-1, argv+1, &opts, std::cerr))
152	{
153		std::cout << argv[0] << " [options]\n";
154		parser.help(std::cout);
155		return false;
156	}
157
158	if (opts.hasOption<opt::StartServer>() && opts.hasOption<opt::Host>())
159	{
160		std::cout << "Invalid command line arguments. Both --start-server and --connect defined." << std::endl;
161		return false;
162	}
163	else if (!opts.hasOption<opt::StartServer>() && !opts.hasOption<opt::Host>())
164	{
165		std::cout << "Invalid command line arguments. Must define either --start-server or --connect." << std::endl;
166		return false;
167	}
168
169	if (!opts.hasOption<opt::TestSet>())
170	{
171		std::cout << "Invalid command line arguments. --testset not defined." << std::endl;
172		return false;
173	}
174
175	if (opts.hasOption<opt::StartServer>())
176	{
177		cmdLine.runMode				= RUNMODE_START_SERVER;
178		cmdLine.serverBinOrAddress	= opts.getOption<opt::StartServer>();
179	}
180	else
181	{
182		cmdLine.runMode				= RUNMODE_CONNECT;
183		cmdLine.serverBinOrAddress	= opts.getOption<opt::Host>();
184	}
185
186	if (opts.hasOption<opt::ContinueFile>())
187	{
188		cmdLine.inFile = opts.getOption<opt::ContinueFile>();
189
190		if (cmdLine.inFile.empty())
191		{
192			std::cout << "Invalid command line arguments. --continue argument is empty." << std::endl;
193			return false;
194		}
195	}
196
197	cmdLine.port					= opts.getOption<opt::Port>();
198	cmdLine.caseListDir				= opts.getOption<opt::CaseListDir>();
199	cmdLine.testset					= opts.getOption<opt::TestSet>();
200	cmdLine.exclude					= opts.getOption<opt::ExcludeSet>();
201	cmdLine.outFile					= opts.getOption<opt::TestLogFile>();
202	cmdLine.infoFile				= opts.getOption<opt::InfoLogFile>();
203	cmdLine.summary					= opts.getOption<opt::Summary>();
204	cmdLine.targetCfg.binaryName	= opts.getOption<opt::BinaryName>();
205	cmdLine.targetCfg.workingDir	= opts.getOption<opt::WorkingDir>();
206	cmdLine.targetCfg.cmdLineArgs	= opts.getOption<opt::CmdLineArgs>();
207
208	return true;
209}
210
211bool checkCasePathPatternMatch (const char* pattern, const char* casePath, bool isTestGroup)
212{
213	int ptrnPos = 0;
214	int casePos = 0;
215
216	for (;;)
217	{
218		char c = casePath[casePos];
219		char p = pattern[ptrnPos];
220
221		if (p == '*')
222		{
223			/* Recurse to rest of positions. */
224			int next = casePos;
225			for (;;)
226			{
227				if (checkCasePathPatternMatch(pattern+ptrnPos+1, casePath+next, isTestGroup))
228					return DE_TRUE;
229
230				if (casePath[next] == 0)
231					return DE_FALSE; /* No match found. */
232				else
233					next += 1;
234			}
235			DE_ASSERT(DE_FALSE);
236		}
237		else if (c == 0 && p == 0)
238			return true;
239		else if (c == 0)
240		{
241			/* Incomplete match is ok for test groups. */
242			return isTestGroup;
243		}
244		else if (c != p)
245			return false;
246
247		casePos += 1;
248		ptrnPos += 1;
249	}
250
251	DE_ASSERT(false);
252	return false;
253}
254
255void readCaseList (xe::TestGroup* root, const char* filename)
256{
257	xe::TestCaseListParser	caseListParser;
258	std::ifstream			in				(filename, std::ios_base::binary);
259	deUint8					buf[1024];
260
261	XE_CHECK(in.good());
262
263	caseListParser.init(root);
264
265	for (;;)
266	{
267		in.read((char*)&buf[0], sizeof(buf));
268		int numRead = (int)in.gcount();
269
270		if (numRead > 0)
271			caseListParser.parse(&buf[0], numRead);
272
273		if (numRead < (int)sizeof(buf))
274			break; // EOF
275	}
276}
277
278void readCaseLists (xe::TestRoot& root, const char* caseListDir)
279{
280	int						testCaseListCount	= 0;
281	de::DirectoryIterator	iter				(caseListDir);
282
283	for (; iter.hasItem(); iter.next())
284	{
285		de::FilePath item = iter.getItem();
286
287		if (item.getType() == de::FilePath::TYPE_FILE)
288		{
289			string baseName = item.getBaseName();
290			if (baseName.find("-cases.xml") == baseName.length()-10)
291			{
292				string		packageName	= baseName.substr(0, baseName.length()-10);
293				xe::TestGroup*	package		= root.createGroup(packageName.c_str());
294
295				readCaseList(package, item.getPath());
296				testCaseListCount++;
297			}
298		}
299	}
300
301	if (testCaseListCount == 0)
302		throw xe::Error("Couldn't find test case lists from test case list directory: '" + string(caseListDir)  + "'");
303}
304
305void addMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
306{
307	for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
308	{
309		const xe::TestNode* child		= group.getChild(childNdx);
310		const bool			isGroup		= child->getNodeType() == xe::TESTNODETYPE_GROUP;
311		const string		fullPath	= child->getFullPath();
312
313		if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
314		{
315			if (isGroup)
316			{
317				// Recurse into group.
318				addMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
319			}
320			else
321			{
322				DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
323				testSet.add(child);
324			}
325		}
326	}
327}
328
329void removeMatchingCases (const xe::TestGroup& group, xe::TestSet& testSet, const char* filter)
330{
331	for (int childNdx = 0; childNdx < group.getNumChildren(); childNdx++)
332	{
333		const xe::TestNode* child		= group.getChild(childNdx);
334		const bool			isGroup		= child->getNodeType() == xe::TESTNODETYPE_GROUP;
335		const string		fullPath	= child->getFullPath();
336
337		if (checkCasePathPatternMatch(filter, fullPath.c_str(), isGroup))
338		{
339			if (isGroup)
340			{
341				// Recurse into group.
342				removeMatchingCases(static_cast<const xe::TestGroup&>(*child), testSet, filter);
343			}
344			else
345			{
346				DE_ASSERT(child->getNodeType() == xe::TESTNODETYPE_TEST_CASE);
347				testSet.remove(child);
348			}
349		}
350	}
351}
352
353class BatchResultHandler : public xe::TestLogHandler
354{
355public:
356	BatchResultHandler (xe::BatchResult* batchResult)
357		: m_batchResult(batchResult)
358	{
359	}
360
361	void setSessionInfo (const xe::SessionInfo& sessionInfo)
362	{
363		m_batchResult->getSessionInfo() = sessionInfo;
364	}
365
366	xe::TestCaseResultPtr startTestCaseResult (const char* casePath)
367	{
368		// \todo [2012-11-01 pyry] What to do with duplicate results?
369		if (m_batchResult->hasTestCaseResult(casePath))
370			return m_batchResult->getTestCaseResult(casePath);
371		else
372			return m_batchResult->createTestCaseResult(casePath);
373	}
374
375	void testCaseResultUpdated (const xe::TestCaseResultPtr&)
376	{
377	}
378
379	void testCaseResultComplete (const xe::TestCaseResultPtr&)
380	{
381	}
382
383private:
384	xe::BatchResult* m_batchResult;
385};
386
387void readLogFile (xe::BatchResult* batchResult, const char* filename)
388{
389	std::ifstream		in		(filename, std::ifstream::binary|std::ifstream::in);
390	BatchResultHandler	handler	(batchResult);
391	xe::TestLogParser	parser	(&handler);
392	deUint8				buf		[1024];
393	int					numRead	= 0;
394
395	for (;;)
396	{
397		in.read((char*)&buf[0], DE_LENGTH_OF_ARRAY(buf));
398		numRead = (int)in.gcount();
399
400		if (numRead <= 0)
401			break;
402
403		parser.parse(&buf[0], numRead);
404	}
405
406	in.close();
407}
408
409void printBatchResultSummary (const xe::TestNode* root, const xe::TestSet& testSet, const xe::BatchResult& batchResult)
410{
411	int countByStatusCode[xe::TESTSTATUSCODE_LAST];
412	std::fill(&countByStatusCode[0], &countByStatusCode[0]+DE_LENGTH_OF_ARRAY(countByStatusCode), 0);
413
414	for (xe::ConstTestNodeIterator iter = xe::ConstTestNodeIterator::begin(root); iter != xe::ConstTestNodeIterator::end(root); ++iter)
415	{
416		const xe::TestNode* node = *iter;
417		if (node->getNodeType() == xe::TESTNODETYPE_TEST_CASE && testSet.hasNode(node))
418		{
419			const xe::TestCase*				testCase		= static_cast<const xe::TestCase*>(node);
420			string							fullPath;
421			xe::TestStatusCode				statusCode		= xe::TESTSTATUSCODE_PENDING;
422			testCase->getFullPath(fullPath);
423
424			// Parse result data if such exists.
425			if (batchResult.hasTestCaseResult(fullPath.c_str()))
426			{
427				xe::ConstTestCaseResultPtr	resultData	= batchResult.getTestCaseResult(fullPath.c_str());
428				xe::TestCaseResult			result;
429				xe::TestResultParser		parser;
430
431				xe::parseTestCaseResultFromData(&parser, &result, *resultData.get());
432				statusCode = result.statusCode;
433			}
434
435			countByStatusCode[statusCode] += 1;
436		}
437	}
438
439	printf("\nTest run summary:\n");
440	int totalCases = 0;
441	for (int code = 0; code < xe::TESTSTATUSCODE_LAST; code++)
442	{
443		if (countByStatusCode[code] > 0)
444			printf("  %20s: %5d\n", xe::getTestStatusCodeName((xe::TestStatusCode)code), countByStatusCode[code]);
445
446		totalCases += countByStatusCode[code];
447	}
448	printf("  %20s: %5d\n", "Total", totalCases);
449}
450
451void writeInfoLog (const xe::InfoLog& log, const char* filename)
452{
453	std::ofstream out(filename, std::ios_base::binary);
454	XE_CHECK(out.good());
455	out.write((const char*)log.getBytes(), log.getSize());
456	out.close();
457}
458
459xe::CommLink* createCommLink (const CommandLine& cmdLine)
460{
461	if (cmdLine.runMode == RUNMODE_START_SERVER)
462	{
463		xe::LocalTcpIpLink* link = new xe::LocalTcpIpLink();
464		try
465		{
466			link->start(cmdLine.serverBinOrAddress.c_str(), DE_NULL, cmdLine.port);
467			return link;
468		}
469		catch (...)
470		{
471			delete link;
472			throw;
473		}
474	}
475	else if (cmdLine.runMode == RUNMODE_CONNECT)
476	{
477		de::SocketAddress address;
478
479		address.setFamily(DE_SOCKETFAMILY_INET4);
480		address.setProtocol(DE_SOCKETPROTOCOL_TCP);
481		address.setHost(cmdLine.serverBinOrAddress.c_str());
482		address.setPort(cmdLine.port);
483
484		xe::TcpIpLink* link = new xe::TcpIpLink();
485		try
486		{
487			std::string error;
488
489			link->connect(address);
490			return link;
491		}
492		catch (const std::exception& error)
493		{
494			delete link;
495			throw xe::Error("Failed to connect to ExecServer at: " + cmdLine.serverBinOrAddress + ":" + de::toString(cmdLine.port) + ", " + error.what());
496		}
497		catch (...)
498		{
499			delete link;
500			throw;
501		}
502	}
503	else
504	{
505		DE_ASSERT(false);
506		return DE_NULL;
507	}
508}
509
510#if (DE_OS == DE_OS_UNIX) || (DE_OS == DE_OS_ANDROID)
511
512static xe::BatchExecutor* s_executor = DE_NULL;
513
514void signalHandler (int, siginfo_t*, void*)
515{
516	if (s_executor)
517		s_executor->cancel();
518}
519
520void setupSignalHandler (xe::BatchExecutor* executor)
521{
522	s_executor = executor;
523	struct sigaction sa;
524
525	sa.sa_sigaction = signalHandler;
526	sa.sa_flags = SA_SIGINFO | SA_RESTART;
527	sigfillset(&sa.sa_mask);
528
529	sigaction(SIGINT, &sa, DE_NULL);
530}
531
532void resetSignalHandler (void)
533{
534	struct sigaction sa;
535
536	sa.sa_handler = SIG_DFL;
537	sa.sa_flags = SA_RESTART;
538	sigfillset(&sa.sa_mask);
539
540	sigaction(SIGINT, &sa, DE_NULL);
541	s_executor = DE_NULL;
542}
543
544#elif (DE_OS == DE_OS_WIN32)
545
546static xe::BatchExecutor* s_executor = DE_NULL;
547
548void signalHandler (int)
549{
550	if (s_executor)
551		s_executor->cancel();
552}
553
554void setupSignalHandler (xe::BatchExecutor* executor)
555{
556	s_executor = executor;
557	signal(SIGINT, signalHandler);
558}
559
560void resetSignalHandler (void)
561{
562	signal(SIGINT, SIG_DFL);
563	s_executor = DE_NULL;
564}
565
566#else
567
568void setupSignalHandler (xe::BatchExecutor*)
569{
570}
571
572void resetSignalHandler (void)
573{
574}
575
576#endif
577
578void runExecutor (const CommandLine& cmdLine)
579{
580	xe::TestRoot root;
581
582	// Read case list definitions.
583	readCaseLists(root, cmdLine.caseListDir.c_str());
584
585	// Build test set.
586	xe::TestSet testSet;
587
588	// Build test set.
589	for (vector<string>::const_iterator filterIter = cmdLine.testset.begin(); filterIter != cmdLine.testset.end(); ++filterIter)
590		addMatchingCases(root, testSet, filterIter->c_str());
591
592	if (testSet.empty())
593		throw xe::Error("None of the test case lists contains tests matching any of the test sets.");
594
595	// Remove excluded cases.
596	for (vector<string>::const_iterator filterIter = cmdLine.exclude.begin(); filterIter != cmdLine.exclude.end(); ++filterIter)
597		removeMatchingCases(root, testSet, filterIter->c_str());
598
599	// Initialize batch result.
600	xe::BatchResult	batchResult;
601	xe::InfoLog		infoLog;
602
603	// Read existing results from input file (if supplied).
604	if (!cmdLine.inFile.empty())
605		readLogFile(&batchResult, cmdLine.inFile.c_str());
606
607	// Initialize commLink.
608	de::UniquePtr<xe::CommLink> commLink(createCommLink(cmdLine));
609
610	xe::BatchExecutor executor(cmdLine.targetCfg, commLink.get(), &root, testSet, &batchResult, &infoLog);
611
612	try
613	{
614		setupSignalHandler(&executor);
615		executor.run();
616		resetSignalHandler();
617	}
618	catch (...)
619	{
620		resetSignalHandler();
621
622		if (!cmdLine.outFile.empty())
623		{
624			xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str());
625			printf("Test log written to %s\n", cmdLine.outFile.c_str());
626		}
627
628		if (!cmdLine.infoFile.empty())
629		{
630			writeInfoLog(infoLog, cmdLine.infoFile.c_str());
631			printf("Info log written to %s\n", cmdLine.infoFile.c_str());
632		}
633
634		if (cmdLine.summary)
635			printBatchResultSummary(&root, testSet, batchResult);
636
637		throw;
638	}
639
640	if (!cmdLine.outFile.empty())
641	{
642		xe::writeBatchResultToFile(batchResult, cmdLine.outFile.c_str());
643		printf("Test log written to %s\n", cmdLine.outFile.c_str());
644	}
645
646	if (!cmdLine.infoFile.empty())
647	{
648		writeInfoLog(infoLog, cmdLine.infoFile.c_str());
649		printf("Info log written to %s\n", cmdLine.infoFile.c_str());
650	}
651
652	if (cmdLine.summary)
653		printBatchResultSummary(&root, testSet, batchResult);
654
655	{
656		string err;
657
658		if (commLink->getState(err) == xe::COMMLINKSTATE_ERROR)
659			throw xe::Error(err);
660	}
661}
662
663} // anonymous
664
665int main (int argc, const char* const* argv)
666{
667	CommandLine cmdLine;
668
669	if (!parseCommandLine(cmdLine, argc, argv))
670		return -1;
671
672	try
673	{
674		runExecutor(cmdLine);
675	}
676	catch (const std::exception& e)
677	{
678		printf("%s\n", e.what());
679		return -1;
680	}
681
682	return 0;
683}
684