xref: /third_party/vk-gl-cts/execserver/tools/xsTest.cpp

/*-------------------------------------------------------------------------
 * drawElements Quality Program Execution Server
 * ---------------------------------------------
 *
 * Copyright 2014 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *//*!
 * \file
 * \brief ExecServer Tests.
 *//*--------------------------------------------------------------------*/

#include "xsDefs.hpp"

#include "xsProtocol.hpp"
#include "deSocket.hpp"
#include "deRingBuffer.hpp"
#include "deFilePath.hpp"
#include "deBlockBuffer.hpp"
#include "deThread.hpp"
#include "deStringUtil.hpp"
#include "deUniquePtr.hpp"

#include "deClock.h"
#include "deProcess.h"
#include "deString.h"
#include "deRandom.h"

#include <memory>
#include <algorithm>

using std::string;
using std::vector;

namespace xs
{

typedef de::UniquePtr<Message> ScopedMsgPtr;

class SocketError : public Error
{
public:
	SocketError (deSocketResult result, const char* message, const char* file, int line)
		: Error		(message, deGetSocketResultName(result), file, line)
		, m_result	(result)
	{
	}

	deSocketResult getResult (void) const
	{
		return m_result;
	}

private:
	deSocketResult m_result;
};

// Helpers.
void sendMessage (de::Socket& socket, const Message& message)
{
	// Format message.
	vector<deUint8> buf;
	message.write(buf);

	// Write to socket.
	size_t pos = 0;
	while (pos < buf.size())
	{
		size_t			numLeft		= buf.size() - pos;
		size_t			numSent		= 0;
		deSocketResult	result		= socket.send(&buf[pos], numLeft, &numSent);

		if (result != DE_SOCKETRESULT_SUCCESS)
			throw SocketError(result, "send() failed", __FILE__, __LINE__);

		pos += numSent;
	}
}

void readBytes (de::Socket& socket, vector<deUint8>& dst, size_t numBytes)
{
	size_t numRead = 0;
	dst.resize(numBytes);
	while (numRead < numBytes)
	{
		size_t			numLeft		= numBytes - numRead;
		size_t			curNumRead	= 0;
		deSocketResult	result		= socket.receive(&dst[numRead], numLeft, &curNumRead);

		if (result != DE_SOCKETRESULT_SUCCESS)
			throw SocketError(result, "receive() failed", __FILE__, __LINE__);

		numRead += curNumRead;
	}
}

Message* readMessage (de::Socket& socket)
{
	// Header.
	vector<deUint8> header;
	readBytes(socket, header, MESSAGE_HEADER_SIZE);

	MessageType	type;
	size_t		messageSize;
	Message::parseHeader(&header[0], (int)header.size(), type, messageSize);

	// Simple messages without any data.
	switch (type)
	{
		case MESSAGETYPE_KEEPALIVE:				return new KeepAliveMessage();
		case MESSAGETYPE_PROCESS_STARTED:		return new ProcessStartedMessage();
		default:
			break; // Read message with data.
	}

	vector<deUint8> messageBuf;
	readBytes(socket, messageBuf, messageSize-MESSAGE_HEADER_SIZE);

	switch (type)
	{
		case MESSAGETYPE_HELLO:					return new HelloMessage(&messageBuf[0], (int)messageBuf.size());
		case MESSAGETYPE_TEST:					return new TestMessage(&messageBuf[0], (int)messageBuf.size());
		case MESSAGETYPE_PROCESS_LOG_DATA:		return new ProcessLogDataMessage(&messageBuf[0], (int)messageBuf.size());
		case MESSAGETYPE_INFO:					return new InfoMessage(&messageBuf[0], (int)messageBuf.size());
		case MESSAGETYPE_PROCESS_LAUNCH_FAILED:	return new ProcessLaunchFailedMessage(&messageBuf[0], (int)messageBuf.size());
		case MESSAGETYPE_PROCESS_FINISHED:		return new ProcessFinishedMessage(&messageBuf[0], (int)messageBuf.size());
		default:
			XS_FAIL("Unknown message");
	}
}

class TestClock
{
public:
	inline TestClock (void)
	{
		reset();
	}

	inline void reset (void)
	{
		m_initTime = deGetMicroseconds();
	}

	inline int getMilliseconds (void)
	{
		return (int)((deGetMicroseconds() - m_initTime) / 1000);
	}

private:
	deUint64 m_initTime;
};

class TestContext
{
public:
						TestContext		(void) : startServer(false) {}

	std::string			serverPath;
	std::string			testerPath;
	de::SocketAddress	address;
	bool				startServer;

	// Passed from execserver.
	std::string			logFileName;
	std::string			caseList;

private:
						TestContext		(const TestContext& other);
	TestContext&		operator=		(const TestContext& other);
};

class TestCase
{
public:
					TestCase		(TestContext& testCtx, const char* name) : m_testCtx(testCtx), m_name(name) {}
	virtual			~TestCase		(void) {}

	const char*		getName			(void) const { return m_name.c_str(); }

	virtual void	runClient		(de::Socket& socket) = DE_NULL;
	virtual void	runProgram		(void) = DE_NULL;

protected:
	TestContext&	m_testCtx;
	std::string		m_name;
};

class TestExecutor
{
public:
					TestExecutor	(TestContext& testCtx);
					~TestExecutor	(void);

	void			runCases		(const std::vector<TestCase*>& testCases);
	bool			runCase			(TestCase* testCase);

private:
	TestContext&	m_testCtx;
};

TestExecutor::TestExecutor (TestContext& testCtx)
	: m_testCtx(testCtx)
{
}

TestExecutor::~TestExecutor (void)
{
}

void TestExecutor::runCases (const std::vector<TestCase*>& testCases)
{
	int numPassed	= 0;
	int numCases	= (int)testCases.size();

	for (std::vector<TestCase*>::const_iterator i = testCases.begin(); i != testCases.end(); i++)
	{
		if (runCase(*i))
			numPassed += 1;
	}

	printf("\n  %d/%d passed!\n", numPassed, numCases);
}

class FilePrinter : public de::Thread
{
public:
	FilePrinter (void)
		: m_curFile(DE_NULL)
	{
	}

	void start (deFile* file)
	{
		DE_ASSERT(!m_curFile);
		m_curFile = file;
		de::Thread::start();
	}

	void run (void)
	{
		char	buf[256];
		deInt64 numRead	= 0;

		while (deFile_read(m_curFile, &buf[0], (deInt64)sizeof(buf), &numRead) == DE_FILERESULT_SUCCESS)
			fwrite(&buf[0], 1, (size_t)numRead, stdout);

		m_curFile = DE_NULL;
	}

private:
	deFile* m_curFile;
};

bool TestExecutor::runCase (TestCase* testCase)
{
	printf("%s\n", testCase->getName());

	bool		success		= false;
	deProcess*	serverProc	= DE_NULL;
	FilePrinter	stdoutPrinter;
	FilePrinter	stderrPrinter;

	try
	{
		if (m_testCtx.startServer)
		{
			string cmdLine = m_testCtx.serverPath + " --port=" + de::toString(m_testCtx.address.getPort());
			serverProc = deProcess_create();
			XS_CHECK(serverProc);

			if (!deProcess_start(serverProc, cmdLine.c_str(), DE_NULL))
			{
				string errMsg = deProcess_getLastError(serverProc);
				deProcess_destroy(serverProc);
				XS_FAIL(errMsg.c_str());
			}

			deSleep(200); /* Give 200ms for server to start. */
			XS_CHECK(deProcess_isRunning(serverProc));

			// Start stdout/stderr printers.
			stdoutPrinter.start(deProcess_getStdOut(serverProc));
			stderrPrinter.start(deProcess_getStdErr(serverProc));
		}

		// Connect.
		de::Socket socket;
		socket.connect(m_testCtx.address);

		// Flags.
		socket.setFlags(DE_SOCKET_CLOSE_ON_EXEC);

		// Run case.
		testCase->runClient(socket);

		// Disconnect.
		if (socket.isConnected())
			socket.shutdown();

		// Kill server.
		if (serverProc && deProcess_isRunning(serverProc))
		{
			XS_CHECK(deProcess_terminate(serverProc));
			deSleep(100);
			XS_CHECK(deProcess_waitForFinish(serverProc));

			stdoutPrinter.join();
			stderrPrinter.join();
		}

		success = true;
	}
	catch (const std::exception& e)
	{
		printf("FAIL: %s\n\n", e.what());
	}

	if (serverProc)
		deProcess_destroy(serverProc);

	return success;
}

class ConnectTest : public TestCase
{
public:
	ConnectTest (TestContext& testCtx)
		: TestCase(testCtx, "connect")
	{
	}

	void runClient (de::Socket& socket)
	{
		DE_UNREF(socket);
	}

	void runProgram (void) { /* nothing */ }
};

class HelloTest : public TestCase
{
public:
	HelloTest (TestContext& testCtx)
		: TestCase(testCtx, "hello")
	{
	}

	void runClient (de::Socket& socket)
	{
		xs::HelloMessage msg;
		sendMessage(socket, (const xs::Message&)msg);
	}

	void runProgram (void) { /* nothing */ }
};

class ExecFailTest : public TestCase
{
public:
	ExecFailTest (TestContext& testCtx)
		: TestCase(testCtx, "exec-fail")
	{
	}

	void runClient (de::Socket& socket)
	{
		xs::ExecuteBinaryMessage execMsg;
		execMsg.name		= "foobar-notfound";
		execMsg.params		= "";
		execMsg.caseList	= "";
		execMsg.workDir		= "";

		sendMessage(socket, execMsg);

		const int		timeout		= 100; // 100ms.
		TestClock		clock;

		for (;;)
		{
			if (clock.getMilliseconds() > timeout)
				XS_FAIL("Didn't receive PROCESS_LAUNCH_FAILED");

			ScopedMsgPtr msg(readMessage(socket));

			if (msg->type == MESSAGETYPE_PROCESS_LAUNCH_FAILED)
				break;
			else if (msg->type == MESSAGETYPE_KEEPALIVE)
				continue;
			else
				XS_FAIL("Invalid message");
		}
	}

	void runProgram (void) { /* nothing */ }
};

class SimpleExecTest : public TestCase
{
public:
	SimpleExecTest (TestContext& testCtx)
		: TestCase(testCtx, "simple-exec")
	{
	}

	void runClient (de::Socket& socket)
	{
		xs::ExecuteBinaryMessage execMsg;
		execMsg.name		= m_testCtx.testerPath;
		execMsg.params		= "--program=simple-exec";
		execMsg.caseList	= "";
		execMsg.workDir		= "";

		sendMessage(socket, execMsg);

		const int		timeout		= 5000; // 5s.
		TestClock		clock;

		bool	gotProcessStarted	= false;
		bool	gotProcessFinished	= false;

		for (;;)
		{
			if (clock.getMilliseconds() > timeout)
				break;

			ScopedMsgPtr msg(readMessage(socket));

			if (msg->type == MESSAGETYPE_PROCESS_STARTED)
				gotProcessStarted = true;
			else if (msg->type == MESSAGETYPE_PROCESS_LAUNCH_FAILED)
				XS_FAIL("Got PROCESS_LAUNCH_FAILED");
			else if (gotProcessStarted && msg->type == MESSAGETYPE_PROCESS_FINISHED)
			{
				gotProcessFinished = true;
				break;
			}
			else if (msg->type == MESSAGETYPE_KEEPALIVE || msg->type == MESSAGETYPE_INFO)
				continue;
			else
				XS_FAIL((string("Invalid message: ") + de::toString(msg->type)).c_str());
		}

		if (!gotProcessStarted)
			XS_FAIL("Did't get PROCESS_STARTED message");

		if (!gotProcessFinished)
			XS_FAIL("Did't get PROCESS_FINISHED message");
	}

	void runProgram (void) { /* print nothing. */ }
};

class InfoTest : public TestCase
{
public:
	std::string infoStr;

	InfoTest (TestContext& testCtx)
		: TestCase	(testCtx, "info")
		, infoStr	("Hello, World")
	{
	}

	void runClient (de::Socket& socket)
	{
		xs::ExecuteBinaryMessage execMsg;
		execMsg.name		= m_testCtx.testerPath;
		execMsg.params		= "--program=info";
		execMsg.caseList	= "";
		execMsg.workDir		= "";

		sendMessage(socket, execMsg);

		const int		timeout		= 10000; // 10s.
		TestClock		clock;

		bool			gotProcessStarted	= false;
		bool			gotProcessFinished	= false;
		std::string		receivedInfo		= "";

		for (;;)
		{
			if (clock.getMilliseconds() > timeout)
				break;

			ScopedMsgPtr msg(readMessage(socket));

			if (msg->type == MESSAGETYPE_PROCESS_STARTED)
				gotProcessStarted = true;
			else if (msg->type == MESSAGETYPE_PROCESS_LAUNCH_FAILED)
				XS_FAIL("Got PROCESS_LAUNCH_FAILED");
			else if (gotProcessStarted && msg->type == MESSAGETYPE_INFO)
				receivedInfo += static_cast<const InfoMessage*>(msg.get())->info;
			else if (gotProcessStarted && msg->type == MESSAGETYPE_PROCESS_FINISHED)
			{
				gotProcessFinished = true;
				break;
			}
			else if (msg->type == MESSAGETYPE_KEEPALIVE)
				continue;
			else
				XS_FAIL("Invalid message");
		}

		if (!gotProcessStarted)
			XS_FAIL("Did't get PROCESS_STARTED message");

		if (!gotProcessFinished)
			XS_FAIL("Did't get PROCESS_FINISHED message");

		if (receivedInfo != infoStr)
			XS_FAIL("Info data doesn't match");
	}

	void runProgram (void) { printf("%s", infoStr.c_str()); }
};

class LogDataTest : public TestCase
{
public:
	LogDataTest (TestContext& testCtx)
		: TestCase(testCtx, "logdata")
	{
	}

	void runClient (de::Socket& socket)
	{
		xs::ExecuteBinaryMessage execMsg;
		execMsg.name		= m_testCtx.testerPath;
		execMsg.params		= "--program=logdata";
		execMsg.caseList	= "";
		execMsg.workDir		= "";

		sendMessage(socket, execMsg);

		const int		timeout		= 10000; // 10s.
		TestClock		clock;

		bool			gotProcessStarted	= false;
		bool			gotProcessFinished	= false;
		std::string		receivedData		= "";

		for (;;)
		{
			if (clock.getMilliseconds() > timeout)
				break;

			ScopedMsgPtr msg(readMessage(socket));

			if (msg->type == MESSAGETYPE_PROCESS_STARTED)
				gotProcessStarted = true;
			else if (msg->type == MESSAGETYPE_PROCESS_LAUNCH_FAILED)
				XS_FAIL("Got PROCESS_LAUNCH_FAILED");
			else if (gotProcessStarted && msg->type == MESSAGETYPE_PROCESS_LOG_DATA)
				receivedData += static_cast<const ProcessLogDataMessage*>(msg.get())->logData;
			else if (gotProcessStarted && msg->type == MESSAGETYPE_PROCESS_FINISHED)
			{
				gotProcessFinished = true;
				break;
			}
			else if (msg->type == MESSAGETYPE_KEEPALIVE)
				continue;
			else if (msg->type == MESSAGETYPE_INFO)
				XS_FAIL(static_cast<const InfoMessage*>(msg.get())->info.c_str());
			else
				XS_FAIL("Invalid message");
		}

		if (!gotProcessStarted)
			XS_FAIL("Did't get PROCESS_STARTED message");

		if (!gotProcessFinished)
			XS_FAIL("Did't get PROCESS_FINISHED message");

		const char* expected = "Foo\nBar\n";
		if (receivedData != expected)
		{
			printf("  received: '%s'\n  expected: '%s'\n", receivedData.c_str(), expected);
			XS_FAIL("Log data doesn't match");
		}
	}

	void runProgram (void)
	{
		deFile* file = deFile_create(m_testCtx.logFileName.c_str(), DE_FILEMODE_OPEN|DE_FILEMODE_CREATE|DE_FILEMODE_TRUNCATE|DE_FILEMODE_WRITE);
		XS_CHECK(file);

		const char line0[] = "Foo\n";
		const char line1[] = "Bar\n";
		deInt64 numWritten = 0;

		// Write first line.
		XS_CHECK(deFile_write(file, line0, sizeof(line0)-1, &numWritten) == DE_FILERESULT_SUCCESS);
		XS_CHECK(numWritten == sizeof(line0)-1);

		// Sleep for 0.5s and write line 2.
		deSleep(500);
		XS_CHECK(deFile_write(file, line1, sizeof(line1)-1, &numWritten) == DE_FILERESULT_SUCCESS);
		XS_CHECK(numWritten == sizeof(line1)-1);

		deFile_destroy(file);
	}
};

class BigLogDataTest : public TestCase
{
public:
	enum
	{
		DATA_SIZE = 100*1024*1024
	};

	BigLogDataTest (TestContext& testCtx)
		: TestCase(testCtx, "biglogdata")
	{
	}

	void runClient (de::Socket& socket)
	{
		xs::ExecuteBinaryMessage execMsg;
		execMsg.name		= m_testCtx.testerPath;
		execMsg.params		= "--program=biglogdata";
		execMsg.caseList	= "";
		execMsg.workDir		= "";

		sendMessage(socket, execMsg);

		const int		timeout		= 30000; // 30s.
		TestClock		clock;

		bool			gotProcessStarted	= false;
		bool			gotProcessFinished	= false;
		int				receivedBytes		= 0;

		for (;;)
		{
			if (clock.getMilliseconds() > timeout)
				break;

			ScopedMsgPtr msg(readMessage(socket));

			if (msg->type == MESSAGETYPE_PROCESS_STARTED)
				gotProcessStarted = true;
			else if (msg->type == MESSAGETYPE_PROCESS_LAUNCH_FAILED)
				XS_FAIL("Got PROCESS_LAUNCH_FAILED");
			else if (gotProcessStarted && msg->type == MESSAGETYPE_PROCESS_LOG_DATA)
				receivedBytes += (int)static_cast<const ProcessLogDataMessage*>(msg.get())->logData.length();
			else if (gotProcessStarted && msg->type == MESSAGETYPE_PROCESS_FINISHED)
			{
				gotProcessFinished = true;
				break;
			}
			else if (msg->type == MESSAGETYPE_KEEPALIVE)
			{
				// Reply with keepalive.
				sendMessage(socket, KeepAliveMessage());
				continue;
			}
			else if (msg->type == MESSAGETYPE_INFO)
				printf("%s", static_cast<const InfoMessage*>(msg.get())->info.c_str());
			else
				XS_FAIL("Invalid message");
		}

		if (!gotProcessStarted)
			XS_FAIL("Did't get PROCESS_STARTED message");

		if (!gotProcessFinished)
			XS_FAIL("Did't get PROCESS_FINISHED message");

		if (receivedBytes != DATA_SIZE)
		{
			printf("  received: %d bytes\n  expected: %d bytes\n", receivedBytes, DATA_SIZE);
			XS_FAIL("Log data size doesn't match");
		}

		int timeMs = clock.getMilliseconds();
		printf("  Streamed %d bytes in %d ms: %.2f MiB/s\n", DATA_SIZE, timeMs, ((float)DATA_SIZE / (float)(1024*1024)) / ((float)timeMs / 1000.0f));
	}

	void runProgram (void)
	{
		deFile* file = deFile_create(m_testCtx.logFileName.c_str(), DE_FILEMODE_OPEN|DE_FILEMODE_CREATE|DE_FILEMODE_TRUNCATE|DE_FILEMODE_WRITE);
		XS_CHECK(file);

		deUint8 tmpBuf[1024*16];
		int numWritten = 0;

		deMemset(&tmpBuf, 'a', sizeof(tmpBuf));

		while (numWritten < DATA_SIZE)
		{
			deInt64 numWrittenInBatch = 0;
			XS_CHECK(deFile_write(file, &tmpBuf[0], de::min((int)sizeof(tmpBuf), DATA_SIZE-numWritten), &numWrittenInBatch) == DE_FILERESULT_SUCCESS);
			numWritten += (int)numWrittenInBatch;
		}

		deFile_destroy(file);
	}
};

class KeepAliveTest : public TestCase
{
public:
	KeepAliveTest (TestContext& testCtx)
		: TestCase(testCtx, "keepalive")
	{
	}

	void runClient (de::Socket& socket)
	{
		// In milliseconds.
		const int	sendInterval			= 5000;
		const int	minReceiveInterval		= 10000;
		const int	testTime				= 30000;
		const int	sleepTime				= 200;
		const int	expectedTimeout			= 40000;
		int			curTime					= 0;
		int			lastSendTime			= 0;
		int			lastReceiveTime			= 0;
		TestClock	clock;

		DE_ASSERT(sendInterval < minReceiveInterval);

		curTime = clock.getMilliseconds();

		while (curTime < testTime)
		{
			bool tryGetKeepalive = false;

			if (curTime-lastSendTime > sendInterval)
			{
				printf("  %d ms: sending keepalive\n", curTime);
				sendMessage(socket, KeepAliveMessage());
				curTime = clock.getMilliseconds();
				lastSendTime = curTime;
				tryGetKeepalive = true;
			}

			if (tryGetKeepalive)
			{
				// Try to acquire keepalive.
				printf("  %d ms: waiting for keepalive\n", curTime);
				ScopedMsgPtr msg(readMessage(socket));
				int recvTime = clock.getMilliseconds();

				if (msg->type != MESSAGETYPE_KEEPALIVE)
					XS_FAIL("Got invalid message");

				printf("  %d ms: got keepalive\n", curTime);

				if (recvTime-lastReceiveTime > minReceiveInterval)
					XS_FAIL("Server doesn't send keepalives");

				lastReceiveTime = recvTime;
			}

			deSleep(sleepTime);
			curTime = clock.getMilliseconds();
		}

		// Verify that server actually kills the connection upon timeout.
		sendMessage(socket, KeepAliveMessage());
		printf("  waiting %d ms for keepalive timeout...\n", expectedTimeout);
		bool isClosed = false;

		try
		{
			// Reset timer.
			clock.reset();
			curTime = clock.getMilliseconds();

			while (curTime < expectedTimeout)
			{
				// Try to get keepalive message.
				ScopedMsgPtr msg(readMessage(socket));
				if (msg->type != MESSAGETYPE_KEEPALIVE)
					XS_FAIL("Got invalid message");

				curTime = clock.getMilliseconds();
				printf("  %d ms: got keepalive\n", curTime);
			}
		}
		catch (const SocketError& e)
		{
			if (e.getResult() == DE_SOCKETRESULT_CONNECTION_CLOSED)
			{
				printf("  %d ms: server closed connection", clock.getMilliseconds());
				isClosed = true;
			}
			else
				throw;
		}

		if (isClosed)
			printf("  ok!\n");
		else
			XS_FAIL("Server didn't close connection");
	}

	void runProgram (void) { /* nothing */ }
};

void printHelp (const char* binName)
{
	printf("%s:\n", binName);
	printf("  --client=[name]       Run test [name]\n");
	printf("  --program=[name]      Run program for test [name]\n");
	printf("  --host=[host]         Connect to host [host]\n");
	printf("  --port=[name]         Use port [port]\n");
	printf("  --tester-cmd=[cmd]    Launch tester with [cmd]\n");
	printf("  --server-cmd=[cmd]    Launch server with [cmd]\n");
	printf("  --start-server        Start server for test execution\n");
}

struct CompareCaseName
{
	std::string name;

	CompareCaseName (const string& name_) : name(name_) {}

	bool operator() (const TestCase* testCase) const
	{
		return name == testCase->getName();
	}
};

void runExecServerTests (int argc, const char* const* argv)
{
	// Construct test context.
	TestContext testCtx;

	testCtx.serverPath	= "execserver";
	testCtx.testerPath	= argv[0];
	testCtx.startServer	= false;
	testCtx.address.setHost("127.0.0.1");
	testCtx.address.setPort(50016);

	std::string runClient = "";
	std::string runProgram = "";

	// Parse command line.
	for (int argNdx = 1; argNdx < argc; argNdx++)
	{
		const char* arg = argv[argNdx];

		if (deStringBeginsWith(arg, "--client="))
			runClient = arg+9;
		else if (deStringBeginsWith(arg, "--program="))
			runProgram = arg+10;
		else if (deStringBeginsWith(arg, "--port="))
			testCtx.address.setPort(atoi(arg+7));
		else if (deStringBeginsWith(arg, "--host="))
			testCtx.address.setHost(arg+7);
		else if (deStringBeginsWith(arg, "--server-cmd="))
			testCtx.serverPath = arg+13;
		else if (deStringBeginsWith(arg, "--tester-cmd="))
			testCtx.testerPath = arg+13;
		else if (deStringBeginsWith(arg, "--deqp-log-filename="))
			testCtx.logFileName = arg+20;
		else if (deStringBeginsWith(arg, "--deqp-caselist="))
			testCtx.caseList = arg+16;
		else if (deStringEqual(arg, "--deqp-stdin-caselist"))
		{
			// \todo [pyry] This is rather brute-force solution...
			char c;
			while (fread(&c, 1, 1, stdin) == 1 && c != 0)
				testCtx.caseList += c;
		}
		else if (deStringEqual(arg, "--start-server"))
			testCtx.startServer = true;
		else
		{
			printHelp(argv[0]);
			return;
		}
	}

	// Test case list.
	std::vector<TestCase*> testCases;
	testCases.push_back(new ConnectTest(testCtx));
	testCases.push_back(new HelloTest(testCtx));
	testCases.push_back(new ExecFailTest(testCtx));
	testCases.push_back(new SimpleExecTest(testCtx));
	testCases.push_back(new InfoTest(testCtx));
	testCases.push_back(new LogDataTest(testCtx));
	testCases.push_back(new KeepAliveTest(testCtx));
	testCases.push_back(new BigLogDataTest(testCtx));

	try
	{
		if (!runClient.empty())
		{
			// Run single case.
			vector<TestCase*>::iterator casePos = std::find_if(testCases.begin(), testCases.end(), CompareCaseName(runClient));
			XS_CHECK(casePos != testCases.end());
			TestExecutor executor(testCtx);
			executor.runCase(*casePos);
		}
		else if (!runProgram.empty())
		{
			// Run program part.
			vector<TestCase*>::iterator casePos = std::find_if(testCases.begin(), testCases.end(), CompareCaseName(runProgram));
			XS_CHECK(casePos != testCases.end());
			(*casePos)->runProgram();
			fflush(stdout);	// Make sure handles are flushed.
			fflush(stderr);
		}
		else
		{
			// Run all tests.
			TestExecutor executor(testCtx);
			executor.runCases(testCases);
		}
	}
	catch (const std::exception& e)
	{
		printf("ERROR: %s\n", e.what());
	}

	// Destroy cases.
	for (std::vector<TestCase*>::const_iterator i = testCases.begin(); i != testCases.end(); i++)
		delete *i;
}

} // xs

#if 0
void testProcFile (void)
{
	/* Test file api. */
	if (deFileExists("test.txt"))
		deDeleteFile("test.txt");
	deFile* file = deFile_create("test.txt", DE_FILEMODE_CREATE|DE_FILEMODE_WRITE);
	const char test[] = "Hello";
	XS_CHECK(deFile_write(file, test, sizeof(test), DE_NULL) == DE_FILERESULT_SUCCESS);
	deFile_destroy(file);

	/* Read. */
	char buf[10] = { 0 };
	file = deFile_create("test.txt", DE_FILEMODE_OPEN|DE_FILEMODE_READ);
	XS_CHECK(deFile_read(file, buf, sizeof(test), DE_NULL) == DE_FILERESULT_SUCCESS);
	printf("buf: %s\n", buf);
	deFile_destroy(file);

	/* Process test. */
	deProcess* proc = deProcess_create("ls -lah /Users/pyry", DE_NULL);
	deFile* out = deProcess_getStdOut(proc);

	deInt64 numRead = 0;
	printf("ls:\n");
	while (deFile_read(out, buf, sizeof(buf)-1, &numRead) == DE_FILERESULT_SUCCESS)
	{
		buf[numRead] = 0;
		printf("%s", buf);
	}
	deProcess_destroy(proc);
}
#endif

#if 0
void testBlockingFile (const char* filename)
{
	deRandom	rnd;
	int			dataSize	= 1024*1024;
	deUint8*	data		= (deUint8*)deCalloc(dataSize);
	deFile*		file;

	deRandom_init(&rnd, 0);

	if (deFileExists(filename))
		DE_VERIFY(deDeleteFile(filename));

	/* Fill in with random data. */
	DE_ASSERT(dataSize % sizeof(int) == 0);
	for (int ndx = 0; ndx < (int)(dataSize/sizeof(int)); ndx++)
		((deUint32*)data)[ndx] = deRandom_getUint32(&rnd);

	/* Write with random-sized blocks. */
	file = deFile_create(filename, DE_FILEMODE_CREATE|DE_FILEMODE_WRITE);
	DE_VERIFY(file);

	int curPos = 0;
	while (curPos < dataSize)
	{
		int				blockSize	= 1 + deRandom_getUint32(&rnd) % (dataSize-curPos);
		deInt64			numWritten	= 0;
		deFileResult	result		= deFile_write(file, &data[curPos], blockSize, &numWritten);

		DE_VERIFY(result == DE_FILERESULT_SUCCESS);
		DE_VERIFY(numWritten == blockSize);

		curPos += blockSize;
	}

	deFile_destroy(file);

	/* Read and verify file. */
	file	= deFile_create(filename, DE_FILEMODE_OPEN|DE_FILEMODE_READ);
	curPos	= 0;
	while (curPos < dataSize)
	{
		deUint8			block[1024];
		int				numToRead	= 1 + deRandom_getUint32(&rnd) % deMin(dataSize-curPos, DE_LENGTH_OF_ARRAY(block));
		deInt64			numRead		= 0;
		deFileResult	result		= deFile_read(file, block, numToRead, &numRead);

		DE_VERIFY(result == DE_FILERESULT_SUCCESS);
		DE_VERIFY((int)numRead == numToRead);
		DE_VERIFY(deMemCmp(block, &data[curPos], numToRead) == 0);

		curPos += numToRead;
	}
	deFile_destroy(file);
}
#endif

int main (int argc, const char* const* argv)
{
	xs::runExecServerTests(argc, argv);
	return 0;
}