1 /*-------------------------------------------------------------------------
2 * OpenGL Conformance Test Suite
3 * -----------------------------
4 *
5 * Copyright (c) 2016 Google Inc.
6 * Copyright (c) 2016 The Khronos Group Inc.
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License");
9 * you may not use this file except in compliance with the License.
10 * You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 *
20 */ /*!
21 * \file
22 * \brief CTS runner.
23 */ /*-------------------------------------------------------------------*/
24
25 #include "glcTestRunner.hpp"
26 #include "deFilePath.hpp"
27 #include "deStringUtil.hpp"
28 #include "deUniquePtr.hpp"
29 #include "glcConfigList.hpp"
30 #include "qpXmlWriter.h"
31 #include "tcuApp.hpp"
32 #include "tcuCommandLine.hpp"
33 #include "tcuTestLog.hpp"
34 #include "tcuTestSessionExecutor.hpp"
35
36 #include <iterator>
37
38 namespace glcts
39 {
40
41 using std::vector;
42 using std::string;
43
44 // RunSession
45
46 class RunSession
47 {
48 public:
RunSession(tcu::Platform& platform, tcu::Archive& archive, const int numArgs, const char* const* args)49 RunSession(tcu::Platform& platform, tcu::Archive& archive, const int numArgs, const char* const* args)
50 : m_cmdLine(numArgs, args)
51 , m_log(m_cmdLine.getLogFileName(), m_cmdLine.getLogFlags())
52 , m_app(platform, archive, m_log, m_cmdLine)
53 {
54 const std::string sessionInfo = "#sessionInfo commandLineParameters \"";
55 m_log.writeSessionInfo(sessionInfo + m_cmdLine.getInitialCmdLine() + "\"\n");
56 }
57
iterate(void)58 inline bool iterate(void)
59 {
60 return m_app.iterate();
61 }
62
getResult(void) const63 inline const tcu::TestRunStatus& getResult(void) const
64 {
65 return m_app.getResult();
66 }
67
68 private:
69 tcu::CommandLine m_cmdLine;
70 tcu::TestLog m_log;
71 tcu::App m_app;
72 };
73
appendConfigArgs(const Config& config, std::vector<std::string>& args, const char* fboConfig)74 static void appendConfigArgs(const Config& config, std::vector<std::string>& args, const char* fboConfig)
75 {
76 if (fboConfig != NULL)
77 {
78 args.push_back(string("--deqp-gl-config-name=") + fboConfig);
79 args.push_back("--deqp-surface-type=fbo");
80 }
81
82 if (config.type != CONFIGTYPE_DEFAULT)
83 {
84 // \todo [2013-05-06 pyry] Test all surface types for some configs?
85 if (fboConfig == NULL)
86 {
87 if (config.surfaceTypes & SURFACETYPE_WINDOW)
88 args.push_back("--deqp-surface-type=window");
89 else if (config.surfaceTypes & SURFACETYPE_PBUFFER)
90 args.push_back("--deqp-surface-type=pbuffer");
91 else if (config.surfaceTypes & SURFACETYPE_PIXMAP)
92 args.push_back("--deqp-surface-type=pixmap");
93 }
94
95 args.push_back(string("--deqp-gl-config-id=") + de::toString(config.id));
96
97 if (config.type == CONFIGTYPE_EGL)
98 args.push_back("--deqp-gl-context-type=egl");
99 else if (config.type == CONFIGTYPE_WGL)
100 args.push_back("--deqp-gl-context-type=wgl");
101 }
102 }
103
104 typedef struct configInfo
105 {
106 deInt32 redBits;
107 deInt32 greenBits;
108 deInt32 blueBits;
109 deInt32 alphaBits;
110 deInt32 depthBits;
111 deInt32 stencilBits;
112 deInt32 samples;
113 } configInfo;
114
parseConfigBitsFromName(const char* configName)115 static configInfo parseConfigBitsFromName(const char* configName)
116 {
117 configInfo cfgInfo;
118 static const struct
119 {
120 const char* name;
121 int redBits;
122 int greenBits;
123 int blueBits;
124 int alphaBits;
125 } colorCfgs[] = {
126 { "rgba8888", 8, 8, 8, 8 }, { "rgb565", 5, 6, 5, 0 },
127 };
128 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(colorCfgs); ndx++)
129 {
130 if (!strncmp(configName, colorCfgs[ndx].name, strlen(colorCfgs[ndx].name)))
131 {
132 cfgInfo.redBits = colorCfgs[ndx].redBits;
133 cfgInfo.greenBits = colorCfgs[ndx].greenBits;
134 cfgInfo.blueBits = colorCfgs[ndx].blueBits;
135 cfgInfo.alphaBits = colorCfgs[ndx].alphaBits;
136
137 configName += strlen(colorCfgs[ndx].name);
138 break;
139 }
140 }
141
142 static const struct
143 {
144 const char* name;
145 int depthBits;
146 } depthCfgs[] = {
147 { "d0", 0 }, { "d24", 24 },
148 };
149 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(depthCfgs); ndx++)
150 {
151 if (!strncmp(configName, depthCfgs[ndx].name, strlen(depthCfgs[ndx].name)))
152 {
153 cfgInfo.depthBits = depthCfgs[ndx].depthBits;
154
155 configName += strlen(depthCfgs[ndx].name);
156 break;
157 }
158 }
159
160 static const struct
161 {
162 const char* name;
163 int stencilBits;
164 } stencilCfgs[] = {
165 { "s0", 0 }, { "s8", 8 },
166 };
167 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(stencilCfgs); ndx++)
168 {
169 if (!strncmp(configName, stencilCfgs[ndx].name, strlen(stencilCfgs[ndx].name)))
170 {
171 cfgInfo.stencilBits = stencilCfgs[ndx].stencilBits;
172
173 configName += strlen(stencilCfgs[ndx].name);
174 break;
175 }
176 }
177
178 static const struct
179 {
180 const char* name;
181 int samples;
182 } multiSampleCfgs[] = {
183 { "ms0", 0 }, { "ms4", 4 },
184 };
185 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(multiSampleCfgs); ndx++)
186 {
187 if (!strncmp(configName, multiSampleCfgs[ndx].name, strlen(multiSampleCfgs[ndx].name)))
188 {
189 cfgInfo.samples = multiSampleCfgs[ndx].samples;
190
191 configName += strlen(multiSampleCfgs[ndx].name);
192 break;
193 }
194 }
195
196 return cfgInfo;
197 }
198
getApiName(glu::ApiType apiType)199 static const char* getApiName(glu::ApiType apiType)
200 {
201 if (apiType == glu::ApiType::es(2, 0))
202 return "gles2";
203 else if (apiType == glu::ApiType::es(3, 0))
204 return "gles3";
205 else if (apiType == glu::ApiType::es(3, 1))
206 return "gles31";
207 else if (apiType == glu::ApiType::es(3, 2))
208 return "gles32";
209 else if (apiType == glu::ApiType::core(3, 0))
210 return "gl30";
211 else if (apiType == glu::ApiType::core(3, 1))
212 return "gl31";
213 else if (apiType == glu::ApiType::core(3, 2))
214 return "gl32";
215 else if (apiType == glu::ApiType::core(3, 3))
216 return "gl33";
217 else if (apiType == glu::ApiType::core(4, 0))
218 return "gl40";
219 else if (apiType == glu::ApiType::core(4, 1))
220 return "gl41";
221 else if (apiType == glu::ApiType::core(4, 2))
222 return "gl42";
223 else if (apiType == glu::ApiType::core(4, 3))
224 return "gl43";
225 else if (apiType == glu::ApiType::core(4, 4))
226 return "gl44";
227 else if (apiType == glu::ApiType::core(4, 5))
228 return "gl45";
229 else if (apiType == glu::ApiType::core(4, 6))
230 return "gl46";
231 else
232 throw std::runtime_error("Unknown context type");
233 }
234
getCaseListFileOption(const char* mustpassDir, const char* apiName, const char* mustpassName)235 static const string getCaseListFileOption(const char* mustpassDir, const char* apiName, const char* mustpassName)
236 {
237 #if DE_OS == DE_OS_ANDROID
238 const string case_list_option = "--deqp-caselist-resource=";
239 #else
240 const string case_list_option = "--deqp-caselist-file=";
241 #endif
242 return case_list_option + mustpassDir + apiName + "-" + mustpassName + ".txt";
243 }
244
getLogFileName(const char* apiName, const char* configName, const int iterId, const int runId, const int width, const int height, const int seed)245 static const string getLogFileName(const char* apiName, const char* configName, const int iterId, const int runId,
246 const int width, const int height, const int seed)
247 {
248 string res = string("config-") + apiName + "-" + configName + "-cfg-" + de::toString(iterId) + "-run-" +
249 de::toString(runId) + "-width-" + de::toString(width) + "-height-" + de::toString(height);
250 if (seed != -1)
251 {
252 res += "-seed-" + de::toString(seed);
253 }
254 res += ".qpa";
255
256 return res;
257 }
258
getBaseOptions(std::vector<std::string>& args, const char* mustpassDir, const char* apiName, const char* configName, const char* screenRotation, int width, int height)259 static void getBaseOptions(std::vector<std::string>& args, const char* mustpassDir, const char* apiName,
260 const char* configName, const char* screenRotation, int width, int height)
261 {
262 args.push_back(getCaseListFileOption(mustpassDir, apiName, configName));
263 args.push_back(string("--deqp-screen-rotation=") + screenRotation);
264 args.push_back(string("--deqp-surface-width=") + de::toString(width));
265 args.push_back(string("--deqp-surface-height=") + de::toString(height));
266 args.push_back("--deqp-watchdog=disable");
267 }
268
isGLConfigCompatible(configInfo cfgInfo, const AOSPConfig& config)269 static bool isGLConfigCompatible(configInfo cfgInfo, const AOSPConfig& config)
270 {
271 return cfgInfo.redBits == config.redBits && cfgInfo.greenBits == config.greenBits &&
272 cfgInfo.blueBits == config.blueBits && cfgInfo.alphaBits == config.alphaBits &&
273 cfgInfo.depthBits == config.depthBits && cfgInfo.stencilBits == config.stencilBits &&
274 cfgInfo.samples == config.samples;
275 }
276
getTestRunsForAOSPEGL(vector<TestRunParams>& runs, const ConfigList& configs)277 static void getTestRunsForAOSPEGL(vector<TestRunParams>& runs, const ConfigList& configs)
278 {
279 #include "glcAospMustpassEgl.hpp"
280
281 for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_egl_first_cfg); ++i)
282 {
283 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_egl_first_cfg[i].glConfigName);
284
285 vector<AOSPConfig>::const_iterator cfgIter;
286 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
287 {
288 // find first compatible config
289 if ((*cfgIter).type == CONFIGTYPE_EGL && isGLConfigCompatible(cfgInfo, *cfgIter))
290 {
291 break;
292 }
293 }
294
295 if (cfgIter == configs.aospConfigs.end())
296 {
297 // No suitable configuration found. Skipping EGL tests
298 continue;
299 }
300
301 const char* apiName = "egl";
302
303 const int width = aosp_mustpass_egl_first_cfg[i].surfaceWidth;
304 const int height = aosp_mustpass_egl_first_cfg[i].surfaceHeight;
305
306 TestRunParams params;
307 params.logFilename =
308 getLogFileName(apiName, aosp_mustpass_egl_first_cfg[i].configName, 1, i, width, height, -1);
309 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_egl_first_cfg[i].configName,
310 aosp_mustpass_egl_first_cfg[i].screenRotation, width, height);
311
312 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_egl_first_cfg[i].glConfigName));
313
314 runs.push_back(params);
315 }
316 }
317
getTestRunsForAOSPES(vector<TestRunParams>& runs, const ConfigList& configs, const glu::ApiType apiType)318 static void getTestRunsForAOSPES(vector<TestRunParams>& runs, const ConfigList& configs, const glu::ApiType apiType)
319 {
320 #include "glcAospMustpassEs.hpp"
321
322 for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_es_first_cfg); ++i)
323 {
324 if (!glu::contextSupports(glu::ContextType(apiType), aosp_mustpass_es_first_cfg[i].apiType))
325 continue;
326
327 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_es_first_cfg[i].glConfigName);
328
329 vector<AOSPConfig>::const_iterator cfgIter;
330 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
331 {
332 // find first compatible config
333 if (isGLConfigCompatible(cfgInfo, *cfgIter))
334 {
335 break;
336 }
337 }
338
339 if (cfgIter == configs.aospConfigs.end())
340 {
341 TCU_FAIL(("No suitable configuration found for GL config " +
342 de::toString(aosp_mustpass_es_first_cfg[i].glConfigName))
343 .c_str());
344 return;
345 }
346
347 const char* apiName = getApiName(aosp_mustpass_es_first_cfg[i].apiType);
348
349 const int width = aosp_mustpass_es_first_cfg[i].surfaceWidth;
350 const int height = aosp_mustpass_es_first_cfg[i].surfaceHeight;
351
352 TestRunParams params;
353 params.logFilename = getLogFileName(apiName, aosp_mustpass_es_first_cfg[i].configName, 1, i, width, height, -1);
354 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_es_first_cfg[i].configName,
355 aosp_mustpass_es_first_cfg[i].screenRotation, width, height);
356
357 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_es_first_cfg[i].glConfigName));
358
359 //set surface type
360 if ((*cfgIter).surfaceTypes & SURFACETYPE_WINDOW)
361 params.args.push_back("--deqp-surface-type=window");
362 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PBUFFER)
363 params.args.push_back("--deqp-surface-type=pbuffer");
364 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PIXMAP)
365 params.args.push_back("--deqp-surface-type=pixmap");
366 runs.push_back(params);
367 }
368 }
369
getTestRunsForNoContext(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs, const RunParams* runParams, const int numRunParams, const char* mustpassDir)370 static void getTestRunsForNoContext(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs, const RunParams* runParams,
371 const int numRunParams, const char* mustpassDir)
372 {
373 vector<Config>::const_iterator cfgIter = configs.configs.begin();
374
375 for (int i = 0; i < numRunParams; ++i)
376 {
377 if (!glu::contextSupports(glu::ContextType(type), runParams[i].apiType))
378 continue;
379
380 const char* apiName = getApiName(runParams[i].apiType);
381
382 const int width = runParams[i].surfaceWidth;
383 const int height = runParams[i].surfaceHeight;
384 const int seed = runParams[i].baseSeed;
385
386 TestRunParams params;
387 params.logFilename = getLogFileName(apiName, runParams[i].configName, 1, i, width, height, seed);
388
389 getBaseOptions(params.args, mustpassDir, apiName, runParams[i].configName, runParams[i].screenRotation, width,
390 height);
391
392 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
393
394 appendConfigArgs(*cfgIter, params.args, runParams[i].fboConfig);
395
396 runs.push_back(params);
397 }
398 }
399
getTestRunsForNoContextES(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)400 static void getTestRunsForNoContextES(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
401 {
402 #include "glcKhronosMustpassEsNocontext.hpp"
403 getTestRunsForNoContext(type, runs, configs, khronos_mustpass_es_nocontext_first_cfg,
404 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_nocontext_first_cfg), mustpassDir);
405 }
406
getTestRunsForSingleConfig(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs, const RunParams* runParams, const int numRunParams, const char* mustpassDir)407 static void getTestRunsForSingleConfig(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs, const RunParams* runParams,
408 const int numRunParams, const char* mustpassDir)
409 {
410 vector<Config>::const_iterator cfgIter = configs.configs.begin();
411
412 for (int i = 0; i < numRunParams; ++i)
413 {
414 if (type != runParams[i].apiType)
415 continue;
416
417 const char* apiName = getApiName(runParams[i].apiType);
418
419 const int width = runParams[i].surfaceWidth;
420 const int height = runParams[i].surfaceHeight;
421 const int seed = runParams[i].baseSeed;
422
423 TestRunParams params;
424 params.logFilename = getLogFileName(apiName, runParams[i].configName, 1, i, width, height, seed);
425
426 getBaseOptions(params.args, mustpassDir, apiName, runParams[i].configName, runParams[i].screenRotation, width,
427 height);
428
429 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
430
431 appendConfigArgs(*cfgIter, params.args, runParams[i].fboConfig);
432
433 runs.push_back(params);
434 }
435 }
getTestRunsForSingleConfigES(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)436 static void getTestRunsForSingleConfigES(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
437 {
438 #include "glcKhronosMustpassEsSingleConfig.hpp"
439 getTestRunsForSingleConfig(type, runs, configs, khronos_mustpass_es_single_config_first_cfg,
440 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_single_config_first_cfg), mustpassDir);
441 }
442
getTestRunsForES(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)443 static void getTestRunsForES(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
444 {
445 getTestRunsForAOSPEGL(runs, configs);
446 getTestRunsForAOSPES(runs, configs, type);
447 getTestRunsForNoContextES(type, runs, configs);
448 getTestRunsForSingleConfigES(type, runs, configs);
449
450 #include "glcKhronosMustpassEs.hpp"
451
452 for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
453 {
454 const bool isFirst = cfgIter == configs.configs.begin();
455 const int numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_es_first_cfg) :
456 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_other_cfg);
457 const RunParams* runParams = isFirst ? khronos_mustpass_es_first_cfg : khronos_mustpass_es_other_cfg;
458
459 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
460 {
461 if (!glu::contextSupports(glu::ContextType(type), runParams[runNdx].apiType))
462 continue;
463
464 const char* apiName = getApiName(runParams[runNdx].apiType);
465
466 const int width = runParams[runNdx].surfaceWidth;
467 const int height = runParams[runNdx].surfaceHeight;
468 const int seed = runParams[runNdx].baseSeed;
469
470 TestRunParams params;
471
472 params.logFilename =
473 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
474
475 getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
476 runParams[runNdx].screenRotation, width, height);
477 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
478
479 appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
480
481 runs.push_back(params);
482 }
483 }
484 }
485
getTestRunsForNoContextGL(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)486 static void getTestRunsForNoContextGL(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
487 {
488 #include "glcKhronosMustpassGlNocontext.hpp"
489 getTestRunsForNoContext(type, runs, configs, khronos_mustpass_gl_nocontext_first_cfg,
490 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_nocontext_first_cfg), mustpassDir);
491 }
getTestRunsForSingleConfigGL(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)492 static void getTestRunsForSingleConfigGL(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
493 {
494 #include "glcKhronosMustpassGlSingleConfig.hpp"
495 getTestRunsForSingleConfig(type, runs, configs, khronos_mustpass_gl_single_config_first_cfg,
496 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_single_config_first_cfg), mustpassDir);
497 }
498
getTestRunsForESForGL(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)499 static void getTestRunsForESForGL(glu::ApiType type, vector<TestRunParams>& runs, const ConfigList& configs)
500 {
501 #include "glcKhronosMustpassAospForGl.hpp"
502
503 vector<Config>::const_iterator cfgIter = configs.configs.begin();
504 const int numRunParams = DE_LENGTH_OF_ARRAY(khronos_mustpass_aosp_for_gl_first_cfg);
505 const RunParams* runParams = khronos_mustpass_aosp_for_gl_first_cfg;
506
507 for (int i = 0; i < numRunParams; ++i)
508 {
509 if (!glu::contextSupports(glu::ContextType(type), runParams[i].apiType))
510 continue;
511
512 const char* apiName = getApiName(runParams[i].apiType);
513
514 const int width = runParams[i].surfaceWidth;
515 const int height = runParams[i].surfaceHeight;
516 const int seed = runParams[i].baseSeed;
517
518 TestRunParams params;
519 params.logFilename = getLogFileName(apiName, runParams[i].configName, 1, i, width, height, seed);
520
521 getBaseOptions(params.args, mustpassDir, apiName, runParams[i].configName, runParams[i].screenRotation, width,
522 height);
523
524 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
525
526 appendConfigArgs(*cfgIter, params.args, runParams[i].fboConfig);
527
528 runs.push_back(params);
529 }
530 }
531
532
getTestRunsForGL(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)533 static void getTestRunsForGL(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
534 {
535 getTestRunsForESForGL(type, runs, configs);
536 getTestRunsForNoContextGL(type, runs, configs);
537 getTestRunsForSingleConfigGL(type, runs, configs);
538 #include "glcKhronosMustpassGl.hpp"
539
540 for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
541 {
542 const bool isFirst = cfgIter == configs.configs.begin();
543 const int numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_first_cfg) :
544 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_other_cfg);
545 const RunParams* runParams = isFirst ? khronos_mustpass_gl_first_cfg : khronos_mustpass_gl_other_cfg;
546
547 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
548 {
549 if (type != runParams[runNdx].apiType)
550 continue;
551
552 const char* apiName = getApiName(runParams[runNdx].apiType);
553
554 const int width = runParams[runNdx].surfaceWidth;
555 const int height = runParams[runNdx].surfaceHeight;
556 const int seed = runParams[runNdx].baseSeed;
557
558 TestRunParams params;
559
560 params.logFilename =
561 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
562
563 getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
564 runParams[runNdx].screenRotation, width, height);
565 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
566
567 appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
568
569 runs.push_back(params);
570 }
571 }
572 }
573
getTestRunParams(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)574 static void getTestRunParams(glu::ApiType type, const ConfigList& configs, vector<TestRunParams>& runs)
575 {
576 switch (type.getProfile())
577 {
578 case glu::PROFILE_CORE:
579 getTestRunsForGL(type, configs, runs);
580 break;
581 case glu::PROFILE_ES:
582 getTestRunsForES(type, configs, runs);
583 break;
584 default:
585 throw std::runtime_error("Unknown context type");
586 }
587 }
588
589 struct FileDeleter
590 {
operator ()glcts::FileDeleter591 void operator()(FILE* file) const
592 {
593 if (file)
594 fclose(file);
595 }
596 };
597
598 struct XmlWriterDeleter
599 {
operator ()glcts::XmlWriterDeleter600 void operator()(qpXmlWriter* writer) const
601 {
602 if (writer)
603 qpXmlWriter_destroy(writer);
604 }
605 };
606
getRunTypeName(glu::ApiType type)607 static const char* getRunTypeName(glu::ApiType type)
608 {
609 if (type == glu::ApiType::es(2, 0))
610 return "es2";
611 else if (type == glu::ApiType::es(3, 0))
612 return "es3";
613 else if (type == glu::ApiType::es(3, 1))
614 return "es31";
615 else if (type == glu::ApiType::es(3, 2))
616 return "es32";
617 else if (type == glu::ApiType::core(3, 0))
618 return "gl30";
619 else if (type == glu::ApiType::core(3, 1))
620 return "gl31";
621 else if (type == glu::ApiType::core(3, 2))
622 return "gl32";
623 else if (type == glu::ApiType::core(3, 3))
624 return "gl33";
625 else if (type == glu::ApiType::core(4, 0))
626 return "gl40";
627 else if (type == glu::ApiType::core(4, 1))
628 return "gl41";
629 else if (type == glu::ApiType::core(4, 2))
630 return "gl42";
631 else if (type == glu::ApiType::core(4, 3))
632 return "gl43";
633 else if (type == glu::ApiType::core(4, 4))
634 return "gl44";
635 else if (type == glu::ApiType::core(4, 5))
636 return "gl45";
637 else if (type == glu::ApiType::core(4, 6))
638 return "gl46";
639 else
640 return DE_NULL;
641 }
642
643 #define XML_CHECK(X) \
644 if (!(X)) \
645 throw tcu::Exception("Writing XML failed")
646
writeRunSummary(const TestRunSummary& summary, const char* filename)647 static void writeRunSummary(const TestRunSummary& summary, const char* filename)
648 {
649 de::UniquePtr<FILE, FileDeleter> out(fopen(filename, "wb"));
650 if (!out)
651 throw tcu::Exception(string("Failed to open ") + filename);
652
653 de::UniquePtr<qpXmlWriter, XmlWriterDeleter> writer(qpXmlWriter_createFileWriter(out.get(), DE_FALSE, DE_FALSE));
654 if (!writer)
655 throw std::bad_alloc();
656
657 XML_CHECK(qpXmlWriter_startDocument(writer.get(), true));
658
659 {
660 qpXmlAttribute attribs[2];
661
662 attribs[0] = qpSetStringAttrib("Type", getRunTypeName(summary.runType));
663 attribs[1] = qpSetBoolAttrib("Conformant", summary.isConformant ? DE_TRUE : DE_FALSE);
664
665 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Summary", DE_LENGTH_OF_ARRAY(attribs), attribs));
666 }
667
668 // Config run
669 {
670 qpXmlAttribute attribs[1];
671 attribs[0] = qpSetStringAttrib("FileName", summary.configLogFilename.c_str());
672 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Configs", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
673 qpXmlWriter_endElement(writer.get(), "Configs"));
674 }
675
676 // Record test run parameters (log filename & command line).
677 for (vector<TestRunParams>::const_iterator runIter = summary.runParams.begin(); runIter != summary.runParams.end();
678 ++runIter)
679 {
680 string cmdLine;
681 qpXmlAttribute attribs[2];
682
683 for (vector<string>::const_iterator argIter = runIter->args.begin(); argIter != runIter->args.end(); ++argIter)
684 {
685 if (argIter != runIter->args.begin())
686 cmdLine += " ";
687 cmdLine += *argIter;
688 }
689
690 attribs[0] = qpSetStringAttrib("FileName", runIter->logFilename.c_str());
691 attribs[1] = qpSetStringAttrib("CmdLine", cmdLine.c_str());
692
693 XML_CHECK(qpXmlWriter_startElement(writer.get(), "TestRun", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
694 qpXmlWriter_endElement(writer.get(), "TestRun"));
695 }
696
697 XML_CHECK(qpXmlWriter_endElement(writer.get(), "Summary"));
698 XML_CHECK(qpXmlWriter_endDocument(writer.get()));
699 }
700
701 #undef XML_CHECK
702
TestRunner(tcu::Platform& platform, tcu::Archive& archive, const char* waiverPath, const char* logDirPath, glu::ApiType type, deUint32 flags)703 TestRunner::TestRunner(tcu::Platform& platform, tcu::Archive& archive, const char* waiverPath,
704 const char* logDirPath, glu::ApiType type, deUint32 flags)
705 : m_platform(platform)
706 , m_archive(archive)
707 , m_waiverPath(waiverPath)
708 , m_logDirPath(logDirPath)
709 , m_type(type)
710 , m_flags(flags)
711 , m_iterState(ITERATE_INIT)
712 , m_curSession(DE_NULL)
713 , m_sessionsExecuted(0)
714 , m_sessionsPassed(0)
715 , m_sessionsFailed(0)
716 {
717 }
718
~TestRunner(void)719 TestRunner::~TestRunner(void)
720 {
721 delete m_curSession;
722 }
723
iterate(void)724 bool TestRunner::iterate(void)
725 {
726 switch (m_iterState)
727 {
728 case ITERATE_INIT:
729 init();
730 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
731 return true;
732
733 case ITERATE_DEINIT:
734 deinit();
735 m_iterState = ITERATE_INIT;
736 return false;
737
738 case ITERATE_INIT_SESSION:
739 DE_ASSERT(m_sessionIter != m_runSessions.end());
740 initSession(*m_sessionIter);
741 if (m_flags & PRINT_SUMMARY)
742 m_iterState = ITERATE_DEINIT_SESSION;
743 else
744 m_iterState = ITERATE_ITERATE_SESSION;
745 return true;
746
747 case ITERATE_DEINIT_SESSION:
748 deinitSession();
749 ++m_sessionIter;
750 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
751 return true;
752
753 case ITERATE_ITERATE_SESSION:
754 if (!iterateSession())
755 m_iterState = ITERATE_DEINIT_SESSION;
756 return true;
757
758 default:
759 DE_ASSERT(false);
760 return false;
761 }
762 }
763
init(void)764 void TestRunner::init(void)
765 {
766 DE_ASSERT(m_runSessions.empty() && m_summary.runParams.empty());
767
768 tcu::print("Running %s conformance\n", glu::getApiTypeDescription(m_type));
769
770 m_summary.runType = m_type;
771
772 // Get list of configs to test.
773 ConfigList configList;
774 getDefaultConfigList(m_platform, m_type, configList);
775
776 tcu::print(" found %d compatible and %d excluded configs\n", (int)configList.configs.size(),
777 (int)configList.excludedConfigs.size());
778
779 // Config list run.
780 {
781 const char* configLogFilename = "configs.qpa";
782 TestRunParams configRun;
783
784 configRun.logFilename = configLogFilename;
785 configRun.args.push_back("--deqp-case=CTS-Configs.*");
786 m_runSessions.push_back(configRun);
787
788 m_summary.configLogFilename = configLogFilename;
789 }
790
791 // Conformance test type specific runs
792 getTestRunParams(m_type, configList, m_runSessions);
793
794 // Record run params for summary.
795 for (std::vector<TestRunParams>::const_iterator runIter = m_runSessions.begin() + 1; runIter != m_runSessions.end();
796 ++runIter)
797 m_summary.runParams.push_back(*runIter);
798
799 // Session iterator
800 m_sessionIter = m_runSessions.begin();
801 }
802
deinit(void)803 void TestRunner::deinit(void)
804 {
805 // Print out totals.
806 bool isConformant_ = m_sessionsExecuted == m_sessionsPassed;
807 DE_ASSERT(m_sessionsExecuted == m_sessionsPassed + m_sessionsFailed);
808 tcu::print("\n%d/%d sessions passed, conformance test %s\n", m_sessionsPassed, m_sessionsExecuted,
809 isConformant_ ? "PASSED" : "FAILED");
810
811 m_summary.isConformant = isConformant_;
812
813 // Write out summary.
814 writeRunSummary(m_summary, de::FilePath::join(m_logDirPath, "cts-run-summary.xml").getPath());
815
816 m_runSessions.clear();
817 }
818
initSession(const TestRunParams& runParams)819 void TestRunner::initSession(const TestRunParams& runParams)
820 {
821 DE_ASSERT(!m_curSession);
822
823 tcu::print("\n Test run %d / %d\n", (int)(m_sessionIter - m_runSessions.begin() + 1), (int)m_runSessions.size());
824
825 // Compute final args for run.
826 vector<string> args(runParams.args);
827 args.push_back(string("--deqp-log-filename=") + de::FilePath::join(m_logDirPath, runParams.logFilename).getPath());
828
829 if (!(m_flags & VERBOSE_IMAGES))
830 args.push_back("--deqp-log-images=disable");
831
832 if (!(m_flags & VERBOSE_SHADERS))
833 args.push_back("--deqp-log-shader-sources=disable");
834
835 if (!m_waiverPath.empty())
836 args.push_back(string("--deqp-waiver-file=") + m_waiverPath);
837
838 std::ostringstream ostr;
839 std::ostream_iterator<string> out_it(ostr, ", ");
840 std::copy(args.begin(), args.end(), out_it);
841 tcu::print("\n Config: %s \n\n", ostr.str().c_str());
842
843 // Translate to argc, argv
844 vector<const char*> argv;
845 argv.push_back("cts-runner"); // Assumed binary name
846 for (vector<string>::const_iterator i = args.begin(); i != args.end(); i++)
847 argv.push_back(i->c_str());
848
849 // Create session
850 m_curSession = new RunSession(m_platform, m_archive, (int)argv.size(), &argv[0]);
851 }
852
deinitSession(void)853 void TestRunner::deinitSession(void)
854 {
855 DE_ASSERT(m_curSession);
856
857 // Collect results.
858 // \note NotSupported is treated as pass.
859 const tcu::TestRunStatus& result = m_curSession->getResult();
860 bool isOk = result.numFailed == 0 && result.isComplete;
861
862 DE_ASSERT(result.numExecuted == result.numPassed + result.numFailed + result.numNotSupported + result.numWarnings + result.numWaived);
863
864 m_sessionsExecuted += 1;
865 (isOk ? m_sessionsPassed : m_sessionsFailed) += 1;
866
867 delete m_curSession;
868 m_curSession = DE_NULL;
869 }
870
iterateSession(void)871 inline bool TestRunner::iterateSession(void)
872 {
873 return m_curSession->iterate();
874 }
875
876 } // glcts
877