1 /*-------------------------------------------------------------------------
2 * OpenGL Conformance Test Suite
3 * -----------------------------
4 *
5 * Copyright (c) 2016 Google Inc.
6 * Copyright (c) 2016 The Khronos Group Inc.
7 *
8 * Licensed under the Apache License, Version 2.0 (the "License");
9 * you may not use this file except in compliance with the License.
10 * You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 *
20 */ /*!
21 * \file
22 * \brief CTS runner.
23 */ /*-------------------------------------------------------------------*/
24
25 #include "glcTestRunner.hpp"
26 #include "deFilePath.hpp"
27 #include "deStringUtil.hpp"
28 #include "deUniquePtr.hpp"
29 #include "glcConfigList.hpp"
30 #include "qpXmlWriter.h"
31 #include "tcuApp.hpp"
32 #include "tcuCommandLine.hpp"
33 #include "tcuTestLog.hpp"
34 #include "tcuTestSessionExecutor.hpp"
35
36 #include <iterator>
37 #include <fstream>
38
39 namespace glcts
40 {
41
42 using std::string;
43 using std::vector;
44
45 // RunSession
46
47 class RunSession
48 {
49 public:
RunSession(tcu::Platform & platform,tcu::Archive & archive,const int numArgs,const char * const * args)50 RunSession(tcu::Platform &platform, tcu::Archive &archive, const int numArgs, const char *const *args)
51 : m_cmdLine(numArgs, args)
52 , m_log(m_cmdLine.getLogFileName(), m_cmdLine.getLogFlags())
53 , m_app(platform, archive, m_log, m_cmdLine)
54 {
55 const std::string sessionInfo = "#sessionInfo commandLineParameters \"";
56 m_log.writeSessionInfo(sessionInfo + m_cmdLine.getInitialCmdLine() + "\"\n");
57 }
58
iterate(void)59 inline bool iterate(void)
60 {
61 return m_app.iterate();
62 }
63
getResult(void) const64 inline const tcu::TestRunStatus &getResult(void) const
65 {
66 return m_app.getResult();
67 }
68
69 private:
70 tcu::CommandLine m_cmdLine;
71 tcu::TestLog m_log;
72 tcu::App m_app;
73 };
74
appendConfigArgs(const Config & config,std::vector<std::string> & args,const char * fboConfig)75 static void appendConfigArgs(const Config &config, std::vector<std::string> &args, const char *fboConfig)
76 {
77 if (fboConfig != NULL)
78 {
79 args.push_back(string("--deqp-gl-config-name=") + fboConfig);
80 args.push_back("--deqp-surface-type=fbo");
81 }
82
83 if (config.type != CONFIGTYPE_DEFAULT)
84 {
85 // \todo [2013-05-06 pyry] Test all surface types for some configs?
86 if (fboConfig == NULL)
87 {
88 if (config.surfaceTypes & SURFACETYPE_WINDOW)
89 args.push_back("--deqp-surface-type=window");
90 else if (config.surfaceTypes & SURFACETYPE_PBUFFER)
91 args.push_back("--deqp-surface-type=pbuffer");
92 else if (config.surfaceTypes & SURFACETYPE_PIXMAP)
93 args.push_back("--deqp-surface-type=pixmap");
94 }
95
96 args.push_back(string("--deqp-gl-config-id=") + de::toString(config.id));
97
98 if (config.type == CONFIGTYPE_EGL)
99 args.push_back("--deqp-gl-context-type=egl");
100 else if (config.type == CONFIGTYPE_WGL)
101 args.push_back("--deqp-gl-context-type=wgl");
102 }
103 }
104
105 typedef struct configInfo
106 {
107 int32_t redBits;
108 int32_t greenBits;
109 int32_t blueBits;
110 int32_t alphaBits;
111 int32_t depthBits;
112 int32_t stencilBits;
113 int32_t samples;
114 } configInfo;
115
parseConfigBitsFromName(const char * configName)116 static configInfo parseConfigBitsFromName(const char *configName)
117 {
118 configInfo cfgInfo;
119 static const struct
120 {
121 const char *name;
122 int redBits;
123 int greenBits;
124 int blueBits;
125 int alphaBits;
126 } colorCfgs[] = {
127 {"rgba8888", 8, 8, 8, 8},
128 {"rgb565", 5, 6, 5, 0},
129 };
130 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(colorCfgs); ndx++)
131 {
132 if (!strncmp(configName, colorCfgs[ndx].name, strlen(colorCfgs[ndx].name)))
133 {
134 cfgInfo.redBits = colorCfgs[ndx].redBits;
135 cfgInfo.greenBits = colorCfgs[ndx].greenBits;
136 cfgInfo.blueBits = colorCfgs[ndx].blueBits;
137 cfgInfo.alphaBits = colorCfgs[ndx].alphaBits;
138
139 configName += strlen(colorCfgs[ndx].name);
140 break;
141 }
142 }
143
144 static const struct
145 {
146 const char *name;
147 int depthBits;
148 } depthCfgs[] = {
149 {"d0", 0},
150 {"d24", 24},
151 };
152 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(depthCfgs); ndx++)
153 {
154 if (!strncmp(configName, depthCfgs[ndx].name, strlen(depthCfgs[ndx].name)))
155 {
156 cfgInfo.depthBits = depthCfgs[ndx].depthBits;
157
158 configName += strlen(depthCfgs[ndx].name);
159 break;
160 }
161 }
162
163 static const struct
164 {
165 const char *name;
166 int stencilBits;
167 } stencilCfgs[] = {
168 {"s0", 0},
169 {"s8", 8},
170 };
171 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(stencilCfgs); ndx++)
172 {
173 if (!strncmp(configName, stencilCfgs[ndx].name, strlen(stencilCfgs[ndx].name)))
174 {
175 cfgInfo.stencilBits = stencilCfgs[ndx].stencilBits;
176
177 configName += strlen(stencilCfgs[ndx].name);
178 break;
179 }
180 }
181
182 static const struct
183 {
184 const char *name;
185 int samples;
186 } multiSampleCfgs[] = {
187 {"ms0", 0},
188 {"ms4", 4},
189 };
190 for (int ndx = 0; ndx < DE_LENGTH_OF_ARRAY(multiSampleCfgs); ndx++)
191 {
192 if (!strncmp(configName, multiSampleCfgs[ndx].name, strlen(multiSampleCfgs[ndx].name)))
193 {
194 cfgInfo.samples = multiSampleCfgs[ndx].samples;
195
196 configName += strlen(multiSampleCfgs[ndx].name);
197 break;
198 }
199 }
200
201 return cfgInfo;
202 }
203
getApiName(glu::ApiType apiType)204 static const char *getApiName(glu::ApiType apiType)
205 {
206 if (apiType == glu::ApiType::es(2, 0))
207 return "gles2";
208 else if (apiType == glu::ApiType::es(3, 0))
209 return "gles3";
210 else if (apiType == glu::ApiType::es(3, 1))
211 return "gles31";
212 else if (apiType == glu::ApiType::es(3, 2))
213 return "gles32";
214 else if (apiType == glu::ApiType::core(3, 0))
215 return "gl30";
216 else if (apiType == glu::ApiType::core(3, 1))
217 return "gl31";
218 else if (apiType == glu::ApiType::core(3, 2))
219 return "gl32";
220 else if (apiType == glu::ApiType::core(3, 3))
221 return "gl33";
222 else if (apiType == glu::ApiType::core(4, 0))
223 return "gl40";
224 else if (apiType == glu::ApiType::core(4, 1))
225 return "gl41";
226 else if (apiType == glu::ApiType::core(4, 2))
227 return "gl42";
228 else if (apiType == glu::ApiType::core(4, 3))
229 return "gl43";
230 else if (apiType == glu::ApiType::core(4, 4))
231 return "gl44";
232 else if (apiType == glu::ApiType::core(4, 5))
233 return "gl45";
234 else if (apiType == glu::ApiType::core(4, 6))
235 return "gl46";
236 else
237 throw std::runtime_error("Unknown context type");
238 }
239
getCaseListFileOption(const char * mustpassDir,const char * apiName,const char * mustpassName)240 static const string getCaseListFileOption(const char *mustpassDir, const char *apiName, const char *mustpassName)
241 {
242 #if DE_OS == DE_OS_ANDROID
243 const string case_list_option = "--deqp-caselist-resource=";
244 #else
245 const string case_list_option = "--deqp-caselist-file=";
246 #endif
247 return case_list_option + mustpassDir + apiName + "-" + mustpassName + ".txt";
248 }
249
getLogFileName(const char * apiName,const char * configName,const int iterId,const int runId,const int width,const int height,const int seed)250 static const string getLogFileName(const char *apiName, const char *configName, const int iterId, const int runId,
251 const int width, const int height, const int seed)
252 {
253 string res = string("config-") + apiName + "-" + configName + "-cfg-" + de::toString(iterId) + "-run-" +
254 de::toString(runId) + "-width-" + de::toString(width) + "-height-" + de::toString(height);
255 if (seed != -1)
256 {
257 res += "-seed-" + de::toString(seed);
258 }
259 res += ".qpa";
260
261 return res;
262 }
263
getBaseOptions(std::vector<std::string> & args,const char * mustpassDir,const char * apiName,const char * configName,const char * screenRotation,int width,int height)264 static void getBaseOptions(std::vector<std::string> &args, const char *mustpassDir, const char *apiName,
265 const char *configName, const char *screenRotation, int width, int height)
266 {
267 args.push_back(getCaseListFileOption(mustpassDir, apiName, configName));
268 args.push_back(string("--deqp-screen-rotation=") + screenRotation);
269 args.push_back(string("--deqp-surface-width=") + de::toString(width));
270 args.push_back(string("--deqp-surface-height=") + de::toString(height));
271 args.push_back("--deqp-watchdog=disable");
272 }
273
isGLConfigCompatible(configInfo cfgInfo,const AOSPConfig & config)274 static bool isGLConfigCompatible(configInfo cfgInfo, const AOSPConfig &config)
275 {
276 return cfgInfo.redBits == config.redBits && cfgInfo.greenBits == config.greenBits &&
277 cfgInfo.blueBits == config.blueBits && cfgInfo.alphaBits == config.alphaBits &&
278 cfgInfo.depthBits == config.depthBits && cfgInfo.stencilBits == config.stencilBits &&
279 cfgInfo.samples == config.samples;
280 }
281
getTestRunsForAOSPEGL(vector<TestRunParams> & runs,const ConfigList & configs)282 static void getTestRunsForAOSPEGL(vector<TestRunParams> &runs, const ConfigList &configs)
283 {
284 #include "glcAospMustpassEgl.hpp"
285
286 for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_egl_first_cfg); ++i)
287 {
288 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_egl_first_cfg[i].glConfigName);
289
290 vector<AOSPConfig>::const_iterator cfgIter;
291 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
292 {
293 // find first compatible config
294 if ((*cfgIter).type == CONFIGTYPE_EGL && isGLConfigCompatible(cfgInfo, *cfgIter))
295 {
296 break;
297 }
298 }
299
300 if (cfgIter == configs.aospConfigs.end())
301 {
302 // No suitable configuration found. Skipping EGL tests
303 continue;
304 }
305
306 const char *apiName = "egl";
307
308 const int width = aosp_mustpass_egl_first_cfg[i].surfaceWidth;
309 const int height = aosp_mustpass_egl_first_cfg[i].surfaceHeight;
310
311 TestRunParams params;
312 params.logFilename =
313 getLogFileName(apiName, aosp_mustpass_egl_first_cfg[i].configName, 1, i, width, height, -1);
314 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_egl_first_cfg[i].configName,
315 aosp_mustpass_egl_first_cfg[i].screenRotation, width, height);
316
317 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_egl_first_cfg[i].glConfigName));
318
319 runs.push_back(params);
320 }
321 }
322
getTestRunsForAOSPES(vector<TestRunParams> & runs,const ConfigList & configs,const glu::ApiType apiType)323 static void getTestRunsForAOSPES(vector<TestRunParams> &runs, const ConfigList &configs, const glu::ApiType apiType)
324 {
325 #include "glcAospMustpassEs.hpp"
326
327 for (int i = 0; i < DE_LENGTH_OF_ARRAY(aosp_mustpass_es_first_cfg); ++i)
328 {
329 if (!glu::contextSupports(glu::ContextType(apiType), aosp_mustpass_es_first_cfg[i].apiType))
330 continue;
331
332 configInfo cfgInfo = parseConfigBitsFromName(aosp_mustpass_es_first_cfg[i].glConfigName);
333
334 vector<AOSPConfig>::const_iterator cfgIter;
335 for (cfgIter = configs.aospConfigs.begin(); cfgIter != configs.aospConfigs.end(); ++cfgIter)
336 {
337 // find first compatible config
338 if (isGLConfigCompatible(cfgInfo, *cfgIter))
339 {
340 break;
341 }
342 }
343
344 if (cfgIter == configs.aospConfigs.end())
345 {
346 TCU_FAIL(("No suitable configuration found for GL config " +
347 de::toString(aosp_mustpass_es_first_cfg[i].glConfigName))
348 .c_str());
349 return;
350 }
351
352 const char *apiName = getApiName(aosp_mustpass_es_first_cfg[i].apiType);
353
354 const int width = aosp_mustpass_es_first_cfg[i].surfaceWidth;
355 const int height = aosp_mustpass_es_first_cfg[i].surfaceHeight;
356
357 TestRunParams params;
358 params.logFilename = getLogFileName(apiName, aosp_mustpass_es_first_cfg[i].configName, 1, i, width, height, -1);
359 getBaseOptions(params.args, mustpassDir, apiName, aosp_mustpass_es_first_cfg[i].configName,
360 aosp_mustpass_es_first_cfg[i].screenRotation, width, height);
361
362 params.args.push_back(string("--deqp-gl-config-name=") + string(aosp_mustpass_es_first_cfg[i].glConfigName));
363
364 //set surface type
365 if ((*cfgIter).surfaceTypes & SURFACETYPE_WINDOW)
366 params.args.push_back("--deqp-surface-type=window");
367 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PBUFFER)
368 params.args.push_back("--deqp-surface-type=pbuffer");
369 else if ((*cfgIter).surfaceTypes & SURFACETYPE_PIXMAP)
370 params.args.push_back("--deqp-surface-type=pixmap");
371 runs.push_back(params);
372 }
373 }
374
getTestRunsForNoContext(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs,const RunParams * runParams,const int numRunParams,const char * mustpassDir)375 static void getTestRunsForNoContext(glu::ApiType type, vector<TestRunParams> &runs, const ConfigList &configs,
376 const RunParams *runParams, const int numRunParams, const char *mustpassDir)
377 {
378 vector<Config>::const_iterator cfgIter = configs.configs.begin();
379
380 for (int i = 0; i < numRunParams; ++i)
381 {
382 if (!glu::contextSupports(glu::ContextType(type), runParams[i].apiType))
383 continue;
384
385 const char *apiName = getApiName(runParams[i].apiType);
386
387 const int width = runParams[i].surfaceWidth;
388 const int height = runParams[i].surfaceHeight;
389 const int seed = runParams[i].baseSeed;
390
391 TestRunParams params;
392 params.logFilename = getLogFileName(apiName, runParams[i].configName, 1, i, width, height, seed);
393
394 getBaseOptions(params.args, mustpassDir, apiName, runParams[i].configName, runParams[i].screenRotation, width,
395 height);
396
397 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
398
399 appendConfigArgs(*cfgIter, params.args, runParams[i].fboConfig);
400
401 runs.push_back(params);
402 }
403 }
404
getTestRunsForNoContextES(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)405 static void getTestRunsForNoContextES(glu::ApiType type, vector<TestRunParams> &runs, const ConfigList &configs)
406 {
407 #include "glcKhronosMustpassEsNocontext.hpp"
408 getTestRunsForNoContext(type, runs, configs, khronos_mustpass_es_nocontext_first_cfg,
409 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_nocontext_first_cfg), mustpassDir);
410 }
411
getTestRunsForSingleConfig(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs,const RunParams * runParams,const int numRunParams,const char * mustpassDir)412 static void getTestRunsForSingleConfig(glu::ApiType type, vector<TestRunParams> &runs, const ConfigList &configs,
413 const RunParams *runParams, const int numRunParams, const char *mustpassDir)
414 {
415 vector<Config>::const_iterator cfgIter = configs.configs.begin();
416
417 for (int i = 0; i < numRunParams; ++i)
418 {
419 if (type != runParams[i].apiType)
420 continue;
421
422 const char *apiName = getApiName(runParams[i].apiType);
423
424 const int width = runParams[i].surfaceWidth;
425 const int height = runParams[i].surfaceHeight;
426 const int seed = runParams[i].baseSeed;
427
428 TestRunParams params;
429 params.logFilename = getLogFileName(apiName, runParams[i].configName, 1, i, width, height, seed);
430
431 getBaseOptions(params.args, mustpassDir, apiName, runParams[i].configName, runParams[i].screenRotation, width,
432 height);
433
434 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
435
436 appendConfigArgs(*cfgIter, params.args, runParams[i].fboConfig);
437
438 runs.push_back(params);
439 }
440 }
getTestRunsForSingleConfigES(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)441 static void getTestRunsForSingleConfigES(glu::ApiType type, vector<TestRunParams> &runs, const ConfigList &configs)
442 {
443 #include "glcKhronosMustpassEsSingleConfig.hpp"
444 getTestRunsForSingleConfig(type, runs, configs, khronos_mustpass_es_single_config_first_cfg,
445 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_single_config_first_cfg), mustpassDir);
446 }
447
getTestRunsForES(glu::ApiType type,const ConfigList & configs,vector<TestRunParams> & runs)448 static void getTestRunsForES(glu::ApiType type, const ConfigList &configs, vector<TestRunParams> &runs)
449 {
450 getTestRunsForAOSPEGL(runs, configs);
451 getTestRunsForAOSPES(runs, configs, type);
452 getTestRunsForNoContextES(type, runs, configs);
453 getTestRunsForSingleConfigES(type, runs, configs);
454
455 #include "glcKhronosMustpassEs.hpp"
456
457 for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
458 {
459 const bool isFirst = cfgIter == configs.configs.begin();
460 const int numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_es_first_cfg) :
461 DE_LENGTH_OF_ARRAY(khronos_mustpass_es_other_cfg);
462 const RunParams *runParams = isFirst ? khronos_mustpass_es_first_cfg : khronos_mustpass_es_other_cfg;
463
464 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
465 {
466 if (!glu::contextSupports(glu::ContextType(type), runParams[runNdx].apiType))
467 continue;
468
469 const char *apiName = getApiName(runParams[runNdx].apiType);
470
471 const int width = runParams[runNdx].surfaceWidth;
472 const int height = runParams[runNdx].surfaceHeight;
473 const int seed = runParams[runNdx].baseSeed;
474
475 TestRunParams params;
476
477 params.logFilename =
478 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
479
480 getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
481 runParams[runNdx].screenRotation, width, height);
482 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
483
484 appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
485
486 runs.push_back(params);
487 }
488 }
489 }
490
getTestRunsForNoContextGL(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)491 static void getTestRunsForNoContextGL(glu::ApiType type, vector<TestRunParams> &runs, const ConfigList &configs)
492 {
493 #include "glcKhronosMustpassGlNocontext.hpp"
494 getTestRunsForNoContext(type, runs, configs, khronos_mustpass_gl_nocontext_first_cfg,
495 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_nocontext_first_cfg), mustpassDir);
496 }
getTestRunsForSingleConfigGL(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)497 static void getTestRunsForSingleConfigGL(glu::ApiType type, vector<TestRunParams> &runs, const ConfigList &configs)
498 {
499 #include "glcKhronosMustpassGlSingleConfig.hpp"
500 getTestRunsForSingleConfig(type, runs, configs, khronos_mustpass_gl_single_config_first_cfg,
501 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_single_config_first_cfg), mustpassDir);
502 }
503
getTestRunsForESForGL(glu::ApiType type,vector<TestRunParams> & runs,const ConfigList & configs)504 static void getTestRunsForESForGL(glu::ApiType type, vector<TestRunParams> &runs, const ConfigList &configs)
505 {
506 #include "glcKhronosMustpassAospForGl.hpp"
507
508 vector<Config>::const_iterator cfgIter = configs.configs.begin();
509 const int numRunParams = DE_LENGTH_OF_ARRAY(khronos_mustpass_aosp_for_gl_first_cfg);
510 const RunParams *runParams = khronos_mustpass_aosp_for_gl_first_cfg;
511
512 for (int i = 0; i < numRunParams; ++i)
513 {
514 if (!glu::contextSupports(glu::ContextType(type), runParams[i].apiType))
515 continue;
516
517 const char *apiName = getApiName(runParams[i].apiType);
518
519 const int width = runParams[i].surfaceWidth;
520 const int height = runParams[i].surfaceHeight;
521 const int seed = runParams[i].baseSeed;
522
523 TestRunParams params;
524 params.logFilename = getLogFileName(apiName, runParams[i].configName, 1, i, width, height, seed);
525
526 getBaseOptions(params.args, mustpassDir, apiName, runParams[i].configName, runParams[i].screenRotation, width,
527 height);
528
529 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
530
531 appendConfigArgs(*cfgIter, params.args, runParams[i].fboConfig);
532
533 runs.push_back(params);
534 }
535 }
536
getTestRunsForGL(glu::ApiType type,const ConfigList & configs,vector<TestRunParams> & runs)537 static void getTestRunsForGL(glu::ApiType type, const ConfigList &configs, vector<TestRunParams> &runs)
538 {
539 getTestRunsForESForGL(type, runs, configs);
540 getTestRunsForNoContextGL(type, runs, configs);
541 getTestRunsForSingleConfigGL(type, runs, configs);
542 #include "glcKhronosMustpassGl.hpp"
543
544 for (vector<Config>::const_iterator cfgIter = configs.configs.begin(); cfgIter != configs.configs.end(); ++cfgIter)
545 {
546 const bool isFirst = cfgIter == configs.configs.begin();
547 const int numRunParams = isFirst ? DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_first_cfg) :
548 DE_LENGTH_OF_ARRAY(khronos_mustpass_gl_other_cfg);
549 const RunParams *runParams = isFirst ? khronos_mustpass_gl_first_cfg : khronos_mustpass_gl_other_cfg;
550
551 for (int runNdx = 0; runNdx < numRunParams; runNdx++)
552 {
553 if (type != runParams[runNdx].apiType)
554 continue;
555
556 const char *apiName = getApiName(runParams[runNdx].apiType);
557
558 const int width = runParams[runNdx].surfaceWidth;
559 const int height = runParams[runNdx].surfaceHeight;
560 const int seed = runParams[runNdx].baseSeed;
561
562 TestRunParams params;
563
564 params.logFilename =
565 getLogFileName(apiName, runParams[runNdx].configName, cfgIter->id, runNdx, width, height, seed);
566
567 getBaseOptions(params.args, mustpassDir, apiName, runParams[runNdx].configName,
568 runParams[runNdx].screenRotation, width, height);
569 params.args.push_back(string("--deqp-base-seed=") + de::toString(seed));
570
571 appendConfigArgs(*cfgIter, params.args, runParams[runNdx].fboConfig);
572
573 runs.push_back(params);
574 }
575 }
576 }
577
getTestRunParams(glu::ApiType type,const ConfigList & configs,vector<TestRunParams> & runs)578 static void getTestRunParams(glu::ApiType type, const ConfigList &configs, vector<TestRunParams> &runs)
579 {
580 switch (type.getProfile())
581 {
582 case glu::PROFILE_CORE:
583 getTestRunsForGL(type, configs, runs);
584 break;
585 case glu::PROFILE_ES:
586 getTestRunsForES(type, configs, runs);
587 break;
588 default:
589 throw std::runtime_error("Unknown context type");
590 }
591 }
592
593 struct FileDeleter
594 {
operator ()glcts::FileDeleter595 void operator()(FILE *file) const
596 {
597 if (file)
598 fclose(file);
599 }
600 };
601
602 struct XmlWriterDeleter
603 {
operator ()glcts::XmlWriterDeleter604 void operator()(qpXmlWriter *writer) const
605 {
606 if (writer)
607 qpXmlWriter_destroy(writer);
608 }
609 };
610
getRunTypeName(glu::ApiType type)611 static const char *getRunTypeName(glu::ApiType type)
612 {
613 if (type == glu::ApiType::es(2, 0))
614 return "es2";
615 else if (type == glu::ApiType::es(3, 0))
616 return "es3";
617 else if (type == glu::ApiType::es(3, 1))
618 return "es31";
619 else if (type == glu::ApiType::es(3, 2))
620 return "es32";
621 else if (type == glu::ApiType::core(3, 0))
622 return "gl30";
623 else if (type == glu::ApiType::core(3, 1))
624 return "gl31";
625 else if (type == glu::ApiType::core(3, 2))
626 return "gl32";
627 else if (type == glu::ApiType::core(3, 3))
628 return "gl33";
629 else if (type == glu::ApiType::core(4, 0))
630 return "gl40";
631 else if (type == glu::ApiType::core(4, 1))
632 return "gl41";
633 else if (type == glu::ApiType::core(4, 2))
634 return "gl42";
635 else if (type == glu::ApiType::core(4, 3))
636 return "gl43";
637 else if (type == glu::ApiType::core(4, 4))
638 return "gl44";
639 else if (type == glu::ApiType::core(4, 5))
640 return "gl45";
641 else if (type == glu::ApiType::core(4, 6))
642 return "gl46";
643 else
644 return DE_NULL;
645 }
646
generateTestSessionParams(tcu::Platform & platform,glu::ApiType type,std::vector<TestRunParams> & runSessionsParams,std::string configLogFilename)647 static void generateTestSessionParams(tcu::Platform &platform, glu::ApiType type,
648 std::vector<TestRunParams> &runSessionsParams, std::string configLogFilename)
649 {
650 // Get list of configs to test.
651 ConfigList configList;
652 getDefaultConfigList(platform, type, configList);
653
654 tcu::print(" found %d compatible and %d excluded configs\n", (int)configList.configs.size(),
655 (int)configList.excludedConfigs.size());
656
657 // Config list run.
658 {
659 TestRunParams configRun;
660
661 configRun.logFilename = configLogFilename;
662 configRun.args.push_back("--deqp-case=CTS-Configs.*");
663 runSessionsParams.push_back(configRun);
664 }
665
666 // Conformance test type specific runs
667 getTestRunParams(type, configList, runSessionsParams);
668 }
669
writeTestsParams(std::ostream & stream,std::vector<TestRunParams> & testRunParams)670 static void writeTestsParams(std::ostream &stream, std::vector<TestRunParams> &testRunParams)
671 {
672 stream << "#beginTestRunParamsCollection"
673 << "\n";
674
675 for (TestRunParams &testRunParam : testRunParams)
676 {
677 stream << "#beginTestRunParams"
678 << " ";
679
680 for (string arg : testRunParam.args)
681 {
682 stream << arg << ",";
683 }
684 stream << "\n";
685
686 stream << "#endTestRunParams"
687 << "\n";
688 }
689
690 stream << "\n#endTestRunParamsCollection"
691 << "\n";
692 }
693
694 #define XML_CHECK(X) \
695 if (!(X)) \
696 throw tcu::Exception("Writing XML failed")
697
writeRunSummary(const TestRunSummary & summary,const char * filename)698 static void writeRunSummary(const TestRunSummary &summary, const char *filename)
699 {
700 de::UniquePtr<FILE, FileDeleter> out(fopen(filename, "wb"));
701 if (!out)
702 throw tcu::Exception(string("Failed to open ") + filename);
703
704 de::UniquePtr<qpXmlWriter, XmlWriterDeleter> writer(qpXmlWriter_createFileWriter(out.get(), false, false));
705 if (!writer)
706 throw std::bad_alloc();
707
708 XML_CHECK(qpXmlWriter_startDocument(writer.get(), true));
709
710 {
711 qpXmlAttribute attribs[2];
712
713 attribs[0] = qpSetStringAttrib("Type", getRunTypeName(summary.runType));
714 attribs[1] = qpSetBoolAttrib("Conformant", summary.isConformant ? true : false);
715
716 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Summary", DE_LENGTH_OF_ARRAY(attribs), attribs));
717 }
718
719 // Config run
720 {
721 qpXmlAttribute attribs[1];
722 attribs[0] = qpSetStringAttrib("FileName", summary.configLogFilename.c_str());
723 XML_CHECK(qpXmlWriter_startElement(writer.get(), "Configs", DE_LENGTH_OF_ARRAY(attribs), attribs) &&
724 qpXmlWriter_endElement(writer.get(), "Configs"));
725 }
726
727 // Record test run parameters (log filename & command line).
728 size_t sessionIndex = 0;
729 for (vector<TestRunParams>::const_iterator runIter = summary.runParams.begin(); runIter != summary.runParams.end();
730 ++runIter)
731 {
732 string cmdLine;
733 qpXmlAttribute attribs[7];
734
735 for (vector<string>::const_iterator argIter = runIter->args.begin(); argIter != runIter->args.end(); ++argIter)
736 {
737 if (argIter != runIter->args.begin())
738 cmdLine += " ";
739 cmdLine += *argIter;
740 }
741
742 attribs[0] = qpSetStringAttrib("FileName", runIter->logFilename.c_str());
743 attribs[1] = qpSetStringAttrib("CmdLine", cmdLine.c_str());
744
745 XML_CHECK(qpXmlWriter_startElement(writer.get(), "TestRun", 2, attribs));
746 if (++sessionIndex < summary.results.size())
747 {
748 const tcu::TestRunStatus &results = summary.results[sessionIndex];
749 attribs[0] = qpSetIntAttrib("Passed", results.numPassed);
750 attribs[1] = qpSetIntAttrib("Failed", results.numFailed);
751 attribs[2] = qpSetIntAttrib("NotSupported", results.numNotSupported);
752 attribs[3] = qpSetIntAttrib("Warnings", results.numWarnings);
753 attribs[4] = qpSetIntAttrib("Waived", results.numWaived);
754 attribs[5] = qpSetIntAttrib("DeviceLost", results.numDeviceLost);
755 attribs[6] = qpSetIntAttrib("Executed", results.numExecuted);
756 XML_CHECK(qpXmlWriter_startElement(writer.get(), "TestResult", DE_LENGTH_OF_ARRAY(attribs), attribs));
757 XML_CHECK(qpXmlWriter_endElement(writer.get(), "TestResult"));
758 }
759 XML_CHECK(qpXmlWriter_endElement(writer.get(), "TestRun"));
760 }
761
762 XML_CHECK(qpXmlWriter_endElement(writer.get(), "Summary"));
763 XML_CHECK(qpXmlWriter_endDocument(writer.get()));
764 }
765
766 #undef XML_CHECK
767
TestRunner(tcu::Platform & platform,tcu::Archive & archive,const char * waiverPath,const char * logDirPath,glu::ApiType type,uint32_t flags)768 TestRunner::TestRunner(tcu::Platform &platform, tcu::Archive &archive, const char *waiverPath, const char *logDirPath,
769 glu::ApiType type, uint32_t flags)
770 : m_platform(platform)
771 , m_archive(archive)
772 , m_waiverPath(waiverPath)
773 , m_logDirPath(logDirPath)
774 , m_type(type)
775 , m_flags(flags)
776 , m_iterState(ITERATE_INIT)
777 , m_curSession(DE_NULL)
778 , m_sessionsExecuted(0)
779 , m_sessionsPassed(0)
780 , m_sessionsFailed(0)
781 {
782 }
783
~TestRunner(void)784 TestRunner::~TestRunner(void)
785 {
786 delete m_curSession;
787 }
788
iterate(void)789 bool TestRunner::iterate(void)
790 {
791 switch (m_iterState)
792 {
793 case ITERATE_INIT:
794 init();
795 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
796 return true;
797
798 case ITERATE_DEINIT:
799 deinit();
800 m_iterState = ITERATE_INIT;
801 return false;
802
803 case ITERATE_INIT_SESSION:
804 DE_ASSERT(m_sessionIter != m_runSessions.end());
805 initSession(*m_sessionIter);
806 if (m_flags & PRINT_SUMMARY)
807 m_iterState = ITERATE_DEINIT_SESSION;
808 else
809 m_iterState = ITERATE_ITERATE_SESSION;
810 return true;
811
812 case ITERATE_DEINIT_SESSION:
813 deinitSession();
814 ++m_sessionIter;
815 m_iterState = (m_sessionIter != m_runSessions.end()) ? ITERATE_INIT_SESSION : ITERATE_DEINIT;
816 return true;
817
818 case ITERATE_ITERATE_SESSION:
819 if (!iterateSession())
820 m_iterState = ITERATE_DEINIT_SESSION;
821 return true;
822
823 default:
824 DE_ASSERT(false);
825 return false;
826 }
827 }
828
init(void)829 void TestRunner::init(void)
830 {
831 DE_ASSERT(m_runSessions.empty() && m_summary.runParams.empty());
832
833 tcu::print("Running %s conformance\n", glu::getApiTypeDescription(m_type));
834
835 m_summary.runType = m_type;
836 m_summary.configLogFilename = "configs.qpa";
837 generateTestSessionParams(m_platform, m_type, m_runSessions, m_summary.configLogFilename);
838
839 // Record run params for summary.
840 for (std::vector<TestRunParams>::const_iterator runIter = m_runSessions.begin() + 1; runIter != m_runSessions.end();
841 ++runIter)
842 m_summary.runParams.push_back(*runIter);
843
844 // Session iterator
845 m_sessionIter = m_runSessions.begin();
846 }
847
deinit(void)848 void TestRunner::deinit(void)
849 {
850 // Print out totals.
851 bool isConformant_ = m_sessionsExecuted == m_sessionsPassed;
852 DE_ASSERT(m_sessionsExecuted == m_sessionsPassed + m_sessionsFailed);
853 tcu::print("\n%d/%d sessions passed, conformance test %s\n", m_sessionsPassed, m_sessionsExecuted,
854 isConformant_ ? "PASSED" : "FAILED");
855
856 m_summary.isConformant = isConformant_;
857
858 // Write out summary.
859 writeRunSummary(m_summary, de::FilePath::join(m_logDirPath, "cts-run-summary.xml").getPath());
860
861 m_runSessions.clear();
862 }
863
initSession(const TestRunParams & runParams)864 void TestRunner::initSession(const TestRunParams &runParams)
865 {
866 DE_ASSERT(!m_curSession);
867
868 tcu::print("\n Test run %d / %d\n", (int)(m_sessionIter - m_runSessions.begin() + 1), (int)m_runSessions.size());
869
870 // Compute final args for run.
871 vector<string> args(runParams.args);
872 args.push_back(string("--deqp-log-filename=") + de::FilePath::join(m_logDirPath, runParams.logFilename).getPath());
873
874 if (!(m_flags & VERBOSE_IMAGES))
875 args.push_back("--deqp-log-images=disable");
876
877 if (!(m_flags & VERBOSE_SHADERS))
878 args.push_back("--deqp-log-shader-sources=disable");
879
880 if (!m_waiverPath.empty())
881 args.push_back(string("--deqp-waiver-file=") + m_waiverPath);
882
883 std::ostringstream ostr;
884 std::ostream_iterator<string> out_it(ostr, ", ");
885 std::copy(args.begin(), args.end(), out_it);
886 tcu::print("\n Config: %s \n\n", ostr.str().c_str());
887
888 // Translate to argc, argv
889 vector<const char *> argv;
890 argv.push_back("cts-runner"); // Assumed binary name
891 for (vector<string>::const_iterator i = args.begin(); i != args.end(); i++)
892 argv.push_back(i->c_str());
893
894 // Create session
895 m_curSession = new RunSession(m_platform, m_archive, (int)argv.size(), &argv[0]);
896 }
897
deinitSession(void)898 void TestRunner::deinitSession(void)
899 {
900 DE_ASSERT(m_curSession);
901
902 // Collect results.
903 // \note NotSupported is treated as pass.
904 const tcu::TestRunStatus &result = m_curSession->getResult();
905 bool isOk = result.numFailed == 0 && result.isComplete;
906
907 DE_ASSERT(result.numExecuted == result.numPassed + result.numFailed + result.numNotSupported + result.numWarnings +
908 result.numWaived + result.numDeviceLost);
909
910 m_sessionsExecuted += 1;
911 (isOk ? m_sessionsPassed : m_sessionsFailed) += 1;
912
913 m_summary.results.push_back(result);
914
915 delete m_curSession;
916 m_curSession = DE_NULL;
917 }
918
iterateSession(void)919 inline bool TestRunner::iterateSession(void)
920 {
921 return m_curSession->iterate();
922 }
923
TestParamCollectorRunner(tcu::Platform & platform,const char * testParamsFilePath,glu::ApiType type)924 TestParamCollectorRunner::TestParamCollectorRunner(tcu::Platform &platform, const char *testParamsFilePath,
925 glu::ApiType type)
926 : m_platform(platform)
927 , m_testParamsFilePath(testParamsFilePath)
928 , m_type(type)
929 {
930 }
931
~TestParamCollectorRunner()932 TestParamCollectorRunner::~TestParamCollectorRunner()
933 {
934 }
935
iterate(void)936 bool TestParamCollectorRunner::iterate(void)
937 {
938 tcu::print("TestParamCollectorRunner iterate\n");
939 DE_ASSERT(m_runSessionsParams.empty());
940
941 tcu::print("Collecting %s conformance test params\n", glu::getApiTypeDescription(m_type));
942
943 generateTestSessionParams(m_platform, m_type, m_runSessionsParams, "configs.qpa");
944
945 // export test run params to a file on device
946 std::ofstream str(m_testParamsFilePath.c_str(), std::ofstream::binary | std::ofstream::trunc);
947 writeTestsParams(str, m_runSessionsParams);
948 str.close();
949 tcu::print("finish writing test run params at %s\n", m_testParamsFilePath.c_str());
950 return false;
951 }
952
953 } // namespace glcts
954