1 /* 2 * Copyright (C) 2024 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package org.khronos.cts.runner; 17 18 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 19 import com.android.ddmlib.IDevice; 20 import com.android.ddmlib.IShellOutputReceiver; 21 import com.android.tradefed.build.IFolderBuildInfo; 22 import com.android.tradefed.config.ConfigurationException; 23 import com.android.tradefed.config.OptionSetter; 24 import com.android.tradefed.device.DeviceNotAvailableException; 25 import com.android.tradefed.device.IManagedTestDevice; 26 import com.android.tradefed.device.ITestDevice; 27 import com.android.tradefed.metrics.proto.MetricMeasurement.Metric; 28 import com.android.tradefed.result.ITestInvocationListener; 29 import com.android.tradefed.result.TestDescription; 30 import com.android.tradefed.testtype.Abi; 31 import com.android.tradefed.testtype.IAbi; 32 import com.android.tradefed.util.AbiUtils; 33 import com.android.tradefed.util.FileUtil; 34 35 import com.drawelements.deqp.runner.DeqpTestRunner; 36 37 import junit.framework.TestCase; 38 39 import org.easymock.EasyMock; 40 import org.easymock.IAnswer; 41 import org.easymock.IMocksControl; 42 43 import java.io.File; 44 import java.io.FileNotFoundException; 45 import java.io.IOException; 46 import java.io.StringWriter; 47 import java.util.ArrayList; 48 import java.util.Collection; 49 import java.util.HashMap; 50 import java.util.HashSet; 51 import java.util.List; 52 import java.util.Set; 53 import java.util.concurrent.TimeUnit; 54 55 /** 56 * Unit tests for {@link KhronosCTSRunner}. 57 * Note: This file is a copy of DeqpTestRunnerTest.java with a few changed types. 58 */ 59 public class KhronosCTSRunnerTests extends TestCase { 60 private static final IAbi ABI = new Abi("armeabi-v7a", "32"); 61 private static final String APP_DIR = "/sdcard/"; 62 private static final String CASE_LIST_FILE_NAME = "dEQP-TestCaseList.txt"; 63 private static final String LOG_FILE_NAME = "TestLog.qpa"; 64 private static final String TEST_RUN_FILE_AND_PARAM = "--deqp-caselist-resource=gles3-caselist.txt,--deqp-screen-rotation=unspecified,--deqp-surface-width=256,--deqp-surface-height=256,--deqp-watchdog=disable,--deqp-gl-config-name=rgba8888d24s8ms0,"; 65 private static final String TEST_FAILURE_MESSAGE_CONFIG = "=== with config --deqp-gl-config-name=rgba8888d24s8ms0 --deqp-screen-rotation=unspecified --deqp-surface-height=256 --deqp-surface-width=256 --deqp-watchdog=disable ==="; 66 private static final String TEST_ID_NAME = "KhronosGLCTS"; 67 68 private File mTestsDir = null; 69 70 private boolean mLogData; 71 72 public static class BuildHelperMock extends CompatibilityBuildHelper { 73 private File mTestsDir = null; BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir)74 public BuildHelperMock(IFolderBuildInfo buildInfo, File testsDir) { 75 super(buildInfo); 76 mTestsDir = testsDir; 77 } 78 @Override getTestsDir()79 public File getTestsDir() throws FileNotFoundException { 80 return mTestsDir; 81 } 82 } 83 getMockBuildHelper(File testsDir)84 private static CompatibilityBuildHelper getMockBuildHelper(File testsDir) { 85 IFolderBuildInfo mockIFolderBuildInfo = EasyMock.createMock(IFolderBuildInfo.class); 86 EasyMock.expect(mockIFolderBuildInfo.getBuildAttributes()).andReturn(new HashMap<>()).anyTimes(); 87 EasyMock.replay(mockIFolderBuildInfo); 88 return new BuildHelperMock(mockIFolderBuildInfo, testsDir); 89 } 90 parseRunParam(String runParam)91 private static KhronosCTSBatchRunConfiguration parseRunParam(String runParam) 92 { 93 int index = 0; 94 HashMap<String, String> runConfigParam = new HashMap<String, String>(); 95 while (index < runParam.length()) 96 { 97 int nextParamArgValIndex = runParam.substring(index).indexOf(','); 98 if (nextParamArgValIndex < 0) 99 { 100 break; 101 } 102 String nextParamArgVal = runParam.substring(index, index + nextParamArgValIndex); 103 int argValDivIndex = nextParamArgVal.indexOf('='); 104 String nextParamArg = nextParamArgVal.substring(0, argValDivIndex); 105 String nextParamVal = nextParamArgVal.substring(argValDivIndex + 1); 106 runConfigParam.put(nextParamArg, nextParamVal); 107 index = index + nextParamArgValIndex+1; 108 } 109 return new KhronosCTSBatchRunConfiguration(runConfigParam); 110 } 111 parseTestRunParams(String testRunParam)112 private static KhronosCTSBatchRunConfiguration parseTestRunParams(String testRunParam) { 113 final String testRunParamArgCaseListResource = "--deqp-caselist-resource"; 114 int indexOfCaseListFileBegin = testRunParam.indexOf(testRunParamArgCaseListResource); 115 if (indexOfCaseListFileBegin == -1) 116 { 117 return new KhronosCTSBatchRunConfiguration(new HashMap<String, String>()); 118 } 119 int indexOfCaseListFileEnd = testRunParam.substring(indexOfCaseListFileBegin).indexOf(','); 120 String runParam = testRunParam.substring(indexOfCaseListFileEnd+1); 121 return parseRunParam(runParam); 122 } 123 124 buildKhronosCTSRunner(Collection<TestDescription> tests, File testsDir)125 private static KhronosCTSRunner buildKhronosCTSRunner(Collection<TestDescription> tests, File testsDir) throws ConfigurationException, IOException { 126 StringWriter testlist = new StringWriter(); 127 for (TestDescription test : tests) { 128 testlist.write(test.getClassName() + "." + test.getTestName() + "\n"); 129 } 130 return buildKhronosCTSRunner(testlist.toString(), testsDir); 131 } 132 buildKhronosCTSRunner(String testlist, File testsDir)133 private static KhronosCTSRunner buildKhronosCTSRunner(String testlist, File testsDir) throws ConfigurationException, IOException { 134 KhronosCTSRunner runner = new KhronosCTSRunner(); 135 final File caselistsFile = new File(testsDir, "gles3-caselist.txt"); 136 FileUtil.writeToFile(testlist, caselistsFile); 137 138 runner.setAbi(ABI); 139 runner.setBuildHelper(getMockBuildHelper(testsDir)); 140 141 return runner; 142 } 143 runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, final String output)144 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, 145 final String output) throws Exception { 146 147 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, null, null, output); 148 } 149 runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, final String testTrie, String cmd, final String output)150 private void runInstrumentationLineAndAnswer(ITestDevice mockDevice, IDevice mockIDevice, 151 final String testTrie, String cmd, final String output) throws Exception { 152 if (cmd==null){ 153 StringBuilder khronosCTSCmdLine = new StringBuilder(); 154 khronosCTSCmdLine.append("--deqp-caselist-file="); 155 khronosCTSCmdLine.append(APP_DIR + CASE_LIST_FILE_NAME); 156 khronosCTSCmdLine.append(" "); 157 khronosCTSCmdLine.append(parseTestRunParams(TEST_RUN_FILE_AND_PARAM).getId()); 158 if(!mLogData) { 159 khronosCTSCmdLine.append(" "); 160 khronosCTSCmdLine.append("--deqp-log-images=disable"); 161 } 162 cmd = khronosCTSCmdLine.toString(); 163 } 164 165 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + APP_DIR + CASE_LIST_FILE_NAME))) 166 .andReturn("").once(); 167 168 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("rm " + APP_DIR + LOG_FILE_NAME))) 169 .andReturn("").once(); 170 171 if (testTrie == null) { 172 mockDevice.pushString((String)EasyMock.anyObject(), EasyMock.eq(APP_DIR + CASE_LIST_FILE_NAME)); 173 } 174 else { 175 mockDevice.pushString(testTrie + "\n", APP_DIR + CASE_LIST_FILE_NAME); 176 } 177 EasyMock.expectLastCall().andReturn(true).once(); 178 179 final String instrumentationName = 180 "org.khronos.gl_cts/org.khronos.cts.testercore.KhronosCTSInstrumentation"; 181 182 final String command = String.format( 183 "am instrument %s -w -e khronosCTSLogFileName \"%s\" -e khronosCTSCmdLine \"%s\" -e deqpLogData \"%s\" %s", 184 AbiUtils.createAbiFlag(ABI.getName()), APP_DIR + LOG_FILE_NAME, cmd, mLogData, instrumentationName); 185 186 EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice); 187 mockIDevice.executeShellCommand(EasyMock.eq(command), 188 EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(), 189 EasyMock.isA(TimeUnit.class)); 190 191 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 192 @Override 193 public Object answer() { 194 IShellOutputReceiver receiver 195 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 196 197 receiver.addOutput(output.getBytes(), 0, output.length()); 198 receiver.flush(); 199 200 return null; 201 } 202 }); 203 } 204 buildTestProcessOutput(List<TestDescription> tests)205 static private String buildTestProcessOutput(List<TestDescription> tests) { 206 /* MultiLineReceiver expects "\r\n" line ending. */ 207 final String outputHeader = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 208 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 209 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 210 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 211 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 212 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 213 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 214 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 215 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 216 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 217 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 218 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 219 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 220 + "INSTRUMENTATION_STATUS_CODE: 0\r\n"; 221 222 final String outputEnd = "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 223 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 224 + "INSTRUMENTATION_CODE: 0\r\n"; 225 226 StringWriter output = new StringWriter(); 227 output.write(outputHeader); 228 for (TestDescription test : tests) { 229 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n"); 230 output.write("INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath="); 231 output.write(test.getClassName()); 232 output.write("."); 233 output.write(test.getTestName()); 234 output.write("\r\n"); 235 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 236 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=Pass\r\n"); 237 output.write("INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Pass\r\n"); 238 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n"); 239 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 240 output.write("INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n"); 241 output.write("INSTRUMENTATION_STATUS_CODE: 0\r\n"); 242 } 243 output.write(outputEnd); 244 return output.toString(); 245 } 246 247 runGetTestsParamsInstrumentation(ITestDevice mockDevice, IDevice mockIDevice)248 private void runGetTestsParamsInstrumentation(ITestDevice mockDevice, IDevice mockIDevice) throws Exception 249 { 250 251 final String getTestsParamsActivity = 252 "org.khronos.gl_cts/org.khronos.cts.ES32GetTestParamActivity"; 253 254 final String testsParamsFileName = 255 "/sdcard/cts-test-params.xml"; 256 257 final String instrumentationName = 258 "org.khronos.gl_cts/org.khronos.cts.testercore.KhronosCTSInstrumentation"; 259 260 String command = String.format( 261 "am instrument %s -w -e khronosCTSTestName \"%s\" -e khronosCTSTestParamFileName \"%s\" %s", 262 AbiUtils.createAbiFlag(ABI.getName()), 263 getTestsParamsActivity, 264 testsParamsFileName, 265 instrumentationName); 266 267 final String output = "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestRunParamsCollection\r\n" 268 +"INSTRUMENTATION_STATUS_CODE: 0\r\n" 269 +"INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestRunParams\r\n" 270 +"INSTRUMENTATION_STATUS: dEQP-TestRunParam="+TEST_RUN_FILE_AND_PARAM+"\r\n" 271 +"INSTRUMENTATION_STATUS_CODE: 0\r\n" 272 +"INSTRUMENTATION_STATUS: dEQP-EventType=EndTestRunParams\r\n" 273 +"INSTRUMENTATION_STATUS_CODE: 0\r\n" 274 +"INSTRUMENTATION_STATUS: dEQP-EventType=EndTestRunParamsCollection\r\n" 275 +"INSTRUMENTATION_STATUS_CODE: 0\r\n" 276 +"INSTRUMENTATION_CODE: 0\r\n"; 277 278 279 EasyMock.expect(mockDevice.getIDevice()).andReturn(mockIDevice); 280 281 mockIDevice.executeShellCommand(EasyMock.eq(command), 282 EasyMock.<IShellOutputReceiver>notNull(), EasyMock.anyLong(), 283 EasyMock.isA(TimeUnit.class)); 284 285 EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() { 286 @Override 287 public Object answer() { 288 IShellOutputReceiver receiver 289 = (IShellOutputReceiver)EasyMock.getCurrentArguments()[1]; 290 291 receiver.addOutput(output.getBytes(), 0, output.length()); 292 receiver.flush(); 293 294 return null; 295 } 296 }); 297 } 298 getTestId()299 private static String getTestId() { 300 return AbiUtils.createId(ABI.getName(), TEST_ID_NAME); 301 } 302 testFiltering(KhronosCTSRunner khronosCTSRunner, String expectedTrie, List<TestDescription> expectedTests)303 private void testFiltering(KhronosCTSRunner khronosCTSRunner, 304 String expectedTrie, 305 List<TestDescription> expectedTests) throws Exception { 306 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 307 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 308 ITestInvocationListener mockListener = EasyMock.createStrictMock(ITestInvocationListener.class); 309 310 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 311 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 312 andReturn("").once(); 313 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 314 andReturn("").once(); 315 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 316 andReturn("").once(); 317 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 318 andReturn("").once(); 319 320 // Expect the runGetTestsParamsActivity() is called once, no matter if there is test to be ran 321 runGetTestsParamsInstrumentation(mockDevice, mockIDevice); 322 323 mockListener.testRunStarted(getTestId(), expectedTests.size()); 324 EasyMock.expectLastCall().once(); 325 326 boolean thereAreTests = !expectedTests.isEmpty(); 327 if (thereAreTests) 328 { 329 String testOut = buildTestProcessOutput(expectedTests); 330 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, expectedTrie, null, testOut); 331 332 for (int i = 0; i < expectedTests.size(); i++) { 333 mockListener.testStarted(EasyMock.eq(expectedTests.get(i))); 334 EasyMock.expectLastCall().once(); 335 336 mockListener.testEnded(EasyMock.eq(expectedTests.get(i)), 337 EasyMock.<HashMap<String, Metric>>notNull()); 338 339 EasyMock.expectLastCall().once(); 340 } 341 } 342 343 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 344 EasyMock.expectLastCall().once(); 345 346 EasyMock.replay(mockDevice, mockIDevice); 347 EasyMock.replay(mockListener); 348 349 khronosCTSRunner.setDevice(mockDevice); 350 khronosCTSRunner.run(mockListener); 351 352 EasyMock.verify(mockListener); 353 EasyMock.verify(mockDevice, mockIDevice); 354 } 355 356 /** 357 * Test running multiple test cases 358 */ testRun_multipleTests()359 public void testRun_multipleTests() throws Exception { 360 final TestDescription[] testIds = { 361 new TestDescription("dEQP-GLES3.info", "vendor"), 362 new TestDescription("dEQP-GLES3.info", "renderer"), 363 new TestDescription("dEQP-GLES3.info", "version"), 364 new TestDescription("dEQP-GLES3.info", "shading_language_version"), 365 new TestDescription("dEQP-GLES3.info", "extensions"), 366 new TestDescription("dEQP-GLES3.info", "render_target") 367 }; 368 369 final String expectedTrie 370 = "{dEQP-GLES3{info{vendor,renderer,version,shading_language_version,extensions,render_target}}}"; 371 372 List<TestDescription> allTests = new ArrayList<TestDescription>(); 373 for (TestDescription id : testIds) { 374 allTests.add(id); 375 } 376 377 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 378 for (TestDescription id: testIds) { 379 activeTests.add(id); 380 } 381 382 KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); 383 384 OptionSetter setter = new OptionSetter(khronosCTSRunner); 385 mLogData = false; 386 setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); 387 388 testFiltering(khronosCTSRunner, expectedTrie, activeTests); 389 } 390 testRun_trivialIncludeFilter()391 public void testRun_trivialIncludeFilter() throws Exception { 392 final TestDescription[] testIds = { 393 new TestDescription("dEQP-GLES3.missing", "no"), 394 new TestDescription("dEQP-GLES3.missing", "nope"), 395 new TestDescription("dEQP-GLES3.missing", "donotwant"), 396 new TestDescription("dEQP-GLES3.pick_me", "yes"), 397 new TestDescription("dEQP-GLES3.pick_me", "ok"), 398 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 399 }; 400 401 List<TestDescription> allTests = new ArrayList<TestDescription>(); 402 for (TestDescription id : testIds) { 403 allTests.add(id); 404 } 405 406 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 407 activeTests.add(testIds[3]); 408 activeTests.add(testIds[4]); 409 activeTests.add(testIds[5]); 410 411 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 412 413 KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); 414 OptionSetter setter = new OptionSetter(khronosCTSRunner); 415 setter.setOptionValue("include-filter", "dEQP-GLES3.pick_me#*"); 416 mLogData = false; 417 setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); 418 419 testFiltering(khronosCTSRunner, expectedTrie, activeTests); 420 } 421 testRun_trivialExcludeFilter()422 public void testRun_trivialExcludeFilter() throws Exception { 423 final TestDescription[] testIds = { 424 new TestDescription("dEQP-GLES3.missing", "no"), 425 new TestDescription("dEQP-GLES3.missing", "nope"), 426 new TestDescription("dEQP-GLES3.missing", "donotwant"), 427 new TestDescription("dEQP-GLES3.pick_me", "yes"), 428 new TestDescription("dEQP-GLES3.pick_me", "ok"), 429 new TestDescription("dEQP-GLES3.pick_me", "accepted"), 430 }; 431 432 List<TestDescription> allTests = new ArrayList<TestDescription>(); 433 for (TestDescription id : testIds) { 434 allTests.add(id); 435 } 436 437 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 438 activeTests.add(testIds[3]); 439 activeTests.add(testIds[4]); 440 activeTests.add(testIds[5]); 441 442 String expectedTrie = "{dEQP-GLES3{pick_me{yes,ok,accepted}}}"; 443 444 KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); 445 OptionSetter setter = new OptionSetter(khronosCTSRunner); 446 setter.setOptionValue("exclude-filter", "dEQP-GLES3.missing#*"); 447 mLogData = false; 448 setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); 449 450 testFiltering(khronosCTSRunner, expectedTrie, activeTests); 451 } 452 testRun_includeAndExcludeFilter()453 public void testRun_includeAndExcludeFilter() throws Exception { 454 final TestDescription[] testIds = { 455 new TestDescription("dEQP-GLES3.group1", "foo"), 456 new TestDescription("dEQP-GLES3.group1", "nope"), 457 new TestDescription("dEQP-GLES3.group1", "donotwant"), 458 new TestDescription("dEQP-GLES3.group2", "foo"), 459 new TestDescription("dEQP-GLES3.group2", "yes"), 460 new TestDescription("dEQP-GLES3.group2", "thoushallnotpass"), 461 }; 462 463 List<TestDescription> allTests = new ArrayList<TestDescription>(); 464 for (TestDescription id : testIds) { 465 allTests.add(id); 466 } 467 468 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 469 activeTests.add(testIds[4]); 470 471 String expectedTrie = "{dEQP-GLES3{group2{yes}}}"; 472 473 KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); 474 475 OptionSetter setter = new OptionSetter(khronosCTSRunner); 476 mLogData = false; 477 setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); 478 479 Set<String> includes = new HashSet<>(); 480 includes.add("dEQP-GLES3.group2#*"); 481 khronosCTSRunner.addAllIncludeFilters(includes); 482 483 Set<String> excludes = new HashSet<>(); 484 excludes.add("*foo"); 485 excludes.add("*thoushallnotpass"); 486 khronosCTSRunner.addAllExcludeFilters(excludes); 487 testFiltering(khronosCTSRunner, expectedTrie, activeTests); 488 } 489 testRun_includeAll()490 public void testRun_includeAll() throws Exception { 491 final TestDescription[] testIds = { 492 new TestDescription("dEQP-GLES3.group1", "mememe"), 493 new TestDescription("dEQP-GLES3.group1", "yeah"), 494 new TestDescription("dEQP-GLES3.group1", "takeitall"), 495 new TestDescription("dEQP-GLES3.group2", "jeba"), 496 new TestDescription("dEQP-GLES3.group2", "yes"), 497 new TestDescription("dEQP-GLES3.group2", "granted"), 498 }; 499 500 List<TestDescription> allTests = new ArrayList<TestDescription>(); 501 for (TestDescription id : testIds) { 502 allTests.add(id); 503 } 504 505 String expectedTrie = "{dEQP-GLES3{group1{mememe,yeah,takeitall},group2{jeba,yes,granted}}}"; 506 507 KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); 508 509 OptionSetter setter = new OptionSetter(khronosCTSRunner); 510 mLogData = false; 511 setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); 512 513 khronosCTSRunner.addIncludeFilter("*"); 514 testFiltering(khronosCTSRunner, expectedTrie, allTests); 515 } 516 testRun_excludeAll()517 public void testRun_excludeAll() throws Exception { 518 final TestDescription[] testIds = { 519 new TestDescription("dEQP-GLES3.group1", "no"), 520 new TestDescription("dEQP-GLES3.group1", "nope"), 521 new TestDescription("dEQP-GLES3.group1", "nottoday"), 522 new TestDescription("dEQP-GLES3.group2", "banned"), 523 new TestDescription("dEQP-GLES3.group2", "notrecognized"), 524 new TestDescription("dEQP-GLES3.group2", "-2"), 525 }; 526 527 List<TestDescription> allTests = new ArrayList<TestDescription>(); 528 for (TestDescription id : testIds) { 529 allTests.add(id); 530 } 531 532 KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); 533 OptionSetter setter = new OptionSetter(khronosCTSRunner); 534 mLogData = false; 535 setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); 536 khronosCTSRunner.addExcludeFilter("*"); 537 String expectedTrie = ""; 538 539 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 540 testFiltering(khronosCTSRunner, expectedTrie, activeTests); 541 } 542 testDotToHashConversionInFilters()543 public void testDotToHashConversionInFilters() throws Exception { 544 final TestDescription[] testIds = { 545 new TestDescription("dEQP-GLES3.missing", "no"), 546 new TestDescription("dEQP-GLES3.pick_me", "donotwant"), 547 new TestDescription("dEQP-GLES3.pick_me", "yes") 548 }; 549 550 List<TestDescription> allTests = new ArrayList<TestDescription>(); 551 for (TestDescription id : testIds) { 552 allTests.add(id); 553 } 554 555 List<TestDescription> activeTests = new ArrayList<TestDescription>(); 556 activeTests.add(testIds[2]); 557 558 String expectedTrie = "{dEQP-GLES3{pick_me{yes}}}"; 559 560 KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); 561 OptionSetter setter = new OptionSetter(khronosCTSRunner); 562 mLogData = false; 563 setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); 564 khronosCTSRunner.addIncludeFilter("dEQP-GLES3.pick_me.yes"); 565 testFiltering(khronosCTSRunner, expectedTrie, activeTests); 566 } 567 568 /** 569 * Test running a unexecutable test. 570 */ testRun_unexecutableTests()571 public void testRun_unexecutableTests() throws Exception { 572 final String instrumentationAnswerNoExecs = 573 "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 574 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 575 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 576 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 577 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 578 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 579 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 580 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 581 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 582 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 583 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 584 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 585 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 586 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 587 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 588 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 589 + "INSTRUMENTATION_CODE: 0\r\n"; 590 591 final TestDescription[] testIds = { 592 new TestDescription("dEQP-GLES3.missing", "no"), 593 new TestDescription("dEQP-GLES3.missing", "nope"), 594 new TestDescription("dEQP-GLES3.missing", "donotwant"), 595 }; 596 597 final String[] testPaths = { 598 "dEQP-GLES3.missing.no", 599 "dEQP-GLES3.missing.nope", 600 "dEQP-GLES3.missing.donotwant", 601 }; 602 603 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 604 ITestInvocationListener mockListener 605 = EasyMock.createStrictMock(ITestInvocationListener.class); 606 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 607 608 Collection<TestDescription> allTests = new ArrayList<TestDescription>(); 609 610 for (TestDescription id : testIds) { 611 allTests.add(id); 612 } 613 614 KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); 615 OptionSetter setter = new OptionSetter(khronosCTSRunner); 616 mLogData = false; 617 setter.setOptionValue("deqp-log-result-details", mLogData ? "true" : "false"); 618 619 // first try 620 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 621 "{dEQP-GLES3{missing{no,nope,donotwant}}}", null, instrumentationAnswerNoExecs); 622 623 // splitting begins 624 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 625 "{dEQP-GLES3{missing{no}}}", null, instrumentationAnswerNoExecs); 626 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 627 "{dEQP-GLES3{missing{nope,donotwant}}}", null, instrumentationAnswerNoExecs); 628 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 629 "{dEQP-GLES3{missing{nope}}}", null, instrumentationAnswerNoExecs); 630 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, 631 "{dEQP-GLES3{missing{donotwant}}}", null, instrumentationAnswerNoExecs); 632 633 mockListener.testRunStarted(getTestId(), testPaths.length); 634 EasyMock.expectLastCall().once(); 635 636 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 637 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 638 andReturn("").once(); 639 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 640 andReturn("").once(); 641 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 642 andReturn("").once(); 643 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 644 andReturn("").once(); 645 646 // Expect the runGetTestsParamsActivity() is called once, no matter if there is test to be ran 647 runGetTestsParamsInstrumentation(mockDevice, mockIDevice); 648 649 for (int i = 0; i < testPaths.length; i++) { 650 mockListener.testStarted(EasyMock.eq(testIds[i])); 651 EasyMock.expectLastCall().once(); 652 653 mockListener.testFailed(EasyMock.eq(testIds[i]), 654 EasyMock.eq(TEST_FAILURE_MESSAGE_CONFIG+"\n" 655 + "Abort: Test cannot be executed")); 656 EasyMock.expectLastCall().once(); 657 658 mockListener.testEnded(EasyMock.eq(testIds[i]), 659 EasyMock.<HashMap<String, Metric>>notNull()); 660 EasyMock.expectLastCall().once(); 661 } 662 663 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 664 EasyMock.expectLastCall().once(); 665 666 EasyMock.replay(mockDevice, mockIDevice); 667 EasyMock.replay(mockListener); 668 669 khronosCTSRunner.setDevice(mockDevice); 670 khronosCTSRunner.run(mockListener); 671 672 EasyMock.verify(mockListener); 673 EasyMock.verify(mockDevice, mockIDevice); 674 } 675 676 /** 677 * Test that result code produces correctly pass or fail. 678 */ testResultCode(final String resultCode, boolean pass)679 private void testResultCode(final String resultCode, boolean pass) throws Exception { 680 final TestDescription testId = new TestDescription("dEQP-GLES3.info", "version"); 681 final String testPath = "dEQP-GLES3.info.version"; 682 final String testTrie = "{dEQP-GLES3{info{version}}}"; 683 684 /* MultiLineReceiver expects "\r\n" line ending. */ 685 final String output = "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseName\r\n" 686 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 687 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=2014.x\r\n" 688 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 689 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=releaseId\r\n" 690 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 691 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=0xcafebabe\r\n" 692 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 693 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Name=targetName\r\n" 694 + "INSTRUMENTATION_STATUS: dEQP-EventType=SessionInfo\r\n" 695 + "INSTRUMENTATION_STATUS: dEQP-SessionInfo-Value=android\r\n" 696 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 697 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginSession\r\n" 698 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 699 + "INSTRUMENTATION_STATUS: dEQP-EventType=BeginTestCase\r\n" 700 + "INSTRUMENTATION_STATUS: dEQP-BeginTestCase-TestCasePath=" + testPath + "\r\n" 701 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 702 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Code=" + resultCode + "\r\n" 703 + "INSTRUMENTATION_STATUS: dEQP-TestCaseResult-Details=Detail" + resultCode + "\r\n" 704 + "INSTRUMENTATION_STATUS: dEQP-EventType=TestCaseResult\r\n" 705 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 706 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndTestCase\r\n" 707 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 708 + "INSTRUMENTATION_STATUS: dEQP-EventType=EndSession\r\n" 709 + "INSTRUMENTATION_STATUS_CODE: 0\r\n" 710 + "INSTRUMENTATION_CODE: 0\r\n"; 711 712 ITestDevice mockDevice = EasyMock.createMock(ITestDevice.class); 713 ITestInvocationListener mockListener 714 = EasyMock.createStrictMock(ITestInvocationListener.class); 715 IDevice mockIDevice = EasyMock.createMock(IDevice.class); 716 717 Collection<TestDescription> allTests = new ArrayList<TestDescription>(); 718 allTests.add(testId); 719 720 KhronosCTSRunner khronosCTSRunner = buildKhronosCTSRunner(allTests, mTestsDir); 721 722 runInstrumentationLineAndAnswer(mockDevice, mockIDevice, testTrie, null, output); 723 724 mockListener.testRunStarted(getTestId(), 1); 725 EasyMock.expectLastCall().once(); 726 727 // Expect the runGetTestsParamsActivity() is called once, no matter if there is test to be ran 728 runGetTestsParamsInstrumentation(mockDevice, mockIDevice); 729 730 // Expect the calls twice: setupTestEnvironment() and teardownTestEnvironment() 731 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 732 andReturn("").once(); 733 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 734 andReturn("").once(); 735 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_pkgs"))). 736 andReturn("").once(); 737 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("settings delete global angle_gl_driver_selection_values"))). 738 andReturn("").once(); 739 740 mockListener.testStarted(EasyMock.eq(testId)); 741 EasyMock.expectLastCall().once(); 742 743 if (!pass) { 744 mockListener.testFailed(testId, 745 TEST_FAILURE_MESSAGE_CONFIG+"\n" 746 + resultCode + ": Detail" + resultCode); 747 748 EasyMock.expectLastCall().once(); 749 } 750 751 mockListener.testEnded(EasyMock.eq(testId), EasyMock.<HashMap<String, Metric>>notNull()); 752 EasyMock.expectLastCall().once(); 753 754 mockListener.testRunEnded(EasyMock.anyLong(), EasyMock.<HashMap<String, Metric>>notNull()); 755 EasyMock.expectLastCall().once(); 756 757 EasyMock.replay(mockDevice, mockIDevice); 758 EasyMock.replay(mockListener); 759 760 khronosCTSRunner.setDevice(mockDevice); 761 khronosCTSRunner.run(mockListener); 762 763 EasyMock.verify(mockListener); 764 EasyMock.verify(mockDevice, mockIDevice); 765 } 766 767 /** 768 * Test Pass result code 769 */ testRun_resultPass()770 public void testRun_resultPass() throws Exception { 771 testResultCode("Pass", true); 772 } 773 774 /** 775 * Test dEQP Fail result code. 776 */ testRun_resultFail()777 public void testRun_resultFail() throws Exception { 778 testResultCode("Fail", false); 779 } 780 781 /** 782 * Test dEQP NotSupported result code. 783 */ testRun_resultNotSupported()784 public void testRun_resultNotSupported() throws Exception { 785 testResultCode("NotSupported", true); 786 } 787 788 /** 789 * Test dEQP QualityWarning result code. 790 */ testRun_resultQualityWarning()791 public void testRun_resultQualityWarning() throws Exception { 792 testResultCode("QualityWarning", true); 793 } 794 795 /** 796 * Test dEQP CompatibilityWarning result code. 797 */ testRun_resultCompatibilityWarning()798 public void testRun_resultCompatibilityWarning() throws Exception { 799 testResultCode("CompatibilityWarning", true); 800 } 801 802 /** 803 * Test dEQP ResourceError result code. 804 */ testRun_resultResourceError()805 public void testRun_resultResourceError() throws Exception { 806 testResultCode("ResourceError", false); 807 } 808 809 /** 810 * Test dEQP InternalError result code. 811 */ testRun_resultInternalError()812 public void testRun_resultInternalError() throws Exception { 813 testResultCode("InternalError", false); 814 } 815 816 /** 817 * Test dEQP Crash result code. 818 */ testRun_resultCrash()819 public void testRun_resultCrash() throws Exception { 820 testResultCode("Crash", false); 821 } 822 823 /** 824 * Test dEQP Timeout result code. 825 */ testRun_resultTimeout()826 public void testRun_resultTimeout() throws Exception { 827 testResultCode("Timeout", false); 828 } 829 830 /** 831 * Test interface to mock Tradefed device types. 832 */ 833 public static interface RecoverableTestDevice extends ITestDevice, IManagedTestDevice { 834 } 835 836 private static enum RecoveryEvent { 837 PROGRESS, 838 FAIL_CONNECTION_REFUSED, 839 FAIL_LINK_KILLED, 840 } 841 runRecoveryWithPattern(KhronosCTSRunner.Recovery recovery, RecoveryEvent[] events)842 private void runRecoveryWithPattern(KhronosCTSRunner.Recovery recovery, RecoveryEvent[] events) 843 throws DeviceNotAvailableException { 844 for (RecoveryEvent event : events) { 845 switch (event) { 846 case PROGRESS: 847 recovery.onExecutionProgressed(); 848 break; 849 case FAIL_CONNECTION_REFUSED: 850 recovery.recoverConnectionRefused(); 851 break; 852 case FAIL_LINK_KILLED: 853 recovery.recoverComLinkKilled(); 854 break; 855 } 856 } 857 } 858 setRecoveryExpectationWait(KhronosCTSRunner.ISleepProvider mockSleepProvider)859 private void setRecoveryExpectationWait(KhronosCTSRunner.ISleepProvider mockSleepProvider) { 860 mockSleepProvider.sleep(EasyMock.gt(0)); 861 EasyMock.expectLastCall().once(); 862 } 863 setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice, KhronosCTSRunner.ISleepProvider mockSleepProvider)864 private void setRecoveryExpectationKillProcess(RecoverableTestDevice mockDevice, 865 KhronosCTSRunner.ISleepProvider mockSleepProvider) throws DeviceNotAvailableException { 866 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 867 andReturn("root 1234 org.khronos.cts").once(); 868 869 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 870 andReturn("").once(); 871 872 // Recovery checks if kill failed 873 mockSleepProvider.sleep(EasyMock.gt(0)); 874 EasyMock.expectLastCall().once(); 875 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 876 andReturn("").once(); 877 } 878 setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice)879 private void setRecoveryExpectationRecovery(RecoverableTestDevice mockDevice) 880 throws DeviceNotAvailableException { 881 EasyMock.expect(mockDevice.recoverDevice()).andReturn(true).once(); 882 } 883 setRecoveryExpectationReboot(RecoverableTestDevice mockDevice)884 private void setRecoveryExpectationReboot(RecoverableTestDevice mockDevice) 885 throws DeviceNotAvailableException { 886 mockDevice.reboot(); 887 EasyMock.expectLastCall().once(); 888 } 889 890 setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice, int numConsecutiveErrors)891 private int setRecoveryExpectationOfAConnFailure(RecoverableTestDevice mockDevice, int numConsecutiveErrors) 892 throws DeviceNotAvailableException { 893 switch (numConsecutiveErrors) { 894 case 0: 895 case 1: 896 setRecoveryExpectationRecovery(mockDevice); 897 return 2; 898 case 2: 899 setRecoveryExpectationReboot(mockDevice); 900 return 3; 901 default: 902 return 4; 903 } 904 } 905 setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice, KhronosCTSRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors)906 private int setRecoveryExpectationOfAComKilled(RecoverableTestDevice mockDevice, 907 KhronosCTSRunner.ISleepProvider mockSleepProvider, int numConsecutiveErrors) 908 throws DeviceNotAvailableException { 909 switch (numConsecutiveErrors) { 910 case 0: 911 setRecoveryExpectationWait(mockSleepProvider); 912 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 913 return 1; 914 case 1: 915 setRecoveryExpectationRecovery(mockDevice); 916 setRecoveryExpectationKillProcess(mockDevice, mockSleepProvider); 917 return 2; 918 case 2: 919 setRecoveryExpectationReboot(mockDevice); 920 return 3; 921 default: 922 return 4; 923 } 924 } 925 setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice, KhronosCTSRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events)926 private void setRecoveryExpectationsOfAPattern(RecoverableTestDevice mockDevice, 927 KhronosCTSRunner.ISleepProvider mockSleepProvider, RecoveryEvent[] events) 928 throws DeviceNotAvailableException { 929 int numConsecutiveErrors = 0; 930 for (RecoveryEvent event : events) { 931 switch (event) { 932 case PROGRESS: 933 numConsecutiveErrors = 0; 934 break; 935 case FAIL_CONNECTION_REFUSED: 936 numConsecutiveErrors = setRecoveryExpectationOfAConnFailure(mockDevice, numConsecutiveErrors); 937 break; 938 case FAIL_LINK_KILLED: 939 numConsecutiveErrors = setRecoveryExpectationOfAComKilled(mockDevice, 940 mockSleepProvider, numConsecutiveErrors); 941 break; 942 } 943 } 944 } 945 946 /** 947 * Test dEQP runner recovery state machine. 948 */ testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern)949 private void testRecoveryWithPattern(boolean expectSuccess, RecoveryEvent...pattern) 950 throws Exception { 951 KhronosCTSRunner.Recovery recovery = new KhronosCTSRunner.Recovery(); 952 IMocksControl orderedControl = EasyMock.createStrictControl(); 953 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); 954 EasyMock.expect(mockDevice.getSerialNumber()).andStubReturn("SERIAL"); 955 KhronosCTSRunner.ISleepProvider mockSleepProvider = 956 orderedControl.createMock(KhronosCTSRunner.ISleepProvider.class); 957 958 setRecoveryExpectationsOfAPattern(mockDevice, mockSleepProvider, pattern); 959 960 orderedControl.replay(); 961 962 recovery.setDevice(mockDevice); 963 recovery.setSleepProvider(mockSleepProvider); 964 try { 965 runRecoveryWithPattern(recovery, pattern); 966 if (!expectSuccess) { 967 fail("Expected DeviceNotAvailableException"); 968 } 969 } catch (DeviceNotAvailableException ex) { 970 if (expectSuccess) { 971 fail("Did not expect DeviceNotAvailableException"); 972 } 973 } 974 975 orderedControl.verify(); 976 } 977 testRecovery_NoEvents()978 public void testRecovery_NoEvents() throws Exception { 979 testRecoveryWithPattern(true); 980 } 981 testRecovery_AllOk()982 public void testRecovery_AllOk() throws Exception { 983 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, RecoveryEvent.PROGRESS); 984 } 985 986 // conn fail patterns 987 testRecovery_OneConnectionFailureBegin()988 public void testRecovery_OneConnectionFailureBegin() throws Exception { 989 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 990 RecoveryEvent.PROGRESS); 991 } 992 testRecovery_TwoConnectionFailuresBegin()993 public void testRecovery_TwoConnectionFailuresBegin() throws Exception { 994 testRecoveryWithPattern(true, RecoveryEvent.FAIL_CONNECTION_REFUSED, 995 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 996 } 997 testRecovery_ThreeConnectionFailuresBegin()998 public void testRecovery_ThreeConnectionFailuresBegin() throws Exception { 999 testRecoveryWithPattern(false, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1000 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED); 1001 } 1002 testRecovery_OneConnectionFailureMid()1003 public void testRecovery_OneConnectionFailureMid() throws Exception { 1004 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1005 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 1006 } 1007 testRecovery_TwoConnectionFailuresMid()1008 public void testRecovery_TwoConnectionFailuresMid() throws Exception { 1009 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1010 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1011 RecoveryEvent.PROGRESS); 1012 } 1013 testRecovery_ThreeConnectionFailuresMid()1014 public void testRecovery_ThreeConnectionFailuresMid() throws Exception { 1015 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, 1016 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1017 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.PROGRESS); 1018 } 1019 1020 // link fail patterns 1021 testRecovery_OneLinkFailureBegin()1022 public void testRecovery_OneLinkFailureBegin() throws Exception { 1023 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1024 RecoveryEvent.PROGRESS); 1025 } 1026 testRecovery_TwoLinkFailuresBegin()1027 public void testRecovery_TwoLinkFailuresBegin() throws Exception { 1028 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1029 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1030 } 1031 testRecovery_ThreeLinkFailuresBegin()1032 public void testRecovery_ThreeLinkFailuresBegin() throws Exception { 1033 testRecoveryWithPattern(true, RecoveryEvent.FAIL_LINK_KILLED, 1034 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1035 RecoveryEvent.PROGRESS); 1036 } 1037 testRecovery_FourLinkFailuresBegin()1038 public void testRecovery_FourLinkFailuresBegin() throws Exception { 1039 testRecoveryWithPattern(false, RecoveryEvent.FAIL_LINK_KILLED, 1040 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1041 RecoveryEvent.FAIL_LINK_KILLED); 1042 } 1043 testRecovery_OneLinkFailureMid()1044 public void testRecovery_OneLinkFailureMid() throws Exception { 1045 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1046 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1047 } 1048 testRecovery_TwoLinkFailuresMid()1049 public void testRecovery_TwoLinkFailuresMid() throws Exception { 1050 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1051 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1052 RecoveryEvent.PROGRESS); 1053 } 1054 testRecovery_ThreeLinkFailuresMid()1055 public void testRecovery_ThreeLinkFailuresMid() throws Exception { 1056 testRecoveryWithPattern(true, RecoveryEvent.PROGRESS, 1057 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1058 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.PROGRESS); 1059 } 1060 testRecovery_FourLinkFailuresMid()1061 public void testRecovery_FourLinkFailuresMid() throws Exception { 1062 testRecoveryWithPattern(false, RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1063 RecoveryEvent.FAIL_LINK_KILLED, RecoveryEvent.FAIL_LINK_KILLED, 1064 RecoveryEvent.FAIL_LINK_KILLED); 1065 } 1066 1067 // mixed patterns 1068 testRecovery_MixedFailuresProgressBetween()1069 public void testRecovery_MixedFailuresProgressBetween() throws Exception { 1070 testRecoveryWithPattern(true, 1071 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1072 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1073 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1074 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_CONNECTION_REFUSED, 1075 RecoveryEvent.PROGRESS); 1076 } 1077 testRecovery_MixedFailuresNoProgressBetween()1078 public void testRecovery_MixedFailuresNoProgressBetween() throws Exception { 1079 testRecoveryWithPattern(true, 1080 RecoveryEvent.PROGRESS, RecoveryEvent.FAIL_LINK_KILLED, 1081 RecoveryEvent.FAIL_CONNECTION_REFUSED, RecoveryEvent.FAIL_LINK_KILLED, 1082 RecoveryEvent.PROGRESS); 1083 } 1084 1085 /** 1086 * Test recovery if process cannot be killed 1087 */ testRecovery_unkillableProcess()1088 public void testRecovery_unkillableProcess () throws Exception { 1089 KhronosCTSRunner.Recovery recovery = new KhronosCTSRunner.Recovery(); 1090 IMocksControl orderedControl = EasyMock.createStrictControl(); 1091 RecoverableTestDevice mockDevice = orderedControl.createMock(RecoverableTestDevice.class); 1092 KhronosCTSRunner.ISleepProvider mockSleepProvider = 1093 orderedControl.createMock(KhronosCTSRunner.ISleepProvider.class); 1094 1095 // recovery attempts to kill the process after a timeout 1096 mockSleepProvider.sleep(EasyMock.gt(0)); 1097 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1098 andReturn("root 1234 com.drawelement.deqp").once(); 1099 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1100 andReturn("").once(); 1101 1102 // Recovery checks if kill failed 1103 mockSleepProvider.sleep(EasyMock.gt(0)); 1104 EasyMock.expectLastCall().once(); 1105 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1106 andReturn("root 1234 com.drawelement.deqp").once(); 1107 1108 // Recovery resets the connection 1109 EasyMock.expect(mockDevice.recoverDevice()).andReturn(true); 1110 1111 // and attempts to kill the process again 1112 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1113 andReturn("root 1234 com.drawelement.deqp").once(); 1114 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.eq("kill -9 1234"))). 1115 andReturn("").once(); 1116 1117 // Recovery checks if kill failed 1118 mockSleepProvider.sleep(EasyMock.gt(0)); 1119 EasyMock.expectLastCall().once(); 1120 EasyMock.expect(mockDevice.executeShellCommand(EasyMock.contains("ps"))). 1121 andReturn("root 1234 com.drawelement.deqp").once(); 1122 1123 // recovery reboots the device 1124 mockDevice.reboot(); 1125 EasyMock.expectLastCall().once(); 1126 1127 orderedControl.replay(); 1128 recovery.setDevice(mockDevice); 1129 recovery.setSleepProvider(mockSleepProvider); 1130 recovery.recoverComLinkKilled(); 1131 orderedControl.verify(); 1132 } 1133 1134 /** 1135 * {@inheritDoc} 1136 */ 1137 @Override setUp()1138 protected void setUp() throws Exception { 1139 super.setUp(); 1140 mTestsDir = FileUtil.createTempDir("khronos-cts-runner-test-cases"); 1141 } 1142 1143 /** 1144 * {@inheritDoc} 1145 */ 1146 @Override tearDown()1147 protected void tearDown() throws Exception { 1148 FileUtil.recursiveDelete(mTestsDir); 1149 super.tearDown(); 1150 } 1151 } 1152