1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.voicerecognition.cts; 18 19 import static android.voicerecognition.cts.CallbackMethod.CALLBACK_METHOD_END_SEGMENTED_SESSION; 20 import static android.voicerecognition.cts.CallbackMethod.CALLBACK_METHOD_ERROR; 21 import static android.voicerecognition.cts.CallbackMethod.CALLBACK_METHOD_LANGUAGE_DETECTION; 22 import static android.voicerecognition.cts.CallbackMethod.CALLBACK_METHOD_RESULTS; 23 import static android.voicerecognition.cts.CallbackMethod.CALLBACK_METHOD_SEGMENTS_RESULTS; 24 import static android.voicerecognition.cts.CallbackMethod.CALLBACK_METHOD_UNSPECIFIED; 25 import static android.voicerecognition.cts.RecognizerMethod.RECOGNIZER_METHOD_CANCEL; 26 import static android.voicerecognition.cts.RecognizerMethod.RECOGNIZER_METHOD_DESTROY; 27 import static android.voicerecognition.cts.RecognizerMethod.RECOGNIZER_METHOD_START_LISTENING; 28 import static android.voicerecognition.cts.RecognizerMethod.RECOGNIZER_METHOD_STOP_LISTENING; 29 import static android.voicerecognition.cts.TestObjects.ERROR_CODE; 30 import static android.voicerecognition.cts.TestObjects.START_LISTENING_INTENT; 31 32 import static com.android.compatibility.common.util.ShellUtils.runShellCommand; 33 34 import static com.google.common.truth.Truth.assertThat; 35 import static com.google.common.truth.Truth.assertWithMessage; 36 37 import static org.junit.Assert.fail; 38 39 import android.content.Intent; 40 import android.os.Looper; 41 import android.os.SystemClock; 42 import android.speech.ModelDownloadListener; 43 import android.speech.RecognitionService; 44 import android.speech.RecognitionSupport; 45 import android.speech.RecognitionSupportCallback; 46 import android.speech.RecognizerIntent; 47 import android.speech.SpeechRecognizer; 48 import android.support.test.uiautomator.UiDevice; 49 import android.util.Log; 50 import android.util.Pair; 51 52 import androidx.annotation.NonNull; 53 import androidx.annotation.Nullable; 54 import androidx.test.platform.app.InstrumentationRegistry; 55 import androidx.test.rule.ActivityTestRule; 56 57 import com.android.compatibility.common.util.PollingCheck; 58 59 import com.google.common.collect.ImmutableList; 60 61 import junitparams.JUnitParamsRunner; 62 import junitparams.Parameters; 63 import junitparams.naming.TestCaseName; 64 65 import org.junit.Before; 66 import org.junit.Rule; 67 import org.junit.Test; 68 import org.junit.runner.RunWith; 69 70 import java.util.ArrayDeque; 71 import java.util.ArrayList; 72 import java.util.List; 73 import java.util.Random; 74 import java.util.concurrent.TimeUnit; 75 import java.util.stream.Collectors; 76 import java.util.stream.IntStream; 77 78 /** Abstract implementation for {@link android.speech.SpeechRecognizer} CTS tests. */ 79 @RunWith(JUnitParamsRunner.class) 80 abstract class AbstractRecognitionServiceTest { 81 private static final String TAG = AbstractRecognitionServiceTest.class.getSimpleName(); 82 83 private static final long INDICATOR_DISMISS_TIMEOUT = 5000L; 84 private static final long WAIT_TIMEOUT_MS = 30000L; // 30 secs 85 private static final long SEQUENCE_TEST_WAIT_TIMEOUT_MS = 5000L; 86 private static final long ACTIVITY_INIT_WAIT_TIMEOUT_MS = 5000L; 87 88 /* package */ static final String CTS_VOICE_RECOGNITION_SERVICE = 89 "android.recognitionservice.service/android.recognitionservice.service" 90 + ".CtsVoiceRecognitionService"; 91 92 /* package */ static final String IN_PACKAGE_RECOGNITION_SERVICE = 93 "android.voicerecognition.cts/android.voicerecognition.cts.CtsRecognitionService"; 94 95 // Expected to create 1 more recognizer than what the concurrency limit is, 96 // so that SpeechRecognizer#ERROR_RECOGNIZER_BUSY scenarios can be tested, too. 97 private static final int EXPECTED_RECOGNIZER_COUNT = 98 CtsRecognitionService.MAX_CONCURRENT_SESSIONS_COUNT + 1; 99 100 @Rule 101 public ActivityTestRule<SpeechRecognitionActivity> mActivityTestRule = 102 new ActivityTestRule<>(SpeechRecognitionActivity.class); 103 104 private UiDevice mUiDevice; 105 private SpeechRecognitionActivity mActivity; 106 107 private final Random mRandom = new Random(); 108 setCurrentRecognizer(SpeechRecognizer recognizer, String component)109 abstract void setCurrentRecognizer(SpeechRecognizer recognizer, String component); 110 isOnDeviceTest()111 abstract boolean isOnDeviceTest(); 112 113 @Nullable customRecognizer()114 abstract String customRecognizer(); 115 116 @Before setup()117 public void setup() { 118 prepareDevice(); 119 mUiDevice = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()); 120 mActivity = mActivityTestRule.getActivity(); 121 mActivity.init(isOnDeviceTest(), customRecognizer(), EXPECTED_RECOGNIZER_COUNT); 122 123 PollingCheck.waitFor(ACTIVITY_INIT_WAIT_TIMEOUT_MS, 124 () -> mActivity.getRecognizerCount() == EXPECTED_RECOGNIZER_COUNT); 125 assertWithMessage("Test activity initialization timed out.") 126 .that(mActivity.getRecognizerCount()).isEqualTo(EXPECTED_RECOGNIZER_COUNT); 127 } 128 129 @Test testStartListening()130 public void testStartListening() throws Throwable { 131 mUiDevice.waitForIdle(); 132 SpeechRecognitionActivity.RecognizerInfo ri = mActivity.getRecognizerInfoDefault(); 133 setCurrentRecognizer(ri.mRecognizer, CTS_VOICE_RECOGNITION_SERVICE); 134 135 mUiDevice.waitForIdle(); 136 mActivity.startListeningDefault(); 137 try { 138 // startListening() will call noteProxyOpNoTrow(). If the permission check passes, 139 // then the RecognitionService.onStartListening() will be called. Otherwise, 140 // a TimeoutException will be thrown. 141 assertThat(ri.mCountDownLatch.await( 142 WAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS)).isTrue(); 143 } catch (InterruptedException e) { 144 assertWithMessage("onStartListening() not called. " + e).fail(); 145 } 146 147 // Wait for the privacy indicator to disappear to avoid the test becoming flaky. 148 SystemClock.sleep(INDICATOR_DISMISS_TIMEOUT); 149 } 150 151 @Test testCanCheckForSupport()152 public void testCanCheckForSupport() throws Throwable { 153 mUiDevice.waitForIdle(); 154 SpeechRecognizer recognizer = mActivity.getRecognizerInfoDefault().mRecognizer; 155 assertThat(recognizer).isNotNull(); 156 setCurrentRecognizer(recognizer, IN_PACKAGE_RECOGNITION_SERVICE); 157 158 mUiDevice.waitForIdle(); 159 List<RecognitionSupport> supportResults = new ArrayList<>(); 160 List<Integer> errors = new ArrayList<>(); 161 RecognitionSupportCallback supportCallback = new RecognitionSupportCallback() { 162 @Override 163 public void onSupportResult(@NonNull RecognitionSupport recognitionSupport) { 164 supportResults.add(recognitionSupport); 165 } 166 167 @Override 168 public void onError(int error) { 169 errors.add(error); 170 } 171 }; 172 Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); 173 mActivity.checkRecognitionSupportDefault(intent, supportCallback); 174 PollingCheck.waitFor(SEQUENCE_TEST_WAIT_TIMEOUT_MS, 175 () -> supportResults.size() + errors.size() > 0); 176 assertThat(supportResults).isEmpty(); 177 assertThat(errors).containsExactly(SpeechRecognizer.ERROR_CANNOT_CHECK_SUPPORT); 178 179 errors.clear(); 180 RecognitionSupport rs = new RecognitionSupport.Builder() 181 .setInstalledOnDeviceLanguages(new ArrayList<>(List.of("es"))) 182 .addInstalledOnDeviceLanguage("en") 183 .setPendingOnDeviceLanguages(new ArrayList<>(List.of("ru"))) 184 .addPendingOnDeviceLanguage("jp") 185 .setSupportedOnDeviceLanguages(new ArrayList<>(List.of("pt"))) 186 .addSupportedOnDeviceLanguage("de") 187 .setOnlineLanguages(new ArrayList<>(List.of("zh"))) 188 .addOnlineLanguage("fr") 189 .build(); 190 CtsRecognitionService.sConsumerQueue.add(c -> c.onSupportResult(rs)); 191 192 mActivity.checkRecognitionSupportDefault(intent, supportCallback); 193 PollingCheck.waitFor(SEQUENCE_TEST_WAIT_TIMEOUT_MS, 194 () -> supportResults.size() + errors.size() > 0); 195 assertThat(errors).isEmpty(); 196 assertThat(supportResults).containsExactly(rs); 197 assertThat(rs.getInstalledOnDeviceLanguages()) 198 .isEqualTo(List.of("es", "en")); 199 assertThat(rs.getPendingOnDeviceLanguages()) 200 .isEqualTo(List.of("ru", "jp")); 201 assertThat(rs.getSupportedOnDeviceLanguages()) 202 .isEqualTo(List.of("pt", "de")); 203 assertThat(rs.getOnlineLanguages()) 204 .isEqualTo(List.of("zh", "fr")); 205 assertThat(CtsRecognitionService.sBindCount.get()).isGreaterThan(0); 206 207 mActivity.destroyRecognizerDefault(); 208 mUiDevice.waitForIdle(); 209 PollingCheck.waitFor(() -> CtsRecognitionService.sBindCount.get() == 0); 210 } 211 212 @Test testCanTriggerModelDownload()213 public void testCanTriggerModelDownload() throws Throwable { 214 mUiDevice.waitForIdle(); 215 SpeechRecognizer recognizer = mActivity.getRecognizerInfoDefault().mRecognizer; 216 assertThat(recognizer).isNotNull(); 217 setCurrentRecognizer(recognizer, IN_PACKAGE_RECOGNITION_SERVICE); 218 219 mUiDevice.waitForIdle(); 220 CtsRecognitionService.sDownloadTriggers.clear(); 221 Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); 222 mActivity.triggerModelDownloadDefault(intent); 223 PollingCheck.waitFor(SEQUENCE_TEST_WAIT_TIMEOUT_MS, 224 () -> CtsRecognitionService.sDownloadTriggers.size() > 0); 225 assertThat(CtsRecognitionService.sDownloadTriggers).hasSize(1); 226 assertThat(CtsRecognitionService.sBindCount.get()).isGreaterThan(0); 227 228 mActivity.destroyRecognizerDefault(); 229 mUiDevice.waitForIdle(); 230 PollingCheck.waitFor(() -> CtsRecognitionService.sBindCount.get() == 0); 231 } 232 233 @Test 234 @Parameters(method = "modelDownloadScenarios") 235 @TestCaseName("{method}_{0}") testCanTriggerModelDownloadWithListener( ModelDownloadExecutionInfo.Scenario scenario)236 public void testCanTriggerModelDownloadWithListener( 237 ModelDownloadExecutionInfo.Scenario scenario) { 238 mUiDevice.waitForIdle(); 239 SpeechRecognizer recognizer = mActivity.getRecognizerInfoDefault().mRecognizer; 240 assertThat(recognizer).isNotNull(); 241 setCurrentRecognizer(recognizer, IN_PACKAGE_RECOGNITION_SERVICE); 242 243 mUiDevice.waitForIdle(); 244 ModelDownloadExecutionInfo mdei = ModelDownloadExecutionInfo.fromScenario(scenario); 245 CtsRecognitionService.sDownloadTriggers.clear(); 246 CtsRecognitionService.sInstructedModelDownloadCallbacks = 247 new ArrayDeque<>(mdei.mInstructedCallbacks); 248 Intent intent = new Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH); 249 ModelDownloadCallbackLogger listener = new ModelDownloadCallbackLogger(); 250 251 mActivity.triggerModelDownloadWithListenerDefault(intent, listener); 252 PollingCheck.waitFor(SEQUENCE_TEST_WAIT_TIMEOUT_MS, 253 () -> CtsRecognitionService.sDownloadTriggers.size() > 0); 254 255 mUiDevice.waitForIdle(); 256 assertThat(listener.mCallbacks) 257 .containsExactlyElementsIn(mdei.mExpectedCallbacks) 258 .inOrder(); 259 assertThat(CtsRecognitionService.sBindCount.get()).isGreaterThan(0); 260 261 mActivity.destroyRecognizerDefault(); 262 mUiDevice.waitForIdle(); 263 PollingCheck.waitFor(() -> CtsRecognitionService.sBindCount.get() == 0); 264 } 265 modelDownloadScenarios()266 static ModelDownloadExecutionInfo.Scenario[] modelDownloadScenarios() { 267 return new ModelDownloadExecutionInfo.Scenario[] { 268 ModelDownloadExecutionInfo.Scenario.PROGRESS_PROGRESS_PROGRESS, 269 ModelDownloadExecutionInfo.Scenario.PROGRESS_SUCCESS_PROGRESS, 270 ModelDownloadExecutionInfo.Scenario.SCHEDULED_ERROR, 271 ModelDownloadExecutionInfo.Scenario.ERROR_SCHEDULED}; 272 } 273 274 @Test 275 @Parameters(method = "singleScenarios") 276 @TestCaseName("{method}_{0}") sequenceTest(SequenceExecutionInfo.Scenario scenario)277 public void sequenceTest(SequenceExecutionInfo.Scenario scenario) { 278 Log.d(TAG, "Running a single sequence: " + scenario.name() + "."); 279 executeSequenceTest(SequenceExecutionInfo.fromScenario(scenario)); 280 } 281 singleScenarios()282 static SequenceExecutionInfo.Scenario[] singleScenarios() { 283 return new SequenceExecutionInfo.Scenario[] { 284 SequenceExecutionInfo.Scenario.START_STOP_RESULTS, 285 SequenceExecutionInfo.Scenario.START_RESULTS_STOP, 286 SequenceExecutionInfo.Scenario.START_RESULTS_CANCEL, 287 SequenceExecutionInfo.Scenario.START_RESULTS_START_RESULTS, 288 SequenceExecutionInfo.Scenario.START_SEGMENT_ENDOFSESSION, 289 SequenceExecutionInfo.Scenario.START_CANCEL, 290 SequenceExecutionInfo.Scenario.START_START, 291 SequenceExecutionInfo.Scenario.START_STOP_CANCEL, 292 SequenceExecutionInfo.Scenario.START_ERROR_CANCEL, 293 SequenceExecutionInfo.Scenario.START_STOP_DESTROY, 294 SequenceExecutionInfo.Scenario.START_ERROR_DESTROY, 295 SequenceExecutionInfo.Scenario.START_DESTROY_DESTROY, 296 SequenceExecutionInfo.Scenario.START_DETECTION_STOP_RESULTS}; 297 } 298 299 @Test 300 @Parameters(method = "doubleScenarios") 301 @TestCaseName("{method}_{0}_x_{1}") concurrentSequenceTest( SequenceExecutionInfo.Scenario scenario1, SequenceExecutionInfo.Scenario scenario2)302 public void concurrentSequenceTest( 303 SequenceExecutionInfo.Scenario scenario1, 304 SequenceExecutionInfo.Scenario scenario2) { 305 Log.d(TAG, "Running a double sequence: " 306 + scenario1.name() + " x " + scenario2.name() + "."); 307 executeSequenceTest(ImmutableList.of( 308 SequenceExecutionInfo.fromScenario(scenario1), 309 SequenceExecutionInfo.fromScenario(scenario2)), 310 /* inOrder */ true); 311 } 312 doubleScenarios()313 static Object[] doubleScenarios() { 314 // Scenarios where the results are not received in the same step when start is called. 315 List<SequenceExecutionInfo.Scenario> concurrencyObservableScenarios = ImmutableList.of( 316 SequenceExecutionInfo.Scenario.START_STOP_RESULTS, 317 SequenceExecutionInfo.Scenario.START_SEGMENT_ENDOFSESSION, 318 SequenceExecutionInfo.Scenario.START_CANCEL, 319 SequenceExecutionInfo.Scenario.START_START, 320 SequenceExecutionInfo.Scenario.START_STOP_CANCEL, 321 SequenceExecutionInfo.Scenario.START_STOP_DESTROY, 322 SequenceExecutionInfo.Scenario.START_DESTROY_DESTROY, 323 SequenceExecutionInfo.Scenario.START_DETECTION_STOP_RESULTS); 324 325 List<Object[]> scenarios = new ArrayList<>(); 326 for (int i = 0; i < concurrencyObservableScenarios.size(); i++) { 327 for (int j = 0; j < concurrencyObservableScenarios.size(); j++) { 328 scenarios.add(new Object[]{ 329 concurrencyObservableScenarios.get(i), 330 concurrencyObservableScenarios.get(j)}); 331 } 332 } 333 return scenarios.toArray(); 334 } 335 336 @Test testRecognitionServiceConcurrencyLimitValidity()337 public void testRecognitionServiceConcurrencyLimitValidity() { 338 // Prepare the looper before creating a RecognitionService object. 339 if (Looper.myLooper() == null) { 340 Looper.prepare(); 341 } 342 343 RecognitionService defaultService = new RecognitionService() { 344 @Override 345 protected void onStartListening(Intent recognizerIntent, Callback listener) {} 346 347 @Override 348 protected void onCancel(Callback listener) {} 349 350 @Override 351 protected void onStopListening(Callback listener) {} 352 }; 353 assertWithMessage("Default recognition service concurrency limit must be positive.") 354 .that(defaultService.getMaxConcurrentSessionsCount()).isGreaterThan(0); 355 356 RecognitionService testService = new CtsRecognitionService(); 357 assertWithMessage("Recognition service implementation concurrency limit must be positive.") 358 .that(testService.getMaxConcurrentSessionsCount()).isGreaterThan(0); 359 } 360 361 @Test testRecognitionServiceBusy()362 public void testRecognitionServiceBusy() { 363 Log.d(TAG, "Running four sequences, one more than the concurrency limit."); 364 executeSequenceTest(ImmutableList.of( 365 SequenceExecutionInfo.fromScenario( 366 SequenceExecutionInfo.Scenario.START_STOP_RESULTS), 367 SequenceExecutionInfo.fromScenario( 368 SequenceExecutionInfo.Scenario.START_SEGMENT_ENDOFSESSION), 369 SequenceExecutionInfo.fromScenario(SequenceExecutionInfo.Scenario.START_CANCEL), 370 371 // This sequence will fail with ERROR_RECOGNIZER_BUSY. 372 SequenceExecutionInfo.fromScenario(SequenceExecutionInfo.Scenario.START_ERROR)), 373 /* inOrder */ true); 374 } 375 executeSequenceTest(SequenceExecutionInfo sei)376 private void executeSequenceTest(SequenceExecutionInfo sei) { 377 executeSequenceTest(ImmutableList.of(sei), /* inOrder */ true); 378 } 379 executeSequenceTest( List<SequenceExecutionInfo> sequenceExecutionInfos, boolean inOrder)380 private void executeSequenceTest( 381 List<SequenceExecutionInfo> sequenceExecutionInfos, 382 boolean inOrder) { 383 mUiDevice.waitForIdle(); 384 385 // Initialize the recognizers to be used and clear their invoked callbacks list. 386 for (int recognizerIndex = 0; 387 recognizerIndex < sequenceExecutionInfos.size(); 388 recognizerIndex++) { 389 SpeechRecognitionActivity.RecognizerInfo ri = 390 mActivity.getRecognizerInfo(recognizerIndex); 391 assertThat(ri.mRecognizer).isNotNull(); 392 setCurrentRecognizer(ri.mRecognizer, IN_PACKAGE_RECOGNITION_SERVICE); 393 ri.mCallbackMethodsInvoked.clear(); 394 } 395 396 // Clear recognition service's invoked recognizer methods list 397 // and callback instruction queue. 398 CtsRecognitionService.sInvokedRecognizerMethods.clear(); 399 CtsRecognitionService.sInstructedCallbackMethods.clear(); 400 401 // Initialize the list of recognizers to be used. 402 List<Integer> remainingRecognizerIndices = IntStream.range(0, sequenceExecutionInfos.size()) 403 .boxed().collect(Collectors.toList()); 404 405 int expectedServiceMethodsRunCount = 0; 406 int nextRemainingRecognizerIndex = 0; 407 while (!remainingRecognizerIndices.isEmpty()) { 408 // If the execution should be in order, select the next recognizer by index. 409 // Else, pick one of the recognizers at random. Start the next step. 410 nextRemainingRecognizerIndex %= remainingRecognizerIndices.size(); 411 int selectedRecognizerIndex = remainingRecognizerIndices.get(inOrder 412 ? nextRemainingRecognizerIndex 413 : mRandom.nextInt(remainingRecognizerIndices.size())); 414 SequenceExecutionInfo sei = sequenceExecutionInfos.get(selectedRecognizerIndex); 415 int executionStep = sei.getNextStep(); 416 417 // If the flag is set, prepare the callback instruction for the service side. 418 if (sei.mExpectedRecognizerServiceMethodsToPropagate.get(executionStep)) { 419 CtsRecognitionService.sInstructedCallbackMethods.add(new Pair<>( 420 selectedRecognizerIndex, 421 sei.mCallbackMethodInstructions.get(executionStep))); 422 } 423 424 // Call the recognizer method. 425 RecognizerMethod recognizerMethod = sei.mRecognizerMethodsToCall.get(executionStep); 426 Log.i(TAG, "Sending service method " + recognizerMethod.name() + "."); 427 switch (recognizerMethod) { 428 case RECOGNIZER_METHOD_START_LISTENING: 429 mActivity.startListening(START_LISTENING_INTENT, selectedRecognizerIndex); 430 break; 431 case RECOGNIZER_METHOD_STOP_LISTENING: 432 mActivity.stopListening(selectedRecognizerIndex); 433 break; 434 case RECOGNIZER_METHOD_CANCEL: 435 mActivity.cancel(selectedRecognizerIndex); 436 break; 437 case RECOGNIZER_METHOD_DESTROY: 438 mActivity.destroyRecognizer(selectedRecognizerIndex); 439 break; 440 case RECOGNIZER_METHOD_UNSPECIFIED: 441 default: 442 fail(); 443 } 444 445 // If the flag is set, wait for the service to propagate the callback. 446 if (sei.mExpectedRecognizerServiceMethodsToPropagate.get(executionStep)) { 447 expectedServiceMethodsRunCount++; 448 int finalExpectedServiceMethodsRunCount = expectedServiceMethodsRunCount; 449 PollingCheck.waitFor(SEQUENCE_TEST_WAIT_TIMEOUT_MS, 450 () -> CtsRecognitionService.totalInvokedRecognizerMethodsCount() 451 == finalExpectedServiceMethodsRunCount); 452 } 453 454 // TODO(kiridza): Make this part of the sequence execution more robust. 455 if (selectedRecognizerIndex >= CtsRecognitionService.MAX_CONCURRENT_SESSIONS_COUNT) { 456 PollingCheck.waitFor(SEQUENCE_TEST_WAIT_TIMEOUT_MS, 457 () -> !mActivity.getRecognizerInfo(selectedRecognizerIndex) 458 .mErrorCodesReceived.isEmpty()); 459 } 460 461 // If this was the last step of the sequence, remove it from the list. 462 if (sei.isFinished()) { 463 remainingRecognizerIndices.remove(Integer.valueOf(selectedRecognizerIndex)); 464 } else { 465 nextRemainingRecognizerIndex++; 466 } 467 } 468 469 // Wait until the service has propagated all callbacks 470 // and the recognizers' listeners have received them. 471 PollingCheck.waitFor(SEQUENCE_TEST_WAIT_TIMEOUT_MS, 472 () -> CtsRecognitionService.sInstructedCallbackMethods.isEmpty()); 473 for (int recognizerIndex = 0; 474 recognizerIndex < sequenceExecutionInfos.size(); 475 recognizerIndex++) { 476 int finalRecognizerIndex = recognizerIndex; 477 PollingCheck.waitFor(SEQUENCE_TEST_WAIT_TIMEOUT_MS, 478 () -> mActivity.getRecognizerInfo(finalRecognizerIndex) 479 .mCallbackMethodsInvoked.size() 480 >= sequenceExecutionInfos.get(finalRecognizerIndex) 481 .mExpectedClientCallbackMethods.size()); 482 } 483 484 // Check for all recognizers that: 485 // - the service has executed the expected methods in the given order; 486 // - the expected client callbacks were invoked in the given order. 487 // - the expected error codes were received in the given order. 488 for (int recognizerIndex = 0; 489 recognizerIndex < sequenceExecutionInfos.size(); 490 recognizerIndex++) { 491 SequenceExecutionInfo sei = sequenceExecutionInfos.get(recognizerIndex); 492 SpeechRecognitionActivity.RecognizerInfo ri = 493 mActivity.getRecognizerInfo(recognizerIndex); 494 495 List<RecognizerMethod> expectedServiceMethods = new ArrayList<>(); 496 for (int step = 0; step < sei.mRecognizerMethodsToCall.size(); step++) { 497 if (sei.mExpectedRecognizerServiceMethodsToPropagate.get(step)) { 498 expectedServiceMethods.add( 499 RECOGNIZER_METHOD_DESTROY != sei.mRecognizerMethodsToCall.get(step) 500 ? sei.mRecognizerMethodsToCall.get(step) 501 : RECOGNIZER_METHOD_CANCEL); 502 } 503 } 504 505 if (expectedServiceMethods.isEmpty()) { 506 assertThat(CtsRecognitionService.sInvokedRecognizerMethods 507 .containsKey(recognizerIndex)).isFalse(); 508 } else { 509 assertThat(CtsRecognitionService.sInvokedRecognizerMethods.get(recognizerIndex)) 510 .isEqualTo(expectedServiceMethods); 511 } 512 assertThat(ri.mCallbackMethodsInvoked).isEqualTo(sei.mExpectedClientCallbackMethods); 513 assertThat(ri.mErrorCodesReceived).isEqualTo(sei.mExpectedErrorCodesReceived); 514 } 515 assertThat(CtsRecognitionService.sInstructedCallbackMethods).isEmpty(); 516 517 mActivity.destroyAllRecognizers(); 518 mUiDevice.waitForIdle(); 519 PollingCheck.waitFor(() -> CtsRecognitionService.sBindCount.get() == 0); 520 } 521 prepareDevice()522 private static void prepareDevice() { 523 // Unlock screen. 524 runShellCommand("input keyevent KEYCODE_WAKEUP"); 525 // Dismiss keyguard, in case it's set as "Swipe to unlock". 526 runShellCommand("wm dismiss-keyguard"); 527 } 528 529 /** 530 * Data class containing information about a recognizer object used in the activity: 531 * <ul> 532 * <li> {@link SequenceExecutionInfo#mRecognizerMethodsToCall} - list of {@link 533 * RecognizerMethod}s to be invoked by the corresponding recognizer; 534 * <li> {@link SequenceExecutionInfo#mCallbackMethodInstructions} - list of {@link 535 * CallbackMethod}s forwarded to the service to be invoked on the corresponding listener; 536 * <li> {@link SequenceExecutionInfo#mExpectedRecognizerServiceMethodsToPropagate} - list of 537 * flags denoting if the callback should be expected after corresponding recognizer methods; 538 * <li> {@link SequenceExecutionInfo#mExpectedClientCallbackMethods} - list of {@link 539 * CallbackMethod}s expected to be run on the corresponding listener. 540 * <li> {@link SequenceExecutionInfo#mNextStep} - next step to be run in the sequence. 541 */ 542 private static class SequenceExecutionInfo { 543 private final List<RecognizerMethod> mRecognizerMethodsToCall; 544 private final List<CallbackMethod> mCallbackMethodInstructions; 545 private final List<Boolean> mExpectedRecognizerServiceMethodsToPropagate; 546 private final List<CallbackMethod> mExpectedClientCallbackMethods; 547 private final List<Integer> mExpectedErrorCodesReceived; 548 private int mNextStep; 549 SequenceExecutionInfo( List<RecognizerMethod> recognizerMethodsToCall, List<CallbackMethod> callbackMethodInstructions, List<Boolean> expectedRecognizerServiceMethodsToPropagate, List<CallbackMethod> expectedClientCallbackMethods, List<Integer> expectedErrorCodesReceived)550 private SequenceExecutionInfo( 551 List<RecognizerMethod> recognizerMethodsToCall, 552 List<CallbackMethod> callbackMethodInstructions, 553 List<Boolean> expectedRecognizerServiceMethodsToPropagate, 554 List<CallbackMethod> expectedClientCallbackMethods, 555 List<Integer> expectedErrorCodesReceived) { 556 mRecognizerMethodsToCall = recognizerMethodsToCall; 557 mCallbackMethodInstructions = callbackMethodInstructions; 558 mExpectedRecognizerServiceMethodsToPropagate = 559 expectedRecognizerServiceMethodsToPropagate; 560 mExpectedClientCallbackMethods = expectedClientCallbackMethods; 561 mExpectedErrorCodesReceived = expectedErrorCodesReceived; 562 mNextStep = 0; 563 } 564 getNextStep()565 private int getNextStep() { 566 return mNextStep++; 567 } 568 isFinished()569 private boolean isFinished() { 570 return mNextStep >= mRecognizerMethodsToCall.size(); 571 } 572 573 enum Scenario { 574 // Happy scenarios. 575 START_STOP_RESULTS, 576 START_RESULTS_STOP, 577 START_RESULTS_CANCEL, 578 START_RESULTS_START_RESULTS, 579 START_SEGMENT_ENDOFSESSION, 580 START_CANCEL, 581 START_START, 582 START_STOP_CANCEL, 583 START_ERROR_CANCEL, 584 START_STOP_DESTROY, 585 START_ERROR_DESTROY, 586 START_DESTROY_DESTROY, 587 START_DETECTION_STOP_RESULTS, 588 589 // Sad scenarios. 590 START_ERROR 591 } 592 fromScenario(Scenario scenario)593 private static SequenceExecutionInfo fromScenario(Scenario scenario) { 594 switch (scenario) { 595 // Happy scenarios. 596 case START_STOP_RESULTS: 597 return new SequenceExecutionInfo( 598 /* service methods to call: */ ImmutableList.of( 599 RECOGNIZER_METHOD_START_LISTENING, 600 RECOGNIZER_METHOD_STOP_LISTENING), 601 /* callback methods to call: */ ImmutableList.of( 602 CALLBACK_METHOD_UNSPECIFIED, 603 CALLBACK_METHOD_RESULTS), 604 /* expected service methods propagated: */ ImmutableList.of( 605 true, 606 true), 607 /* expected callback methods invoked: */ ImmutableList.of( 608 CALLBACK_METHOD_RESULTS), 609 /* expected error codes received: */ ImmutableList.of()); 610 case START_RESULTS_STOP: 611 return new SequenceExecutionInfo( 612 /* service methods to call: */ ImmutableList.of( 613 RECOGNIZER_METHOD_START_LISTENING, 614 RECOGNIZER_METHOD_STOP_LISTENING), 615 /* callback methods to call: */ ImmutableList.of( 616 CALLBACK_METHOD_RESULTS), 617 /* expected service methods propagated: */ ImmutableList.of( 618 true, 619 false), 620 /* expected callback methods invoked: */ ImmutableList.of( 621 CALLBACK_METHOD_RESULTS, 622 CALLBACK_METHOD_ERROR), 623 /* expected error codes received: */ ImmutableList.of( 624 SpeechRecognizer.ERROR_CLIENT)); 625 case START_RESULTS_CANCEL: 626 return new SequenceExecutionInfo( 627 /* service methods to call: */ ImmutableList.of( 628 RECOGNIZER_METHOD_START_LISTENING, 629 RECOGNIZER_METHOD_CANCEL), 630 /* callback methods to call: */ ImmutableList.of( 631 CALLBACK_METHOD_RESULTS), 632 /* expected service methods propagated: */ ImmutableList.of( 633 true, 634 false), 635 /* expected callback methods invoked: */ ImmutableList.of( 636 CALLBACK_METHOD_RESULTS), 637 /* expected error codes received: */ ImmutableList.of()); 638 case START_RESULTS_START_RESULTS: 639 return new SequenceExecutionInfo( 640 /* service methods to call: */ ImmutableList.of( 641 RECOGNIZER_METHOD_START_LISTENING, 642 RECOGNIZER_METHOD_START_LISTENING), 643 /* callback methods to call: */ ImmutableList.of( 644 CALLBACK_METHOD_RESULTS, 645 CALLBACK_METHOD_RESULTS), 646 /* expected service methods propagated: */ ImmutableList.of( 647 true, 648 true), 649 /* expected callback methods invoked: */ ImmutableList.of( 650 CALLBACK_METHOD_RESULTS, 651 CALLBACK_METHOD_RESULTS), 652 /* expected error codes received: */ ImmutableList.of()); 653 case START_SEGMENT_ENDOFSESSION: 654 return new SequenceExecutionInfo( 655 /* service methods to call: */ ImmutableList.of( 656 RECOGNIZER_METHOD_START_LISTENING, 657 RECOGNIZER_METHOD_STOP_LISTENING), 658 /* callback methods to call: */ ImmutableList.of( 659 CALLBACK_METHOD_SEGMENTS_RESULTS, 660 CALLBACK_METHOD_END_SEGMENTED_SESSION), 661 /* expected service methods propagated: */ ImmutableList.of( 662 true, 663 true, 664 true), 665 /* expected callback methods invoked: */ ImmutableList.of( 666 CALLBACK_METHOD_SEGMENTS_RESULTS, 667 CALLBACK_METHOD_END_SEGMENTED_SESSION), 668 /* expected error codes received: */ ImmutableList.of()); 669 case START_CANCEL: 670 return new SequenceExecutionInfo( 671 /* service methods to call: */ ImmutableList.of( 672 RECOGNIZER_METHOD_START_LISTENING, 673 RECOGNIZER_METHOD_CANCEL), 674 /* callback methods to call: */ ImmutableList.of( 675 CALLBACK_METHOD_UNSPECIFIED, 676 CALLBACK_METHOD_UNSPECIFIED), 677 /* expected service methods propagated: */ ImmutableList.of( 678 true, 679 true), 680 /* expected callback methods invoked: */ ImmutableList.of(), 681 /* expected error codes received: */ ImmutableList.of()); 682 case START_START: 683 return new SequenceExecutionInfo( 684 /* service methods to call: */ ImmutableList.of( 685 RECOGNIZER_METHOD_START_LISTENING, 686 RECOGNIZER_METHOD_START_LISTENING), 687 /* callback methods to call: */ ImmutableList.of( 688 CALLBACK_METHOD_UNSPECIFIED), 689 /* expected service methods propagated: */ ImmutableList.of( 690 true, 691 false), 692 /* expected callback methods invoked: */ ImmutableList.of( 693 CALLBACK_METHOD_ERROR), 694 /* expected error codes received: */ ImmutableList.of( 695 SpeechRecognizer.ERROR_CLIENT)); 696 case START_STOP_CANCEL: 697 return new SequenceExecutionInfo( 698 /* service methods to call: */ ImmutableList.of( 699 RECOGNIZER_METHOD_START_LISTENING, 700 RECOGNIZER_METHOD_STOP_LISTENING, 701 RECOGNIZER_METHOD_CANCEL), 702 /* callback methods to call: */ ImmutableList.of( 703 CALLBACK_METHOD_UNSPECIFIED, 704 CALLBACK_METHOD_UNSPECIFIED, 705 CALLBACK_METHOD_UNSPECIFIED), 706 /* expected service methods propagated: */ ImmutableList.of( 707 true, 708 true, 709 true), 710 /* expected callback methods invoked: */ ImmutableList.of(), 711 /* expected error codes received: */ ImmutableList.of()); 712 case START_ERROR_CANCEL: 713 return new SequenceExecutionInfo( 714 /* service methods to call: */ ImmutableList.of( 715 RECOGNIZER_METHOD_START_LISTENING, 716 RECOGNIZER_METHOD_CANCEL), 717 /* callback methods to call: */ ImmutableList.of( 718 CALLBACK_METHOD_ERROR), 719 /* expected service methods propagated: */ ImmutableList.of( 720 true, 721 false), 722 /* expected callback methods invoked: */ ImmutableList.of( 723 CALLBACK_METHOD_ERROR), 724 /* expected error codes received: */ ImmutableList.of( 725 ERROR_CODE)); 726 case START_STOP_DESTROY: 727 return new SequenceExecutionInfo( 728 /* service methods to call: */ ImmutableList.of( 729 RECOGNIZER_METHOD_START_LISTENING, 730 RECOGNIZER_METHOD_STOP_LISTENING, 731 RECOGNIZER_METHOD_DESTROY), 732 /* callback methods to call: */ ImmutableList.of( 733 CALLBACK_METHOD_UNSPECIFIED, 734 CALLBACK_METHOD_UNSPECIFIED, 735 CALLBACK_METHOD_UNSPECIFIED), 736 /* expected service methods propagated: */ ImmutableList.of( 737 true, 738 true, 739 true), 740 /* expected callback methods invoked: */ ImmutableList.of(), 741 /* expected error codes received: */ ImmutableList.of()); 742 case START_ERROR_DESTROY: 743 return new SequenceExecutionInfo( 744 /* service methods to call: */ ImmutableList.of( 745 RECOGNIZER_METHOD_START_LISTENING, 746 RECOGNIZER_METHOD_DESTROY), 747 /* callback methods to call: */ ImmutableList.of( 748 CALLBACK_METHOD_ERROR), 749 /* expected service methods propagated: */ ImmutableList.of( 750 true, 751 false), 752 /* expected callback methods invoked: */ ImmutableList.of( 753 CALLBACK_METHOD_ERROR), 754 /* expected error codes received: */ ImmutableList.of( 755 ERROR_CODE)); 756 case START_DESTROY_DESTROY: 757 return new SequenceExecutionInfo( 758 /* service methods to call: */ ImmutableList.of( 759 RECOGNIZER_METHOD_START_LISTENING, 760 RECOGNIZER_METHOD_DESTROY, 761 RECOGNIZER_METHOD_DESTROY), 762 /* callback methods to call: */ ImmutableList.of( 763 CALLBACK_METHOD_UNSPECIFIED, 764 CALLBACK_METHOD_UNSPECIFIED), 765 /* expected service methods propagated: */ ImmutableList.of( 766 true, 767 true, 768 false), 769 /* expected callback methods invoked: */ ImmutableList.of(), 770 /* expected error codes received: */ ImmutableList.of()); 771 case START_DETECTION_STOP_RESULTS: 772 return new SequenceExecutionInfo( 773 /* service methods to call: */ ImmutableList.of( 774 RECOGNIZER_METHOD_START_LISTENING, 775 RECOGNIZER_METHOD_STOP_LISTENING), 776 /* callback methods to call: */ ImmutableList.of( 777 CALLBACK_METHOD_LANGUAGE_DETECTION, 778 CALLBACK_METHOD_RESULTS), 779 /* expected service methods propagated: */ ImmutableList.of( 780 true, 781 true), 782 /* expected callback methods invoked: */ ImmutableList.of( 783 CALLBACK_METHOD_LANGUAGE_DETECTION, 784 CALLBACK_METHOD_RESULTS), 785 /* expected error codes received: */ ImmutableList.of()); 786 787 // Sad scenarios. 788 case START_ERROR: 789 return new SequenceExecutionInfo( 790 /* service methods to call: */ ImmutableList.of( 791 RECOGNIZER_METHOD_START_LISTENING), 792 /* callback methods to call: */ ImmutableList.of(), 793 /* expected service methods propagated: */ ImmutableList.of( 794 false), 795 /* expected callback methods invoked: */ ImmutableList.of( 796 CALLBACK_METHOD_ERROR), 797 /* expected error codes received: */ ImmutableList.of( 798 SpeechRecognizer.ERROR_RECOGNIZER_BUSY)); 799 800 default: 801 return new SequenceExecutionInfo(ImmutableList.of(), ImmutableList.of(), 802 ImmutableList.of(), ImmutableList.of(), ImmutableList.of()); 803 } 804 } 805 } 806 807 /** 808 * Data class containing information about model download listener callback sequence: 809 * <ul> 810 * <li> {@link ModelDownloadExecutionInfo#mInstructedCallbacks} - list of {@link 811 * ModelDownloadCallback}s instructed to be invoked by the service on the given listener; 812 * <li> {@link ModelDownloadExecutionInfo#mExpectedCallbacks} - list of {@link 813 * ModelDownloadCallback}s expected to be received at the client's end by the given listener. 814 */ 815 private static class ModelDownloadExecutionInfo { 816 private final List<ModelDownloadCallback> mInstructedCallbacks; 817 private final List<ModelDownloadCallback> mExpectedCallbacks; 818 ModelDownloadExecutionInfo( List<ModelDownloadCallback> instructedCallbacks, List<ModelDownloadCallback> expectedCallbacks)819 private ModelDownloadExecutionInfo( 820 List<ModelDownloadCallback> instructedCallbacks, 821 List<ModelDownloadCallback> expectedCallbacks) { 822 mInstructedCallbacks = instructedCallbacks; 823 mExpectedCallbacks = expectedCallbacks; 824 } 825 826 enum Scenario { 827 PROGRESS_PROGRESS_PROGRESS, 828 PROGRESS_SUCCESS_PROGRESS, 829 SCHEDULED_ERROR, 830 ERROR_SCHEDULED 831 } 832 fromScenario(Scenario scenario)833 private static ModelDownloadExecutionInfo fromScenario(Scenario scenario) { 834 switch (scenario) { 835 case PROGRESS_PROGRESS_PROGRESS: 836 return new ModelDownloadExecutionInfo( 837 /* callbacks to be invoked by the service: */ ImmutableList.of( 838 ModelDownloadCallback.ON_PROGRESS, 839 ModelDownloadCallback.ON_PROGRESS, 840 ModelDownloadCallback.ON_PROGRESS), 841 /* callbacks to be received by the client: */ ImmutableList.of( 842 ModelDownloadCallback.ON_PROGRESS, 843 ModelDownloadCallback.ON_PROGRESS, 844 ModelDownloadCallback.ON_PROGRESS)); 845 case PROGRESS_SUCCESS_PROGRESS: 846 return new ModelDownloadExecutionInfo( 847 /* callbacks to be invoked by the service: */ ImmutableList.of( 848 ModelDownloadCallback.ON_PROGRESS, 849 ModelDownloadCallback.ON_SUCCESS, 850 ModelDownloadCallback.ON_PROGRESS), 851 /* callbacks to be received by the client: */ ImmutableList.of( 852 ModelDownloadCallback.ON_PROGRESS, 853 ModelDownloadCallback.ON_SUCCESS)); 854 case SCHEDULED_ERROR: 855 return new ModelDownloadExecutionInfo( 856 /* callbacks to be invoked by the service: */ ImmutableList.of( 857 ModelDownloadCallback.ON_SCHEDULED, 858 ModelDownloadCallback.ON_ERROR), 859 /* callbacks to be received by the client: */ ImmutableList.of( 860 ModelDownloadCallback.ON_SCHEDULED)); 861 case ERROR_SCHEDULED: 862 return new ModelDownloadExecutionInfo( 863 /* callbacks to be invoked by the service: */ ImmutableList.of( 864 ModelDownloadCallback.ON_ERROR, 865 ModelDownloadCallback.ON_SCHEDULED), 866 /* callbacks to be received by the client: */ ImmutableList.of( 867 ModelDownloadCallback.ON_ERROR)); 868 default: 869 return new ModelDownloadExecutionInfo(ImmutableList.of(), ImmutableList.of()); 870 } 871 } 872 } 873 874 private static class ModelDownloadCallbackLogger implements ModelDownloadListener { 875 private List<ModelDownloadCallback> mCallbacks = new ArrayList<>(); 876 877 @Override onProgress(int completedPercent)878 public void onProgress(int completedPercent) { 879 mCallbacks.add(ModelDownloadCallback.ON_PROGRESS); 880 } 881 882 @Override onSuccess()883 public void onSuccess() { 884 mCallbacks.add(ModelDownloadCallback.ON_SUCCESS); 885 } 886 887 @Override onScheduled()888 public void onScheduled() { 889 mCallbacks.add(ModelDownloadCallback.ON_SCHEDULED); 890 } 891 892 @Override onError(int error)893 public void onError(int error) { 894 mCallbacks.add(ModelDownloadCallback.ON_ERROR); 895 } 896 } 897 } 898