1version: 0.1 2 3# Phases are collection of commands that get executed on Device Farm. 4phases: 5 # The install phase includes commands that install dependencies that your tests use. 6 # Default dependencies for testing frameworks supported on Device Farm are already installed. 7 install: 8 commands: 9 10 # The pre-test phase includes commands that setup your test environment. 11 pre_test: 12 commands: 13 14 # The test phase includes commands that run your test suite execution. 15 test: 16 commands: 17 # By default, the following ADB command is used by Device Farm to run your Instrumentation test. 18 # Alternatively, you may specify a customized command. 19 # Please refer to Android's documentation (https://developer.android.com/studio/test/command-line#run-tests-with-adb) 20 # for more options on running instrumentation tests from the command line. 21 - echo "Start Instrumentation test" 22 - | 23 adb -s $DEVICEFARM_DEVICE_UDID shell "am instrument -r -w --no-window-animation \ 24 $DEVICEFARM_TEST_PACKAGE_NAME/$DEVICEFARM_TEST_PACKAGE_RUNNER 2>&1 || echo \": -1\"" | 25 tee $DEVICEFARM_LOG_DIR/instrument.log 26 27 # Parse the output of JUnit4 instrumentation tests, and check how many tests experienced each possible status code 28 # Note that this parsing is designed for the AndroidJUnitRunner, and thus different test runners may require customized 29 # parsing code specific to those runners specific test reporting output. 30 # For more information on instrumentation status codes, please see https://android.googlesource.com/platform/tools/base/+/refs/tags/android-mainline-12.0.0_r58/ddmlib/src/main/java/com/android/ddmlib/testrunner/InstrumentationResultParser.java#49 31 # as well as https://android.googlesource.com/platform/tools/base/+/refs/tags/android-mainline-12.0.0_r58/ddmlib/src/main/java/com/android/ddmlib/testrunner/InstrumentationResultParser.java#100 32 # Note that tests that give the "Ignored" (-3) and "Assumption Failure" (-4) status coded are treated as "passed" 33 # but you can change this behavior for your tests by modifying the "if" statement below. 34 - |- 35 DID_ANY_TESTS_START=$(grep "INSTRUMENTATION_STATUS_CODE: 1" $DEVICEFARM_LOG_DIR/instrument.log | wc -l); 36 TESTS_PASSED=$(grep "INSTRUMENTATION_STATUS_CODE: 0" $DEVICEFARM_LOG_DIR/instrument.log | wc -l); 37 TESTS_ERRORED=$(grep "INSTRUMENTATION_STATUS_CODE: -1" $DEVICEFARM_LOG_DIR/instrument.log | wc -l); 38 TESTS_FAILED=$(grep "INSTRUMENTATION_STATUS_CODE: -2" $DEVICEFARM_LOG_DIR/instrument.log | wc -l); 39 TESTS_IGNORED=$(grep "INSTRUMENTATION_STATUS_CODE: -3" $DEVICEFARM_LOG_DIR/instrument.log | wc -l); 40 TESTS_ASSUMPTION_FAILED=$(grep "INSTRUMENTATION_STATUS_CODE: -4" $DEVICEFARM_LOG_DIR/instrument.log | wc -l); 41 TESTS_PROCESSES_CRASHED=$(grep "INSTRUMENTATION_RESULT: shortMsg=Process crashed." $DEVICEFARM_LOG_DIR/instrument.log | wc -l); 42 43 - >- 44 echo "\n\nFrom your instrumentation output, it appears \n 45 $TESTS_PASSED tests PASSED \n 46 $TESTS_ASSUMPTION_FAILED tests were SKIPPED \n 47 $TESTS_FAILED tests FAILED \n 48 $TESTS_ERRORED tests ERRORED \n 49 $TESTS_IGNORED tests were IGNORED"; 50 51 echo "These counts are calculated directly from the raw instrumentation output, thus any 52 redundant lines of output (e.g. a test case that was retried multiple times) could cause 53 the numbers to not align exactly with your expected number of test cases."; 54 55 - | 56 if [ $DID_ANY_TESTS_START -eq 0 ]; 57 then 58 echo "Marking the test suite as failed because no tests started!"; 59 false; 60 elif [ $TESTS_FAILED -ne 0 ]; 61 then 62 echo "Marking the test suite as failed because $TESTS_FAILED tests failed!"; 63 false; 64 elif [ $TESTS_ERRORED -ne 0 ]; 65 then 66 echo "Marking the test suite as failed because $TESTS_ERRORED tests errored!"; 67 false; 68 elif [ $TESTS_PROCESSES_CRASHED -ne 0 ]; 69 then 70 echo "Marking the test suite as failed because the application-under-test crashed during the test!"; 71 false; 72 else 73 echo "Passed with $TESTS_PASSED tests passed and $TESTS_ASSUMPTION_FAILED tests skipped!"; 74 fi; 75 76 # # To run your tests using "Android Test Orchestrator", you can use the following commands instead. 77 # # Note: you will need to include the orchestrator APK and test services APK as auxiliary apps with your ScheduleRun request. 78 # # For more information, please see the Android Test Orchestrator documentation: https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#use-android 79 # - | 80 # adb -s $DEVICEFARM_DEVICE_UDID shell "CLASSPATH=\$(pm path androidx.test.services) app_process / \ 81 # androidx.test.services.shellexecutor.ShellMain am instrument -w --no-window-animation -e clearPackageData true \ 82 # -e targetInstrumentation $DEVICEFARM_TEST_PACKAGE_NAME/$DEVICEFARM_TEST_PACKAGE_RUNNER \ 83 # androidx.test.orchestrator/.AndroidTestOrchestrator || echo \"FAILURES...\"" 2>&1 | tee $DEVICEFARM_LOG_DIR/instrument.log 84 # 85 # # If Android Test Orchestrator produces any reports on the device, they will be pulled 86 # # into your customer artifacts directory using the following commands 87 # - | 88 # adb -s $DEVICEFARM_DEVICE_UDID pull /sdcard/odo >/dev/null && 89 # mv odo/* $DEVICEFARM_LOG_DIR || true 90 # 91 # # The below command is used to detect if any of the Orchestrator tests have failed 92 # - | 93 # if [ $(cat $DEVICEFARM_LOG_DIR/instrument.log | egrep "^(FAILURES...|INSTRUMENTATION_RESULT: shortMsg=Process crashed.)$" | wc -l) -ge 1 ]; 94 # then 95 # echo "Marking the test suite as failed because Android Orchestrator found that some of the tests failed"; 96 # false; 97 # fi; 98 99 # The post test phase includes are commands that are run after your tests are executed. 100 post_test: 101 commands: 102 103# The artifacts phase lets you specify the location where your tests logs, device logs will be stored. 104# And also let you specify the location of your test logs and artifacts which you want to be collected by Device Farm. 105# These logs and artifacts will be available through ListArtifacts API in Device Farm. 106artifacts: 107 # By default, Device Farm will collect your artifacts from following directories 108 - $DEVICEFARM_LOG_DIR