1 /*
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 package android.platform.test.rule
17
18 import android.os.Build
19 import android.platform.test.rule.DeviceProduct.CF_PHONE
20 import android.platform.test.rule.DeviceProduct.CF_TABLET
21 import android.util.Log
22 import androidx.test.platform.app.InstrumentationRegistry
23 import java.lang.annotation.Inherited
24 import kotlin.annotation.AnnotationRetention.RUNTIME
25 import kotlin.annotation.AnnotationTarget.CLASS
26 import kotlin.annotation.AnnotationTarget.FUNCTION
27 import org.junit.AssumptionViolatedException
28 import org.junit.rules.TestRule
29 import org.junit.runner.Description
30 import org.junit.runners.model.Statement
31
32 /** Limits the test to run on devices specified by [allowed] */
33 @Retention(RUNTIME)
34 @Target(FUNCTION, CLASS)
35 @Inherited
36 annotation class AllowedDevices(vararg val allowed: DeviceProduct)
37
38 /** Does not run the test on device specified by [denied], */
39 @Retention(RUNTIME)
40 @Target(FUNCTION, CLASS)
41 @Inherited
42 annotation class DeniedDevices(vararg val denied: DeviceProduct)
43
44 /** Limits the test on default screenshot devices, or [allowed] devices if specified. */
45 @Retention(RUNTIME)
46 @Target(FUNCTION, CLASS)
47 @Inherited
48 annotation class ScreenshotTestDevices(vararg val allowed: DeviceProduct = [CF_PHONE, CF_TABLET])
49
50 /**
51 * Only runs the test on [flakyProducts] if this configuration is running flaky tests (see
52 * runningFlakyTests parameter on [LimitDevicesRule] constructor Runs it normally on all other
53 * devices.
54 */
55 @Retention(RUNTIME)
56 @Target(FUNCTION, CLASS)
57 @Inherited
58 annotation class FlakyDevices(vararg val flaky: DeviceProduct)
59
60 /**
61 * Ignore LimitDevicesRule constraints when [ignoreLimit] is true. Main use case is to allow local
62 * builds to bypass [LimitDevicesRule] and be able to run on any devices.
63 */
64 @Retention(RUNTIME)
65 @Target(FUNCTION, CLASS)
66 @Inherited
67 annotation class IgnoreLimit(val ignoreLimit: Boolean)
68
69 /**
70 * Limits a test to run specified devices.
71 *
72 * Devices are specified by [AllowedDevices], [DeniedDevices], [ScreenshotTestDevices], and
73 * [FlakyDevices] annotations. Only one annotation on class or one per test is supported. Values are
74 * matched against [thisDevice].
75 *
76 * To read the instrumentation args to determine whether to run on [FlakyDevices] (recommended), use
77 * [readParamsFromInstrumentation] to construct the rule
78 *
79 * NOTE: It's not encouraged to use this to filter if it's possible to filter based on other device
80 * characteristics. For example, to run a test only only on large screens or foldable,
81 * [DeviceTypeRule] is encouraged. This rule should **never** be used to avoid running a test on a
82 * tablet when the test is broken.
83 */
84 class LimitDevicesRule(
85 private val thisDevice: String = Build.PRODUCT,
86 private val runningFlakyTests: Boolean = false,
87 ) : TestRule {
88 val scanner = TestAnnotationScanner()
89
applynull90 override fun apply(base: Statement, description: Description): Statement {
91 val skipReason = skipReasonIfAny(description)
92 if (skipReason == null) {
93 return base
94 } else {
95 return makeAssumptionViolatedStatement(skipReason)
96 }
97 }
98
skipReasonIfAnynull99 fun skipReasonIfAny(description: Description): String? {
100 if (description.ignoreLimit()) {
101 return null
102 }
103
104 val limitDevicesAnnotations = description.limitDevicesAnnotation()
105 if (limitDevicesAnnotations.count() > 1) {
106 return "Only one LimitDeviceRule annotation is supported. Found $limitDevicesAnnotations"
107 }
108 val deniedDevices = description.deniedDevices()
109 if (thisDevice in deniedDevices) {
110 return "Skipping test as $thisDevice is in $deniedDevices"
111 }
112
113 val flakyDevices = description.flakyDevices()
114 if (thisDevice in flakyDevices) {
115 if (!runningFlakyTests) {
116 return "Skipping test as $thisDevice is flaky and this config excludes flakes"
117 }
118 }
119
120 val allowedDevices = description.allowedDevices()
121 if (allowedDevices.isEmpty() || thisDevice in allowedDevices) {
122 return null
123 }
124 return "Skipping test as $thisDevice in not in $allowedDevices"
125 }
126
allowedDevicesnull127 private fun Description.allowedDevices(): List<String> =
128 listOf(
129 getMostSpecificAnnotation<AllowedDevices>()?.allowed,
130 getMostSpecificAnnotation<ScreenshotTestDevices>()?.allowed,
131 )
132 .collectProducts()
133
134 private fun Description.deniedDevices(): List<String> =
135 listOf(getMostSpecificAnnotation<DeniedDevices>()?.denied).collectProducts()
136
137 private fun Description.flakyDevices(): List<String> =
138 listOf(getMostSpecificAnnotation<FlakyDevices>()?.flaky).collectProducts()
139
140 private fun Description.limitDevicesAnnotation(): Set<Annotation> =
141 listOfNotNull(
142 getMostSpecificAnnotation<AllowedDevices>(),
143 getMostSpecificAnnotation<DeniedDevices>(),
144 getMostSpecificAnnotation<ScreenshotTestDevices>(),
145 getMostSpecificAnnotation<FlakyDevices>(),
146 )
147 .toSet()
148
149 private fun Description.ignoreLimit(): Boolean =
150 getMostSpecificAnnotation<IgnoreLimit>()?.ignoreLimit == true
151
152 private inline fun <reified T : Annotation> Description.getMostSpecificAnnotation() =
153 scanner.find<T>(this)
154
155 private fun List<Array<out DeviceProduct>?>.collectProducts() =
156 filterNotNull().flatMap { it.toList() }.map { it.product }
157
158 companion object {
isRunningFlakyTestsnull159 private fun isRunningFlakyTests(): Boolean {
160 val args = InstrumentationRegistry.getArguments()
161 val isRunning = args.getString(RUNNING_FLAKY_TESTS_KEY, "false").toBoolean()
162 if (isRunning) {
163 Log.d(TAG, "Running on flaky devices, due to $RUNNING_FLAKY_TESTS_KEY param.")
164 }
165 return isRunning
166 }
167
168 @JvmOverloads
169 @JvmStatic
readParamsFromInstrumentationnull170 fun readParamsFromInstrumentation(thisDevice: String = Build.PRODUCT) =
171 LimitDevicesRule(thisDevice, isRunningFlakyTests())
172
173 private const val RUNNING_FLAKY_TESTS_KEY = "running-flaky-tests"
174 private const val TAG = "LimitDevicesRule"
175 }
176 }
177
178 enum class DeviceProduct(val product: String) {
179 CF_PHONE("cf_x86_64_phone"),
180 CF_TABLET("cf_x86_64_tablet"),
181 CF_FOLDABLE("cf_x86_64_foldable"),
182 CF_COMET("cf_x86_64_comet"),
183 CF_AUTO("cf_x86_64_auto"),
184 CF_ARM_PHONE("cf_arm64_only_phone"),
185 TANGORPRO("tangorpro"),
186 FELIX("felix"),
187 ROBOLECTRIC("robolectric"),
188 COMET("comet"),
189 CHEETAH("cheetah"),
190 }
191
makeAssumptionViolatedStatementnull192 private fun makeAssumptionViolatedStatement(errorMessage: String): Statement =
193 object : Statement() {
194 override fun evaluate() {
195 throw AssumptionViolatedException(errorMessage)
196 }
197 }
198