1 /*
<lambda>null2  * Copyright (C) 2024 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.tools.flicker
18 
19 import android.app.Instrumentation
20 import android.device.collectors.BaseMetricListener
21 import android.device.collectors.DataRecord
22 import android.tools.FLICKER_TAG
23 import android.tools.Scenario
24 import android.tools.ScenarioBuilder
25 import android.tools.flicker.assertions.AssertionResult
26 import android.tools.flicker.config.FlickerServiceConfig
27 import android.tools.flicker.config.ScenarioId
28 import android.tools.io.Reader
29 import android.tools.io.RunStatus
30 import android.tools.traces.getDefaultFlickerOutputDir
31 import android.util.Log
32 import androidx.test.platform.app.InstrumentationRegistry
33 import com.android.internal.annotations.VisibleForTesting
34 import java.io.File
35 import org.junit.runner.Description
36 import org.junit.runner.Result
37 import org.junit.runner.notification.Failure
38 
39 /**
40  * Collects all the Flicker Service's metrics which are then uploaded for analysis and monitoring to
41  * the CrystalBall database.
42  */
43 class FlickerServiceResultsCollector
44 @JvmOverloads
45 constructor(
46     private val tracesCollector: TracesCollector,
47     private val flickerService: FlickerService =
48         FlickerService(FlickerConfig().use(FlickerServiceConfig.DEFAULT)),
49     instrumentation: Instrumentation = InstrumentationRegistry.getInstrumentation(),
50     private val collectMetricsPerTest: Boolean = true,
51     private val reportOnlyForPassingTests: Boolean = true,
52 ) : BaseMetricListener(), IFlickerServiceResultsCollector {
53     private var hasFailedTest = false
54     private var testSkipped = false
55 
56     private val _executionErrors = mutableListOf<Throwable>()
57     override val executionErrors
58         get() = _executionErrors
59 
60     @VisibleForTesting val assertionResults = mutableListOf<AssertionResult>()
61 
62     @VisibleForTesting
63     val assertionResultsByTest = mutableMapOf<Description, Collection<AssertionResult>>()
64 
65     @VisibleForTesting
66     val detectedScenariosByTest = mutableMapOf<Description, Collection<ScenarioId>>()
67 
68     private var testRunScenario: Scenario? = null
69     private var testScenario: Scenario? = null
70 
71     init {
72         setInstrumentation(instrumentation)
73     }
74 
75     override fun onTestRunStart(runData: DataRecord, description: Description) {
76         errorReportingBlock {
77             tracesCollector.cleanup() // Cleanup any trace archives from previous runs
78 
79             Log.i(LOG_TAG, "onTestRunStart :: collectMetricsPerTest = $collectMetricsPerTest")
80             if (!collectMetricsPerTest) {
81                 hasFailedTest = false
82                 val scenario =
83                     ScenarioBuilder()
84                         .forClass(description.testClass.canonicalName)
85                         .withDescriptionOverride("")
86                         .build()
87                 testRunScenario = scenario
88                 tracesCollector.start(scenario)
89             }
90         }
91     }
92 
93     override fun onTestStart(testData: DataRecord, description: Description) {
94         errorReportingBlock {
95             Log.i(LOG_TAG, "onTestStart :: collectMetricsPerTest = $collectMetricsPerTest")
96             if (collectMetricsPerTest) {
97                 hasFailedTest = false
98                 val scenario =
99                     ScenarioBuilder()
100                         .forClass(
101                             "${description.testClass.canonicalName}#${description.methodName}"
102                         )
103                         .withDescriptionOverride("")
104                         .build()
105                 testScenario = scenario
106                 tracesCollector.start(scenario)
107             }
108             testSkipped = false
109         }
110     }
111 
112     override fun onTestFail(testData: DataRecord, description: Description, failure: Failure) {
113         errorReportingBlock {
114             Log.i(LOG_TAG, "onTestFail")
115             hasFailedTest = true
116         }
117     }
118 
119     override fun testAssumptionFailure(failure: Failure?) {
120         errorReportingBlock {
121             Log.i(LOG_TAG, "testAssumptionFailure")
122             testSkipped = true
123         }
124     }
125 
126     override fun testSkipped(description: Description) {
127         errorReportingBlock {
128             Log.i(LOG_TAG, "testSkipped")
129             testSkipped = true
130         }
131     }
132 
133     override fun onTestEnd(testData: DataRecord, description: Description) {
134         Log.i(LOG_TAG, "onTestEnd :: collectMetricsPerTest = $collectMetricsPerTest")
135         if (collectMetricsPerTest) {
136             val results = errorReportingBlock {
137                 Log.i(LOG_TAG, "Stopping trace collection")
138                 val reader = tracesCollector.stop()
139                 Log.i(LOG_TAG, "Stopped trace collection")
140 
141                 if (reportOnlyForPassingTests && hasFailedTest) {
142                     return@errorReportingBlock null
143                 }
144 
145                 if (testSkipped) {
146                     return@errorReportingBlock null
147                 }
148 
149                 return@errorReportingBlock collectFlickerMetrics(testData, reader, description)
150             }
151 
152             reportFlickerServiceStatus(
153                 testData,
154                 results,
155                 testScenario ?: error("Test scenario should not be null"),
156                 testData,
157             )
158         }
159     }
160 
161     override fun onTestRunEnd(runData: DataRecord, result: Result) {
162         Log.i(LOG_TAG, "onTestRunEnd :: collectMetricsPerTest = $collectMetricsPerTest")
163         if (!collectMetricsPerTest) {
164             val results = errorReportingBlock {
165                 Log.i(LOG_TAG, "Stopping trace collection")
166                 val reader = tracesCollector.stop()
167                 Log.i(LOG_TAG, "Stopped trace collection")
168 
169                 if (reportOnlyForPassingTests && hasFailedTest) {
170                     return@errorReportingBlock null
171                 }
172 
173                 return@errorReportingBlock collectFlickerMetrics(runData, reader)
174             }
175 
176             reportFlickerServiceStatus(
177                 runData,
178                 results,
179                 testRunScenario ?: error("Test run scenario should not be null"),
180                 runData,
181             )
182         }
183     }
184 
185     private fun collectFlickerMetrics(
186         dataRecord: DataRecord,
187         reader: Reader,
188         description: Description? = null,
189     ): Collection<AssertionResult>? {
190         return errorReportingBlock {
191             return@errorReportingBlock try {
192                 Log.i(LOG_TAG, "Processing traces")
193                 val scenarios = flickerService.detectScenarios(reader)
194                 val results = scenarios.flatMap { it.generateAssertions() }.map { it.execute() }
195                 reader.artifact.updateStatus(RunStatus.RUN_EXECUTED)
196                 Log.i(LOG_TAG, "Got ${results.size} results")
197                 assertionResults.addAll(results)
198                 if (description != null) {
199                     require(assertionResultsByTest[description] == null) {
200                         "Test description ($description) already contains flicker assertion results"
201                     }
202                     require(detectedScenariosByTest[description] == null) {
203                         "Test description ($description) already contains detected scenarios"
204                     }
205                     assertionResultsByTest[description] = results
206                     detectedScenariosByTest[description] = scenarios.map { it.type }.distinct()
207                 }
208                 if (results.any { it.status == AssertionResult.Status.FAIL }) {
209                     reader.artifact.updateStatus(RunStatus.ASSERTION_FAILED)
210                 } else {
211                     reader.artifact.updateStatus(RunStatus.ASSERTION_SUCCESS)
212                 }
213 
214                 Log.v(LOG_TAG, "Adding metric $FLICKER_ASSERTIONS_COUNT_KEY = ${results.size}")
215                 dataRecord.addStringMetric(FLICKER_ASSERTIONS_COUNT_KEY, "${results.size}")
216 
217                 val aggregatedResults = processFlickerResults(results)
218                 collectMetrics(dataRecord, aggregatedResults)
219 
220                 results
221             } finally {
222                 Log.v(LOG_TAG, "Adding metric $WINSCOPE_FILE_PATH_KEY = ${reader.artifactPath}")
223                 dataRecord.addStringMetric(WINSCOPE_FILE_PATH_KEY, reader.artifactPath)
224             }
225         }
226     }
227 
228     private fun processFlickerResults(
229         results: Collection<AssertionResult>
230     ): Map<String, AggregatedFlickerResult> {
231         val aggregatedResults = mutableMapOf<String, AggregatedFlickerResult>()
232         for (result in results) {
233             val key = getKeyForAssertionResult(result)
234             if (!aggregatedResults.containsKey(key)) {
235                 aggregatedResults[key] = AggregatedFlickerResult()
236             }
237             aggregatedResults[key]!!.addResult(result)
238         }
239         return aggregatedResults
240     }
241 
242     private fun collectMetrics(
243         data: DataRecord,
244         aggregatedResults: Map<String, AggregatedFlickerResult>,
245     ) {
246         val it = aggregatedResults.entries.iterator()
247 
248         while (it.hasNext()) {
249             val (key, aggregatedResult) = it.next()
250             aggregatedResult.results.forEachIndexed { index, result ->
251                 if (result.status == AssertionResult.Status.ASSUMPTION_VIOLATION) {
252                     // skip
253                     return@forEachIndexed
254                 }
255 
256                 val resultStatus = if (result.status == AssertionResult.Status.PASS) 0 else 1
257                 Log.v(LOG_TAG, "Adding metric ${key}_$index = $resultStatus")
258                 data.addStringMetric("${key}_$index", "$resultStatus")
259             }
260         }
261     }
262 
263     private fun <T> errorReportingBlock(function: () -> T): T? {
264         return try {
265             function()
266         } catch (e: Throwable) {
267             Log.e(FLICKER_TAG, "Error executing in FlickerServiceResultsCollector", e)
268             _executionErrors.add(e)
269             null
270         }
271     }
272 
273     override fun resultsForTest(description: Description): Collection<AssertionResult> {
274         val resultsForTest = assertionResultsByTest[description]
275         requireNotNull(resultsForTest) { "No results set for test $description" }
276         return resultsForTest
277     }
278 
279     override fun detectedScenariosForTest(description: Description): Collection<ScenarioId> {
280         val scenariosForTest = detectedScenariosByTest[description]
281         requireNotNull(scenariosForTest) { "No detected scenarios set for test $description" }
282         return scenariosForTest
283     }
284 
285     private fun reportFlickerServiceStatus(
286         record: DataRecord,
287         results: Collection<AssertionResult>?,
288         scenario: Scenario,
289         dataRecord: DataRecord,
290     ) {
291         val status = if (executionErrors.isEmpty()) OK_STATUS_CODE else EXECUTION_ERROR_STATUS_CODE
292         record.addStringMetric(FAAS_STATUS_KEY, status.toString())
293 
294         val maxLineLength = 120
295         val statusFile = createFlickerServiceStatusFile(scenario)
296         val flickerResultString = buildString {
297             appendLine(
298                 "FAAS_STATUS: ${if (executionErrors.isEmpty()) "OK" else "EXECUTION_ERROR"}\n"
299             )
300 
301             appendLine("EXECUTION ERRORS:\n")
302             if (executionErrors.isEmpty()) {
303                 appendLine("None".prependIndent())
304             } else {
305                 appendLine(
306                     executionErrors
307                         .joinToString("\n\n${"-".repeat(maxLineLength / 2)}\n\n") {
308                             it.stackTraceToString()
309                         }
310                         .prependIndent()
311                 )
312             }
313 
314             appendLine()
315             appendLine("FLICKER RESULTS:\n")
316             val executionErrorsString =
317                 buildString {
318                         results?.forEach {
319                             append("${it.name} (${it.stabilityGroup}) :: ")
320                             append("${it.status}\n")
321                             appendLine(
322                                 it.assertionErrors
323                                     .joinToString("\n${"-".repeat(maxLineLength / 2)}\n\n") { error
324                                         ->
325                                         error.message
326                                     }
327                                     .prependIndent()
328                             )
329                         }
330                     }
331                     .prependIndent()
332             appendLine(executionErrorsString)
333         }
334 
335         statusFile.writeText(flickerResultString.replace(Regex("(.{$maxLineLength})"), "$1\n"))
336 
337         Log.v(LOG_TAG, "Adding metric $FAAS_RESULTS_FILE_PATH_KEY = ${statusFile.absolutePath}")
338         dataRecord.addStringMetric(FAAS_RESULTS_FILE_PATH_KEY, statusFile.absolutePath)
339     }
340 
341     private fun createFlickerServiceStatusFile(scenario: Scenario): File {
342         val fileName = "FAAS_RESULTS_$scenario"
343 
344         val outputDir = getDefaultFlickerOutputDir()
345         // Ensure output directory exists
346         outputDir.mkdirs()
347         return outputDir.resolve(fileName)
348     }
349 
350     companion object {
351         // Unique prefix to add to all FaaS metrics to identify them
352         const val FAAS_METRICS_PREFIX = "FAAS"
353         private const val LOG_TAG = "$FLICKER_TAG-Collector"
354         const val FAAS_STATUS_KEY = "${FAAS_METRICS_PREFIX}_STATUS"
355         const val WINSCOPE_FILE_PATH_KEY = "winscope_file_path"
356         const val FAAS_RESULTS_FILE_PATH_KEY = "faas_results_file_path"
357         const val FLICKER_ASSERTIONS_COUNT_KEY = "flicker_assertions_count"
358         const val OK_STATUS_CODE = 0
359         const val EXECUTION_ERROR_STATUS_CODE = 1
360 
361         fun getKeyForAssertionResult(result: AssertionResult): String {
362             return "$FAAS_METRICS_PREFIX::${result.name}"
363         }
364 
365         class AggregatedFlickerResult {
366             val results = mutableListOf<AssertionResult>()
367             var failures = 0
368             var passes = 0
369             var assumptionViolations = 0
370             val errors = mutableListOf<String>()
371             var invocationGroup: AssertionInvocationGroup? = null
372 
373             fun addResult(result: AssertionResult) {
374                 results.add(result)
375 
376                 when (result.status) {
377                     AssertionResult.Status.PASS -> passes++
378                     AssertionResult.Status.FAIL -> {
379                         failures++
380                         result.assertionErrors.forEach { errors.add(it.message) }
381                     }
382                     AssertionResult.Status.ASSUMPTION_VIOLATION -> assumptionViolations++
383                 }
384 
385                 if (invocationGroup == null) {
386                     invocationGroup = result.stabilityGroup
387                 }
388 
389                 if (invocationGroup != result.stabilityGroup) {
390                     error("Unexpected assertion group mismatch")
391                 }
392             }
393         }
394     }
395 }
396