1 /* 2 * Copyright (C) 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.compat.cts; 18 19 import static com.android.tradefed.targetprep.UserHelper.getRunTestsAsUser; 20 21 import static com.google.common.truth.Truth.assertThat; 22 import static com.google.common.truth.Truth.assertWithMessage; 23 24 import android.cts.statsdatom.lib.ReportUtils; 25 26 import com.android.compatibility.common.tradefed.build.CompatibilityBuildHelper; 27 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; 28 import com.android.ddmlib.testrunner.TestResult.TestStatus; 29 import com.android.internal.os.StatsdConfigProto; 30 import com.android.os.AtomsProto; 31 import com.android.os.AtomsProto.Atom; 32 import com.android.os.StatsLog; 33 import com.android.os.StatsLog.ConfigMetricsReportList; 34 import com.android.tradefed.build.IBuildInfo; 35 import com.android.tradefed.device.CollectingByteOutputReceiver; 36 import com.android.tradefed.device.CollectingOutputReceiver; 37 import com.android.tradefed.device.DeviceNotAvailableException; 38 import com.android.tradefed.device.ITestDevice; 39 import com.android.tradefed.invoker.TestInformation; 40 import com.android.tradefed.log.LogUtil.CLog; 41 import com.android.tradefed.result.CollectingTestListener; 42 import com.android.tradefed.result.ITestInvocationListener; 43 import com.android.tradefed.result.TestDescription; 44 import com.android.tradefed.result.TestResult; 45 import com.android.tradefed.result.TestRunResult; 46 import com.android.tradefed.testtype.DeviceTestCase; 47 import com.android.tradefed.testtype.IBuildReceiver; 48 49 import com.google.common.io.Files; 50 import com.google.protobuf.InvalidProtocolBufferException; 51 52 import java.io.File; 53 import java.io.FileNotFoundException; 54 import java.io.IOException; 55 import java.util.Arrays; 56 import java.util.List; 57 import java.util.Map; 58 import java.util.Objects; 59 import java.util.Set; 60 import java.util.stream.Collectors; 61 62 import javax.annotation.Nonnull; 63 64 // Shamelessly plagiarised from incident's ProtoDumpTestCase and statsd's BaseTestCase family 65 public class CompatChangeGatingTestCase extends DeviceTestCase implements IBuildReceiver { 66 protected IBuildInfo mCtsBuild; 67 68 private static final String UPDATE_CONFIG_CMD = "cat %s | cmd stats config update %d"; 69 private static final String DUMP_REPORT_CMD = 70 "cmd stats dump-report %d --include_current_bucket --proto"; 71 private static final String REMOVE_CONFIG_CMD = "cmd stats config remove %d"; 72 73 private static final String TEST_RUNNER = "androidx.test.runner.AndroidJUnitRunner"; 74 75 private int mTestRunningUserId; 76 77 @Override run(TestInformation testInfo, ITestInvocationListener listener)78 public void run(TestInformation testInfo, ITestInvocationListener listener) 79 throws DeviceNotAvailableException { 80 // The test runs as the current user in most cases. For secondary_user_on_secondary_display 81 // case, we set mTestRunningUserId from RUN_TEST_AS_USER. 82 mTestRunningUserId = getDevice().getCurrentUser(); 83 if (getDevice().isVisibleBackgroundUsersSupported()) { 84 mTestRunningUserId = getRunTestsAsUser(testInfo); 85 } 86 super.run(testInfo, listener); 87 } 88 89 @Override setUp()90 protected void setUp() throws Exception { 91 super.setUp(); 92 assertThat(mCtsBuild).isNotNull(); 93 } 94 95 @Override setBuild(IBuildInfo buildInfo)96 public void setBuild(IBuildInfo buildInfo) { 97 mCtsBuild = buildInfo; 98 } 99 100 /** 101 * Install a device side test package. 102 * 103 * @param appFileName Apk file name, such as "CtsNetStatsApp.apk". 104 * @param grantPermissions whether to give runtime permissions. 105 */ installPackage(String appFileName, boolean grantPermissions)106 protected void installPackage(String appFileName, boolean grantPermissions) 107 throws FileNotFoundException, DeviceNotAvailableException { 108 CLog.d("Installing app " + appFileName); 109 CompatibilityBuildHelper buildHelper = new CompatibilityBuildHelper(mCtsBuild); 110 final String result = getDevice().installPackage(buildHelper.getTestFile(appFileName), true, 111 grantPermissions, "-t"); 112 assertWithMessage("Failed to install %s: %s", appFileName, result).that(result).isNull(); 113 } 114 115 /** 116 * Uninstall a device side test package. 117 * 118 * @param appFileName Apk file name, such as "CtsNetStatsApp.apk". 119 * @param shouldSucceed Whether to assert on failure. 120 */ uninstallPackage(String packageName, boolean shouldSucceed)121 protected void uninstallPackage(String packageName, boolean shouldSucceed) 122 throws DeviceNotAvailableException { 123 final String result = getDevice().uninstallPackage(packageName); 124 if (shouldSucceed) { 125 assertWithMessage("uninstallPackage(%s) failed: %s", packageName, result) 126 .that(result).isNull(); 127 } 128 } 129 130 /** 131 * Run a device side compat test. 132 * 133 * @param pkgName Test package name, such as 134 * "com.android.server.cts.netstats". 135 * @param testClassName Test class name; either a fully qualified name, or "." 136 * + a class name. 137 * @param testMethodName Test method name. 138 * @param enabledChanges Set of compat changes to enable. 139 * @param disabledChanges Set of compat changes to disable. 140 */ runDeviceCompatTest(@onnull String pkgName, @Nonnull String testClassName, @Nonnull String testMethodName, Set<Long> enabledChanges, Set<Long> disabledChanges)141 protected void runDeviceCompatTest(@Nonnull String pkgName, @Nonnull String testClassName, 142 @Nonnull String testMethodName, 143 Set<Long> enabledChanges, Set<Long> disabledChanges) 144 throws DeviceNotAvailableException { 145 runDeviceCompatTestReported(pkgName, testClassName, testMethodName, enabledChanges, 146 disabledChanges, enabledChanges, disabledChanges); 147 } 148 149 /** 150 * Run a device side compat test where not all changes are reported through statsd. 151 * 152 * @param pkgName Test package name, such as 153 * "com.android.server.cts.netstats". 154 * @param testClassName Test class name; either a fully qualified name, or "." 155 * + a class name. 156 * @param testMethodName Test method name. 157 * @param enabledChanges Set of compat changes to enable. 158 * @param disabledChanges Set of compat changes to disable. 159 * @param reportedEnabledChanges Expected enabled changes in statsd report. 160 * @param reportedDisabledChanges Expected disabled changes in statsd report. 161 */ runDeviceCompatTestReported(@onnull String pkgName, @Nonnull String testClassName, @Nonnull String testMethodName, Set<Long> enabledChanges, Set<Long> disabledChanges, Set<Long> reportedEnabledChanges, Set<Long> reportedDisabledChanges)162 protected void runDeviceCompatTestReported(@Nonnull String pkgName, @Nonnull String testClassName, 163 @Nonnull String testMethodName, 164 Set<Long> enabledChanges, Set<Long> disabledChanges, 165 Set<Long> reportedEnabledChanges, Set<Long> reportedDisabledChanges) 166 throws DeviceNotAvailableException { 167 168 // Set compat overrides 169 setCompatConfig(enabledChanges, disabledChanges, pkgName); 170 // Send statsd config 171 final long configId = getClass().getCanonicalName().hashCode(); 172 createAndUploadStatsdConfig(configId, pkgName); 173 174 try { 175 // Run device-side test 176 if (testClassName.startsWith(".")) { 177 testClassName = pkgName + testClassName; 178 } 179 RemoteAndroidTestRunner testRunner = new RemoteAndroidTestRunner(pkgName, TEST_RUNNER, 180 getDevice().getIDevice()); 181 testRunner.setMethodName(testClassName, testMethodName); 182 CollectingTestListener listener = new CollectingTestListener(); 183 assertThat(getDevice().runInstrumentationTestsAsUser( 184 testRunner, mTestRunningUserId, listener)).isTrue(); 185 186 // Check that device side test occurred as expected 187 final TestRunResult result = listener.getCurrentRunResults(); 188 assertWithMessage("Failed to successfully run device tests for %s: %s", 189 result.getName(), result.getRunFailureMessage()) 190 .that(result.isRunFailure()).isFalse(); 191 assertWithMessage("Should run only exactly one test method!") 192 .that(result.getNumTests()).isEqualTo(1); 193 if (result.hasFailedTests()) { 194 // build a meaningful error message 195 StringBuilder errorBuilder = new StringBuilder("On-device test failed:\n"); 196 for (Map.Entry<TestDescription, TestResult> resultEntry : 197 result.getTestResults().entrySet()) { 198 if (!resultEntry.getValue().getStatus().equals(TestStatus.PASSED)) { 199 errorBuilder.append(resultEntry.getKey().toString()); 200 errorBuilder.append(":\n"); 201 errorBuilder.append(resultEntry.getValue().getStackTrace()); 202 } 203 } 204 throw new AssertionError(errorBuilder.toString()); 205 } 206 207 } finally { 208 // Cleanup compat overrides 209 resetCompatConfig(pkgName, enabledChanges, disabledChanges); 210 // Validate statsd report 211 validatePostRunStatsdReport(configId, pkgName, reportedEnabledChanges, 212 reportedDisabledChanges); 213 } 214 215 } 216 217 /** 218 * Gets the statsd report. Note that this also deletes that report from statsd. 219 */ getReportList(long configId)220 private ConfigMetricsReportList getReportList(long configId) 221 throws DeviceNotAvailableException { 222 try { 223 final CollectingByteOutputReceiver receiver = new CollectingByteOutputReceiver(); 224 getDevice().executeShellCommand(String.format(DUMP_REPORT_CMD, configId), receiver); 225 return ConfigMetricsReportList.parser() 226 .parseFrom(receiver.getOutput()); 227 } catch (InvalidProtocolBufferException e) { 228 throw new IllegalStateException("Failed to fetch and parse the statsd output report.", 229 e); 230 } 231 } 232 getEventMetricDataList( ConfigMetricsReportList reportList)233 private static List<StatsLog.EventMetricData> getEventMetricDataList( 234 ConfigMetricsReportList reportList) { 235 try { 236 return ReportUtils.getEventMetricDataList(reportList); 237 } catch (Exception e) { 238 throw new IllegalStateException("Failed to parse ConfigMetrisReportList", e); 239 } 240 } 241 242 /** 243 * Creates and uploads a statsd config that matches the AppCompatibilityChangeReported atom 244 * logged by a given package name. 245 * 246 * @param configId A unique config id. 247 * @param pkgName The package name of the app that is expected to report the atom. It will be 248 * the only allowed log source. 249 */ createAndUploadStatsdConfig(long configId, String pkgName)250 protected void createAndUploadStatsdConfig(long configId, String pkgName) 251 throws DeviceNotAvailableException { 252 final String atomName = "Atom" + System.nanoTime(); 253 final String eventName = "Event" + System.nanoTime(); 254 final ITestDevice device = getDevice(); 255 256 StatsdConfigProto.StatsdConfig.Builder configBuilder = 257 StatsdConfigProto.StatsdConfig.newBuilder() 258 .setId(configId) 259 .addAllowedLogSource(pkgName) 260 .addWhitelistedAtomIds(Atom.APP_COMPATIBILITY_CHANGE_REPORTED_FIELD_NUMBER); 261 StatsdConfigProto.SimpleAtomMatcher.Builder simpleAtomMatcherBuilder = 262 StatsdConfigProto.SimpleAtomMatcher 263 .newBuilder().setAtomId( 264 Atom.APP_COMPATIBILITY_CHANGE_REPORTED_FIELD_NUMBER); 265 configBuilder.addAtomMatcher( 266 StatsdConfigProto.AtomMatcher.newBuilder() 267 .setId(atomName.hashCode()) 268 .setSimpleAtomMatcher(simpleAtomMatcherBuilder)); 269 configBuilder.addEventMetric( 270 StatsdConfigProto.EventMetric.newBuilder() 271 .setId(eventName.hashCode()) 272 .setWhat(atomName.hashCode())); 273 StatsdConfigProto.StatsdConfig config = configBuilder.build(); 274 try { 275 File configFile = File.createTempFile("statsdconfig", ".config"); 276 configFile.deleteOnExit(); 277 Files.write(config.toByteArray(), configFile); 278 String remotePath = "/data/local/tmp/" + configFile.getName(); 279 device.pushFile(configFile, remotePath); 280 device.executeShellCommand(String.format(UPDATE_CONFIG_CMD, remotePath, configId)); 281 device.executeShellCommand("rm " + remotePath); 282 } catch (IOException e) { 283 throw new RuntimeException("IO error when writing to temp file.", e); 284 } 285 // Purge data 286 getReportList(configId); 287 } 288 289 /** 290 * Gets the uid of the test app. 291 */ getUid(@onnull String packageName)292 protected int getUid(@Nonnull String packageName) throws DeviceNotAvailableException { 293 String uidLines = getDevice() 294 .executeShellCommand( 295 "cmd package list packages -U --user " + mTestRunningUserId + " " 296 + packageName); 297 for (String uidLine : uidLines.split("\n")) { 298 if (uidLine.startsWith("package:" + packageName + " uid:")) { 299 String[] uidLineParts = uidLine.split(":"); 300 // 3rd entry is package uid 301 assertThat(uidLineParts.length).isGreaterThan(2); 302 int uid = Integer.parseInt(uidLineParts[2].trim()); 303 assertThat(uid).isGreaterThan(10000); 304 return uid; 305 } 306 } 307 throw new IllegalStateException("Failed to find the test app on the device"); 308 } 309 310 /** 311 * Set the compat config using adb. 312 * 313 * @param enabledChanges Changes to be enabled. 314 * @param disabledChanges Changes to be disabled. 315 * @param packageName Package name for the app whose config is being changed. 316 */ setCompatConfig(Set<Long> enabledChanges, Set<Long> disabledChanges, @Nonnull String packageName)317 protected void setCompatConfig(Set<Long> enabledChanges, Set<Long> disabledChanges, 318 @Nonnull String packageName) throws DeviceNotAvailableException { 319 for (Long enabledChange : enabledChanges) { 320 runCommand("am compat enable " + enabledChange + " " + packageName); 321 } 322 for (Long disabledChange : disabledChanges) { 323 runCommand("am compat disable " + disabledChange + " " + packageName); 324 } 325 } 326 327 /** 328 * Reset changes to default for a package. 329 */ resetCompatChanges(Set<Long> changes, @Nonnull String packageName)330 protected void resetCompatChanges(Set<Long> changes, @Nonnull String packageName) 331 throws DeviceNotAvailableException { 332 for (Long change : changes) { 333 runCommand("am compat reset " + change + " " + packageName); 334 } 335 } 336 337 /** 338 * Remove statsd config for a given id. 339 */ removeStatsdConfig(long configId)340 private void removeStatsdConfig(long configId) throws DeviceNotAvailableException { 341 getDevice().executeShellCommand( 342 String.join(" ", REMOVE_CONFIG_CMD, String.valueOf(configId))); 343 } 344 345 /** 346 * Get the compat changes that were logged. 347 */ getReportedChanges(long configId, String pkgName)348 private Map<Long, Boolean> getReportedChanges(long configId, String pkgName) 349 throws DeviceNotAvailableException { 350 final int packageUid = getUid(pkgName); 351 return getEventMetricDataList(getReportList(configId)).stream() 352 .filter(eventMetricData -> eventMetricData.hasAtom()) 353 .map(eventMetricData -> eventMetricData.getAtom()) 354 .map(atom -> atom.getAppCompatibilityChangeReported()) 355 .filter(atom -> atom != null && atom.getUid() == packageUid) // Should be redundant 356 .collect(Collectors.toMap( 357 atom -> atom.getChangeId(), // Key 358 atom -> atom.getState() == // Value 359 AtomsProto.AppCompatibilityChangeReported.State.ENABLED, 360 (a, b) -> { 361 if (!Objects.equals(a, b)) { 362 throw new IllegalStateException( 363 "inconsistent compatibility states"); 364 } 365 return a; 366 })); 367 } 368 369 /** 370 * Cleanup the altered change ids under test. 371 * 372 * @param pkgName Package name of the app under test. 373 * @param enabledChanges Set of changes that were enabled during the test and need to be 374 * reset to the default value. 375 * @param disabledChanges Set of changes that were disabled during the test and need to 376 * be reset to the default value. 377 */ 378 protected void resetCompatConfig( String pkgName, Set<Long> enabledChanges, 379 Set<Long> disabledChanges) throws DeviceNotAvailableException { 380 // Clear overrides. 381 resetCompatChanges(enabledChanges, pkgName); 382 resetCompatChanges(disabledChanges, pkgName); 383 } 384 385 /** 386 * Validate that all overridden changes were logged while running the test. 387 * 388 * @param configId The unique config id used to track change id queries. 389 * @param pkgName Package name of the app under test. 390 * @param loggedEnabledChanges Changes expected to be logged as enabled during the test. 391 * @param loggedDisabledChanges Changes expected to be logged as disabled during the test. 392 */ 393 protected void validatePostRunStatsdReport(long configId, String pkgName, 394 Set<Long> loggedEnabledChanges, Set<Long> loggedDisabledChanges) 395 throws DeviceNotAvailableException { 396 // Clear statsd report data and remove config 397 Map<Long, Boolean> reportedChanges = getReportedChanges(configId, pkgName); 398 removeStatsdConfig(configId); 399 400 for (Long enabledChange : loggedEnabledChanges) { 401 assertThat(reportedChanges) 402 .containsEntry(enabledChange, true); 403 } 404 for (Long disabledChange : loggedDisabledChanges) { 405 assertThat(reportedChanges) 406 .containsEntry(disabledChange, false); 407 } 408 } 409 410 /** 411 * Execute the given command, and returns the output. 412 */ 413 protected String runCommand(String command) throws DeviceNotAvailableException { 414 final CollectingOutputReceiver receiver = new CollectingOutputReceiver(); 415 getDevice().executeShellCommand(command, receiver); 416 return receiver.getOutput(); 417 } 418 419 /** 420 * Get the on device compat config. 421 */ 422 protected List<Change> getOnDeviceCompatConfig() throws Exception { 423 String config = runCommand("dumpsys platform_compat"); 424 return Arrays.stream(config.split("\n")) 425 .map(Change::fromString) 426 .collect(Collectors.toList()); 427 } 428 429 protected Change getOnDeviceChangeIdConfig(long changeId) throws Exception { 430 List<Change> changes = getOnDeviceCompatConfig(); 431 for (Change change : changes) { 432 if (change.changeId == changeId) { 433 return change; 434 } 435 } 436 return null; 437 } 438 } 439