1 /* 2 * Copyright (C) 2024 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package android.media.projection.cts; 17 18 import static android.content.pm.PackageManager.FEATURE_SCREEN_LANDSCAPE; 19 import static android.content.pm.PackageManager.FEATURE_SCREEN_PORTRAIT; 20 import static android.server.wm.CtsWindowInfoUtils.assertAndDumpWindowState; 21 import static android.server.wm.CtsWindowInfoUtils.waitForStableWindowGeometry; 22 import static android.server.wm.CtsWindowInfoUtils.waitForWindowInfo; 23 import static android.view.Surface.ROTATION_270; 24 25 import static com.android.compatibility.common.util.SystemUtil.runWithShellPermissionIdentity; 26 27 import static com.google.common.truth.Truth.assertThat; 28 29 import static org.junit.Assume.assumeTrue; 30 31 import android.annotation.NonNull; 32 import android.annotation.Nullable; 33 import android.app.Activity; 34 import android.app.ActivityOptions; 35 import android.content.Context; 36 import android.content.Intent; 37 import android.graphics.Bitmap; 38 import android.graphics.PixelFormat; 39 import android.graphics.Point; 40 import android.graphics.Rect; 41 import android.hardware.display.VirtualDisplay; 42 import android.media.Image; 43 import android.media.ImageReader; 44 import android.media.projection.MediaProjection; 45 import android.os.Bundle; 46 import android.os.Environment; 47 import android.os.Handler; 48 import android.os.IBinder; 49 import android.os.Looper; 50 import android.os.UserHandle; 51 import android.server.wm.MediaProjectionHelper; 52 import android.server.wm.RotationSession; 53 import android.server.wm.WindowManagerStateHelper; 54 import android.util.DisplayMetrics; 55 import android.util.Log; 56 import android.view.Surface; 57 import android.view.WindowMetrics; 58 import android.window.WindowInfosListenerForTest.WindowInfo; 59 60 import androidx.test.core.app.ActivityScenario; 61 import androidx.test.platform.app.InstrumentationRegistry; 62 63 import com.android.compatibility.common.util.FrameworkSpecificTest; 64 65 import org.junit.After; 66 import org.junit.Before; 67 import org.junit.Test; 68 69 import java.io.File; 70 import java.io.FileOutputStream; 71 import java.nio.ByteBuffer; 72 import java.time.Duration; 73 import java.util.concurrent.CountDownLatch; 74 import java.util.concurrent.TimeUnit; 75 import java.util.function.Predicate; 76 import java.util.function.Supplier; 77 78 /** 79 * Test {@link MediaProjection} successfully mirrors the display contents. 80 * 81 * <p>Validate that mirrored views are the expected size, for both full display and single app 82 * capture (if offered). Instead of examining the pixels match exactly (which is historically a 83 * flaky way of validating mirroring), examine the structure of the mirrored hierarchy, to ensure 84 * that mirroring is initiated correctly, and any transformations are applied as expected. 85 * 86 * <p>Run with: 87 * atest CtsMediaProjectionTestCases:MediaProjectionMirroringTest 88 */ 89 @FrameworkSpecificTest 90 public class MediaProjectionMirroringTest { 91 private static final String TAG = "MediaProjectionMirroringTest-FOO"; 92 private static final int SCREENSHOT_TIMEOUT_MS = 1000; 93 private static final int TOLERANCE = 1; 94 // Enable debug mode to save screenshots from MediaProjection session. 95 private static final boolean DEBUG_MODE = false; 96 private static final String VIRTUAL_DISPLAY = "MirroringTestVD"; 97 private Context mContext; 98 // Manage a MediaProjection capture session. 99 private final MediaProjectionHelper mMediaProjectionHelper = new MediaProjectionHelper(); 100 101 private MediaProjection mMediaProjection; 102 private MediaProjection.Callback mMediaProjectionCallback = 103 new MediaProjection.Callback() { 104 @Override 105 public void onStop() { 106 super.onStop(); 107 } 108 109 @Override 110 public void onCapturedContentResize(int width, int height) { 111 super.onCapturedContentResize(width, height); 112 } 113 114 @Override 115 public void onCapturedContentVisibilityChanged(boolean isVisible) { 116 super.onCapturedContentVisibilityChanged(isVisible); 117 } 118 }; 119 private ImageReader mImageReader; 120 private CountDownLatch mScreenshotCountDownLatch; 121 private VirtualDisplay mVirtualDisplay; 122 private final ActivityOptions.LaunchCookie mLaunchCookie = new ActivityOptions.LaunchCookie(); 123 public ActivityScenario<TestRotationActivity> mTestRotationActivityActivityScenario; 124 private Activity mActivity; 125 private final WindowManagerStateHelper mWmState = new WindowManagerStateHelper(); 126 /** 127 * Whether to wait for the rotation to be stable state after testing. It can be set if the 128 * display rotation may be changed by test. 129 */ 130 private boolean mWaitForRotationOnTearDown; 131 132 @Before setUp()133 public void setUp() { 134 mContext = InstrumentationRegistry.getInstrumentation().getContext(); 135 runWithShellPermissionIdentity(() -> { 136 mContext.getPackageManager().revokeRuntimePermission( 137 mContext.getPackageName(), 138 android.Manifest.permission.SYSTEM_ALERT_WINDOW, 139 new UserHandle(mContext.getUserId())); 140 }); 141 mMediaProjection = null; 142 if (DEBUG_MODE) { 143 mScreenshotCountDownLatch = new CountDownLatch(1); 144 } 145 } 146 147 @After tearDown()148 public void tearDown() { 149 if (mMediaProjection != null) { 150 if (mMediaProjectionCallback != null) { 151 mMediaProjection.unregisterCallback(mMediaProjectionCallback); 152 mMediaProjectionCallback = null; 153 } 154 mMediaProjection.stop(); 155 mMediaProjection = null; 156 } 157 if (mImageReader != null) { 158 mImageReader = null; 159 } 160 if (mVirtualDisplay != null) { 161 mVirtualDisplay.getSurface().release(); 162 mVirtualDisplay.release(); 163 mVirtualDisplay = null; 164 } 165 if (mWaitForRotationOnTearDown) { 166 mWmState.waitForDisplayUnfrozen(); 167 } 168 } 169 170 // Validate that the mirrored hierarchy is the expected size. 171 @Test testDisplayCapture()172 public void testDisplayCapture() { 173 ActivityScenario<Activity> activityScenario = 174 ActivityScenario.launch(new Intent(mContext, Activity.class)); 175 activityScenario.onActivity(activity -> mActivity = activity); 176 177 final WindowMetrics maxWindowMetrics = 178 mActivity.getWindowManager().getMaximumWindowMetrics(); 179 final Rect activityRect = new Rect(); 180 181 // Select full screen capture. 182 mMediaProjectionHelper.authorizeMediaProjection(); 183 184 // Start capture of the entire display. 185 mMediaProjection = mMediaProjectionHelper.startMediaProjection(); 186 mVirtualDisplay = createVirtualDisplay(maxWindowMetrics.getBounds(), "testDisplayCapture"); 187 waitForLatestScreenshot(); 188 189 // Get the bounds of the activity on screen - use getGlobalVisibleRect to account for 190 // possible insets caused by DisplayCutout 191 mActivity.getWindow().getDecorView().getGlobalVisibleRect(activityRect); 192 193 validateMirroredHierarchy(mActivity, 194 mVirtualDisplay.getDisplay().getDisplayId(), 195 new Point(activityRect.width(), activityRect.height())); 196 activityScenario.close(); 197 } 198 199 // Validate that the mirrored hierarchy is the expected size after rotating the default display. 200 @Test testDisplayCapture_rotation()201 public void testDisplayCapture_rotation() { 202 assumeTrue("Skipping test: no rotation support", supportsRotation()); 203 204 mTestRotationActivityActivityScenario = 205 ActivityScenario.launch(new Intent(mContext, TestRotationActivity.class)); 206 mTestRotationActivityActivityScenario.onActivity(activity -> mActivity = activity); 207 208 final RotationSession rotationSession = createManagedRotationSession(); 209 final WindowMetrics maxWindowMetrics = 210 mActivity.getWindowManager().getMaximumWindowMetrics(); 211 final Rect activityRect = new Rect(); 212 final int initialRotation = mActivity.getDisplay().getRotation(); 213 214 // Select full screen capture. 215 mMediaProjectionHelper.authorizeMediaProjection(); 216 217 // Start capture of the entire display. 218 mMediaProjection = mMediaProjectionHelper.startMediaProjection(); 219 mVirtualDisplay = createVirtualDisplay(maxWindowMetrics.getBounds(), 220 "testDisplayCapture_rotation"); 221 222 rotateDeviceAndWaitForActivity(rotationSession, initialRotation); 223 224 // Get the bounds of the activity on screen - use getGlobalVisibleRect to account for 225 // possible insets caused by DisplayCutout 226 mActivity.getWindow().getDecorView().getGlobalVisibleRect(activityRect); 227 228 final Point mirroredSize = calculateScaledMirroredActivitySize( 229 mActivity.getWindowManager().getCurrentWindowMetrics(), mVirtualDisplay, 230 new Point(activityRect.width(), activityRect.height())); 231 validateMirroredHierarchy(mActivity, mVirtualDisplay.getDisplay().getDisplayId(), 232 mirroredSize); 233 234 rotationSession.close(); 235 mTestRotationActivityActivityScenario.close(); 236 } 237 238 // Validate that the mirrored hierarchy is the expected size. 239 @Test testSingleAppCapture()240 public void testSingleAppCapture() { 241 final ActivityScenario<Activity> activityScenario = ActivityScenario.launch( 242 new Intent(mContext, Activity.class), 243 createActivityScenarioWithLaunchCookie(mLaunchCookie) 244 ); 245 activityScenario.onActivity(activity -> mActivity = activity); 246 final WindowMetrics maxWindowMetrics = 247 mActivity.getWindowManager().getMaximumWindowMetrics(); 248 final Rect activityRect = new Rect(); 249 250 // Select single app capture if supported. 251 mMediaProjectionHelper.authorizeMediaProjection(mLaunchCookie); 252 253 // Start capture of the single app. 254 mMediaProjection = mMediaProjectionHelper.startMediaProjection(); 255 mVirtualDisplay = createVirtualDisplay(maxWindowMetrics.getBounds(), 256 "testSingleAppCapture"); 257 waitForLatestScreenshot(); 258 259 // Get the bounds of the activity on screen - use getGlobalVisibleRect to account for 260 // possible insets caused by DisplayCutout 261 mActivity.getWindow().getDecorView().getGlobalVisibleRect(activityRect); 262 263 validateMirroredHierarchy(mActivity, 264 mVirtualDisplay.getDisplay().getDisplayId(), 265 new Point(activityRect.width(), activityRect.height())); 266 activityScenario.close(); 267 } 268 269 // TODO (b/284968776): test single app capture in split screen 270 271 /** 272 * Returns ActivityOptions with the given launch cookie set. 273 */ createActivityScenarioWithLaunchCookie( @onNull ActivityOptions.LaunchCookie launchCookie)274 private static Bundle createActivityScenarioWithLaunchCookie( 275 @NonNull ActivityOptions.LaunchCookie launchCookie) { 276 ActivityOptions activityOptions = ActivityOptions.makeBasic(); 277 activityOptions.setLaunchCookie(launchCookie); 278 return activityOptions.toBundle(); 279 } 280 createVirtualDisplay(Rect displayBounds, String methodName)281 private VirtualDisplay createVirtualDisplay(Rect displayBounds, String methodName) { 282 mImageReader = ImageReader.newInstance(displayBounds.width(), displayBounds.height(), 283 PixelFormat.RGBA_8888, /* maxImages= */ 1); 284 if (DEBUG_MODE) { 285 ScreenshotListener screenshotListener = new ScreenshotListener(methodName, 286 mScreenshotCountDownLatch); 287 mImageReader.setOnImageAvailableListener(screenshotListener, 288 new Handler(Looper.getMainLooper())); 289 } 290 mMediaProjection.registerCallback(mMediaProjectionCallback, 291 new Handler(Looper.getMainLooper())); 292 return mMediaProjection.createVirtualDisplay(VIRTUAL_DISPLAY + "_" + methodName, 293 displayBounds.width(), displayBounds.height(), 294 DisplayMetrics.DENSITY_HIGH, /* flags= */ 0, 295 mImageReader.getSurface(), /* callback= */ 296 null, new Handler(Looper.getMainLooper())); 297 } 298 299 /** 300 * Rotates the device 90 degrees & waits for the display & activity configuration to stabilize. 301 */ rotateDeviceAndWaitForActivity( @onNull RotationSession rotationSession, @Surface.Rotation int initialRotation)302 private void rotateDeviceAndWaitForActivity( 303 @NonNull RotationSession rotationSession, @Surface.Rotation int initialRotation) { 304 // Rotate the device by 90 degrees 305 rotationSession.set((initialRotation + 1) % (ROTATION_270 + 1), 306 /* waitForDeviceRotation=*/ true); 307 waitForLatestScreenshot(); 308 try { 309 waitForStableWindowGeometry(Duration.ofMillis(SCREENSHOT_TIMEOUT_MS)); 310 } catch (InterruptedException e) { 311 Log.e(TAG, "Unable to wait for window to stabilize after rotation: " + e.getMessage()); 312 } 313 // Re-fetch the activity since reference may have been modified during rotation. 314 mTestRotationActivityActivityScenario.onActivity(activity -> mActivity = activity); 315 } 316 317 /** 318 * Calculate the size of the activity, scaled to fit on the VirtualDisplay. 319 * 320 * @param currentWindowMetrics The size of the source activity, before it is mirrored 321 * @param virtualDisplay The VirtualDisplay the mirrored content is sent to and scaled to 322 * fit 323 * @return The expected size of the mirrored activity on the VirtualDisplay 324 */ calculateScaledMirroredActivitySize( @onNull WindowMetrics currentWindowMetrics, @NonNull VirtualDisplay virtualDisplay, @Nullable Point visibleBounds)325 private static Point calculateScaledMirroredActivitySize( 326 @NonNull WindowMetrics currentWindowMetrics, 327 @NonNull VirtualDisplay virtualDisplay, @Nullable Point visibleBounds) { 328 // Calculate the aspect ratio of the original activity. 329 final Point currentBounds = new Point(currentWindowMetrics.getBounds().width(), 330 currentWindowMetrics.getBounds().height()); 331 final float aspectRatio = currentBounds.x * 1f / currentBounds.y; 332 // Find the size of the surface we are mirroring to. 333 final Point surfaceSize = virtualDisplay.getSurface().getDefaultSize(); 334 int mirroredWidth; 335 int mirroredHeight; 336 337 // Calculate any width & height deltas caused by DisplayCutout insets 338 Point sizeDifference = new Point(); 339 if (visibleBounds != null) { 340 int widthDifference = currentBounds.x - visibleBounds.x; 341 int heightDifference = currentBounds.y - visibleBounds.y; 342 sizeDifference.set(widthDifference, heightDifference); 343 } 344 345 if (surfaceSize.x < surfaceSize.y) { 346 // Output surface is portrait, so its width constrains. The mirrored activity is 347 // scaled down to fill the width entirely, and will have horizontal black bars at the 348 // top and bottom. 349 // Also apply scaled insets, to handle case where device has a display cutout which 350 // shifts the content horizontally when landscape. 351 int adjustedHorizontalInsets = Math.round(sizeDifference.x / aspectRatio); 352 int adjustedVerticalInsets = Math.round(sizeDifference.y / aspectRatio); 353 mirroredWidth = surfaceSize.x - adjustedHorizontalInsets; 354 mirroredHeight = Math.round(surfaceSize.x / aspectRatio) - adjustedVerticalInsets; 355 } else { 356 // Output surface is landscape, so its height constrains. The mirrored activity is 357 // scaled down to fill the height entirely, and will have horizontal black bars on the 358 // left and right. 359 // Also apply scaled insets, to handle case where device has a display cutout which 360 // shifts the content vertically when portrait. 361 int adjustedHorizontalInsets = Math.round(sizeDifference.x * aspectRatio); 362 int adjustedVerticalInsets = Math.round(sizeDifference.y * aspectRatio); 363 mirroredWidth = Math.round(surfaceSize.y * aspectRatio) - adjustedHorizontalInsets; 364 mirroredHeight = surfaceSize.y - adjustedVerticalInsets; 365 } 366 return new Point(mirroredWidth, mirroredHeight); 367 } 368 369 /** 370 * Validate the given activity is in the hierarchy mirrored to the VirtualDisplay. 371 * 372 * <p>Note that the hierarchy is present on the VirtualDisplay because the hierarchy is mirrored 373 * to the Surface provided to #createVirtualDisplay. 374 * 375 * @param activity The activity that we expect to be mirrored 376 * @param virtualDisplayId The id of the virtual display we are mirroring to 377 * @param expectedWindowSize The expected size of the mirrored activity 378 */ validateMirroredHierarchy( Activity activity, int virtualDisplayId, @NonNull Point expectedWindowSize)379 private static void validateMirroredHierarchy( 380 Activity activity, int virtualDisplayId, 381 @NonNull Point expectedWindowSize) { 382 Predicate<WindowInfo> hasExpectedDimensions = windowInfo -> { 383 int widthDiff = Math.abs(windowInfo.bounds.width() - expectedWindowSize.x); 384 int heightDiff = Math.abs(windowInfo.bounds.height() - expectedWindowSize.y); 385 return widthDiff <= TOLERANCE && heightDiff <= TOLERANCE; 386 }; 387 Supplier<IBinder> taskWindowTokenSupplier = 388 activity.getWindow().getDecorView()::getWindowToken; 389 try { 390 boolean condition = waitForWindowInfo(hasExpectedDimensions, Duration.ofSeconds(5), 391 taskWindowTokenSupplier, virtualDisplayId); 392 assertAndDumpWindowState(TAG, 393 "Mirrored activity isn't the expected size of " + expectedWindowSize, 394 condition); 395 } catch (InterruptedException e) { 396 throw new RuntimeException(e); 397 } 398 } 399 createManagedRotationSession()400 private RotationSession createManagedRotationSession() { 401 mWaitForRotationOnTearDown = true; 402 return new RotationSession(mWmState); 403 } 404 405 /** 406 * Rotation support is indicated by explicitly having both landscape and portrait 407 * features or not listing either at all. 408 */ supportsRotation()409 protected boolean supportsRotation() { 410 final boolean supportsLandscape = hasDeviceFeature(FEATURE_SCREEN_LANDSCAPE); 411 final boolean supportsPortrait = hasDeviceFeature(FEATURE_SCREEN_PORTRAIT); 412 return (supportsLandscape && supportsPortrait) 413 || (!supportsLandscape && !supportsPortrait); 414 } 415 hasDeviceFeature(final String requiredFeature)416 protected boolean hasDeviceFeature(final String requiredFeature) { 417 return mContext.getPackageManager() 418 .hasSystemFeature(requiredFeature); 419 } 420 421 /** 422 * Stub activity for launching an activity meant to be rotated. 423 */ 424 public static class TestRotationActivity extends Activity { 425 // Stub 426 } 427 428 /** 429 * Wait for any screenshot that has been received already. Assumes that the countdown 430 * latch is already set. 431 */ waitForLatestScreenshot()432 private void waitForLatestScreenshot() { 433 if (DEBUG_MODE) { 434 // wait until we've received a screenshot 435 try { 436 assertThat(mScreenshotCountDownLatch.await(SCREENSHOT_TIMEOUT_MS, 437 TimeUnit.MILLISECONDS)).isTrue(); 438 } catch (InterruptedException e) { 439 Log.e(TAG, e.toString()); 440 } 441 } 442 } 443 444 /** 445 * Save MediaProjection's screenshots to the device to help debug test failures. 446 */ 447 public static class ScreenshotListener implements ImageReader.OnImageAvailableListener { 448 private final CountDownLatch mCountDownLatch; 449 private final String mMethodName; 450 private int mCurrentScreenshot = 0; 451 // How often to save an image 452 private static final int SCREENSHOT_FREQUENCY = 5; 453 ScreenshotListener(@onNull String methodName, @NonNull CountDownLatch latch)454 public ScreenshotListener(@NonNull String methodName, 455 @NonNull CountDownLatch latch) { 456 mMethodName = methodName; 457 mCountDownLatch = latch; 458 } 459 460 @Override onImageAvailable(ImageReader reader)461 public void onImageAvailable(ImageReader reader) { 462 if (mCurrentScreenshot % SCREENSHOT_FREQUENCY != 0) { 463 Log.d(TAG, "onImageAvailable - skip this one"); 464 return; 465 } 466 Log.d(TAG, "onImageAvailable - processing"); 467 if (mCountDownLatch != null) { 468 mCountDownLatch.countDown(); 469 } 470 mCurrentScreenshot++; 471 472 final Image image = reader.acquireLatestImage(); 473 474 assertThat(image).isNotNull(); 475 476 final Image.Plane plane = image.getPlanes()[0]; 477 478 assertThat(plane).isNotNull(); 479 480 final int rowPadding = 481 plane.getRowStride() - plane.getPixelStride() * image.getWidth(); 482 final Bitmap bitmap = Bitmap.createBitmap( 483 /* width= */ image.getWidth() + rowPadding / plane.getPixelStride(), 484 /* height= */ image.getHeight(), Bitmap.Config.ARGB_8888); 485 final ByteBuffer buffer = plane.getBuffer(); 486 487 assertThat(buffer).isNotNull(); 488 assertThat(bitmap).isNotNull(); // why null? 489 490 bitmap.copyPixelsFromBuffer(plane.getBuffer()); 491 assertThat(bitmap).isNotNull(); // why null? 492 493 try { 494 // save to virtual sdcard 495 final File outputDirectory = new File(Environment.getExternalStorageDirectory(), 496 "cts." + TAG); 497 Log.d(TAG, "Had to create the directory? " + outputDirectory.mkdir()); 498 final File screenshot = new File(outputDirectory, 499 mMethodName + "_screenshot_" + mCurrentScreenshot + "_" 500 + System.currentTimeMillis() + ".jpg"); 501 final FileOutputStream stream = new FileOutputStream(screenshot); 502 assertThat(stream).isNotNull(); 503 bitmap.compress(Bitmap.CompressFormat.JPEG, 100, stream); 504 stream.close(); 505 image.close(); 506 } catch (Exception e) { 507 Log.e(TAG, "Unable to write out screenshot", e); 508 } 509 } 510 } 511 } 512