1 /* 2 * Copyright 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 package org.webrtc; 12 13 import static org.junit.Assert.assertEquals; 14 import static org.junit.Assert.assertTrue; 15 import static org.junit.Assert.fail; 16 17 import android.opengl.GLES20; 18 import android.os.SystemClock; 19 import androidx.annotation.Nullable; 20 import androidx.test.filters.MediumTest; 21 import androidx.test.filters.SmallTest; 22 import java.nio.ByteBuffer; 23 import java.util.concurrent.CountDownLatch; 24 import org.junit.Before; 25 import org.junit.Test; 26 27 public class SurfaceTextureHelperTest { 28 /** 29 * Mock texture listener with blocking wait functionality. 30 */ 31 public static final class MockTextureListener implements VideoSink { 32 private final Object lock = new Object(); 33 private @Nullable VideoFrame.TextureBuffer textureBuffer; 34 // Thread where frames are expected to be received on. 35 private final @Nullable Thread expectedThread; 36 MockTextureListener()37 MockTextureListener() { 38 this.expectedThread = null; 39 } 40 MockTextureListener(Thread expectedThread)41 MockTextureListener(Thread expectedThread) { 42 this.expectedThread = expectedThread; 43 } 44 45 @Override onFrame(VideoFrame frame)46 public void onFrame(VideoFrame frame) { 47 if (expectedThread != null && Thread.currentThread() != expectedThread) { 48 throw new IllegalStateException("onTextureFrameAvailable called on wrong thread."); 49 } 50 synchronized (lock) { 51 this.textureBuffer = (VideoFrame.TextureBuffer) frame.getBuffer(); 52 textureBuffer.retain(); 53 lock.notifyAll(); 54 } 55 } 56 57 /** Wait indefinitely for a new textureBuffer. */ waitForTextureBuffer()58 public VideoFrame.TextureBuffer waitForTextureBuffer() throws InterruptedException { 59 synchronized (lock) { 60 while (true) { 61 final VideoFrame.TextureBuffer textureBufferToReturn = textureBuffer; 62 if (textureBufferToReturn != null) { 63 textureBuffer = null; 64 return textureBufferToReturn; 65 } 66 lock.wait(); 67 } 68 } 69 } 70 71 /** Make sure we get no frame in the specified time period. */ assertNoFrameIsDelivered(final long waitPeriodMs)72 public void assertNoFrameIsDelivered(final long waitPeriodMs) throws InterruptedException { 73 final long startTimeMs = SystemClock.elapsedRealtime(); 74 long timeRemainingMs = waitPeriodMs; 75 synchronized (lock) { 76 while (textureBuffer == null && timeRemainingMs > 0) { 77 lock.wait(timeRemainingMs); 78 final long elapsedTimeMs = SystemClock.elapsedRealtime() - startTimeMs; 79 timeRemainingMs = waitPeriodMs - elapsedTimeMs; 80 } 81 assertTrue(textureBuffer == null); 82 } 83 } 84 } 85 86 /** Assert that two integers are close, with difference at most 87 * {@code threshold}. */ assertClose(int threshold, int expected, int actual)88 public static void assertClose(int threshold, int expected, int actual) { 89 if (Math.abs(expected - actual) <= threshold) 90 return; 91 fail("Not close enough, threshold " + threshold + ". Expected: " + expected + " Actual: " 92 + actual); 93 } 94 95 @Before setUp()96 public void setUp() { 97 // Load the JNI library for textureToYuv. 98 NativeLibrary.initialize(new NativeLibrary.DefaultLoader(), TestConstants.NATIVE_LIBRARY); 99 } 100 101 /** 102 * Test normal use by receiving three uniform texture frames. Texture frames are returned as early 103 * as possible. The texture pixel values are inspected by drawing the texture frame to a pixel 104 * buffer and reading it back with glReadPixels(). 105 */ 106 @Test 107 @MediumTest testThreeConstantColorFrames()108 public void testThreeConstantColorFrames() throws InterruptedException { 109 final int width = 16; 110 final int height = 16; 111 // Create EGL base with a pixel buffer as display output. 112 final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PIXEL_BUFFER); 113 eglBase.createPbufferSurface(width, height); 114 final GlRectDrawer drawer = new GlRectDrawer(); 115 116 // Create SurfaceTextureHelper and listener. 117 final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create( 118 "SurfaceTextureHelper test" /* threadName */, eglBase.getEglBaseContext()); 119 final MockTextureListener listener = new MockTextureListener(); 120 surfaceTextureHelper.startListening(listener); 121 surfaceTextureHelper.setTextureSize(width, height); 122 123 // Create resources for stubbing an OES texture producer. `eglOesBase` has the SurfaceTexture in 124 // `surfaceTextureHelper` as the target EGLSurface. 125 final EglBase eglOesBase = EglBase.create(eglBase.getEglBaseContext(), EglBase.CONFIG_PLAIN); 126 eglOesBase.createSurface(surfaceTextureHelper.getSurfaceTexture()); 127 assertEquals(eglOesBase.surfaceWidth(), width); 128 assertEquals(eglOesBase.surfaceHeight(), height); 129 130 final int red[] = new int[] {79, 144, 185}; 131 final int green[] = new int[] {66, 210, 162}; 132 final int blue[] = new int[] {161, 117, 158}; 133 // Draw three frames. 134 for (int i = 0; i < 3; ++i) { 135 // Draw a constant color frame onto the SurfaceTexture. 136 eglOesBase.makeCurrent(); 137 GLES20.glClearColor(red[i] / 255.0f, green[i] / 255.0f, blue[i] / 255.0f, 1.0f); 138 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 139 // swapBuffers() will ultimately trigger onTextureFrameAvailable(). 140 eglOesBase.swapBuffers(); 141 142 // Wait for an OES texture to arrive and draw it onto the pixel buffer. 143 final VideoFrame.TextureBuffer textureBuffer = listener.waitForTextureBuffer(); 144 eglBase.makeCurrent(); 145 drawer.drawOes(textureBuffer.getTextureId(), 146 RendererCommon.convertMatrixFromAndroidGraphicsMatrix(textureBuffer.getTransformMatrix()), 147 width, height, 0, 0, width, height); 148 textureBuffer.release(); 149 150 // Download the pixels in the pixel buffer as RGBA. Not all platforms support RGB, e.g. 151 // Nexus 9. 152 final ByteBuffer rgbaData = ByteBuffer.allocateDirect(width * height * 4); 153 GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, rgbaData); 154 GlUtil.checkNoGLES2Error("glReadPixels"); 155 156 // Assert rendered image is expected constant color. 157 while (rgbaData.hasRemaining()) { 158 assertEquals(rgbaData.get() & 0xFF, red[i]); 159 assertEquals(rgbaData.get() & 0xFF, green[i]); 160 assertEquals(rgbaData.get() & 0xFF, blue[i]); 161 assertEquals(rgbaData.get() & 0xFF, 255); 162 } 163 } 164 165 drawer.release(); 166 surfaceTextureHelper.dispose(); 167 eglBase.release(); 168 } 169 170 /** 171 * Test disposing the SurfaceTextureHelper while holding a pending texture frame. The pending 172 * texture frame should still be valid, and this is tested by drawing the texture frame to a pixel 173 * buffer and reading it back with glReadPixels(). 174 */ 175 @Test 176 @MediumTest testLateReturnFrame()177 public void testLateReturnFrame() throws InterruptedException { 178 final int width = 16; 179 final int height = 16; 180 // Create EGL base with a pixel buffer as display output. 181 final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PIXEL_BUFFER); 182 eglBase.createPbufferSurface(width, height); 183 184 // Create SurfaceTextureHelper and listener. 185 final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create( 186 "SurfaceTextureHelper test" /* threadName */, eglBase.getEglBaseContext()); 187 final MockTextureListener listener = new MockTextureListener(); 188 surfaceTextureHelper.startListening(listener); 189 surfaceTextureHelper.setTextureSize(width, height); 190 191 // Create resources for stubbing an OES texture producer. `eglOesBase` has the SurfaceTexture in 192 // `surfaceTextureHelper` as the target EGLSurface. 193 final EglBase eglOesBase = EglBase.create(eglBase.getEglBaseContext(), EglBase.CONFIG_PLAIN); 194 eglOesBase.createSurface(surfaceTextureHelper.getSurfaceTexture()); 195 assertEquals(eglOesBase.surfaceWidth(), width); 196 assertEquals(eglOesBase.surfaceHeight(), height); 197 198 final int red = 79; 199 final int green = 66; 200 final int blue = 161; 201 // Draw a constant color frame onto the SurfaceTexture. 202 eglOesBase.makeCurrent(); 203 GLES20.glClearColor(red / 255.0f, green / 255.0f, blue / 255.0f, 1.0f); 204 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 205 // swapBuffers() will ultimately trigger onTextureFrameAvailable(). 206 eglOesBase.swapBuffers(); 207 eglOesBase.release(); 208 209 // Wait for OES texture frame. 210 final VideoFrame.TextureBuffer textureBuffer = listener.waitForTextureBuffer(); 211 // Diconnect while holding the frame. 212 surfaceTextureHelper.dispose(); 213 214 // Draw the pending texture frame onto the pixel buffer. 215 eglBase.makeCurrent(); 216 final GlRectDrawer drawer = new GlRectDrawer(); 217 drawer.drawOes(textureBuffer.getTextureId(), 218 RendererCommon.convertMatrixFromAndroidGraphicsMatrix(textureBuffer.getTransformMatrix()), 219 width, height, 0, 0, width, height); 220 drawer.release(); 221 222 // Download the pixels in the pixel buffer as RGBA. Not all platforms support RGB, e.g. Nexus 9. 223 final ByteBuffer rgbaData = ByteBuffer.allocateDirect(width * height * 4); 224 GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, rgbaData); 225 GlUtil.checkNoGLES2Error("glReadPixels"); 226 eglBase.release(); 227 228 // Assert rendered image is expected constant color. 229 while (rgbaData.hasRemaining()) { 230 assertEquals(rgbaData.get() & 0xFF, red); 231 assertEquals(rgbaData.get() & 0xFF, green); 232 assertEquals(rgbaData.get() & 0xFF, blue); 233 assertEquals(rgbaData.get() & 0xFF, 255); 234 } 235 // Late frame return after everything has been disposed and released. 236 textureBuffer.release(); 237 } 238 239 /** 240 * Test disposing the SurfaceTextureHelper, but keep trying to produce more texture frames. No 241 * frames should be delivered to the listener. 242 */ 243 @Test 244 @MediumTest testDispose()245 public void testDispose() throws InterruptedException { 246 // Create SurfaceTextureHelper and listener. 247 final SurfaceTextureHelper surfaceTextureHelper = 248 SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null); 249 final MockTextureListener listener = new MockTextureListener(); 250 surfaceTextureHelper.startListening(listener); 251 // Create EglBase with the SurfaceTexture as target EGLSurface. 252 final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PLAIN); 253 surfaceTextureHelper.setTextureSize(/* textureWidth= */ 32, /* textureHeight= */ 32); 254 eglBase.createSurface(surfaceTextureHelper.getSurfaceTexture()); 255 eglBase.makeCurrent(); 256 // Assert no frame has been received yet. 257 listener.assertNoFrameIsDelivered(/* waitPeriodMs= */ 1); 258 // Draw and wait for one frame. 259 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 260 // swapBuffers() will ultimately trigger onTextureFrameAvailable(). 261 eglBase.swapBuffers(); 262 listener.waitForTextureBuffer().release(); 263 264 // Dispose - we should not receive any textures after this. 265 surfaceTextureHelper.dispose(); 266 267 // Draw one frame. 268 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 269 eglBase.swapBuffers(); 270 // swapBuffers() should not trigger onTextureFrameAvailable() because disposed has been called. 271 // Assert that no OES texture was delivered. 272 listener.assertNoFrameIsDelivered(/* waitPeriodMs= */ 500); 273 274 eglBase.release(); 275 } 276 277 /** 278 * Test disposing the SurfaceTextureHelper immediately after is has been setup to use a 279 * shared context. No frames should be delivered to the listener. 280 */ 281 @Test 282 @SmallTest testDisposeImmediately()283 public void testDisposeImmediately() { 284 final SurfaceTextureHelper surfaceTextureHelper = 285 SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null); 286 surfaceTextureHelper.dispose(); 287 } 288 289 /** 290 * Call stopListening(), but keep trying to produce more texture frames. No frames should be 291 * delivered to the listener. 292 */ 293 @Test 294 @MediumTest testStopListening()295 public void testStopListening() throws InterruptedException { 296 // Create SurfaceTextureHelper and listener. 297 final SurfaceTextureHelper surfaceTextureHelper = 298 SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null); 299 surfaceTextureHelper.setTextureSize(/* textureWidth= */ 32, /* textureHeight= */ 32); 300 final MockTextureListener listener = new MockTextureListener(); 301 surfaceTextureHelper.startListening(listener); 302 // Create EglBase with the SurfaceTexture as target EGLSurface. 303 final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PLAIN); 304 eglBase.createSurface(surfaceTextureHelper.getSurfaceTexture()); 305 eglBase.makeCurrent(); 306 // Assert no frame has been received yet. 307 listener.assertNoFrameIsDelivered(/* waitPeriodMs= */ 1); 308 // Draw and wait for one frame. 309 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 310 // swapBuffers() will ultimately trigger onTextureFrameAvailable(). 311 eglBase.swapBuffers(); 312 listener.waitForTextureBuffer().release(); 313 314 // Stop listening - we should not receive any textures after this. 315 surfaceTextureHelper.stopListening(); 316 317 // Draw one frame. 318 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 319 eglBase.swapBuffers(); 320 // swapBuffers() should not trigger onTextureFrameAvailable() because disposed has been called. 321 // Assert that no OES texture was delivered. 322 listener.assertNoFrameIsDelivered(/* waitPeriodMs= */ 500); 323 324 surfaceTextureHelper.dispose(); 325 eglBase.release(); 326 } 327 328 /** 329 * Test stopListening() immediately after the SurfaceTextureHelper has been setup. 330 */ 331 @Test 332 @SmallTest testStopListeningImmediately()333 public void testStopListeningImmediately() throws InterruptedException { 334 final SurfaceTextureHelper surfaceTextureHelper = 335 SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null); 336 final MockTextureListener listener = new MockTextureListener(); 337 surfaceTextureHelper.startListening(listener); 338 surfaceTextureHelper.stopListening(); 339 surfaceTextureHelper.dispose(); 340 } 341 342 /** 343 * Test stopListening() immediately after the SurfaceTextureHelper has been setup on the handler 344 * thread. 345 */ 346 @Test 347 @SmallTest testStopListeningImmediatelyOnHandlerThread()348 public void testStopListeningImmediatelyOnHandlerThread() throws InterruptedException { 349 final SurfaceTextureHelper surfaceTextureHelper = 350 SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null); 351 final MockTextureListener listener = new MockTextureListener(); 352 353 final CountDownLatch stopListeningBarrier = new CountDownLatch(1); 354 final CountDownLatch stopListeningBarrierDone = new CountDownLatch(1); 355 // Start by posting to the handler thread to keep it occupied. 356 surfaceTextureHelper.getHandler().post(new Runnable() { 357 @Override 358 public void run() { 359 ThreadUtils.awaitUninterruptibly(stopListeningBarrier); 360 surfaceTextureHelper.stopListening(); 361 stopListeningBarrierDone.countDown(); 362 } 363 }); 364 365 // startListening() is asynchronous and will post to the occupied handler thread. 366 surfaceTextureHelper.startListening(listener); 367 // Wait for stopListening() to be called on the handler thread. 368 stopListeningBarrier.countDown(); 369 stopListeningBarrierDone.await(); 370 // Wait until handler thread is idle to try to catch late startListening() call. 371 final CountDownLatch barrier = new CountDownLatch(1); 372 surfaceTextureHelper.getHandler().post(new Runnable() { 373 @Override 374 public void run() { 375 barrier.countDown(); 376 } 377 }); 378 ThreadUtils.awaitUninterruptibly(barrier); 379 // Previous startListening() call should never have taken place and it should be ok to call it 380 // again. 381 surfaceTextureHelper.startListening(listener); 382 383 surfaceTextureHelper.dispose(); 384 } 385 386 /** 387 * Test calling startListening() with a new listener after stopListening() has been called. 388 */ 389 @Test 390 @MediumTest testRestartListeningWithNewListener()391 public void testRestartListeningWithNewListener() throws InterruptedException { 392 // Create SurfaceTextureHelper and listener. 393 final SurfaceTextureHelper surfaceTextureHelper = 394 SurfaceTextureHelper.create("SurfaceTextureHelper test" /* threadName */, null); 395 surfaceTextureHelper.setTextureSize(/* textureWidth= */ 32, /* textureHeight= */ 32); 396 final MockTextureListener listener1 = new MockTextureListener(); 397 surfaceTextureHelper.startListening(listener1); 398 // Create EglBase with the SurfaceTexture as target EGLSurface. 399 final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PLAIN); 400 eglBase.createSurface(surfaceTextureHelper.getSurfaceTexture()); 401 eglBase.makeCurrent(); 402 // Assert no frame has been received yet. 403 listener1.assertNoFrameIsDelivered(/* waitPeriodMs= */ 1); 404 // Draw and wait for one frame. 405 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 406 // swapBuffers() will ultimately trigger onTextureFrameAvailable(). 407 eglBase.swapBuffers(); 408 listener1.waitForTextureBuffer().release(); 409 410 // Stop listening - `listener1` should not receive any textures after this. 411 surfaceTextureHelper.stopListening(); 412 413 // Connect different listener. 414 final MockTextureListener listener2 = new MockTextureListener(); 415 surfaceTextureHelper.startListening(listener2); 416 // Assert no frame has been received yet. 417 listener2.assertNoFrameIsDelivered(/* waitPeriodMs= */ 1); 418 419 // Draw one frame. 420 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 421 eglBase.swapBuffers(); 422 423 // Check that `listener2` received the frame, and not `listener1`. 424 listener2.waitForTextureBuffer().release(); 425 listener1.assertNoFrameIsDelivered(/* waitPeriodMs= */ 1); 426 427 surfaceTextureHelper.dispose(); 428 eglBase.release(); 429 } 430 431 @Test 432 @MediumTest testTexturetoYuv()433 public void testTexturetoYuv() throws InterruptedException { 434 final int width = 16; 435 final int height = 16; 436 437 final EglBase eglBase = EglBase.create(null, EglBase.CONFIG_PLAIN); 438 439 // Create SurfaceTextureHelper and listener. 440 final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create( 441 "SurfaceTextureHelper test" /* threadName */, eglBase.getEglBaseContext()); 442 final MockTextureListener listener = new MockTextureListener(); 443 surfaceTextureHelper.startListening(listener); 444 surfaceTextureHelper.setTextureSize(width, height); 445 446 // Create resources for stubbing an OES texture producer. `eglBase` has the SurfaceTexture in 447 // `surfaceTextureHelper` as the target EGLSurface. 448 449 eglBase.createSurface(surfaceTextureHelper.getSurfaceTexture()); 450 assertEquals(eglBase.surfaceWidth(), width); 451 assertEquals(eglBase.surfaceHeight(), height); 452 453 final int red[] = new int[] {79, 144, 185}; 454 final int green[] = new int[] {66, 210, 162}; 455 final int blue[] = new int[] {161, 117, 158}; 456 457 final int ref_y[] = new int[] {85, 170, 161}; 458 final int ref_u[] = new int[] {168, 97, 123}; 459 final int ref_v[] = new int[] {127, 106, 138}; 460 461 // Draw three frames. 462 for (int i = 0; i < 3; ++i) { 463 // Draw a constant color frame onto the SurfaceTexture. 464 eglBase.makeCurrent(); 465 GLES20.glClearColor(red[i] / 255.0f, green[i] / 255.0f, blue[i] / 255.0f, 1.0f); 466 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); 467 // swapBuffers() will ultimately trigger onTextureFrameAvailable(). 468 eglBase.swapBuffers(); 469 470 // Wait for an OES texture to arrive. 471 final VideoFrame.TextureBuffer textureBuffer = listener.waitForTextureBuffer(); 472 final VideoFrame.I420Buffer i420 = textureBuffer.toI420(); 473 textureBuffer.release(); 474 475 // Memory layout: Lines are 16 bytes. First 16 lines are 476 // the Y data. These are followed by 8 lines with 8 bytes of U 477 // data on the left and 8 bytes of V data on the right. 478 // 479 // Offset 480 // 0 YYYYYYYY YYYYYYYY 481 // 16 YYYYYYYY YYYYYYYY 482 // ... 483 // 240 YYYYYYYY YYYYYYYY 484 // 256 UUUUUUUU VVVVVVVV 485 // 272 UUUUUUUU VVVVVVVV 486 // ... 487 // 368 UUUUUUUU VVVVVVVV 488 // 384 buffer end 489 490 // Allow off-by-one differences due to different rounding. 491 final ByteBuffer dataY = i420.getDataY(); 492 final int strideY = i420.getStrideY(); 493 for (int y = 0; y < height; y++) { 494 for (int x = 0; x < width; x++) { 495 assertClose(1, ref_y[i], dataY.get(y * strideY + x) & 0xFF); 496 } 497 } 498 499 final int chromaWidth = width / 2; 500 final int chromaHeight = height / 2; 501 502 final ByteBuffer dataU = i420.getDataU(); 503 final ByteBuffer dataV = i420.getDataV(); 504 final int strideU = i420.getStrideU(); 505 final int strideV = i420.getStrideV(); 506 for (int y = 0; y < chromaHeight; y++) { 507 for (int x = 0; x < chromaWidth; x++) { 508 assertClose(1, ref_u[i], dataU.get(y * strideU + x) & 0xFF); 509 assertClose(1, ref_v[i], dataV.get(y * strideV + x) & 0xFF); 510 } 511 } 512 i420.release(); 513 } 514 515 surfaceTextureHelper.dispose(); 516 eglBase.release(); 517 } 518 } 519