1 /* 2 * Copyright 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <android/hardware_buffer.h> 20 #include <gui/BufferQueueDefs.h> 21 #include <gui/ConsumerBase.h> 22 23 #include <gui/IGraphicBufferProducer.h> 24 #include <sys/cdefs.h> 25 #include <system/graphics.h> 26 #include <ui/FenceTime.h> 27 #include <ui/GraphicBuffer.h> 28 #include <utils/Mutex.h> 29 #include <utils/String8.h> 30 31 #include "EGLConsumer.h" 32 #include "ImageConsumer.h" 33 34 namespace android { 35 36 /* 37 * SurfaceTexture consumes buffers of graphics data from a BufferQueue, 38 * and makes them available to HWUI render thread as a SkImage and to 39 * an application GL render thread as an OpenGL texture. 40 * 41 * When attached to an application GL render thread, a typical usage 42 * pattern is to set up the SurfaceTexture with the 43 * desired options, and call updateTexImage() when a new frame is desired. 44 * If a new frame is available, the texture will be updated. If not, 45 * the previous contents are retained. 46 * 47 * When attached to a HWUI render thread, the TextureView implementation 48 * calls dequeueBuffer, which either pulls a new buffer or returns the 49 * last cached buffer if BufferQueue is empty. 50 * When attached to HWUI render thread, SurfaceTexture is compatible to 51 * both Vulkan and GL drawing pipelines. 52 */ 53 class ANDROID_API SurfaceTexture : public ConsumerBase { 54 public: 55 /** 56 * Callback function needed by dequeueBuffer. It creates a fence, 57 * that is signalled, when the previous buffer is no longer in use by HWUI 58 * and can be used by written by the producer. 59 */ 60 typedef status_t (*SurfaceTexture_createReleaseFence)(bool useFenceSync, EGLSyncKHR* eglFence, 61 EGLDisplay* display, int* releaseFence, 62 void* passThroughHandle); 63 64 /** 65 * Callback function needed by dequeueBuffer. It waits for the new buffer 66 * fence to signal, before issuing any draw commands. 67 */ 68 typedef status_t (*SurfaceTexture_fenceWait)(int fence, void* passThroughHandle); 69 70 enum { TEXTURE_EXTERNAL = 0x8D65 }; // GL_TEXTURE_EXTERNAL_OES 71 typedef ConsumerBase::FrameAvailableListener FrameAvailableListener; 72 73 /** 74 * SurfaceTexture constructs a new SurfaceTexture object. If the constructor 75 * with the tex parameter is used, tex indicates the name of the OpenGL ES 76 * texture to which images are to be streamed. texTarget specifies the 77 * OpenGL ES texture target to which the texture will be bound in 78 * updateTexImage. useFenceSync specifies whether fences should be used to 79 * synchronize access to buffers if that behavior is enabled at 80 * compile-time. 81 * 82 * A SurfaceTexture may be detached from one OpenGL ES context and then 83 * attached to a different context using the detachFromContext and 84 * attachToContext methods, respectively. The intention of these methods is 85 * purely to allow a SurfaceTexture to be transferred from one consumer 86 * context to another. If such a transfer is not needed there is no 87 * requirement that either of these methods be called. 88 * 89 * If the constructor with the tex parameter is used, the SurfaceTexture is 90 * created in a state where it is considered attached to an OpenGL ES 91 * context for the purposes of the attachToContext and detachFromContext 92 * methods. However, despite being considered "attached" to a context, the 93 * specific OpenGL ES context doesn't get latched until the first call to 94 * updateTexImage. After that point, all calls to updateTexImage must be 95 * made with the same OpenGL ES context current. 96 * 97 * If the constructor without the tex parameter is used, the SurfaceTexture 98 * is created in a detached state, and attachToContext must be called before 99 * calls to updateTexImage. 100 */ 101 #if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) 102 SurfaceTexture(uint32_t tex, uint32_t textureTarget, bool useFenceSync, bool isControlledByApp); 103 104 SurfaceTexture(uint32_t textureTarget, bool useFenceSync, bool isControlledByApp); 105 106 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t tex, uint32_t textureTarget, 107 bool useFenceSync, bool isControlledByApp) 108 __attribute((deprecated("Prefer ctors that create their own surface and consumer."))); 109 110 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t textureTarget, bool useFenceSync, 111 bool isControlledByApp) 112 __attribute((deprecated("Prefer ctors that create their own surface and consumer."))); 113 #else 114 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t tex, uint32_t textureTarget, 115 bool useFenceSync, bool isControlledByApp); 116 117 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t textureTarget, bool useFenceSync, 118 bool isControlledByApp); 119 #endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ) 120 121 /** 122 * updateTexImage acquires the most recently queued buffer, and sets the 123 * image contents of the target texture to it. 124 * 125 * This call may only be made while the OpenGL ES context to which the 126 * target texture belongs is bound to the calling thread. 127 * 128 * This calls doGLFenceWait to ensure proper synchronization. 129 */ 130 status_t updateTexImage(); 131 132 /** 133 * releaseTexImage releases the texture acquired in updateTexImage(). 134 * This is intended to be used in single buffer mode. 135 * 136 * This call may only be made while the OpenGL ES context to which the 137 * target texture belongs is bound to the calling thread. 138 */ 139 status_t releaseTexImage(); 140 141 /** 142 * getTransformMatrix retrieves the 4x4 texture coordinate transform matrix 143 * associated with the texture image set by the most recent call to 144 * updateTexImage. 145 * 146 * This transform matrix maps 2D homogeneous texture coordinates of the form 147 * (s, t, 0, 1) with s and t in the inclusive range [0, 1] to the texture 148 * coordinate that should be used to sample that location from the texture. 149 * Sampling the texture outside of the range of this transform is undefined. 150 * 151 * This transform is necessary to compensate for transforms that the stream 152 * content producer may implicitly apply to the content. By forcing users of 153 * a SurfaceTexture to apply this transform we avoid performing an extra 154 * copy of the data that would be needed to hide the transform from the 155 * user. 156 * 157 * The matrix is stored in column-major order so that it may be passed 158 * directly to OpenGL ES via the glLoadMatrixf or glUniformMatrix4fv 159 * functions. 160 */ 161 void getTransformMatrix(float mtx[16]); 162 163 /** 164 * Computes the transform matrix documented by getTransformMatrix 165 * from the BufferItem sub parts. 166 */ 167 static void computeTransformMatrix(float outTransform[16], const sp<GraphicBuffer>& buf, 168 const Rect& cropRect, uint32_t transform, bool filtering); 169 170 /** 171 * Scale the crop down horizontally or vertically such that it has the 172 * same aspect ratio as the buffer does. 173 */ 174 static Rect scaleDownCrop(const Rect& crop, uint32_t bufferWidth, uint32_t bufferHeight); 175 176 /** 177 * getTimestamp retrieves the timestamp associated with the texture image 178 * set by the most recent call to updateTexImage. 179 * 180 * The timestamp is in nanoseconds, and is monotonically increasing. Its 181 * other semantics (zero point, etc) are source-dependent and should be 182 * documented by the source. 183 */ 184 int64_t getTimestamp(); 185 186 /** 187 * getDataSpace retrieves the DataSpace associated with the texture image 188 * set by the most recent call to updateTexImage. 189 */ 190 android_dataspace getCurrentDataSpace(); 191 192 /** 193 * getFrameNumber retrieves the frame number associated with the texture 194 * image set by the most recent call to updateTexImage. 195 * 196 * The frame number is an incrementing counter set to 0 at the creation of 197 * the BufferQueue associated with this consumer. 198 */ 199 uint64_t getFrameNumber(); 200 201 /** 202 * setDefaultBufferSize is used to set the size of buffers returned by 203 * requestBuffers when a with and height of zero is requested. 204 * A call to setDefaultBufferSize() may trigger requestBuffers() to 205 * be called from the client. 206 * The width and height parameters must be no greater than the minimum of 207 * GL_MAX_VIEWPORT_DIMS and GL_MAX_TEXTURE_SIZE (see: glGetIntegerv). 208 * An error due to invalid dimensions might not be reported until 209 * updateTexImage() is called. 210 */ 211 status_t setDefaultBufferSize(uint32_t width, uint32_t height); 212 213 /** 214 * setFilteringEnabled sets whether the transform matrix should be computed 215 * for use with bilinear filtering. 216 */ 217 void setFilteringEnabled(bool enabled); 218 219 /** 220 * getCurrentTextureTarget returns the texture target of the current 221 * texture as returned by updateTexImage(). 222 */ 223 uint32_t getCurrentTextureTarget() const; 224 225 /** 226 * getCurrentCrop returns the cropping rectangle of the current buffer. 227 */ 228 Rect getCurrentCrop() const; 229 230 /** 231 * getCurrentTransform returns the transform of the current buffer. 232 */ 233 uint32_t getCurrentTransform() const; 234 235 /** 236 * getCurrentScalingMode returns the scaling mode of the current buffer. 237 */ 238 uint32_t getCurrentScalingMode() const; 239 240 /** 241 * getCurrentFence returns the fence indicating when the current buffer is 242 * ready to be read from. 243 */ 244 sp<Fence> getCurrentFence() const; 245 246 /** 247 * getCurrentFence returns the FenceTime indicating when the current 248 * buffer is ready to be read from. 249 */ 250 std::shared_ptr<FenceTime> getCurrentFenceTime() const; 251 252 /** 253 * setConsumerUsageBits overrides the ConsumerBase method to OR 254 * DEFAULT_USAGE_FLAGS to usage. 255 */ 256 status_t setConsumerUsageBits(uint64_t usage); 257 258 /** 259 * detachFromContext detaches the SurfaceTexture from the calling thread's 260 * current OpenGL ES context. This context must be the same as the context 261 * that was current for previous calls to updateTexImage. 262 * 263 * Detaching a SurfaceTexture from an OpenGL ES context will result in the 264 * deletion of the OpenGL ES texture object into which the images were being 265 * streamed. After a SurfaceTexture has been detached from the OpenGL ES 266 * context calls to updateTexImage will fail returning INVALID_OPERATION 267 * until the SurfaceTexture is attached to a new OpenGL ES context using the 268 * attachToContext method. 269 */ 270 status_t detachFromContext(); 271 272 /** 273 * attachToContext attaches a SurfaceTexture that is currently in the 274 * 'detached' state to the current OpenGL ES context. A SurfaceTexture is 275 * in the 'detached' state iff detachFromContext has successfully been 276 * called and no calls to attachToContext have succeeded since the last 277 * detachFromContext call. Calls to attachToContext made on a 278 * SurfaceTexture that is not in the 'detached' state will result in an 279 * INVALID_OPERATION error. 280 * 281 * The tex argument specifies the OpenGL ES texture object name in the 282 * new context into which the image contents will be streamed. A successful 283 * call to attachToContext will result in this texture object being bound to 284 * the texture target and populated with the image contents that were 285 * current at the time of the last call to detachFromContext. 286 */ 287 status_t attachToContext(uint32_t tex); 288 289 sp<GraphicBuffer> dequeueBuffer(int* outSlotid, android_dataspace* outDataspace, 290 HdrMetadata* outHdrMetadata, float* outTransformMatrix, 291 uint32_t* outTransform, bool* outQueueEmpty, 292 SurfaceTexture_createReleaseFence createFence, 293 SurfaceTexture_fenceWait fenceWait, 294 void* fencePassThroughHandle, ARect* currentCrop); 295 296 /** 297 * takeConsumerOwnership attaches a SurfaceTexture that is currently in the 298 * 'detached' state to a consumer context (usually HWUI RenderThread). 299 */ 300 void takeConsumerOwnership(); 301 302 /** 303 * releaseConsumerOwnership detaches a SurfaceTexture from a consumer 304 * context (usually HWUI RenderThread). 305 */ 306 void releaseConsumerOwnership(); 307 308 /** 309 * Interface for SurfaceTexture callback(s). 310 */ 311 struct SurfaceTextureListener : public RefBase { 312 virtual void onFrameAvailable(const BufferItem& item) = 0; 313 virtual void onSetFrameRate(float frameRate, int8_t compatibility, 314 int8_t changeFrameRateStrategy) = 0; 315 }; 316 317 /** 318 * setSurfaceTextureListener registers a SurfaceTextureListener. 319 */ 320 void setSurfaceTextureListener(const sp<SurfaceTextureListener>&); 321 322 protected: 323 /** 324 * abandonLocked overrides the ConsumerBase method to clear 325 * mCurrentTextureImage in addition to the ConsumerBase behavior. 326 */ 327 virtual void abandonLocked(); 328 329 /** 330 * dumpLocked overrides the ConsumerBase method to dump SurfaceTexture- 331 * specific info in addition to the ConsumerBase behavior. 332 */ 333 virtual void dumpLocked(String8& result, const char* prefix) const override; 334 335 /** 336 * acquireBufferLocked overrides the ConsumerBase method to update the 337 * mEglSlots array in addition to the ConsumerBase behavior. 338 */ 339 virtual status_t acquireBufferLocked(BufferItem* item, nsecs_t presentWhen, 340 uint64_t maxFrameNumber = 0) override; 341 342 /** 343 * releaseBufferLocked overrides the ConsumerBase method to update the 344 * mEglSlots array in addition to the ConsumerBase. 345 */ 346 virtual status_t releaseBufferLocked(int slot, const sp<GraphicBuffer> graphicBuffer, 347 EGLDisplay display = EGL_NO_DISPLAY, 348 EGLSyncKHR eglFence = EGL_NO_SYNC_KHR) override; 349 350 /** 351 * freeBufferLocked frees up the given buffer slot. If the slot has been 352 * initialized this will release the reference to the GraphicBuffer in that 353 * slot and destroy the EGLImage in that slot. Otherwise it has no effect. 354 * 355 * This method must be called with mMutex locked. 356 */ 357 virtual void freeBufferLocked(int slotIndex); 358 359 /** 360 * computeCurrentTransformMatrixLocked computes the transform matrix for the 361 * current texture. It uses mCurrentTransform and the current GraphicBuffer 362 * to compute this matrix and stores it in mCurrentTransformMatrix. 363 * mCurrentTextureImage must not be NULL. 364 */ 365 void computeCurrentTransformMatrixLocked(); 366 367 /** 368 * onSetFrameRate Notifies the consumer of a setFrameRate call from the producer side. 369 */ 370 #if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BQ_SETFRAMERATE) 371 void onSetFrameRate(float frameRate, int8_t compatibility, 372 int8_t changeFrameRateStrategy) override; 373 #endif 374 375 /** 376 * The default consumer usage flags that SurfaceTexture always sets on its 377 * BufferQueue instance; these will be OR:d with any additional flags passed 378 * from the SurfaceTexture user. In particular, SurfaceTexture will always 379 * consume buffers as hardware textures. 380 */ 381 static const uint64_t DEFAULT_USAGE_FLAGS = GraphicBuffer::USAGE_HW_TEXTURE; 382 383 /** 384 * mCurrentCrop is the crop rectangle that applies to the current texture. 385 * It gets set each time updateTexImage is called. 386 */ 387 Rect mCurrentCrop; 388 389 /** 390 * mCurrentTransform is the transform identifier for the current texture. It 391 * gets set each time updateTexImage is called. 392 */ 393 uint32_t mCurrentTransform; 394 395 /** 396 * mCurrentScalingMode is the scaling mode for the current texture. It gets 397 * set each time updateTexImage is called. 398 */ 399 uint32_t mCurrentScalingMode; 400 401 /** 402 * mCurrentFence is the fence received from BufferQueue in updateTexImage. 403 */ 404 sp<Fence> mCurrentFence; 405 406 /** 407 * The FenceTime wrapper around mCurrentFence. 408 */ 409 std::shared_ptr<FenceTime> mCurrentFenceTime{FenceTime::NO_FENCE}; 410 411 /** 412 * mCurrentTransformMatrix is the transform matrix for the current texture. 413 * It gets computed by computeTransformMatrix each time updateTexImage is 414 * called. 415 */ 416 float mCurrentTransformMatrix[16]; 417 418 /** 419 * mCurrentTimestamp is the timestamp for the current texture. It 420 * gets set each time updateTexImage is called. 421 */ 422 int64_t mCurrentTimestamp; 423 424 /** 425 * mCurrentDataSpace is the dataspace for the current texture. It 426 * gets set each time updateTexImage is called. 427 */ 428 android_dataspace mCurrentDataSpace; 429 430 /** 431 * mCurrentFrameNumber is the frame counter for the current texture. 432 * It gets set each time updateTexImage is called. 433 */ 434 uint64_t mCurrentFrameNumber; 435 436 uint32_t mDefaultWidth, mDefaultHeight; 437 438 /** 439 * mFilteringEnabled indicates whether the transform matrix is computed for 440 * use with bilinear filtering. It defaults to true and is changed by 441 * setFilteringEnabled(). 442 */ 443 bool mFilteringEnabled; 444 445 /** 446 * mTexName is the name of the OpenGL texture to which streamed images will 447 * be bound when updateTexImage is called. It is set at construction time 448 * and can be changed with a call to attachToContext. 449 */ 450 uint32_t mTexName; 451 452 /** 453 * mUseFenceSync indicates whether creation of the EGL_KHR_fence_sync 454 * extension should be used to prevent buffers from being dequeued before 455 * it's safe for them to be written. It gets set at construction time and 456 * never changes. 457 */ 458 const bool mUseFenceSync; 459 460 /** 461 * mTexTarget is the GL texture target with which the GL texture object is 462 * associated. It is set in the constructor and never changed. It is 463 * almost always GL_TEXTURE_EXTERNAL_OES except for one use case in Android 464 * Browser. In that case it is set to GL_TEXTURE_2D to allow 465 * glCopyTexSubImage to read from the texture. This is a hack to work 466 * around a GL driver limitation on the number of FBO attachments, which the 467 * browser's tile cache exceeds. 468 */ 469 const uint32_t mTexTarget; 470 471 /** 472 * mCurrentTexture is the buffer slot index of the buffer that is currently 473 * bound to the OpenGL texture. It is initialized to INVALID_BUFFER_SLOT, 474 * indicating that no buffer slot is currently bound to the texture. Note, 475 * however, that a value of INVALID_BUFFER_SLOT does not necessarily mean 476 * that no buffer is bound to the texture. A call to setBufferCount will 477 * reset mCurrentTexture to INVALID_BUFFER_SLOT. 478 */ 479 int mCurrentTexture; 480 481 enum class OpMode { detached, attachedToConsumer, attachedToGL }; 482 /** 483 * mOpMode indicates whether the SurfaceTexture is currently attached to 484 * an OpenGL ES context or the consumer context. For legacy reasons, this 485 * is initialized to, "attachedToGL" indicating that the SurfaceTexture is 486 * considered to be attached to whatever GL context is current at the time 487 * of the first updateTexImage call. 488 * It is set to "detached" by detachFromContext, and then set to 489 * "attachedToGL" again by attachToContext. 490 * takeConsumerOwnership/releaseConsumerOwnership are used to attach/detach 491 * from a consumer context - usually HWUI RenderThread. 492 */ 493 OpMode mOpMode; 494 495 /** 496 * mEGLConsumer has SurfaceTexture logic used when attached to GL context. 497 */ 498 EGLConsumer mEGLConsumer; 499 500 /** 501 * mImageConsumer has SurfaceTexture logic used when attached to a consumer 502 * context (usually HWUI RenderThread). 503 */ 504 ImageConsumer mImageConsumer; 505 506 /** 507 * mSurfaceTextureListener holds the registered SurfaceTextureListener. 508 * Note that SurfaceTexture holds the lister with an sp<>, which means that the listener 509 * must only hold a wp<> to SurfaceTexture and not an sp<>. 510 */ 511 sp<SurfaceTextureListener> mSurfaceTextureListener; 512 513 friend class ImageConsumer; 514 friend class EGLConsumer; 515 516 private: 517 void initialize(); 518 519 // Proxy listener to avoid having SurfaceTexture directly implement FrameAvailableListener as it 520 // is extending ConsumerBase which also implements FrameAvailableListener. 521 class FrameAvailableListenerProxy : public ConsumerBase::FrameAvailableListener { 522 public: FrameAvailableListenerProxy(const wp<SurfaceTextureListener> & listener)523 FrameAvailableListenerProxy(const wp<SurfaceTextureListener>& listener) 524 : mSurfaceTextureListener(listener) {} 525 526 private: 527 void onFrameAvailable(const BufferItem& item) override; 528 529 const wp<SurfaceTextureListener> mSurfaceTextureListener; 530 }; 531 sp<FrameAvailableListenerProxy> mFrameAvailableListenerProxy; 532 }; 533 534 // ---------------------------------------------------------------------------- 535 } // namespace android 536