nativedisplay/surfacetexture/SurfaceTexture.cpp

/*
 * Copyright 2019 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <cutils/compiler.h>
#include <gui/BufferQueue.h>
#include <surfacetexture/ImageConsumer.h>
#include <surfacetexture/SurfaceTexture.h>
#include <surfacetexture/surface_texture_platform.h>
#include <math/mat4.h>
#include <system/window.h>
#include <utils/Trace.h>

#include <com_android_graphics_libgui_flags.h>

namespace android {

// Macros for including the SurfaceTexture name in log messages
#define SFT_LOGV(x, ...) ALOGV("[%s] " x, mName.c_str(), ##__VA_ARGS__)
#define SFT_LOGD(x, ...) ALOGD("[%s] " x, mName.c_str(), ##__VA_ARGS__)
#define SFT_LOGW(x, ...) ALOGW("[%s] " x, mName.c_str(), ##__VA_ARGS__)
#define SFT_LOGE(x, ...) ALOGE("[%s] " x, mName.c_str(), ##__VA_ARGS__)

static const mat4 mtxIdentity;

#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ)
SurfaceTexture::SurfaceTexture(uint32_t tex, uint32_t texTarget, bool useFenceSync,
                               bool isControlledByApp)
      : ConsumerBase(isControlledByApp),
        mCurrentCrop(Rect::EMPTY_RECT),
        mCurrentTransform(0),
        mCurrentScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE),
        mCurrentFence(Fence::NO_FENCE),
        mCurrentTimestamp(0),
        mCurrentDataSpace(HAL_DATASPACE_UNKNOWN),
        mCurrentFrameNumber(0),
        mDefaultWidth(1),
        mDefaultHeight(1),
        mFilteringEnabled(true),
        mTexName(tex),
        mUseFenceSync(useFenceSync),
        mTexTarget(texTarget),
        mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT),
        mOpMode(OpMode::attachedToGL) {
    initialize();
}

SurfaceTexture::SurfaceTexture(uint32_t texTarget, bool useFenceSync, bool isControlledByApp)
      : ConsumerBase(isControlledByApp),
        mCurrentCrop(Rect::EMPTY_RECT),
        mCurrentTransform(0),
        mCurrentScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE),
        mCurrentFence(Fence::NO_FENCE),
        mCurrentTimestamp(0),
        mCurrentDataSpace(HAL_DATASPACE_UNKNOWN),
        mCurrentFrameNumber(0),
        mDefaultWidth(1),
        mDefaultHeight(1),
        mFilteringEnabled(true),
        mTexName(0),
        mUseFenceSync(useFenceSync),
        mTexTarget(texTarget),
        mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT),
        mOpMode(OpMode::detached) {
    initialize();
}
#endif // COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(WB_CONSUMER_BASE_OWNS_BQ)

SurfaceTexture::SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t tex,
                               uint32_t texTarget, bool useFenceSync, bool isControlledByApp)
      : ConsumerBase(bq, isControlledByApp),
        mCurrentCrop(Rect::EMPTY_RECT),
        mCurrentTransform(0),
        mCurrentScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE),
        mCurrentFence(Fence::NO_FENCE),
        mCurrentTimestamp(0),
        mCurrentDataSpace(HAL_DATASPACE_UNKNOWN),
        mCurrentFrameNumber(0),
        mDefaultWidth(1),
        mDefaultHeight(1),
        mFilteringEnabled(true),
        mTexName(tex),
        mUseFenceSync(useFenceSync),
        mTexTarget(texTarget),
        mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT),
        mOpMode(OpMode::attachedToGL) {
    initialize();
}

SurfaceTexture::SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t texTarget,
                               bool useFenceSync, bool isControlledByApp)
      : ConsumerBase(bq, isControlledByApp),
        mCurrentCrop(Rect::EMPTY_RECT),
        mCurrentTransform(0),
        mCurrentScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE),
        mCurrentFence(Fence::NO_FENCE),
        mCurrentTimestamp(0),
        mCurrentDataSpace(HAL_DATASPACE_UNKNOWN),
        mCurrentFrameNumber(0),
        mDefaultWidth(1),
        mDefaultHeight(1),
        mFilteringEnabled(true),
        mTexName(0),
        mUseFenceSync(useFenceSync),
        mTexTarget(texTarget),
        mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT),
        mOpMode(OpMode::detached) {
    initialize();
}

status_t SurfaceTexture::setDefaultBufferSize(uint32_t w, uint32_t h) {
    Mutex::Autolock lock(mMutex);
    if (mAbandoned) {
        SFT_LOGE("setDefaultBufferSize: SurfaceTexture is abandoned!");
        return NO_INIT;
    }
    mDefaultWidth = w;
    mDefaultHeight = h;
    return mConsumer->setDefaultBufferSize(w, h);
}

status_t SurfaceTexture::updateTexImage() {
    ATRACE_CALL();
    SFT_LOGV("updateTexImage");
    Mutex::Autolock lock(mMutex);

    if (mAbandoned) {
        SFT_LOGE("updateTexImage: SurfaceTexture is abandoned!");
        return NO_INIT;
    }

    return mEGLConsumer.updateTexImage(*this);
}

status_t SurfaceTexture::releaseTexImage() {
    // releaseTexImage can be invoked even when not attached to a GL context.
    ATRACE_CALL();
    SFT_LOGV("releaseTexImage");
    Mutex::Autolock lock(mMutex);

    if (mAbandoned) {
        SFT_LOGE("releaseTexImage: SurfaceTexture is abandoned!");
        return NO_INIT;
    }

    return mEGLConsumer.releaseTexImage(*this);
}

status_t SurfaceTexture::acquireBufferLocked(BufferItem* item, nsecs_t presentWhen,
                                             uint64_t maxFrameNumber) {
    status_t err = ConsumerBase::acquireBufferLocked(item, presentWhen, maxFrameNumber);
    if (err != NO_ERROR) {
        return err;
    }

    switch (mOpMode) {
        case OpMode::attachedToConsumer:
            break;
        case OpMode::attachedToGL:
            mEGLConsumer.onAcquireBufferLocked(item, *this);
            break;
        case OpMode::detached:
            break;
    }

    return NO_ERROR;
}

status_t SurfaceTexture::releaseBufferLocked(int buf, sp<GraphicBuffer> graphicBuffer,
                                             EGLDisplay display, EGLSyncKHR eglFence) {
    // release the buffer if it hasn't already been discarded by the
    // BufferQueue. This can happen, for example, when the producer of this
    // buffer has reallocated the original buffer slot after this buffer
    // was acquired.
    status_t err = ConsumerBase::releaseBufferLocked(buf, graphicBuffer, display, eglFence);
    // We could be releasing an EGL/Vulkan buffer, even if not currently
    // attached to a GL context.
    mImageConsumer.onReleaseBufferLocked(buf);
    mEGLConsumer.onReleaseBufferLocked(buf);
    return err;
}

status_t SurfaceTexture::detachFromContext() {
    ATRACE_CALL();
    SFT_LOGV("detachFromContext");
    Mutex::Autolock lock(mMutex);

    if (mAbandoned) {
        SFT_LOGE("detachFromContext: abandoned SurfaceTexture");
        return NO_INIT;
    }

    if (mOpMode != OpMode::attachedToGL) {
        SFT_LOGE("detachFromContext: SurfaceTexture is not attached to a GL context");
        return INVALID_OPERATION;
    }

    status_t err = mEGLConsumer.detachFromContext(*this);
    if (err == OK) {
        mOpMode = OpMode::detached;
    }

    return err;
}

status_t SurfaceTexture::attachToContext(uint32_t tex) {
    ATRACE_CALL();
    SFT_LOGV("attachToContext");
    Mutex::Autolock lock(mMutex);

    if (mAbandoned) {
        SFT_LOGE("attachToContext: abandoned SurfaceTexture");
        return NO_INIT;
    }

    if (mOpMode != OpMode::detached) {
        SFT_LOGE("attachToContext: SurfaceTexture is already attached to a "
                 "context");
        return INVALID_OPERATION;
    }

    return mEGLConsumer.attachToContext(tex, *this);
}

void SurfaceTexture::takeConsumerOwnership() {
    ATRACE_CALL();
    Mutex::Autolock _l(mMutex);
    if (mAbandoned) {
        SFT_LOGE("attachToView: abandoned SurfaceTexture");
        return;
    }
    if (mOpMode == OpMode::detached) {
        mOpMode = OpMode::attachedToConsumer;

        if (mCurrentTexture != BufferQueue::INVALID_BUFFER_SLOT) {
            // release possible EGLConsumer texture cache
            mEGLConsumer.onFreeBufferLocked(mCurrentTexture);
            mEGLConsumer.onAbandonLocked();
        }
    } else {
        SFT_LOGE("attachToView: already attached");
    }
}

void SurfaceTexture::releaseConsumerOwnership() {
    ATRACE_CALL();
    Mutex::Autolock _l(mMutex);

    if (mAbandoned) {
        SFT_LOGE("detachFromView: abandoned SurfaceTexture");
        return;
    }

    if (mOpMode == OpMode::attachedToConsumer) {
        mOpMode = OpMode::detached;
    } else {
        SFT_LOGE("detachFromView: not attached to View");
    }
}

uint32_t SurfaceTexture::getCurrentTextureTarget() const {
    return mTexTarget;
}

void SurfaceTexture::getTransformMatrix(float mtx[16]) {
    Mutex::Autolock lock(mMutex);
    memcpy(mtx, mCurrentTransformMatrix, sizeof(mCurrentTransformMatrix));
}

void SurfaceTexture::setFilteringEnabled(bool enabled) {
    Mutex::Autolock lock(mMutex);
    if (mAbandoned) {
        SFT_LOGE("setFilteringEnabled: SurfaceTexture is abandoned!");
        return;
    }
    bool needsRecompute = mFilteringEnabled != enabled;
    mFilteringEnabled = enabled;

    if (needsRecompute && mCurrentTexture == BufferQueue::INVALID_BUFFER_SLOT) {
        SFT_LOGD("setFilteringEnabled called with no current item");
    }

    if (needsRecompute && mCurrentTexture != BufferQueue::INVALID_BUFFER_SLOT) {
        computeCurrentTransformMatrixLocked();
    }
}

void SurfaceTexture::computeCurrentTransformMatrixLocked() {
    SFT_LOGV("computeCurrentTransformMatrixLocked");
    sp<GraphicBuffer> buf = (mCurrentTexture == BufferQueue::INVALID_BUFFER_SLOT)
            ? nullptr
            : mSlots[mCurrentTexture].mGraphicBuffer;
    if (buf == nullptr) {
        SFT_LOGD("computeCurrentTransformMatrixLocked: no current item");
    }
    computeTransformMatrix(mCurrentTransformMatrix, buf, mCurrentCrop, mCurrentTransform,
                           mFilteringEnabled);
}

void SurfaceTexture::computeTransformMatrix(float outTransform[16], const sp<GraphicBuffer>& buf,
                                            const Rect& cropRect, uint32_t transform,
                                            bool filtering) {
    // Transform matrices
    static const mat4 mtxFlipH(-1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1);
    static const mat4 mtxFlipV(1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1);
    static const mat4 mtxRot90(0, 1, 0, 0, -1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1);

    mat4 xform;
    if (transform & NATIVE_WINDOW_TRANSFORM_FLIP_H) {
        xform *= mtxFlipH;
    }
    if (transform & NATIVE_WINDOW_TRANSFORM_FLIP_V) {
        xform *= mtxFlipV;
    }
    if (transform & NATIVE_WINDOW_TRANSFORM_ROT_90) {
        xform *= mtxRot90;
    }

    if (!cropRect.isEmpty() && buf.get()) {
        float tx = 0.0f, ty = 0.0f, sx = 1.0f, sy = 1.0f;
        float bufferWidth = buf->getWidth();
        float bufferHeight = buf->getHeight();
        float shrinkAmount = 0.0f;
        if (filtering) {
            // In order to prevent bilinear sampling beyond the edge of the
            // crop rectangle we may need to shrink it by 2 texels in each
            // dimension.  Normally this would just need to take 1/2 a texel
            // off each end, but because the chroma channels of YUV420 images
            // are subsampled we may need to shrink the crop region by a whole
            // texel on each side.
            switch (buf->getPixelFormat()) {
                case PIXEL_FORMAT_RGBA_8888:
                case PIXEL_FORMAT_RGBX_8888:
                case PIXEL_FORMAT_RGBA_FP16:
                case PIXEL_FORMAT_RGBA_1010102:
                case PIXEL_FORMAT_RGB_888:
                case PIXEL_FORMAT_RGB_565:
                case PIXEL_FORMAT_BGRA_8888:
                    // We know there's no subsampling of any channels, so we
                    // only need to shrink by a half a pixel.
                    shrinkAmount = 0.5;
                    break;

                default:
                    // If we don't recognize the format, we must assume the
                    // worst case (that we care about), which is YUV420.
                    shrinkAmount = 1.0;
                    break;
            }
        }

        // Only shrink the dimensions that are not the size of the buffer.
        if (cropRect.width() < bufferWidth) {
            tx = (float(cropRect.left) + shrinkAmount) / bufferWidth;
            sx = (float(cropRect.width()) - (2.0f * shrinkAmount)) / bufferWidth;
        }
        if (cropRect.height() < bufferHeight) {
            ty = (float(bufferHeight - cropRect.bottom) + shrinkAmount) / bufferHeight;
            sy = (float(cropRect.height()) - (2.0f * shrinkAmount)) / bufferHeight;
        }

        mat4 crop(sx, 0, 0, 0, 0, sy, 0, 0, 0, 0, 1, 0, tx, ty, 0, 1);
        xform = crop * xform;
    }

    // SurfaceFlinger expects the top of its window textures to be at a Y
    // coordinate of 0, so SurfaceTexture must behave the same way.  We don't
    // want to expose this to applications, however, so we must add an
    // additional vertical flip to the transform after all the other transforms.
    xform = mtxFlipV * xform;

    memcpy(outTransform, xform.asArray(), sizeof(xform));
}

Rect SurfaceTexture::scaleDownCrop(const Rect& crop, uint32_t bufferWidth, uint32_t bufferHeight) {
    Rect outCrop = crop;

    uint32_t newWidth = static_cast<uint32_t>(crop.width());
    uint32_t newHeight = static_cast<uint32_t>(crop.height());

    if (newWidth * bufferHeight > newHeight * bufferWidth) {
        newWidth = newHeight * bufferWidth / bufferHeight;
        ALOGV("too wide: newWidth = %d", newWidth);
    } else if (newWidth * bufferHeight < newHeight * bufferWidth) {
        newHeight = newWidth * bufferHeight / bufferWidth;
        ALOGV("too tall: newHeight = %d", newHeight);
    }

    uint32_t currentWidth = static_cast<uint32_t>(crop.width());
    uint32_t currentHeight = static_cast<uint32_t>(crop.height());

    // The crop is too wide
    if (newWidth < currentWidth) {
        uint32_t dw = currentWidth - newWidth;
        auto halfdw = dw / 2;
        outCrop.left += halfdw;
        // Not halfdw because it would subtract 1 too few when dw is odd
        outCrop.right -= (dw - halfdw);
        // The crop is too tall
    } else if (newHeight < currentHeight) {
        uint32_t dh = currentHeight - newHeight;
        auto halfdh = dh / 2;
        outCrop.top += halfdh;
        // Not halfdh because it would subtract 1 too few when dh is odd
        outCrop.bottom -= (dh - halfdh);
    }

    ALOGV("getCurrentCrop final crop [%d,%d,%d,%d]", outCrop.left, outCrop.top, outCrop.right,
          outCrop.bottom);

    return outCrop;
}

nsecs_t SurfaceTexture::getTimestamp() {
    SFT_LOGV("getTimestamp");
    Mutex::Autolock lock(mMutex);
    return mCurrentTimestamp;
}

android_dataspace SurfaceTexture::getCurrentDataSpace() {
    SFT_LOGV("getCurrentDataSpace");
    Mutex::Autolock lock(mMutex);
    return mCurrentDataSpace;
}

uint64_t SurfaceTexture::getFrameNumber() {
    SFT_LOGV("getFrameNumber");
    Mutex::Autolock lock(mMutex);
    return mCurrentFrameNumber;
}

Rect SurfaceTexture::getCurrentCrop() const {
    Mutex::Autolock lock(mMutex);
    return (mCurrentScalingMode == NATIVE_WINDOW_SCALING_MODE_SCALE_CROP)
            ? scaleDownCrop(mCurrentCrop, mDefaultWidth, mDefaultHeight)
            : mCurrentCrop;
}

uint32_t SurfaceTexture::getCurrentTransform() const {
    Mutex::Autolock lock(mMutex);
    return mCurrentTransform;
}

uint32_t SurfaceTexture::getCurrentScalingMode() const {
    Mutex::Autolock lock(mMutex);
    return mCurrentScalingMode;
}

sp<Fence> SurfaceTexture::getCurrentFence() const {
    Mutex::Autolock lock(mMutex);
    return mCurrentFence;
}

std::shared_ptr<FenceTime> SurfaceTexture::getCurrentFenceTime() const {
    Mutex::Autolock lock(mMutex);
    return mCurrentFenceTime;
}

void SurfaceTexture::freeBufferLocked(int slotIndex) {
    SFT_LOGV("freeBufferLocked: slotIndex=%d", slotIndex);
    if (slotIndex == mCurrentTexture) {
        mCurrentTexture = BufferQueue::INVALID_BUFFER_SLOT;
    }
    // The slotIndex buffer could have EGL cache, but there is no way to tell
    // for sure. Buffers can be freed after SurfaceTexture has detached from GL
    // context or View.
    mEGLConsumer.onFreeBufferLocked(slotIndex);
    ConsumerBase::freeBufferLocked(slotIndex);
}

void SurfaceTexture::abandonLocked() {
    SFT_LOGV("abandonLocked");
    mEGLConsumer.onAbandonLocked();
    ConsumerBase::abandonLocked();
}

status_t SurfaceTexture::setConsumerUsageBits(uint64_t usage) {
    return ConsumerBase::setConsumerUsageBits(usage | DEFAULT_USAGE_FLAGS);
}

void SurfaceTexture::dumpLocked(String8& result, const char* prefix) const {
    result.appendFormat("%smTexName=%d mCurrentTexture=%d\n"
                        "%smCurrentCrop=[%d,%d,%d,%d] mCurrentTransform=%#x\n",
                        prefix, mTexName, mCurrentTexture, prefix, mCurrentCrop.left,
                        mCurrentCrop.top, mCurrentCrop.right, mCurrentCrop.bottom,
                        mCurrentTransform);

    ConsumerBase::dumpLocked(result, prefix);
}

sp<GraphicBuffer> SurfaceTexture::dequeueBuffer(int* outSlotid, android_dataspace* outDataspace,
                                                HdrMetadata* outHdrMetadata,
                                                float* outTransformMatrix, uint32_t* outTransform,
                                                bool* outQueueEmpty,
                                                SurfaceTexture_createReleaseFence createFence,
                                                SurfaceTexture_fenceWait fenceWait,
                                                void* fencePassThroughHandle, ARect* currentCrop) {
    Mutex::Autolock _l(mMutex);
    sp<GraphicBuffer> buffer;

    if (mAbandoned) {
        SFT_LOGE("dequeueImage: SurfaceTexture is abandoned!");
        return buffer;
    }

    if (mOpMode != OpMode::attachedToConsumer) {
        SFT_LOGE("dequeueImage: SurfaceTexture is not attached to a View");
        return buffer;
    }

    buffer = mImageConsumer.dequeueBuffer(outSlotid, outDataspace, outHdrMetadata, outQueueEmpty,
                                          *this, createFence, fenceWait, fencePassThroughHandle);
    memcpy(outTransformMatrix, mCurrentTransformMatrix, sizeof(mCurrentTransformMatrix));
    *outTransform = mCurrentTransform;
    *currentCrop = mCurrentCrop;
    return buffer;
}

void SurfaceTexture::setSurfaceTextureListener(
        const sp<android::SurfaceTexture::SurfaceTextureListener>& listener) {
    SFT_LOGV("setSurfaceTextureListener");

    Mutex::Autolock _l(mMutex);
    mSurfaceTextureListener = listener;
    if (mSurfaceTextureListener != nullptr) {
        mFrameAvailableListenerProxy =
                sp<FrameAvailableListenerProxy>::make(mSurfaceTextureListener);
        setFrameAvailableListener(mFrameAvailableListenerProxy);
    } else {
        mFrameAvailableListenerProxy.clear();
    }
}

void SurfaceTexture::FrameAvailableListenerProxy::onFrameAvailable(const BufferItem& item) {
    const auto listener = mSurfaceTextureListener.promote();
    if (listener) {
        listener->onFrameAvailable(item);
    }
}

#if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BQ_SETFRAMERATE)
void SurfaceTexture::onSetFrameRate(float frameRate, int8_t compatibility,
                                    int8_t changeFrameRateStrategy) {
    SFT_LOGV("onSetFrameRate: %.2f", frameRate);

    auto listener = [&] {
        Mutex::Autolock _l(mMutex);
        return mSurfaceTextureListener;
    }();

    if (listener) {
        listener->onSetFrameRate(frameRate, compatibility, changeFrameRateStrategy);
    }
}
#endif

void SurfaceTexture::initialize() {
    SFT_LOGV("SurfaceTexture");

    memcpy(mCurrentTransformMatrix, mtxIdentity.asArray(), sizeof(mCurrentTransformMatrix));

    mConsumer->setConsumerUsageBits(DEFAULT_USAGE_FLAGS);
}

} // namespace android