1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "HardwareBitmapUploader.h"
18
19 #include <EGL/egl.h>
20 #include <EGL/eglext.h>
21 #include <GLES2/gl2.h>
22 #include <GLES2/gl2ext.h>
23 #include <GLES3/gl3.h>
24 #include <SkBitmap.h>
25 #include <SkCanvas.h>
26 #include <SkImage.h>
27 #include <SkImageAndroid.h>
28 #include <SkImageInfo.h>
29 #include <SkRefCnt.h>
30 #include <gui/TraceUtils.h>
31 #include <include/gpu/ganesh/GrDirectContext.h>
32 #include <include/gpu/ganesh/GrTypes.h>
33 #include <utils/GLUtils.h>
34 #include <utils/NdkUtils.h>
35 #include <utils/Trace.h>
36
37 #include <thread>
38
39 #include "hwui/Bitmap.h"
40 #include "renderthread/EglManager.h"
41 #include "renderthread/VulkanManager.h"
42 #include "thread/ThreadBase.h"
43 #include "utils/TimeUtils.h"
44
45 namespace android::uirenderer {
46
47 static constexpr auto kThreadTimeout = 60000_ms;
48
49 class AHBUploader;
50 // This helper uploader classes allows us to upload using either EGL or Vulkan using the same
51 // interface.
52 static sp<AHBUploader> sUploader = nullptr;
53
54 struct FormatInfo {
55 AHardwareBuffer_Format bufferFormat;
56 GLint format, type;
57 VkFormat vkFormat;
58 bool isSupported = false;
59 bool valid = true;
60 };
61
62 class AHBUploader : public RefBase {
63 public:
~AHBUploader()64 virtual ~AHBUploader() {}
65
destroy()66 void destroy() {
67 std::lock_guard _lock{mLock};
68 LOG_ALWAYS_FATAL_IF(mPendingUploads, "terminate called while uploads in progress");
69 if (mUploadThread) {
70 mUploadThread->requestExit();
71 mUploadThread->join();
72 mUploadThread = nullptr;
73 }
74 onDestroy();
75 }
76
uploadHardwareBitmap(const SkBitmap & bitmap,const FormatInfo & format,AHardwareBuffer * ahb)77 bool uploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
78 AHardwareBuffer* ahb) {
79 ATRACE_CALL();
80 beginUpload();
81 bool result = onUploadHardwareBitmap(bitmap, format, ahb);
82 endUpload();
83 return result;
84 }
85
postIdleTimeoutCheck()86 void postIdleTimeoutCheck() {
87 mUploadThread->queue().postDelayed(kThreadTimeout, [this]() { this->idleTimeoutCheck(); });
88 }
89
90 protected:
91 std::mutex mLock;
92 sp<ThreadBase> mUploadThread = nullptr;
93
94 private:
95 virtual void onIdle() = 0;
96 virtual void onDestroy() = 0;
97
98 virtual bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
99 AHardwareBuffer* ahb) = 0;
100 virtual void onBeginUpload() = 0;
101
shouldTimeOutLocked()102 bool shouldTimeOutLocked() {
103 nsecs_t durationSince = systemTime() - mLastUpload;
104 return durationSince > kThreadTimeout;
105 }
106
idleTimeoutCheck()107 void idleTimeoutCheck() {
108 std::lock_guard _lock{mLock};
109 if (mPendingUploads == 0 && shouldTimeOutLocked()) {
110 onIdle();
111 } else {
112 this->postIdleTimeoutCheck();
113 }
114 }
115
beginUpload()116 void beginUpload() {
117 std::lock_guard _lock{mLock};
118 mPendingUploads++;
119
120 if (!mUploadThread) {
121 mUploadThread = new ThreadBase{};
122 }
123 if (!mUploadThread->isRunning()) {
124 mUploadThread->start("GrallocUploadThread");
125 }
126
127 onBeginUpload();
128 }
129
endUpload()130 void endUpload() {
131 std::lock_guard _lock{mLock};
132 mPendingUploads--;
133 mLastUpload = systemTime();
134 }
135
136 int mPendingUploads = 0;
137 nsecs_t mLastUpload = 0;
138 };
139
140 #define FENCE_TIMEOUT 2000000000
141
142 class EGLUploader : public AHBUploader {
143 private:
onDestroy()144 void onDestroy() override {
145 mEglManager.destroy();
146 }
onIdle()147 void onIdle() override {
148 mEglManager.destroy();
149 }
150
onBeginUpload()151 void onBeginUpload() override {
152 if (!mEglManager.hasEglContext()) {
153 mUploadThread->queue().runSync([this]() {
154 this->mEglManager.initialize();
155 glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
156 });
157
158 this->postIdleTimeoutCheck();
159 }
160 }
161
162
getUploadEglDisplay()163 EGLDisplay getUploadEglDisplay() {
164 std::lock_guard _lock{mLock};
165 LOG_ALWAYS_FATAL_IF(!mEglManager.hasEglContext(), "Forgot to begin an upload?");
166 return mEglManager.eglDisplay();
167 }
168
onUploadHardwareBitmap(const SkBitmap & bitmap,const FormatInfo & format,AHardwareBuffer * ahb)169 bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
170 AHardwareBuffer* ahb) override {
171 ATRACE_CALL();
172
173 EGLDisplay display = getUploadEglDisplay();
174
175 LOG_ALWAYS_FATAL_IF(display == EGL_NO_DISPLAY, "Failed to get EGL_DEFAULT_DISPLAY! err=%s",
176 uirenderer::renderthread::EglManager::eglErrorString());
177 // We use an EGLImage to access the content of the buffer
178 // The EGL image is later bound to a 2D texture
179 const EGLClientBuffer clientBuffer = eglGetNativeClientBufferANDROID(ahb);
180 AutoEglImage autoImage(display, clientBuffer);
181 if (autoImage.image == EGL_NO_IMAGE_KHR) {
182 ALOGW("Could not create EGL image, err =%s",
183 uirenderer::renderthread::EglManager::eglErrorString());
184 return false;
185 }
186
187 {
188 ATRACE_FORMAT("CPU -> gralloc transfer (%dx%d)", bitmap.width(), bitmap.height());
189 EGLSyncKHR fence = mUploadThread->queue().runSync([&]() -> EGLSyncKHR {
190 AutoSkiaGlTexture glTexture;
191 glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, autoImage.image);
192 if (GLUtils::dumpGLErrors()) {
193 return EGL_NO_SYNC_KHR;
194 }
195
196 // glTexSubImage2D is synchronous in sense that it memcpy() from pointer that we
197 // provide.
198 // But asynchronous in sense that driver may upload texture onto hardware buffer
199 // when we first use it in drawing
200 glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, bitmap.width(), bitmap.height(),
201 format.format, format.type, bitmap.getPixels());
202 if (GLUtils::dumpGLErrors()) {
203 return EGL_NO_SYNC_KHR;
204 }
205
206 EGLSyncKHR uploadFence =
207 eglCreateSyncKHR(eglGetCurrentDisplay(), EGL_SYNC_FENCE_KHR, NULL);
208 if (uploadFence == EGL_NO_SYNC_KHR) {
209 ALOGW("Could not create sync fence %#x", eglGetError());
210 };
211 glFlush();
212 GLUtils::dumpGLErrors();
213 return uploadFence;
214 });
215
216 if (fence == EGL_NO_SYNC_KHR) {
217 return false;
218 }
219 EGLint waitStatus = eglClientWaitSyncKHR(display, fence, 0, FENCE_TIMEOUT);
220 ALOGE_IF(waitStatus != EGL_CONDITION_SATISFIED_KHR,
221 "Failed to wait for the fence %#x", eglGetError());
222
223 eglDestroySyncKHR(display, fence);
224 }
225 return true;
226 }
227
228 renderthread::EglManager mEglManager;
229 };
230
231 class VkUploader : public AHBUploader {
232 private:
onDestroy()233 void onDestroy() override {
234 std::lock_guard _lock{mVkLock};
235 mGrContext.reset();
236 mVulkanManagerStrong.clear();
237 }
onIdle()238 void onIdle() override {
239 onDestroy();
240 }
241
onBeginUpload()242 void onBeginUpload() override {}
243
onUploadHardwareBitmap(const SkBitmap & bitmap,const FormatInfo & format,AHardwareBuffer * ahb)244 bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
245 AHardwareBuffer* ahb) override {
246 bool uploadSucceeded = false;
247 mUploadThread->queue().runSync([this, &uploadSucceeded, bitmap, ahb]() {
248 ATRACE_CALL();
249 std::lock_guard _lock{mVkLock};
250
251 renderthread::VulkanManager* vkManager = getVulkanManager();
252 if (!vkManager->hasVkContext()) {
253 LOG_ALWAYS_FATAL_IF(mGrContext,
254 "GrContext exists with no VulkanManager for vulkan uploads");
255 vkManager->initialize();
256 }
257
258 if (!mGrContext) {
259 GrContextOptions options;
260 mGrContext = vkManager->createContext(options,
261 renderthread::VulkanManager::ContextType::kUploadThread);
262 LOG_ALWAYS_FATAL_IF(!mGrContext, "failed to create GrContext for vulkan uploads");
263 this->postIdleTimeoutCheck();
264 }
265
266 sk_sp<SkImage> image =
267 SkImages::TextureFromAHardwareBufferWithData(mGrContext.get(), bitmap.pixmap(),
268 ahb);
269 mGrContext->submit(GrSyncCpu::kYes);
270
271 uploadSucceeded = (image.get() != nullptr);
272 });
273 return uploadSucceeded;
274 }
275
276 /* must be called on the upload thread after the vkLock has been acquired */
getVulkanManager()277 renderthread::VulkanManager* getVulkanManager() {
278 if (!mVulkanManagerStrong) {
279 mVulkanManagerStrong = mVulkanManagerWeak.promote();
280
281 // create a new manager if we couldn't promote the weak ref
282 if (!mVulkanManagerStrong) {
283 mVulkanManagerStrong = renderthread::VulkanManager::getInstance();
284 mGrContext.reset();
285 mVulkanManagerWeak = mVulkanManagerStrong;
286 }
287 }
288
289 return mVulkanManagerStrong.get();
290 }
291
292 sk_sp<GrDirectContext> mGrContext;
293 sp<renderthread::VulkanManager> mVulkanManagerStrong;
294 wp<renderthread::VulkanManager> mVulkanManagerWeak;
295 std::mutex mVkLock;
296 };
297
checkSupport(AHardwareBuffer_Format format)298 static bool checkSupport(AHardwareBuffer_Format format) {
299 AHardwareBuffer_Desc desc = {
300 .width = 1,
301 .height = 1,
302 .layers = 1,
303 .format = format,
304 .usage = AHARDWAREBUFFER_USAGE_CPU_READ_NEVER | AHARDWAREBUFFER_USAGE_CPU_WRITE_NEVER |
305 AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE,
306 };
307 UniqueAHardwareBuffer buffer = allocateAHardwareBuffer(desc);
308 return buffer != nullptr;
309 }
310
hasFP16Support()311 bool HardwareBitmapUploader::hasFP16Support() {
312 static bool hasFP16Support = checkSupport(AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT);
313 return hasFP16Support;
314 }
315
has1010102Support()316 bool HardwareBitmapUploader::has1010102Support() {
317 static bool has101012Support = checkSupport(AHARDWAREBUFFER_FORMAT_R10G10B10A2_UNORM);
318 return has101012Support;
319 }
320
has10101010Support()321 bool HardwareBitmapUploader::has10101010Support() {
322 static bool has1010110Support = checkSupport(AHARDWAREBUFFER_FORMAT_R10G10B10A10_UNORM);
323 return has1010110Support;
324 }
325
hasAlpha8Support()326 bool HardwareBitmapUploader::hasAlpha8Support() {
327 static bool hasAlpha8Support = checkSupport(AHARDWAREBUFFER_FORMAT_R8_UNORM);
328 return hasAlpha8Support;
329 }
330
determineFormat(const SkBitmap & skBitmap,bool usingGL)331 static FormatInfo determineFormat(const SkBitmap& skBitmap, bool usingGL) {
332 FormatInfo formatInfo;
333 switch (skBitmap.info().colorType()) {
334 case kRGBA_8888_SkColorType:
335 formatInfo.isSupported = true;
336 [[fallthrough]];
337 // ARGB_4444 is upconverted to RGBA_8888
338 case kARGB_4444_SkColorType:
339 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
340 formatInfo.format = GL_RGBA;
341 formatInfo.type = GL_UNSIGNED_BYTE;
342 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
343 break;
344 case kRGBA_F16_SkColorType:
345 formatInfo.isSupported = HardwareBitmapUploader::hasFP16Support();
346 if (formatInfo.isSupported) {
347 formatInfo.type = GL_HALF_FLOAT;
348 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT;
349 formatInfo.vkFormat = VK_FORMAT_R16G16B16A16_SFLOAT;
350 } else {
351 formatInfo.type = GL_UNSIGNED_BYTE;
352 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
353 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
354 }
355 formatInfo.format = GL_RGBA;
356 break;
357 case kRGB_565_SkColorType:
358 formatInfo.isSupported = true;
359 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R5G6B5_UNORM;
360 formatInfo.format = GL_RGB;
361 formatInfo.type = GL_UNSIGNED_SHORT_5_6_5;
362 formatInfo.vkFormat = VK_FORMAT_R5G6B5_UNORM_PACK16;
363 break;
364 case kGray_8_SkColorType:
365 formatInfo.isSupported = usingGL;
366 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
367 formatInfo.format = GL_LUMINANCE;
368 formatInfo.type = GL_UNSIGNED_BYTE;
369 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
370 break;
371 case kRGBA_1010102_SkColorType:
372 formatInfo.isSupported = HardwareBitmapUploader::has1010102Support();
373 if (formatInfo.isSupported) {
374 formatInfo.type = GL_UNSIGNED_INT_2_10_10_10_REV;
375 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R10G10B10A2_UNORM;
376 formatInfo.vkFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32;
377 } else {
378 formatInfo.type = GL_UNSIGNED_BYTE;
379 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
380 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
381 }
382 formatInfo.format = GL_RGBA;
383 break;
384 case kRGBA_10x6_SkColorType:
385 formatInfo.isSupported = HardwareBitmapUploader::has10101010Support();
386 if (formatInfo.isSupported) {
387 formatInfo.type = 0; // Not supported in GL
388 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R10G10B10A10_UNORM;
389 formatInfo.vkFormat = VK_FORMAT_R10X6G10X6B10X6A10X6_UNORM_4PACK16;
390 } else {
391 formatInfo.type = GL_UNSIGNED_BYTE;
392 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
393 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
394 }
395 formatInfo.format = 0; // Not supported in GL
396 break;
397 case kAlpha_8_SkColorType:
398 formatInfo.isSupported = HardwareBitmapUploader::hasAlpha8Support();
399 if (formatInfo.isSupported) {
400 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8_UNORM;
401 formatInfo.format = GL_RED;
402 formatInfo.type = GL_UNSIGNED_BYTE;
403 formatInfo.vkFormat = VK_FORMAT_R8_UNORM;
404 } else {
405 formatInfo.type = GL_UNSIGNED_BYTE;
406 formatInfo.bufferFormat = AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM;
407 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
408 formatInfo.format = GL_RGBA;
409 }
410 break;
411 default:
412 ALOGW("unable to create hardware bitmap of colortype: %d", skBitmap.info().colorType());
413 formatInfo.valid = false;
414 }
415 return formatInfo;
416 }
417
makeHwCompatible(const FormatInfo & format,const SkBitmap & source)418 static SkBitmap makeHwCompatible(const FormatInfo& format, const SkBitmap& source) {
419 if (format.isSupported) {
420 return source;
421 } else {
422 SkBitmap bitmap;
423 bitmap.allocPixels(source.info().makeColorType(kN32_SkColorType));
424 bitmap.writePixels(source.pixmap());
425 return bitmap;
426 }
427 }
428
429
createUploader(bool usingGL)430 static void createUploader(bool usingGL) {
431 static std::mutex lock;
432 std::lock_guard _lock{lock};
433 if (!sUploader.get()) {
434 if (usingGL) {
435 sUploader = new EGLUploader();
436 } else {
437 sUploader = new VkUploader();
438 }
439 }
440 }
441
allocateHardwareBitmap(const SkBitmap & sourceBitmap)442 sk_sp<Bitmap> HardwareBitmapUploader::allocateHardwareBitmap(const SkBitmap& sourceBitmap) {
443 ATRACE_CALL();
444
445 bool usingGL = uirenderer::Properties::getRenderPipelineType() ==
446 uirenderer::RenderPipelineType::SkiaGL;
447
448 FormatInfo format = determineFormat(sourceBitmap, usingGL);
449 if (!format.valid) {
450 return nullptr;
451 }
452
453 SkBitmap bitmap = makeHwCompatible(format, sourceBitmap);
454 AHardwareBuffer_Desc desc = {
455 .width = static_cast<uint32_t>(bitmap.width()),
456 .height = static_cast<uint32_t>(bitmap.height()),
457 .layers = 1,
458 .format = format.bufferFormat,
459 .usage = AHARDWAREBUFFER_USAGE_CPU_READ_NEVER | AHARDWAREBUFFER_USAGE_CPU_WRITE_NEVER |
460 AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE,
461 };
462 UniqueAHardwareBuffer ahb = allocateAHardwareBuffer(desc);
463 if (!ahb) {
464 ALOGW("allocateHardwareBitmap() failed in AHardwareBuffer_allocate()");
465 return nullptr;
466 };
467
468 createUploader(usingGL);
469
470 if (!sUploader->uploadHardwareBitmap(bitmap, format, ahb.get())) {
471 return nullptr;
472 }
473 return Bitmap::createFrom(ahb.get(), bitmap.colorType(), bitmap.refColorSpace(),
474 bitmap.alphaType(), Bitmap::computePalette(bitmap));
475 }
476
initialize()477 void HardwareBitmapUploader::initialize() {
478 bool usingGL = uirenderer::Properties::getRenderPipelineType() ==
479 uirenderer::RenderPipelineType::SkiaGL;
480 createUploader(usingGL);
481 }
482
terminate()483 void HardwareBitmapUploader::terminate() {
484 if (sUploader) {
485 sUploader->destroy();
486 }
487 }
488
489 } // namespace android::uirenderer
490