1 // Copyright (C) 2020 The Android Open Source Project
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "host-common/MediaH264DecoderGeneric.h"
16 #include "aemu/base/system/System.h"
17 #include "host-common/H264PingInfoParser.h"
18 #include "host-common/MediaFfmpegVideoHelper.h"
19 #include "android/main-emugl.h"
20 
21 #ifndef __APPLE__
22 // for Linux and Window, Cuvid is available
23 #include "host-common/MediaCudaDriverHelper.h"
24 #include "host-common/MediaCudaVideoHelper.h"
25 #else
26 #include "host-common/MediaVideoToolBoxVideoHelper.h"
27 #endif
28 
29 #include <cstdint>
30 #include <string>
31 #include <vector>
32 
33 #include <stdio.h>
34 #include <string.h>
35 
36 #define MEDIA_H264_DEBUG 0
37 
38 #if MEDIA_H264_DEBUG
39 #define H264_DPRINT(fmt, ...)                                                \
40     fprintf(stderr, "h264-dec-generic: %s:%d " fmt "\n", __func__, __LINE__, \
41             ##__VA_ARGS__);
42 #else
43 #define H264_DPRINT(fmt, ...)
44 #endif
45 
46 namespace android {
47 namespace emulation {
48 
49 using InitContextParam = H264PingInfoParser::InitContextParam;
50 using DecodeFrameParam = H264PingInfoParser::DecodeFrameParam;
51 using ResetParam = H264PingInfoParser::ResetParam;
52 using GetImageParam = H264PingInfoParser::GetImageParam;
53 using TextureFrame = MediaHostRenderer::TextureFrame;
54 
55 namespace {
56 
canUseCudaDecoder()57 bool canUseCudaDecoder() {
58 #ifndef __APPLE__
59     if (MediaCudaDriverHelper::initCudaDrivers()) {
60         H264_DPRINT("Using Cuvid decoder on Linux/Windows");
61         return true;
62     } else {
63         H264_DPRINT(
64                 "ERROR: cannot use cuvid decoder: failed to init cuda driver");
65         return false;
66     }
67 #else
68     return false;
69 #endif
70 }
71 
canDecodeToGpuTexture()72 bool canDecodeToGpuTexture() {
73     if (emuglConfig_get_current_renderer() == SELECTED_RENDERER_HOST) {
74         return true;
75     } else {
76         return false;
77     }
78 }
79 };  // end namespace
80 
MediaH264DecoderGeneric(uint64_t id,H264PingInfoParser parser)81 MediaH264DecoderGeneric::MediaH264DecoderGeneric(uint64_t id,
82                                                  H264PingInfoParser parser)
83     : mId(id), mParser(parser) {
84     H264_DPRINT("allocated MediaH264DecoderGeneric %p with version %d", this,
85                 (int)mParser.version());
86     mUseGpuTexture = canDecodeToGpuTexture();
87 }
88 
~MediaH264DecoderGeneric()89 MediaH264DecoderGeneric::~MediaH264DecoderGeneric() {
90     H264_DPRINT("destroyed MediaH264DecoderGeneric %p", this);
91     destroyH264Context();
92 }
93 
reset(void * ptr)94 void MediaH264DecoderGeneric::reset(void* ptr) {
95     destroyH264Context();
96     ResetParam param{};
97     mParser.parseResetParams(ptr, param);
98     initH264ContextInternal(param.width, param.height, param.outputWidth,
99                             param.outputHeight, param.outputPixelFormat);
100 }
101 
initH264Context(void * ptr)102 void MediaH264DecoderGeneric::initH264Context(void* ptr) {
103     InitContextParam param{};
104     mParser.parseInitContextParams(ptr, param);
105     initH264ContextInternal(param.width, param.height, param.outputWidth,
106                             param.outputHeight, param.outputPixelFormat);
107 }
108 
initH264ContextInternal(unsigned int width,unsigned int height,unsigned int outWidth,unsigned int outHeight,PixelFormat outPixFmt)109 void MediaH264DecoderGeneric::initH264ContextInternal(unsigned int width,
110                                                       unsigned int height,
111                                                       unsigned int outWidth,
112                                                       unsigned int outHeight,
113                                                       PixelFormat outPixFmt) {
114     H264_DPRINT("%s(w=%u h=%u out_w=%u out_h=%u pixfmt=%u)", __func__, width,
115                 height, outWidth, outHeight, (uint8_t)outPixFmt);
116     mWidth = width;
117     mHeight = height;
118     mOutputWidth = outWidth;
119     mOutputHeight = outHeight;
120     mOutPixFmt = outPixFmt;
121 
122 #ifndef __APPLE__
123     if (canUseCudaDecoder() && mParser.version() >= 200) {
124         MediaCudaVideoHelper::OutputTreatmentMode oMode =
125                 MediaCudaVideoHelper::OutputTreatmentMode::SAVE_RESULT;
126 
127         MediaCudaVideoHelper::FrameStorageMode fMode =
128                 mUseGpuTexture ? MediaCudaVideoHelper::FrameStorageMode::
129                                          USE_GPU_TEXTURE
130                                : MediaCudaVideoHelper::FrameStorageMode::
131                                          USE_BYTE_BUFFER;
132 
133         auto cudavid =
134                 new MediaCudaVideoHelper(oMode, fMode, cudaVideoCodec_H264);
135 
136         if (mUseGpuTexture) {
137             H264_DPRINT("use gpu texture");
138             cudavid->resetTexturePool(mRenderer.getTexturePool());
139         }
140         mHwVideoHelper.reset(cudavid);
141         if (!mHwVideoHelper->init()) {
142             mHwVideoHelper.reset(nullptr);
143             H264_DPRINT("failed to init cuda decoder");
144         } else {
145             H264_DPRINT("succeeded to init cuda decoder");
146         }
147     }
148 #else
149     //TODO: once all the CTS passed with VTB, remove this
150     const bool is_vtb_allowed = android::base::System::getEnvironmentVariable(
151                             "ANDROID_EMU_MEDIA_DECODER_VTB") == "1";
152 
153     if (is_vtb_allowed) {
154         MediaVideoToolBoxVideoHelper::FrameStorageMode fMode =
155             (mParser.version() >= 200 && mUseGpuTexture)
156                     ? MediaVideoToolBoxVideoHelper::FrameStorageMode::
157                               USE_GPU_TEXTURE
158                     : MediaVideoToolBoxVideoHelper::FrameStorageMode::
159                               USE_BYTE_BUFFER;
160         auto macDecoder = new MediaVideoToolBoxVideoHelper(
161             mOutputWidth, mOutputHeight,
162             MediaVideoToolBoxVideoHelper::OutputTreatmentMode::SAVE_RESULT,
163             fMode);
164 
165         if (mUseGpuTexture && mParser.version() >= 200) {
166             H264_DPRINT("use gpu texture on OSX");
167             macDecoder->resetTexturePool(mRenderer.getTexturePool());
168         }
169         mHwVideoHelper.reset(macDecoder);
170         mHwVideoHelper->init();
171     }
172 #endif
173 
174     mSnapshotHelper.reset(
175             new MediaSnapshotHelper(MediaSnapshotHelper::CodecType::H264));
176 
177     H264_DPRINT("Successfully created h264 decoder context %p", this);
178 }
179 
createAndInitSoftVideoHelper()180 void MediaH264DecoderGeneric::createAndInitSoftVideoHelper() {
181     mSwVideoHelper.reset(
182             new MediaFfmpegVideoHelper(264, mParser.version() < 200 ? 1 : 4));
183     mUseGpuTexture = false;
184     mSwVideoHelper->init();
185 }
186 
clone()187 MediaH264DecoderPlugin* MediaH264DecoderGeneric::clone() {
188     H264_DPRINT("clone MediaH264DecoderGeneric %p with version %d", this,
189                 (int)mParser.version());
190     return new MediaH264DecoderGeneric(mId, mParser);
191 }
192 
destroyH264Context()193 void MediaH264DecoderGeneric::destroyH264Context() {
194     H264_DPRINT("Destroy context %p", this);
195     while (mSnapshotHelper != nullptr) {
196         MediaSnapshotState::FrameInfo* pFrame = mSnapshotHelper->frontFrame();
197         if (!pFrame) {
198             break;
199         }
200         if (mUseGpuTexture && pFrame->texture[0] > 0 &&
201             pFrame->texture[1] > 0) {
202             mRenderer.putTextureFrame(
203                     TextureFrame{pFrame->texture[0], pFrame->texture[1]});
204         }
205         mSnapshotHelper->discardFrontFrame();
206     }
207     mRenderer.cleanUpTextures();
208 
209     if (mHwVideoHelper != nullptr) {
210         mHwVideoHelper->deInit();
211         mHwVideoHelper.reset(nullptr);
212     }
213     if (mVideoHelper != nullptr) {
214         mVideoHelper->deInit();
215         mVideoHelper.reset(nullptr);
216     }
217 }
218 
decodeFrame(void * ptr)219 void MediaH264DecoderGeneric::decodeFrame(void* ptr) {
220     DecodeFrameParam param{};
221     mParser.parseDecodeFrameParams(ptr, param);
222     const uint8_t* frame = param.pData;
223     size_t szBytes = param.size;
224     uint64_t inputPts = param.pts;
225 
226     H264_DPRINT("%s(frame=%p, sz=%zu pts %lld)", __func__, frame, szBytes,
227                 (long long)inputPts);
228     size_t* retSzBytes = param.pConsumedBytes;
229     int32_t* retErr = param.pDecoderErrorCode;
230 
231     decodeFrameInternal(frame, szBytes, inputPts);
232 
233     mSnapshotHelper->savePacket(frame, szBytes, inputPts);
234     fetchAllFrames();
235 
236     *retSzBytes = szBytes;
237     *retErr = (int32_t)Err::NoErr;
238 
239     H264_DPRINT("done decoding this frame");
240 }
241 
decodeFrameInternal(const uint8_t * data,size_t len,uint64_t pts)242 void MediaH264DecoderGeneric::decodeFrameInternal(const uint8_t* data,
243                                                   size_t len,
244                                                   uint64_t pts) {
245     if (mTrialPeriod) {
246         H264_DPRINT("still in trial period");
247         try_decode(data, len, pts);
248     } else {
249         mVideoHelper->decode(data, len, pts);
250     }
251 }
252 
try_decode(const uint8_t * data,size_t len,uint64_t pts)253 void MediaH264DecoderGeneric::try_decode(const uint8_t* data,
254                                          size_t len,
255                                          uint64_t pts) {
256     // for h264, it needs sps/pps to decide whether decoding is
257     // possible;
258     if (mHwVideoHelper != nullptr) {
259         mHwVideoHelper->decode(data, len, pts);
260         if (mHwVideoHelper->good()) {
261             return;
262         } else {
263             H264_DPRINT("Failed to decode with HW decoder %d; switch to SW",
264                         mHwVideoHelper->error());
265             mHwVideoHelper.reset(nullptr);
266         }
267     }
268 
269     createAndInitSoftVideoHelper();
270     mVideoHelper = std::move(mSwVideoHelper);
271 
272     mVideoHelper->setIgnoreDecodedFrames();
273     std::function<void(const uint8_t*, size_t, uint64_t)> func =
274             [=](const uint8_t* data, size_t len, uint64_t pts) {
275                 this->oneShotDecode(data, len, pts);
276             };
277 
278     mSnapshotHelper->replay(func);
279 
280     mVideoHelper->setSaveDecodedFrames();
281 
282     mVideoHelper->decode(data, len, pts);
283     mTrialPeriod = false;
284 }
285 
fetchAllFrames()286 void MediaH264DecoderGeneric::fetchAllFrames() {
287     while (true) {
288         MediaSnapshotState::FrameInfo frame;
289         bool success =
290                 mHwVideoHelper
291                         ? mHwVideoHelper->receiveFrame(&frame)
292                         : (mVideoHelper ? mVideoHelper->receiveFrame(&frame)
293                                         : false);
294         if (!success) {
295             break;
296         }
297         mSnapshotHelper->saveDecodedFrame(std::move(frame));
298     }
299 }
300 
flush(void * ptr)301 void MediaH264DecoderGeneric::flush(void* ptr) {
302     H264_DPRINT("Flushing...");
303     if (mHwVideoHelper) {
304         mHwVideoHelper->flush();
305     } else if (mVideoHelper) {
306         mVideoHelper->flush();
307     }
308     fetchAllFrames();
309     H264_DPRINT("Flushing done");
310 }
311 
getImage(void * ptr)312 void MediaH264DecoderGeneric::getImage(void* ptr) {
313     H264_DPRINT("getImage %p", ptr);
314     GetImageParam param{};
315     mParser.parseGetImageParams(ptr, param);
316 
317     int* retErr = param.pDecoderErrorCode;
318     uint32_t* retWidth = param.pRetWidth;
319     uint32_t* retHeight = param.pRetHeight;
320     uint64_t* retPts = param.pRetPts;
321     uint32_t* retColorPrimaries = param.pRetColorPrimaries;
322     uint32_t* retColorRange = param.pRetColorRange;
323     uint32_t* retColorTransfer = param.pRetColorTransfer;
324     uint32_t* retColorSpace = param.pRetColorSpace;
325     uint8_t* dst = param.pDecodedFrame;
326 
327     static int numbers = 0;
328     H264_DPRINT("calling getImage %d colorbuffer %d", numbers++,
329                 (int)param.hostColorBufferId);
330 
331     MediaSnapshotState::FrameInfo* pFrame = mSnapshotHelper->frontFrame();
332     if (pFrame == nullptr) {
333         H264_DPRINT("there is no image");
334         *retErr = static_cast<int>(Err::NoDecodedFrame);
335         return;
336     }
337 
338     bool needToCopyToGuest = true;
339     if (mParser.version() == 200) {
340         if (mUseGpuTexture && pFrame->texture[0] > 0 && pFrame->texture[1] > 0) {
341             H264_DPRINT(
342                     "calling rendering to host side color buffer with id %d "
343                     "tex %d tex %d",
344                     param.hostColorBufferId, pFrame->texture[0],
345                     pFrame->texture[1]);
346             mRenderer.renderToHostColorBufferWithTextures(
347                     param.hostColorBufferId, pFrame->width, pFrame->height,
348                     TextureFrame{pFrame->texture[0], pFrame->texture[1]});
349         } else {
350             H264_DPRINT(
351                     "calling rendering to host side color buffer with id %d",
352                     param.hostColorBufferId);
353             mRenderer.renderToHostColorBuffer(param.hostColorBufferId,
354                                               pFrame->width, pFrame->height,
355                                               pFrame->data.data());
356         }
357         needToCopyToGuest = false;
358     }
359 
360     if (needToCopyToGuest) {
361         memcpy(param.pDecodedFrame, pFrame->data.data(),
362                pFrame->width * pFrame->height * 3 / 2);
363     }
364 
365     *retErr = pFrame->width * pFrame->height * 3 / 2;
366     *retWidth = pFrame->width;
367     *retHeight = pFrame->height;
368     *retPts = pFrame->pts;
369     *retColorPrimaries = pFrame->color.primaries;
370     *retColorRange = pFrame->color.range;
371     *retColorSpace = pFrame->color.space;
372     *retColorTransfer = pFrame->color.transfer;
373 
374     mSnapshotHelper->discardFrontFrame();
375 
376     H264_DPRINT("Copying completed pts %lld w %d h %d ow %d oh %d",
377                 (long long)*retPts, (int)*retWidth, (int)*retHeight,
378                 (int)mOutputWidth, (int)mOutputHeight);
379 }
380 
save(base::Stream * stream) const381 void MediaH264DecoderGeneric::save(base::Stream* stream) const {
382     stream->putBe32(mParser.version());
383     stream->putBe32(mWidth);
384     stream->putBe32(mHeight);
385     stream->putBe32((int)mOutPixFmt);
386 
387     const int hasContext = (mVideoHelper || mHwVideoHelper) ? 1 : 0;
388     stream->putBe32(hasContext);
389 
390     mSnapshotHelper->save(stream);
391 }
392 
oneShotDecode(const uint8_t * data,size_t len,uint64_t pts)393 void MediaH264DecoderGeneric::oneShotDecode(const uint8_t* data,
394                                             size_t len,
395                                             uint64_t pts) {
396     if (!mHwVideoHelper && !mSwVideoHelper && !mVideoHelper) {
397         return;
398     }
399 
400     decodeFrameInternal(data, len, pts);
401     while (true) {
402         MediaSnapshotState::FrameInfo frame;
403         bool success = mHwVideoHelper ? mHwVideoHelper->receiveFrame(&frame)
404                                       : mVideoHelper->receiveFrame(&frame);
405         if (!success) {
406             break;
407         }
408     }
409 }
410 
load(base::Stream * stream)411 bool MediaH264DecoderGeneric::load(base::Stream* stream) {
412     uint32_t version = stream->getBe32();
413     mParser = H264PingInfoParser{version};
414 
415     mWidth = stream->getBe32();
416     mHeight = stream->getBe32();
417     mOutPixFmt = (PixelFormat)stream->getBe32();
418 
419     const int hasContext = stream->getBe32();
420     if (hasContext) {
421         initH264ContextInternal(mWidth, mHeight, mWidth, mHeight, mOutPixFmt);
422     }
423 
424     if (mVideoHelper) {
425         mVideoHelper->setIgnoreDecodedFrames();
426     }
427     std::function<void(const uint8_t*, size_t, uint64_t)> func =
428             [=](const uint8_t* data, size_t len, uint64_t pts) {
429                 this->oneShotDecode(data, len, pts);
430             };
431 
432     mSnapshotHelper->load(stream, func);
433 
434     if (mVideoHelper) {
435         mVideoHelper->setSaveDecodedFrames();
436     }
437 
438     H264_DPRINT("Done loading snapshots frames\n\n");
439     return true;
440 }
441 
442 }  // namespace emulation
443 }  // namespace android
444