1 // Copyright 2023 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 //#define LOG_NDEBUG 0
6 #define ATRACE_TAG ATRACE_TAG_VIDEO
7 #define LOG_TAG "DecodeComponent"
8
9 #include <v4l2_codec2/components/DecodeComponent.h>
10
11 #include <inttypes.h>
12 #include <linux/videodev2.h>
13 #include <stdint.h>
14
15 #include <memory>
16
17 #include <C2.h>
18 #include <C2PlatformSupport.h>
19 #include <Codec2Mapper.h>
20 #include <SimpleC2Interface.h>
21 #include <base/bind.h>
22 #include <base/callback_helpers.h>
23 #include <base/strings/stringprintf.h>
24 #include <base/time/time.h>
25 #include <cutils/properties.h>
26 #include <log/log.h>
27 #include <media/stagefright/foundation/ColorUtils.h>
28 #include <utils/Trace.h>
29
30 #include <v4l2_codec2/common/Common.h>
31 #include <v4l2_codec2/common/H264NalParser.h>
32 #include <v4l2_codec2/common/HEVCNalParser.h>
33 #include <v4l2_codec2/common/VideoTypes.h>
34 #include <v4l2_codec2/components/BitstreamBuffer.h>
35 #include <v4l2_codec2/components/VideoFramePool.h>
36
37 namespace android {
38 namespace {
39
40 // Mask against 30 bits to avoid (undefined) wraparound on signed integer.
frameIndexToBitstreamId(c2_cntr64_t frameIndex)41 int32_t frameIndexToBitstreamId(c2_cntr64_t frameIndex) {
42 return static_cast<int32_t>(frameIndex.peeku() & 0x3FFFFFFF);
43 }
44
parseCodedColorAspects(const C2ConstLinearBlock & input,std::optional<VideoCodec> codec,C2StreamColorAspectsInfo::input * codedAspects)45 bool parseCodedColorAspects(const C2ConstLinearBlock& input, std::optional<VideoCodec> codec,
46 C2StreamColorAspectsInfo::input* codedAspects) {
47 C2ReadView view = input.map().get();
48 NalParser::ColorAspects aspects;
49 std::unique_ptr<NalParser> parser;
50 if (codec == VideoCodec::H264) {
51 parser = std::make_unique<H264NalParser>(view.data(), view.capacity());
52 } else if (codec == VideoCodec::HEVC) {
53 parser = std::make_unique<HEVCNalParser>(view.data(), view.capacity());
54 } else {
55 ALOGV("Unsupported codec for finding color aspects");
56 return false;
57 }
58
59 if (!parser->locateSPS()) {
60 ALOGV("Couldn't find SPS");
61 return false;
62 }
63
64 if (!parser->findCodedColorAspects(&aspects)) {
65 ALOGV("Couldn't find color description in SPS");
66 return false;
67 }
68
69 // Convert ISO color aspects to ColorUtils::ColorAspects.
70 ColorAspects colorAspects;
71 ColorUtils::convertIsoColorAspectsToCodecAspects(
72 aspects.primaries, aspects.transfer, aspects.coeffs, aspects.fullRange, colorAspects);
73 ALOGV("Parsed ColorAspects from bitstream: (R:%d, P:%d, M:%d, T:%d)", colorAspects.mRange,
74 colorAspects.mPrimaries, colorAspects.mMatrixCoeffs, colorAspects.mTransfer);
75
76 // Map ColorUtils::ColorAspects to C2StreamColorAspectsInfo::input parameter.
77 if (!C2Mapper::map(colorAspects.mPrimaries, &codedAspects->primaries)) {
78 codedAspects->primaries = C2Color::PRIMARIES_UNSPECIFIED;
79 }
80 if (!C2Mapper::map(colorAspects.mRange, &codedAspects->range)) {
81 codedAspects->range = C2Color::RANGE_UNSPECIFIED;
82 }
83 if (!C2Mapper::map(colorAspects.mMatrixCoeffs, &codedAspects->matrix)) {
84 codedAspects->matrix = C2Color::MATRIX_UNSPECIFIED;
85 }
86 if (!C2Mapper::map(colorAspects.mTransfer, &codedAspects->transfer)) {
87 codedAspects->transfer = C2Color::TRANSFER_UNSPECIFIED;
88 }
89
90 return true;
91 }
92
isWorkDone(const C2Work & work)93 bool isWorkDone(const C2Work& work) {
94 const int32_t bitstreamId = frameIndexToBitstreamId(work.input.ordinal.frameIndex);
95
96 // Exception: EOS work should be processed by reportEOSWork().
97 // Always return false here no matter the work is actually done.
98 if (work.input.flags & C2FrameData::FLAG_END_OF_STREAM) return false;
99
100 // Work is done when all conditions meet:
101 // 1. mDecoder has released the work's input buffer.
102 // 2. mDecoder has returned the work's output buffer in normal case,
103 // or the input buffer is CSD, or we decide to drop the frame.
104 bool inputReleased = (work.input.buffers.front() == nullptr);
105 bool outputReturned = !work.worklets.front()->output.buffers.empty();
106 bool ignoreOutput = (work.input.flags & C2FrameData::FLAG_CODEC_CONFIG) ||
107 (work.worklets.front()->output.flags & C2FrameData::FLAG_DROP_FRAME);
108 ALOGV("work(%d): inputReleased: %d, outputReturned: %d, ignoreOutput: %d", bitstreamId,
109 inputReleased, outputReturned, ignoreOutput);
110 return inputReleased && (outputReturned || ignoreOutput);
111 }
112
isNoShowFrameWork(const C2Work & work,const C2WorkOrdinalStruct & currOrdinal)113 bool isNoShowFrameWork(const C2Work& work, const C2WorkOrdinalStruct& currOrdinal) {
114 // We consider Work contains no-show frame when all conditions meet:
115 // 1. Work's ordinal is smaller than current ordinal.
116 // 2. Work's output buffer is not returned.
117 // 3. Work is not EOS, CSD, or marked with dropped frame.
118 bool smallOrdinal = (work.input.ordinal.timestamp < currOrdinal.timestamp) &&
119 (work.input.ordinal.frameIndex < currOrdinal.frameIndex);
120 bool outputReturned = !work.worklets.front()->output.buffers.empty();
121 bool specialWork = (work.input.flags & C2FrameData::FLAG_END_OF_STREAM) ||
122 (work.input.flags & C2FrameData::FLAG_CODEC_CONFIG) ||
123 (work.worklets.front()->output.flags & C2FrameData::FLAG_DROP_FRAME);
124 return smallOrdinal && !outputReturned && !specialWork;
125 }
126
127 } // namespace
128
DecodeComponent(uint32_t debugStreamId,const std::string & name,c2_node_id_t id,const std::shared_ptr<DecodeInterface> & intfImpl)129 DecodeComponent::DecodeComponent(uint32_t debugStreamId, const std::string& name, c2_node_id_t id,
130 const std::shared_ptr<DecodeInterface>& intfImpl)
131 : mDebugStreamId(debugStreamId),
132 mIntfImpl(intfImpl),
133 mIntf(std::make_shared<SimpleInterface<DecodeInterface>>(name.c_str(), id, mIntfImpl)) {
134 ALOGV("%s(%s)", __func__, name.c_str());
135 mIsSecure = name.find(".secure") != std::string::npos;
136 }
137
~DecodeComponent()138 DecodeComponent::~DecodeComponent() {
139 ALOGV("%s()", __func__);
140 if (mDecoderThread.IsRunning() && !mDecoderTaskRunner->RunsTasksInCurrentSequence()) {
141 mDecoderTaskRunner->PostTask(FROM_HERE,
142 ::base::BindOnce(&DecodeComponent::releaseTask, mWeakThis));
143 mDecoderThread.Stop();
144 }
145 ALOGV("%s() done", __func__);
146 }
147
start()148 c2_status_t DecodeComponent::start() {
149 ALOGV("%s()", __func__);
150 std::lock_guard<std::mutex> lock(mStartStopLock);
151
152 auto currentState = mComponentState.load();
153 if (currentState != ComponentState::STOPPED) {
154 ALOGE("Could not start at %s state", ComponentStateToString(currentState));
155 return C2_BAD_STATE;
156 }
157
158 if (!mDecoderThread.Start()) {
159 ALOGE("Decoder thread failed to start.");
160 return C2_CORRUPTED;
161 }
162 mDecoderTaskRunner = mDecoderThread.task_runner();
163 mWeakThis = mWeakThisFactory.GetWeakPtr();
164
165 c2_status_t status = C2_CORRUPTED;
166 ::base::WaitableEvent done;
167 mDecoderTaskRunner->PostTask(
168 FROM_HERE, ::base::BindOnce(&DecodeComponent::startTask, mWeakThis,
169 ::base::Unretained(&status), ::base::Unretained(&done)));
170 done.Wait();
171
172 if (status == C2_OK) mComponentState.store(ComponentState::RUNNING);
173 return status;
174 }
175
getVideoFramePool(const ui::Size & size,HalPixelFormat pixelFormat,size_t numBuffers)176 std::unique_ptr<VideoFramePool> DecodeComponent::getVideoFramePool(const ui::Size& size,
177 HalPixelFormat pixelFormat,
178 size_t numBuffers) {
179 ALOGV("%s()", __func__);
180 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
181
182 auto sharedThis = weak_from_this().lock();
183 if (sharedThis == nullptr) {
184 ALOGE("%s(): DecodeComponent instance is destroyed.", __func__);
185 return nullptr;
186 }
187
188 // (b/157113946): Prevent malicious dynamic resolution change exhausts system memory.
189 constexpr int kMaximumSupportedArea = 4096 * 4096;
190 if (getArea(size).value_or(INT_MAX) > kMaximumSupportedArea) {
191 ALOGE("The output size (%dx%d) is larger than supported size (4096x4096)", size.width,
192 size.height);
193 reportError(C2_BAD_VALUE);
194 return nullptr;
195 }
196
197 // Get block pool ID configured from the client.
198 auto poolId = mIntfImpl->getBlockPoolId();
199 ALOGI("Using C2BlockPool ID = %" PRIu64 " for allocating output buffers", poolId);
200 std::shared_ptr<C2BlockPool> blockPool;
201 auto status = GetCodec2BlockPool(poolId, std::move(sharedThis), &blockPool);
202 if (status != C2_OK) {
203 ALOGE("Graphic block allocator is invalid: %d", status);
204 reportError(status);
205 return nullptr;
206 }
207
208 return VideoFramePool::Create(std::move(blockPool), numBuffers, size, pixelFormat, mIsSecure,
209 mDecoderTaskRunner);
210 }
211
stop()212 c2_status_t DecodeComponent::stop() {
213 ALOGV("%s()", __func__);
214 std::lock_guard<std::mutex> lock(mStartStopLock);
215
216 auto currentState = mComponentState.load();
217 if (currentState != ComponentState::RUNNING && currentState != ComponentState::ERROR) {
218 ALOGE("Could not stop at %s state", ComponentStateToString(currentState));
219 return C2_BAD_STATE;
220 }
221
222 if (mDecoderThread.IsRunning()) {
223 mDecoderTaskRunner->PostTask(FROM_HERE,
224 ::base::BindOnce(&DecodeComponent::stopTask, mWeakThis));
225 mDecoderThread.Stop();
226 mDecoderTaskRunner = nullptr;
227 }
228
229 mComponentState.store(ComponentState::STOPPED);
230 return C2_OK;
231 }
232
stopTask()233 void DecodeComponent::stopTask() {
234 ATRACE_CALL();
235 ALOGV("%s()", __func__);
236 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
237
238 reportAbandonedWorks();
239 mIsDraining = false;
240
241 releaseTask();
242 }
243
reset()244 c2_status_t DecodeComponent::reset() {
245 ALOGV("%s()", __func__);
246 auto currentState = mComponentState.load();
247
248 if (currentState == ComponentState::STOPPED)
249 return C2_OK;
250
251 return stop();
252 }
253
release()254 c2_status_t DecodeComponent::release() {
255 ALOGV("%s()", __func__);
256 std::lock_guard<std::mutex> lock(mStartStopLock);
257
258 if (mDecoderThread.IsRunning()) {
259 mDecoderTaskRunner->PostTask(FROM_HERE,
260 ::base::BindOnce(&DecodeComponent::releaseTask, mWeakThis));
261 mDecoderThread.Stop();
262 mDecoderTaskRunner = nullptr;
263 }
264
265 mComponentState.store(ComponentState::RELEASED);
266 return C2_OK;
267 }
268
releaseTask()269 void DecodeComponent::releaseTask() {
270 ATRACE_CALL();
271 ALOGV("%s()", __func__);
272 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
273
274 mWeakThisFactory.InvalidateWeakPtrs();
275 mDecoder = nullptr;
276 }
277
setListener_vb(const std::shared_ptr<C2Component::Listener> & listener,c2_blocking_t mayBlock)278 c2_status_t DecodeComponent::setListener_vb(const std::shared_ptr<C2Component::Listener>& listener,
279 c2_blocking_t mayBlock) {
280 ALOGV("%s()", __func__);
281
282 auto currentState = mComponentState.load();
283 if (currentState == ComponentState::RELEASED ||
284 (currentState == ComponentState::RUNNING && listener)) {
285 ALOGE("Could not set listener at %s state", ComponentStateToString(currentState));
286 return C2_BAD_STATE;
287 }
288 if (currentState == ComponentState::RUNNING && mayBlock != C2_MAY_BLOCK) {
289 ALOGE("Could not set listener at %s state non-blocking",
290 ComponentStateToString(currentState));
291 return C2_BLOCKING;
292 }
293
294 // If the decoder thread is not running it's safe to update the listener directly.
295 if (!mDecoderThread.IsRunning()) {
296 mListener = listener;
297 return C2_OK;
298 }
299
300 ::base::WaitableEvent done;
301 mDecoderTaskRunner->PostTask(
302 FROM_HERE, ::base::Bind(&DecodeComponent::setListenerTask, mWeakThis, listener, &done));
303 done.Wait();
304 return C2_OK;
305 }
306
setListenerTask(const std::shared_ptr<Listener> & listener,::base::WaitableEvent * done)307 void DecodeComponent::setListenerTask(const std::shared_ptr<Listener>& listener,
308 ::base::WaitableEvent* done) {
309 ALOGV("%s()", __func__);
310 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
311
312 mListener = listener;
313 done->Signal();
314 }
315
queue_nb(std::list<std::unique_ptr<C2Work>> * const items)316 c2_status_t DecodeComponent::queue_nb(std::list<std::unique_ptr<C2Work>>* const items) {
317 ALOGV("%s()", __func__);
318
319 auto currentState = mComponentState.load();
320 if (currentState != ComponentState::RUNNING) {
321 ALOGE("Could not queue at state: %s", ComponentStateToString(currentState));
322 return C2_BAD_STATE;
323 }
324
325 while (!items->empty()) {
326 if (ATRACE_ENABLED()) {
327 const std::string atraceLabel = ::base::StringPrintf("#%u C2Work", mDebugStreamId);
328 ATRACE_ASYNC_BEGIN(atraceLabel.c_str(),
329 items->front()->input.ordinal.frameIndex.peekull());
330 }
331
332 mDecoderTaskRunner->PostTask(FROM_HERE,
333 ::base::BindOnce(&DecodeComponent::queueTask, mWeakThis,
334 std::move(items->front())));
335 items->pop_front();
336 }
337 return C2_OK;
338 }
339
queueTask(std::unique_ptr<C2Work> work)340 void DecodeComponent::queueTask(std::unique_ptr<C2Work> work) {
341 ATRACE_CALL();
342 ALOGV("%s(): flags=0x%x, index=%llu, timestamp=%llu", __func__, work->input.flags,
343 work->input.ordinal.frameIndex.peekull(), work->input.ordinal.timestamp.peekull());
344 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
345
346 if (work->worklets.size() != 1u || work->input.buffers.size() > 1u) {
347 ALOGE("Invalid work: worklets.size()=%zu, input.buffers.size()=%zu", work->worklets.size(),
348 work->input.buffers.size());
349 work->result = C2_CORRUPTED;
350 reportWork(std::move(work));
351 return;
352 }
353
354 work->worklets.front()->output.flags = static_cast<C2FrameData::flags_t>(0);
355 work->worklets.front()->output.buffers.clear();
356 work->worklets.front()->output.ordinal = work->input.ordinal;
357 if (work->input.buffers.empty()) {
358 // Client may queue a work with no input buffer for either it's EOS or empty CSD, otherwise
359 // every work must have one input buffer.
360 if ((work->input.flags & C2FrameData::FLAG_END_OF_STREAM) == 0 &&
361 (work->input.flags & C2FrameData::FLAG_CODEC_CONFIG) == 0) {
362 ALOGE("Invalid work: work with no input buffer should be EOS or CSD.");
363 reportError(C2_BAD_VALUE);
364 return;
365 }
366
367 // Emplace a nullptr to unify the check for work done.
368 ALOGV("Got a work with no input buffer! Emplace a nullptr inside.");
369 work->input.buffers.emplace_back(nullptr);
370 }
371
372 mPendingWorks.push(std::move(work));
373 pumpPendingWorks();
374 }
375
pumpPendingWorks()376 void DecodeComponent::pumpPendingWorks() {
377 ATRACE_CALL();
378 ALOGV("%s()", __func__);
379 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
380
381 auto currentState = mComponentState.load();
382 if (currentState != ComponentState::RUNNING) {
383 ALOGW("Could not pump C2Work at state: %s", ComponentStateToString(currentState));
384 return;
385 }
386
387 while (!mPendingWorks.empty() && !mIsDraining) {
388 std::unique_ptr<C2Work> pendingWork(std::move(mPendingWorks.front()));
389 mPendingWorks.pop();
390
391 const int32_t bitstreamId = frameIndexToBitstreamId(pendingWork->input.ordinal.frameIndex);
392 const bool isCSDWork = pendingWork->input.flags & C2FrameData::FLAG_CODEC_CONFIG;
393 const bool isEmptyWork = pendingWork->input.buffers.front() == nullptr;
394 const bool isEOSWork = pendingWork->input.flags & C2FrameData::FLAG_END_OF_STREAM;
395 const C2Work* work = pendingWork.get();
396 ALOGV("Process C2Work bitstreamId=%d isCSDWork=%d, isEmptyWork=%d", bitstreamId, isCSDWork,
397 isEmptyWork);
398
399 auto res = mWorksAtDecoder.insert(std::make_pair(bitstreamId, std::move(pendingWork)));
400 ALOGW_IF(!res.second, "We already inserted bitstreamId %d to decoder?", bitstreamId);
401
402 if (!isEmptyWork) {
403 if (isCSDWork) {
404 processCSDWork(bitstreamId, work);
405 } else {
406 processWork(bitstreamId, work);
407 }
408 }
409
410 if (isEOSWork) {
411 mDecoder->drain(::base::BindOnce(&DecodeComponent::onDrainDone, mWeakThis));
412 mIsDraining = true;
413 }
414
415 // Directly report the empty CSD work as finished.
416 if (isCSDWork && isEmptyWork) reportWorkIfFinished(bitstreamId);
417 }
418 }
419
processCSDWork(const int32_t bitstreamId,const C2Work * work)420 void DecodeComponent::processCSDWork(const int32_t bitstreamId, const C2Work* work) {
421 // If input.buffers is not empty, the buffer should have meaningful content inside.
422 C2ConstLinearBlock linearBlock = work->input.buffers.front()->data().linearBlocks().front();
423 ALOG_ASSERT(linearBlock.size() > 0u, "Input buffer of work(%d) is empty.", bitstreamId);
424
425 if (mIntfImpl->getVideoCodec() == VideoCodec::VP9) {
426 // The VP9 decoder does not support and does not need the Codec Specific Data (CSD):
427 // https://www.webmproject.org/docs/container/#vp9-codec-feature-metadata-codecprivate.
428 // The most of its content (profile, level, bit depth and chroma subsampling)
429 // can be extracted directly from VP9 bitstream. Ignore CSD if it was passed.
430 reportWorkIfFinished(bitstreamId);
431 return;
432 } else if ((!mIsSecure && mIntfImpl->getVideoCodec() == VideoCodec::H264) ||
433 mIntfImpl->getVideoCodec() == VideoCodec::HEVC) {
434 // Try to parse color aspects from bitstream for CSD work of non-secure H264 codec or HEVC
435 // codec (HEVC will only be CENCv3 which is parseable for secure).
436 C2StreamColorAspectsInfo::input codedAspects = {0u};
437 if (parseCodedColorAspects(linearBlock, mIntfImpl->getVideoCodec(), &codedAspects)) {
438 std::vector<std::unique_ptr<C2SettingResult>> failures;
439 c2_status_t status = mIntfImpl->config({&codedAspects}, C2_MAY_BLOCK, &failures);
440 if (status != C2_OK) {
441 ALOGE("Failed to config color aspects to interface: %d", status);
442 reportError(status);
443 return;
444 }
445 // Record current frame index, color aspects should be updated only for output
446 // buffers whose frame indices are not less than this one.
447 mPendingColorAspectsChange = true;
448 mPendingColorAspectsChangeFrameIndex = work->input.ordinal.frameIndex.peeku();
449 }
450 }
451
452 processWorkBuffer(bitstreamId, linearBlock);
453 }
454
processWork(const int32_t bitstreamId,const C2Work * work)455 void DecodeComponent::processWork(const int32_t bitstreamId, const C2Work* work) {
456 // If input.buffers is not empty, the buffer should have meaningful content inside.
457 C2ConstLinearBlock linearBlock = work->input.buffers.front()->data().linearBlocks().front();
458 ALOG_ASSERT(linearBlock.size() > 0u, "Input buffer of work(%d) is empty.", bitstreamId);
459
460 processWorkBuffer(bitstreamId, linearBlock);
461 }
462
processWorkBuffer(const int32_t bitstreamId,const C2ConstLinearBlock & linearBlock)463 void DecodeComponent::processWorkBuffer(const int32_t bitstreamId,
464 const C2ConstLinearBlock& linearBlock) {
465 std::unique_ptr<ConstBitstreamBuffer> buffer = std::make_unique<ConstBitstreamBuffer>(
466 bitstreamId, linearBlock, linearBlock.offset(), linearBlock.size());
467 if (!buffer) {
468 reportError(C2_CORRUPTED);
469 return;
470 }
471 mDecoder->decode(std::move(buffer),
472 ::base::BindOnce(&DecodeComponent::onDecodeDone, mWeakThis, bitstreamId));
473 }
474
onDecodeDone(int32_t bitstreamId,VideoDecoder::DecodeStatus status)475 void DecodeComponent::onDecodeDone(int32_t bitstreamId, VideoDecoder::DecodeStatus status) {
476 ATRACE_CALL();
477 ALOGV("%s(bitstreamId=%d, status=%s)", __func__, bitstreamId,
478 VideoDecoder::DecodeStatusToString(status));
479 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
480
481 auto it = mWorksAtDecoder.find(bitstreamId);
482 ALOG_ASSERT(it != mWorksAtDecoder.end());
483 C2Work* work = it->second.get();
484
485 switch (status) {
486 case VideoDecoder::DecodeStatus::kAborted:
487 work->input.buffers.front().reset();
488 work->worklets.front()->output.flags = static_cast<C2FrameData::flags_t>(
489 work->worklets.front()->output.flags & C2FrameData::FLAG_DROP_FRAME);
490 mOutputBitstreamIds.push(bitstreamId);
491
492 pumpReportWork();
493 return;
494
495 case VideoDecoder::DecodeStatus::kError:
496 reportError(C2_CORRUPTED);
497 return;
498
499 case VideoDecoder::DecodeStatus::kOk:
500 // Release the input buffer.
501 work->input.buffers.front().reset();
502
503 // CSD Work doesn't have output buffer, the corresponding onOutputFrameReady() won't be
504 // called. Push the bitstreamId here.
505 if (work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)
506 mOutputBitstreamIds.push(bitstreamId);
507
508 pumpReportWork();
509 return;
510 }
511 }
512
onOutputFrameReady(std::unique_ptr<VideoFrame> frame)513 void DecodeComponent::onOutputFrameReady(std::unique_ptr<VideoFrame> frame) {
514 ALOGV("%s(bitstreamId=%d)", __func__, frame->getBitstreamId());
515 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
516
517 const int32_t bitstreamId = frame->getBitstreamId();
518 auto it = mWorksAtDecoder.find(bitstreamId);
519 if (it == mWorksAtDecoder.end()) {
520 ALOGE("Work with bitstreamId=%d not found, already abandoned?", bitstreamId);
521 reportError(C2_CORRUPTED);
522 return;
523 }
524 C2Work* work = it->second.get();
525
526 C2ConstGraphicBlock constBlock = std::move(frame)->getGraphicBlock();
527 std::shared_ptr<C2Buffer> buffer = C2Buffer::CreateGraphicBuffer(std::move(constBlock));
528 if (mPendingColorAspectsChange &&
529 work->input.ordinal.frameIndex.peeku() >= mPendingColorAspectsChangeFrameIndex) {
530 mIntfImpl->queryColorAspects(&mCurrentColorAspects);
531 mPendingColorAspectsChange = false;
532 }
533 if (mCurrentColorAspects) {
534 buffer->setInfo(mCurrentColorAspects);
535 }
536 work->worklets.front()->output.buffers.emplace_back(std::move(buffer));
537
538 // Check no-show frame by timestamps for VP8/VP9 cases before reporting the current work.
539 if (mIntfImpl->getVideoCodec() == VideoCodec::VP8 ||
540 mIntfImpl->getVideoCodec() == VideoCodec::VP9) {
541 detectNoShowFrameWorksAndReportIfFinished(work->input.ordinal);
542 }
543
544 mOutputBitstreamIds.push(bitstreamId);
545 pumpReportWork();
546 }
547
detectNoShowFrameWorksAndReportIfFinished(const C2WorkOrdinalStruct & currOrdinal)548 void DecodeComponent::detectNoShowFrameWorksAndReportIfFinished(
549 const C2WorkOrdinalStruct& currOrdinal) {
550 ATRACE_CALL();
551 ALOGV("%s()", __func__);
552 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
553
554 std::vector<int32_t> noShowFrameBitstreamIds;
555 for (auto& kv : mWorksAtDecoder) {
556 const int32_t bitstreamId = kv.first;
557 const C2Work* work = kv.second.get();
558
559 // A work in mWorksAtDecoder would be considered to have no-show frame if there is no
560 // corresponding output buffer returned while the one of the work with latter timestamp is
561 // already returned. (VD is outputted in display order.)
562 if (isNoShowFrameWork(*work, currOrdinal)) {
563 work->worklets.front()->output.flags = C2FrameData::FLAG_DROP_FRAME;
564
565 // We need to call reportWorkIfFinished() for all detected no-show frame works. However,
566 // we should do it after the detection loop since reportWorkIfFinished() may erase
567 // entries in |mWorksAtDecoder|.
568 noShowFrameBitstreamIds.push_back(bitstreamId);
569 ALOGV("Detected no-show frame work index=%llu timestamp=%llu",
570 work->input.ordinal.frameIndex.peekull(),
571 work->input.ordinal.timestamp.peekull());
572 }
573 }
574
575 // Try to report works with no-show frame.
576 for (const int32_t bitstreamId : noShowFrameBitstreamIds) reportWorkIfFinished(bitstreamId);
577 }
578
pumpReportWork()579 void DecodeComponent::pumpReportWork() {
580 ATRACE_CALL();
581 ALOGV("%s()", __func__);
582 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
583
584 while (!mOutputBitstreamIds.empty()) {
585 if (!reportWorkIfFinished(mOutputBitstreamIds.front())) break;
586 mOutputBitstreamIds.pop();
587 }
588 }
589
reportWorkIfFinished(int32_t bitstreamId)590 bool DecodeComponent::reportWorkIfFinished(int32_t bitstreamId) {
591 ATRACE_CALL();
592 ALOGV("%s(bitstreamId = %d)", __func__, bitstreamId);
593 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
594
595 // EOS work will not be reported here. reportEOSWork() does it.
596 if (mIsDraining && mWorksAtDecoder.size() == 1u) {
597 ALOGV("work(bitstreamId = %d) is EOS Work.", bitstreamId);
598 return false;
599 }
600
601 auto it = mWorksAtDecoder.find(bitstreamId);
602 if (it == mWorksAtDecoder.end()) {
603 ALOGI("work(bitstreamId = %d) is dropped, skip.", bitstreamId);
604 return true;
605 }
606
607 if (!isWorkDone(*(it->second))) {
608 ALOGV("work(bitstreamId = %d) is not done yet.", bitstreamId);
609 return false;
610 }
611
612 std::unique_ptr<C2Work> work = std::move(it->second);
613 mWorksAtDecoder.erase(it);
614
615 work->result = C2_OK;
616 work->workletsProcessed = static_cast<uint32_t>(work->worklets.size());
617 // A work with neither flags nor output buffer would be treated as no-corresponding
618 // output by C2 framework, and regain pipeline capacity immediately.
619 if (work->worklets.front()->output.flags & C2FrameData::FLAG_DROP_FRAME)
620 work->worklets.front()->output.flags = static_cast<C2FrameData::flags_t>(0);
621
622 return reportWork(std::move(work));
623 }
624
reportEOSWork()625 bool DecodeComponent::reportEOSWork() {
626 ATRACE_CALL();
627 ALOGV("%s()", __func__);
628 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
629
630 const auto it =
631 std::find_if(mWorksAtDecoder.begin(), mWorksAtDecoder.end(), [](const auto& kv) {
632 return kv.second->input.flags & C2FrameData::FLAG_END_OF_STREAM;
633 });
634 if (it == mWorksAtDecoder.end()) {
635 ALOGE("Failed to find EOS work.");
636 return false;
637 }
638
639 std::unique_ptr<C2Work> eosWork(std::move(it->second));
640 mWorksAtDecoder.erase(it);
641
642 eosWork->result = C2_OK;
643 eosWork->workletsProcessed = static_cast<uint32_t>(eosWork->worklets.size());
644 eosWork->worklets.front()->output.flags = C2FrameData::FLAG_END_OF_STREAM;
645 if (!eosWork->input.buffers.empty()) eosWork->input.buffers.front().reset();
646
647 if (!mWorksAtDecoder.empty()) {
648 ALOGW("There are remaining works except EOS work. abandon them.");
649 for (const auto& kv : mWorksAtDecoder) {
650 ALOGW("bitstreamId(%d) => Work index=%llu, timestamp=%llu", kv.first,
651 kv.second->input.ordinal.frameIndex.peekull(),
652 kv.second->input.ordinal.timestamp.peekull());
653 }
654 reportAbandonedWorks();
655 }
656
657 return reportWork(std::move(eosWork));
658 }
659
reportWork(std::unique_ptr<C2Work> work)660 bool DecodeComponent::reportWork(std::unique_ptr<C2Work> work) {
661 ATRACE_CALL();
662 ALOGV("%s(work=%llu)", __func__, work->input.ordinal.frameIndex.peekull());
663 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
664
665 if (!mListener) {
666 ALOGE("mListener is nullptr, setListener_vb() not called?");
667 return false;
668 }
669
670 if (ATRACE_ENABLED()) {
671 const std::string atraceLabel = ::base::StringPrintf("#%u C2Work", mDebugStreamId);
672 ATRACE_ASYNC_END(atraceLabel.c_str(), work->input.ordinal.frameIndex.peekull());
673 }
674 std::list<std::unique_ptr<C2Work>> finishedWorks;
675 finishedWorks.emplace_back(std::move(work));
676 mListener->onWorkDone_nb(weak_from_this(), std::move(finishedWorks));
677 return true;
678 }
679
flush_sm(flush_mode_t mode,std::list<std::unique_ptr<C2Work>> * const)680 c2_status_t DecodeComponent::flush_sm(flush_mode_t mode,
681 std::list<std::unique_ptr<C2Work>>* const /* flushedWork */) {
682 ATRACE_CALL();
683 ALOGV("%s()", __func__);
684
685 auto currentState = mComponentState.load();
686 if (currentState != ComponentState::RUNNING) {
687 ALOGE("Could not flush at state: %s", ComponentStateToString(currentState));
688 return C2_BAD_STATE;
689 }
690 if (mode != FLUSH_COMPONENT) {
691 return C2_OMITTED; // Tunneling is not supported by now
692 }
693
694 mDecoderTaskRunner->PostTask(FROM_HERE,
695 ::base::BindOnce(&DecodeComponent::flushTask, mWeakThis));
696 return C2_OK;
697 }
698
flushTask()699 void DecodeComponent::flushTask() {
700 ATRACE_CALL();
701 ALOGV("%s()", __func__);
702 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
703
704 mDecoder->flush();
705 reportAbandonedWorks();
706
707 // Pending EOS work will be abandoned here due to component flush if any.
708 mIsDraining = false;
709 }
710
reportAbandonedWorks()711 void DecodeComponent::reportAbandonedWorks() {
712 ALOGV("%s()", __func__);
713 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
714
715 std::list<std::unique_ptr<C2Work>> abandonedWorks;
716 while (!mPendingWorks.empty()) {
717 abandonedWorks.emplace_back(std::move(mPendingWorks.front()));
718 mPendingWorks.pop();
719 }
720 for (auto& kv : mWorksAtDecoder) {
721 abandonedWorks.emplace_back(std::move(kv.second));
722 }
723 mWorksAtDecoder.clear();
724
725 for (auto& work : abandonedWorks) {
726 // TODO: correlate the definition of flushed work result to framework.
727 work->result = C2_NOT_FOUND;
728 // When the work is abandoned, buffer in input.buffers shall reset by component.
729 if (!work->input.buffers.empty()) {
730 work->input.buffers.front().reset();
731 }
732
733 if (ATRACE_ENABLED()) {
734 const std::string atraceLabel = ::base::StringPrintf("#%u C2Work", mDebugStreamId);
735 ATRACE_ASYNC_END(atraceLabel.c_str(), work->input.ordinal.frameIndex.peekull());
736 }
737 }
738 if (!abandonedWorks.empty()) {
739 if (!mListener) {
740 ALOGE("mListener is nullptr, setListener_vb() not called?");
741 return;
742 }
743 mListener->onWorkDone_nb(weak_from_this(), std::move(abandonedWorks));
744 }
745 }
746
drain_nb(drain_mode_t mode)747 c2_status_t DecodeComponent::drain_nb(drain_mode_t mode) {
748 ALOGV("%s(mode=%u)", __func__, mode);
749
750 auto currentState = mComponentState.load();
751 if (currentState != ComponentState::RUNNING) {
752 ALOGE("Could not drain at state: %s", ComponentStateToString(currentState));
753 return C2_BAD_STATE;
754 }
755
756 switch (mode) {
757 case DRAIN_CHAIN:
758 return C2_OMITTED; // Tunneling is not supported.
759
760 case DRAIN_COMPONENT_NO_EOS:
761 return C2_OK; // Do nothing special.
762
763 case DRAIN_COMPONENT_WITH_EOS:
764 mDecoderTaskRunner->PostTask(FROM_HERE,
765 ::base::BindOnce(&DecodeComponent::drainTask, mWeakThis));
766 return C2_OK;
767 }
768 }
769
drainTask()770 void DecodeComponent::drainTask() {
771 ATRACE_CALL();
772 ALOGV("%s()", __func__);
773 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
774
775 if (!mPendingWorks.empty()) {
776 ALOGV("Set EOS flag at last queued work.");
777 auto& flags = mPendingWorks.back()->input.flags;
778 flags = static_cast<C2FrameData::flags_t>(flags | C2FrameData::FLAG_END_OF_STREAM);
779 return;
780 }
781
782 if (!mWorksAtDecoder.empty()) {
783 ALOGV("Drain the pending works at the decoder.");
784 mDecoder->drain(::base::BindOnce(&DecodeComponent::onDrainDone, mWeakThis));
785 mIsDraining = true;
786 }
787 }
788
onDrainDone(VideoDecoder::DecodeStatus status)789 void DecodeComponent::onDrainDone(VideoDecoder::DecodeStatus status) {
790 ALOGV("%s(status=%s)", __func__, VideoDecoder::DecodeStatusToString(status));
791 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
792
793 switch (status) {
794 case VideoDecoder::DecodeStatus::kAborted:
795 return;
796
797 case VideoDecoder::DecodeStatus::kError:
798 reportError(C2_CORRUPTED);
799 return;
800
801 case VideoDecoder::DecodeStatus::kOk:
802 mIsDraining = false;
803 if (!reportEOSWork()) {
804 reportError(C2_CORRUPTED);
805 return;
806 }
807
808 mDecoderTaskRunner->PostTask(
809 FROM_HERE, ::base::BindOnce(&DecodeComponent::pumpPendingWorks, mWeakThis));
810 return;
811 }
812 }
813
reportError(c2_status_t error)814 void DecodeComponent::reportError(c2_status_t error) {
815 ALOGE("%s(error=%u)", __func__, static_cast<uint32_t>(error));
816 ALOG_ASSERT(mDecoderTaskRunner->RunsTasksInCurrentSequence());
817
818 if (mComponentState.load() == ComponentState::ERROR) return;
819 mComponentState.store(ComponentState::ERROR);
820
821 if (!mListener) {
822 ALOGE("mListener is nullptr, setListener_vb() not called?");
823 return;
824 }
825 mListener->onError_nb(weak_from_this(), static_cast<uint32_t>(error));
826 }
827
announce_nb(const std::vector<C2WorkOutline> &)828 c2_status_t DecodeComponent::announce_nb(const std::vector<C2WorkOutline>& /* items */) {
829 return C2_OMITTED; // Tunneling is not supported by now
830 }
831
intf()832 std::shared_ptr<C2ComponentInterface> DecodeComponent::intf() {
833 return mIntf;
834 }
835
836 // static
ComponentStateToString(ComponentState state)837 const char* DecodeComponent::ComponentStateToString(ComponentState state) {
838 switch (state) {
839 case ComponentState::STOPPED:
840 return "STOPPED";
841 case ComponentState::RUNNING:
842 return "RUNNING";
843 case ComponentState::RELEASED:
844 return "RELEASED";
845 case ComponentState::ERROR:
846 return "ERROR";
847 }
848 }
849
850 } // namespace android
851