1 /*
2 * Copyright 2017, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #include <utils/Errors.h>
19 #define LOG_TAG "CCodecBufferChannel"
20 #define ATRACE_TAG ATRACE_TAG_VIDEO
21 #include <utils/Log.h>
22 #include <utils/Trace.h>
23
24 #include <algorithm>
25 #include <atomic>
26 #include <list>
27 #include <numeric>
28 #include <thread>
29 #include <chrono>
30
31 #include <android_media_codec.h>
32
33 #include <C2AllocatorGralloc.h>
34 #include <C2PlatformSupport.h>
35 #include <C2BlockInternal.h>
36 #include <C2Config.h>
37 #include <C2Debug.h>
38
39 #include <android/hardware/cas/native/1.0/IDescrambler.h>
40 #include <android/hardware/drm/1.0/types.h>
41 #include <android/sysprop/MediaProperties.sysprop.h>
42 #include <android-base/parseint.h>
43 #include <android-base/properties.h>
44 #include <android-base/no_destructor.h>
45 #include <android-base/stringprintf.h>
46 #include <binder/MemoryBase.h>
47 #include <binder/MemoryDealer.h>
48 #include <cutils/properties.h>
49 #include <gui/Surface.h>
50 #include <hidlmemory/FrameworkUtils.h>
51 #include <media/openmax/OMX_Core.h>
52 #include <media/stagefright/foundation/ABuffer.h>
53 #include <media/stagefright/foundation/ALookup.h>
54 #include <media/stagefright/foundation/AMessage.h>
55 #include <media/stagefright/foundation/AUtils.h>
56 #include <media/stagefright/foundation/hexdump.h>
57 #include <media/stagefright/MediaCodecConstants.h>
58 #include <media/stagefright/SkipCutBuffer.h>
59 #include <media/stagefright/SurfaceUtils.h>
60 #include <media/MediaCodecBuffer.h>
61 #include <mediadrm/ICrypto.h>
62 #include <server_configurable_flags/get_flags.h>
63 #include <system/window.h>
64
65 #include "CCodecBufferChannel.h"
66 #include "Codec2Buffer.h"
67
68 namespace android {
69
70 using android::base::StringPrintf;
71 using hardware::hidl_handle;
72 using hardware::hidl_string;
73 using hardware::hidl_vec;
74 using hardware::fromHeap;
75 using hardware::HidlMemory;
76 using server_configurable_flags::GetServerConfigurableFlag;
77
78 using namespace hardware::cas::V1_0;
79 using namespace hardware::cas::native::V1_0;
80
81 using CasStatus = hardware::cas::V1_0::Status;
82 using DrmBufferType = hardware::drm::V1_0::BufferType;
83
84 namespace {
85
86 constexpr size_t kSmoothnessFactor = 4;
87
88 // This is for keeping IGBP's buffer dropping logic in legacy mode other
89 // than making it non-blocking. Do not change this value.
90 const static size_t kDequeueTimeoutNs = 0;
91
areRenderMetricsEnabled()92 static bool areRenderMetricsEnabled() {
93 std::string v = GetServerConfigurableFlag("media_native", "render_metrics_enabled", "false");
94 return v == "true";
95 }
96
97 // Flags can come with individual BufferInfos
98 // when used with large frame audio
99 constexpr static std::initializer_list<std::pair<uint32_t, uint32_t>> flagList = {
100 {BUFFER_FLAG_CODEC_CONFIG, C2FrameData::FLAG_CODEC_CONFIG},
101 {BUFFER_FLAG_END_OF_STREAM, C2FrameData::FLAG_END_OF_STREAM},
102 {BUFFER_FLAG_DECODE_ONLY, C2FrameData::FLAG_DROP_FRAME}
103 };
104
convertFlags(uint32_t flags,bool toC2)105 static uint32_t convertFlags(uint32_t flags, bool toC2) {
106 return std::transform_reduce(
107 flagList.begin(), flagList.end(),
108 0u,
109 std::bit_or{},
110 [flags, toC2](const std::pair<uint32_t, uint32_t> &entry) {
111 if (toC2) {
112 return (flags & entry.first) ? entry.second : 0;
113 } else {
114 return (flags & entry.second) ? entry.first : 0;
115 }
116 });
117 }
118
119 class SurfaceCallbackHandler {
120 public:
121 enum callback_type_t {
122 ON_BUFFER_RELEASED = 0,
123 ON_BUFFER_ATTACHED
124 };
125
post(callback_type_t callback,std::shared_ptr<Codec2Client::Component> component,uint32_t generation)126 void post(callback_type_t callback,
127 std::shared_ptr<Codec2Client::Component> component,
128 uint32_t generation) {
129 if (!component) {
130 ALOGW("surface callback psoted for invalid component");
131 }
132 std::shared_ptr<SurfaceCallbackItem> item =
133 std::make_shared<SurfaceCallbackItem>(callback, component, generation);
134 std::unique_lock<std::mutex> lock(mMutex);
135 mItems.emplace_back(std::move(item));
136 mCv.notify_one();
137 }
138
~SurfaceCallbackHandler()139 ~SurfaceCallbackHandler() {
140 {
141 std::unique_lock<std::mutex> lock(mMutex);
142 mDone = true;
143 mCv.notify_all();
144 }
145 if (mThread.joinable()) {
146 mThread.join();
147 }
148 }
149
GetInstance()150 static SurfaceCallbackHandler& GetInstance() {
151 static base::NoDestructor<SurfaceCallbackHandler> sSurfaceCallbackHandler{};
152 return *sSurfaceCallbackHandler;
153 }
154
155 private:
156 struct SurfaceCallbackItem {
157 callback_type_t mCallback;
158 std::weak_ptr<Codec2Client::Component> mComp;
159 uint32_t mGeneration;
160
SurfaceCallbackItemandroid::__anon154d4c550111::SurfaceCallbackHandler::SurfaceCallbackItem161 SurfaceCallbackItem(
162 callback_type_t callback,
163 std::shared_ptr<Codec2Client::Component> comp,
164 uint32_t generation)
165 : mCallback(callback), mComp(comp), mGeneration(generation) {}
166 };
167
SurfaceCallbackHandler()168 SurfaceCallbackHandler() { mThread = std::thread(&SurfaceCallbackHandler::run, this); }
169
run()170 void run() {
171 std::unique_lock<std::mutex> lock(mMutex);
172 while (!mDone) {
173 while (!mItems.empty()) {
174 std::deque<std::shared_ptr<SurfaceCallbackItem>> items = std::move(mItems);
175 mItems.clear();
176 lock.unlock();
177 handle(items);
178 lock.lock();
179 }
180 mCv.wait(lock);
181 }
182 }
183
handle(std::deque<std::shared_ptr<SurfaceCallbackItem>> & items)184 void handle(std::deque<std::shared_ptr<SurfaceCallbackItem>> &items) {
185 while (!items.empty()) {
186 std::shared_ptr<SurfaceCallbackItem> item = items.front();
187 items.pop_front();
188 switch (item->mCallback) {
189 case ON_BUFFER_RELEASED: {
190 std::shared_ptr<Codec2Client::Component> comp = item->mComp.lock();;
191 if (comp) {
192 comp->onBufferReleasedFromOutputSurface(item->mGeneration);
193 }
194 break;
195 }
196 case ON_BUFFER_ATTACHED: {
197 std::shared_ptr<Codec2Client::Component> comp = item->mComp.lock();
198 if (comp) {
199 comp->onBufferAttachedToOutputSurface(item->mGeneration);
200 }
201 break;
202 }
203 default:
204 ALOGE("Non defined surface callback message");
205 break;
206 }
207 }
208 }
209
210 std::thread mThread;
211 bool mDone = false;
212 std::deque<std::shared_ptr<SurfaceCallbackItem>> mItems;
213 std::mutex mMutex;
214 std::condition_variable mCv;
215
216
217 friend class base::NoDestructor<SurfaceCallbackHandler>;
218
219 DISALLOW_EVIL_CONSTRUCTORS(SurfaceCallbackHandler);
220 };
221
222 } // namespace
223
QueueGuard(CCodecBufferChannel::QueueSync & sync)224 CCodecBufferChannel::QueueGuard::QueueGuard(
225 CCodecBufferChannel::QueueSync &sync) : mSync(sync) {
226 Mutex::Autolock l(mSync.mGuardLock);
227 // At this point it's guaranteed that mSync is not under state transition,
228 // as we are holding its mutex.
229
230 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
231 if (count->value == -1) {
232 mRunning = false;
233 } else {
234 ++count->value;
235 mRunning = true;
236 }
237 }
238
~QueueGuard()239 CCodecBufferChannel::QueueGuard::~QueueGuard() {
240 if (mRunning) {
241 // We are not holding mGuardLock at this point so that QueueSync::stop() can
242 // keep holding the lock until mCount reaches zero.
243 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
244 --count->value;
245 count->cond.broadcast();
246 }
247 }
248
start()249 void CCodecBufferChannel::QueueSync::start() {
250 Mutex::Autolock l(mGuardLock);
251 // If stopped, it goes to running state; otherwise no-op.
252 Mutexed<Counter>::Locked count(mCount);
253 if (count->value == -1) {
254 count->value = 0;
255 }
256 }
257
stop()258 void CCodecBufferChannel::QueueSync::stop() {
259 Mutex::Autolock l(mGuardLock);
260 Mutexed<Counter>::Locked count(mCount);
261 if (count->value == -1) {
262 // no-op
263 return;
264 }
265 // Holding mGuardLock here blocks creation of additional QueueGuard objects, so
266 // mCount can only decrement. In other words, threads that acquired the lock
267 // are allowed to finish execution but additional threads trying to acquire
268 // the lock at this point will block, and then get QueueGuard at STOPPED
269 // state.
270 while (count->value != 0) {
271 count.waitForCondition(count->cond);
272 }
273 count->value = -1;
274 }
275
276 // Input
277
Input()278 CCodecBufferChannel::Input::Input() : extraBuffers("extra") {}
279
280 // CCodecBufferChannel
281
CCodecBufferChannel(const std::shared_ptr<CCodecCallback> & callback)282 CCodecBufferChannel::CCodecBufferChannel(
283 const std::shared_ptr<CCodecCallback> &callback)
284 : mHeapSeqNum(-1),
285 mCCodecCallback(callback),
286 mFrameIndex(0u),
287 mFirstValidFrameIndex(0u),
288 mAreRenderMetricsEnabled(areRenderMetricsEnabled()),
289 mIsSurfaceToDisplay(false),
290 mHasPresentFenceTimes(false),
291 mRenderingDepth(3u),
292 mMetaMode(MODE_NONE),
293 mInputMetEos(false),
294 mSendEncryptedInfoBuffer(false) {
295 {
296 Mutexed<Input>::Locked input(mInput);
297 input->buffers.reset(new DummyInputBuffers(""));
298 input->extraBuffers.flush();
299 input->inputDelay = 0u;
300 input->pipelineDelay = 0u;
301 input->numSlots = kSmoothnessFactor;
302 input->numExtraSlots = 0u;
303 input->lastFlushIndex = 0u;
304 }
305 {
306 Mutexed<Output>::Locked output(mOutput);
307 output->outputDelay = 0u;
308 output->numSlots = kSmoothnessFactor;
309 output->bounded = false;
310 }
311 {
312 Mutexed<BlockPools>::Locked pools(mBlockPools);
313 pools->outputPoolId = C2BlockPool::BASIC_LINEAR;
314 }
315 if (android::media::codec::provider_->rendering_depth_removal()) {
316 constexpr int kAndroidApi202404 = 202404;
317 int vendorVersion = ::android::base::GetIntProperty("ro.vendor.api_level", -1);
318 using ::android::sysprop::MediaProperties::codec2_remove_rendering_depth;
319 if (vendorVersion > kAndroidApi202404 || codec2_remove_rendering_depth().value_or(false)) {
320 mRenderingDepth = 0;
321 }
322 } else {
323 std::string value = GetServerConfigurableFlag(
324 "media_native", "ccodec_rendering_depth", "3");
325 android::base::ParseInt(value, &mRenderingDepth);
326 }
327 mOutputSurface.lock()->maxDequeueBuffers = kSmoothnessFactor + mRenderingDepth;
328 }
329
~CCodecBufferChannel()330 CCodecBufferChannel::~CCodecBufferChannel() {
331 if (mCrypto != nullptr && mHeapSeqNum >= 0) {
332 mCrypto->unsetHeap(mHeapSeqNum);
333 }
334 }
335
setComponent(const std::shared_ptr<Codec2Client::Component> & component)336 void CCodecBufferChannel::setComponent(
337 const std::shared_ptr<Codec2Client::Component> &component) {
338 std::atomic_store(&mComponent, component);
339 mComponentName = component->getName() + StringPrintf("#%d", int(uintptr_t(component.get()) % 997));
340 mName = mComponentName.c_str();
341 }
342
setInputSurface(const std::shared_ptr<InputSurfaceWrapper> & surface)343 status_t CCodecBufferChannel::setInputSurface(
344 const std::shared_ptr<InputSurfaceWrapper> &surface) {
345 ALOGV("[%s] setInputSurface", mName);
346 if (!surface) {
347 ALOGE("[%s] setInputSurface: surface must not be null", mName);
348 return BAD_VALUE;
349 }
350 Mutexed<InputSurface>::Locked inputSurface(mInputSurface);
351 inputSurface->numProcessingBuffersBalance = 0;
352 inputSurface->surface = surface;
353 mHasInputSurface = true;
354 return inputSurface->surface->connect(std::atomic_load(&mComponent));
355 }
356
signalEndOfInputStream()357 status_t CCodecBufferChannel::signalEndOfInputStream() {
358 Mutexed<InputSurface>::Locked inputSurface(mInputSurface);
359 if (inputSurface->surface == nullptr) {
360 return INVALID_OPERATION;
361 }
362 return inputSurface->surface->signalEndOfInputStream();
363 }
364
queueInputBufferInternal(sp<MediaCodecBuffer> buffer,std::shared_ptr<C2LinearBlock> encryptedBlock,size_t blockSize)365 status_t CCodecBufferChannel::queueInputBufferInternal(
366 sp<MediaCodecBuffer> buffer,
367 std::shared_ptr<C2LinearBlock> encryptedBlock,
368 size_t blockSize) {
369 int64_t timeUs;
370 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
371
372 if (mInputMetEos) {
373 ALOGD("[%s] buffers after EOS ignored (%lld us)", mName, (long long)timeUs);
374 return OK;
375 }
376
377 int32_t flags = 0;
378 int32_t tmp = 0;
379 bool eos = false;
380 bool tunnelFirstFrame = false;
381 if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
382 eos = true;
383 mInputMetEos = true;
384 ALOGV("[%s] input EOS", mName);
385 }
386 if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
387 flags |= C2FrameData::FLAG_CODEC_CONFIG;
388 }
389 if (buffer->meta()->findInt32("tunnel-first-frame", &tmp) && tmp) {
390 tunnelFirstFrame = true;
391 }
392 if (buffer->meta()->findInt32("decode-only", &tmp) && tmp) {
393 flags |= C2FrameData::FLAG_DROP_FRAME;
394 }
395 ALOGV("[%s] queueInputBuffer: buffer->size() = %zu time: %lld",
396 mName, buffer->size(), (long long)timeUs);
397 std::list<std::unique_ptr<C2Work>> items;
398 std::unique_ptr<C2Work> work(new C2Work);
399 work->input.ordinal.timestamp = timeUs;
400 work->input.ordinal.frameIndex = mFrameIndex++;
401 // WORKAROUND: until codecs support handling work after EOS and max output sizing, use timestamp
402 // manipulation to achieve image encoding via video codec, and to constrain encoded output.
403 // Keep client timestamp in customOrdinal
404 work->input.ordinal.customOrdinal = timeUs;
405 work->input.buffers.clear();
406
407 sp<Codec2Buffer> copy;
408 bool usesFrameReassembler = false;
409
410 if (buffer->size() > 0u) {
411 Mutexed<Input>::Locked input(mInput);
412 std::shared_ptr<C2Buffer> c2buffer;
413 if (!input->buffers->releaseBuffer(buffer, &c2buffer, false)) {
414 return -ENOENT;
415 }
416 // TODO: we want to delay copying buffers.
417 if (input->extraBuffers.numComponentBuffers() < input->numExtraSlots) {
418 copy = input->buffers->cloneAndReleaseBuffer(buffer);
419 if (copy != nullptr) {
420 (void)input->extraBuffers.assignSlot(copy);
421 if (!input->extraBuffers.releaseSlot(copy, &c2buffer, false)) {
422 return UNKNOWN_ERROR;
423 }
424 bool released = input->buffers->releaseBuffer(buffer, nullptr, true);
425 ALOGV("[%s] queueInputBuffer: buffer copied; %sreleased",
426 mName, released ? "" : "not ");
427 buffer = copy;
428 } else {
429 ALOGW("[%s] queueInputBuffer: failed to copy a buffer; this may cause input "
430 "buffer starvation on component.", mName);
431 }
432 }
433 if (input->frameReassembler) {
434 usesFrameReassembler = true;
435 input->frameReassembler.process(buffer, &items);
436 } else {
437 int32_t cvo = 0;
438 if (buffer->meta()->findInt32("cvo", &cvo)) {
439 int32_t rotation = cvo % 360;
440 // change rotation to counter-clock wise.
441 rotation = ((rotation <= 0) ? 0 : 360) - rotation;
442
443 Mutexed<OutputSurface>::Locked output(mOutputSurface);
444 uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
445 output->rotation[frameIndex] = rotation;
446 }
447 sp<RefBase> obj;
448 if (buffer->meta()->findObject("accessUnitInfo", &obj)) {
449 ALOGV("Filling C2Info from multiple access units");
450 sp<WrapperObject<std::vector<AccessUnitInfo>>> infos{
451 (decltype(infos.get()))obj.get()};
452 std::vector<AccessUnitInfo> &accessUnitInfoVec = infos->value;
453 std::vector<C2AccessUnitInfosStruct> multipleAccessUnitInfos;
454 uint32_t outFlags = 0;
455 for (int i = 0; i < accessUnitInfoVec.size(); i++) {
456 outFlags = 0;
457 outFlags = convertFlags(accessUnitInfoVec[i].mFlags, true);
458 if (eos && (outFlags & C2FrameData::FLAG_END_OF_STREAM)) {
459 outFlags &= (~C2FrameData::FLAG_END_OF_STREAM);
460 }
461 multipleAccessUnitInfos.emplace_back(
462 outFlags,
463 accessUnitInfoVec[i].mSize,
464 accessUnitInfoVec[i].mTimestamp);
465 ALOGV("%d) flags: %d, size: %d, time: %llu",
466 i, outFlags, accessUnitInfoVec[i].mSize,
467 (long long)accessUnitInfoVec[i].mTimestamp);
468
469 }
470 const std::shared_ptr<C2AccessUnitInfos::input> c2AccessUnitInfos =
471 C2AccessUnitInfos::input::AllocShared(
472 multipleAccessUnitInfos.size(), 0u, multipleAccessUnitInfos);
473 c2buffer->setInfo(c2AccessUnitInfos);
474 }
475 work->input.buffers.push_back(c2buffer);
476 if (encryptedBlock) {
477 work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(
478 kParamIndexEncryptedBuffer,
479 encryptedBlock->share(0, blockSize, C2Fence())));
480 }
481 }
482 } else if (eos) {
483 Mutexed<Input>::Locked input(mInput);
484 if (input->frameReassembler) {
485 usesFrameReassembler = true;
486 // drain any pending items with eos
487 input->frameReassembler.process(buffer, &items);
488 }
489 flags |= C2FrameData::FLAG_END_OF_STREAM;
490 }
491 if (usesFrameReassembler) {
492 if (!items.empty()) {
493 items.front()->input.configUpdate = std::move(mParamsToBeSet);
494 mFrameIndex = (items.back()->input.ordinal.frameIndex + 1).peek();
495 }
496 } else {
497 work->input.flags = (C2FrameData::flags_t)flags;
498
499 // TODO: fill info's
500 if (android::media::codec::provider_->region_of_interest()
501 && android::media::codec::provider_->region_of_interest_support()) {
502 if (mInfoBuffers.size()) {
503 for (auto infoBuffer : mInfoBuffers) {
504 work->input.infoBuffers.emplace_back(*infoBuffer);
505 }
506 mInfoBuffers.clear();
507 }
508 }
509
510 work->input.configUpdate = std::move(mParamsToBeSet);
511 if (tunnelFirstFrame) {
512 C2StreamTunnelHoldRender::input tunnelHoldRender{
513 0u /* stream */,
514 C2_TRUE /* value */
515 };
516 work->input.configUpdate.push_back(C2Param::Copy(tunnelHoldRender));
517 }
518 work->worklets.clear();
519 work->worklets.emplace_back(new C2Worklet);
520
521 items.push_back(std::move(work));
522
523 eos = eos && buffer->size() > 0u;
524 }
525 if (eos) {
526 work.reset(new C2Work);
527 work->input.ordinal.timestamp = timeUs;
528 work->input.ordinal.frameIndex = mFrameIndex++;
529 // WORKAROUND: keep client timestamp in customOrdinal
530 work->input.ordinal.customOrdinal = timeUs;
531 work->input.buffers.clear();
532 work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
533 work->worklets.emplace_back(new C2Worklet);
534 items.push_back(std::move(work));
535 }
536 c2_status_t err = C2_OK;
537 if (!items.empty()) {
538 ScopedTrace trace(ATRACE_TAG, android::base::StringPrintf(
539 "CCodecBufferChannel::queue(%s@ts=%lld)", mName, (long long)timeUs).c_str());
540 {
541 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
542 PipelineWatcher::Clock::time_point now = PipelineWatcher::Clock::now();
543 for (const std::unique_ptr<C2Work> &work : items) {
544 watcher->onWorkQueued(
545 work->input.ordinal.frameIndex.peeku(),
546 std::vector(work->input.buffers),
547 now);
548 }
549 }
550 err = std::atomic_load(&mComponent)->queue(&items);
551 }
552 if (err != C2_OK) {
553 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
554 for (const std::unique_ptr<C2Work> &work : items) {
555 watcher->onWorkDone(work->input.ordinal.frameIndex.peeku());
556 }
557 } else {
558 Mutexed<Input>::Locked input(mInput);
559 bool released = false;
560 if (copy) {
561 released = input->extraBuffers.releaseSlot(copy, nullptr, true);
562 } else if (buffer) {
563 released = input->buffers->releaseBuffer(buffer, nullptr, true);
564 }
565 ALOGV("[%s] queueInputBuffer: buffer%s %sreleased",
566 mName, (buffer == nullptr) ? "(copy)" : "", released ? "" : "not ");
567 }
568
569 feedInputBufferIfAvailableInternal();
570 return err;
571 }
572
setParameters(std::vector<std::unique_ptr<C2Param>> & params)573 status_t CCodecBufferChannel::setParameters(std::vector<std::unique_ptr<C2Param>> ¶ms) {
574 QueueGuard guard(mSync);
575 if (!guard.isRunning()) {
576 ALOGD("[%s] setParameters is only supported in the running state.", mName);
577 return -ENOSYS;
578 }
579 mParamsToBeSet.insert(mParamsToBeSet.end(),
580 std::make_move_iterator(params.begin()),
581 std::make_move_iterator(params.end()));
582 params.clear();
583 return OK;
584 }
585
attachBuffer(const std::shared_ptr<C2Buffer> & c2Buffer,const sp<MediaCodecBuffer> & buffer)586 status_t CCodecBufferChannel::attachBuffer(
587 const std::shared_ptr<C2Buffer> &c2Buffer,
588 const sp<MediaCodecBuffer> &buffer) {
589 if (!buffer->copy(c2Buffer)) {
590 return -ENOSYS;
591 }
592 return OK;
593 }
594
ensureDecryptDestination(size_t size)595 void CCodecBufferChannel::ensureDecryptDestination(size_t size) {
596 if (!mDecryptDestination || mDecryptDestination->size() < size) {
597 sp<IMemoryHeap> heap{new MemoryHeapBase(size * 2)};
598 if (mDecryptDestination && mCrypto && mHeapSeqNum >= 0) {
599 mCrypto->unsetHeap(mHeapSeqNum);
600 }
601 mDecryptDestination = new MemoryBase(heap, 0, size * 2);
602 if (mCrypto) {
603 mHeapSeqNum = mCrypto->setHeap(hardware::fromHeap(heap));
604 }
605 }
606 }
607
getHeapSeqNum(const sp<HidlMemory> & memory)608 int32_t CCodecBufferChannel::getHeapSeqNum(const sp<HidlMemory> &memory) {
609 CHECK(mCrypto);
610 auto it = mHeapSeqNumMap.find(memory);
611 int32_t heapSeqNum = -1;
612 if (it == mHeapSeqNumMap.end()) {
613 heapSeqNum = mCrypto->setHeap(memory);
614 mHeapSeqNumMap.emplace(memory, heapSeqNum);
615 } else {
616 heapSeqNum = it->second;
617 }
618 return heapSeqNum;
619 }
620
621 typedef WrapperObject<std::vector<AccessUnitInfo>> BufferInfosWrapper;
622 typedef WrapperObject<std::vector<std::unique_ptr<CodecCryptoInfo>>> CryptoInfosWrapper;
attachEncryptedBuffers(const sp<hardware::HidlMemory> & memory,size_t offset,const sp<MediaCodecBuffer> & buffer,bool secure,AString * errorDetailMsg)623 status_t CCodecBufferChannel::attachEncryptedBuffers(
624 const sp<hardware::HidlMemory> &memory,
625 size_t offset,
626 const sp<MediaCodecBuffer> &buffer,
627 bool secure,
628 AString* errorDetailMsg) {
629 static const C2MemoryUsage kDefaultReadWriteUsage{
630 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
631 if (!hasCryptoOrDescrambler()) {
632 ALOGE("attachEncryptedBuffers requires Crypto/descrambler object");
633 return -ENOSYS;
634 }
635 size_t size = 0;
636 CHECK(buffer->meta()->findSize("ssize", &size));
637 if (size == 0) {
638 buffer->setRange(0, 0);
639 return OK;
640 }
641 sp<RefBase> obj;
642 CHECK(buffer->meta()->findObject("cryptoInfos", &obj));
643 sp<CryptoInfosWrapper> cryptoInfos{(CryptoInfosWrapper *)obj.get()};
644 CHECK(buffer->meta()->findObject("accessUnitInfo", &obj));
645 sp<BufferInfosWrapper> bufferInfos{(BufferInfosWrapper *)obj.get()};
646 if (secure || (mCrypto == nullptr)) {
647 if (cryptoInfos->value.size() != 1) {
648 ALOGE("Cannot decrypt multiple access units");
649 return -ENOSYS;
650 }
651 // we are dealing with just one cryptoInfo or descrambler.
652 std::unique_ptr<CodecCryptoInfo> info = std::move(cryptoInfos->value[0]);
653 if (info == nullptr) {
654 ALOGE("Cannot decrypt, CryptoInfos are null.");
655 return -ENOSYS;
656 }
657 return attachEncryptedBuffer(
658 memory,
659 secure,
660 info->mKey,
661 info->mIv,
662 info->mMode,
663 info->mPattern,
664 offset,
665 info->mSubSamples,
666 info->mNumSubSamples,
667 buffer,
668 errorDetailMsg);
669 }
670 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
671 std::shared_ptr<C2LinearBlock> block;
672 c2_status_t err = pool->fetchLinearBlock(
673 size,
674 kDefaultReadWriteUsage,
675 &block);
676 if (err != C2_OK) {
677 ALOGI("[%s] attachEncryptedBuffers: fetchLinearBlock failed: size = %zu (%s) err = %d",
678 mName, size, secure ? "secure" : "non-secure", err);
679 return NO_MEMORY;
680 }
681 ensureDecryptDestination(size);
682 C2WriteView wView = block->map().get();
683 if (wView.error() != C2_OK) {
684 ALOGI("[%s] attachEncryptedBuffers: block map error: %d (non-secure)",
685 mName, wView.error());
686 return UNKNOWN_ERROR;
687 }
688
689 ssize_t result = -1;
690 size_t srcOffset = offset;
691 size_t outBufferSize = 0;
692 uint32_t cryptoInfoIdx = 0;
693 int32_t heapSeqNum = getHeapSeqNum(memory);
694 hardware::drm::V1_0::SharedBuffer src{(uint32_t)heapSeqNum, offset, size};
695 hardware::drm::V1_0::DestinationBuffer dst;
696 dst.type = DrmBufferType::SHARED_MEMORY;
697 IMemoryToSharedBuffer(
698 mDecryptDestination, mHeapSeqNum, &dst.nonsecureMemory);
699 for (int i = 0; i < bufferInfos->value.size(); i++) {
700 if (bufferInfos->value[i].mSize > 0) {
701 std::unique_ptr<CodecCryptoInfo> info = std::move(cryptoInfos->value[cryptoInfoIdx++]);
702 src.offset = srcOffset;
703 src.size = bufferInfos->value[i].mSize;
704 result = mCrypto->decrypt(
705 (uint8_t*)info->mKey,
706 (uint8_t*)info->mIv,
707 info->mMode,
708 info->mPattern,
709 src,
710 0,
711 info->mSubSamples,
712 info->mNumSubSamples,
713 dst,
714 errorDetailMsg);
715 srcOffset += bufferInfos->value[i].mSize;
716 if (result < 0) {
717 ALOGI("[%s] attachEncryptedBuffers: decrypt failed: result = %zd",
718 mName, result);
719 return result;
720 }
721 if (wView.error() == C2_OK) {
722 if (wView.size() < result) {
723 ALOGI("[%s] attachEncryptedBuffers: block size too small:"
724 "size=%u result=%zd (non-secure)", mName, wView.size(), result);
725 return UNKNOWN_ERROR;
726 }
727 memcpy(wView.data(), mDecryptDestination->unsecurePointer(), result);
728 bufferInfos->value[i].mSize = result;
729 wView.setOffset(wView.offset() + result);
730 }
731 outBufferSize += result;
732 }
733 }
734 if (wView.error() == C2_OK) {
735 wView.setOffset(0);
736 }
737 std::shared_ptr<C2Buffer> c2Buffer{C2Buffer::CreateLinearBuffer(
738 block->share(0, outBufferSize, C2Fence{}))};
739 if (!buffer->copy(c2Buffer)) {
740 ALOGI("[%s] attachEncryptedBuffers: buffer copy failed", mName);
741 return -ENOSYS;
742 }
743 return OK;
744 }
745
attachEncryptedBuffer(const sp<hardware::HidlMemory> & memory,bool secure,const uint8_t * key,const uint8_t * iv,CryptoPlugin::Mode mode,CryptoPlugin::Pattern pattern,size_t offset,const CryptoPlugin::SubSample * subSamples,size_t numSubSamples,const sp<MediaCodecBuffer> & buffer,AString * errorDetailMsg)746 status_t CCodecBufferChannel::attachEncryptedBuffer(
747 const sp<hardware::HidlMemory> &memory,
748 bool secure,
749 const uint8_t *key,
750 const uint8_t *iv,
751 CryptoPlugin::Mode mode,
752 CryptoPlugin::Pattern pattern,
753 size_t offset,
754 const CryptoPlugin::SubSample *subSamples,
755 size_t numSubSamples,
756 const sp<MediaCodecBuffer> &buffer,
757 AString* errorDetailMsg) {
758 static const C2MemoryUsage kSecureUsage{C2MemoryUsage::READ_PROTECTED, 0};
759 static const C2MemoryUsage kDefaultReadWriteUsage{
760 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
761
762 size_t size = 0;
763 for (size_t i = 0; i < numSubSamples; ++i) {
764 size += subSamples[i].mNumBytesOfClearData + subSamples[i].mNumBytesOfEncryptedData;
765 }
766 if (size == 0) {
767 buffer->setRange(0, 0);
768 return OK;
769 }
770 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
771 std::shared_ptr<C2LinearBlock> block;
772 c2_status_t err = pool->fetchLinearBlock(
773 size,
774 secure ? kSecureUsage : kDefaultReadWriteUsage,
775 &block);
776 if (err != C2_OK) {
777 ALOGI("[%s] attachEncryptedBuffer: fetchLinearBlock failed: size = %zu (%s) err = %d",
778 mName, size, secure ? "secure" : "non-secure", err);
779 return NO_MEMORY;
780 }
781 if (!secure) {
782 ensureDecryptDestination(size);
783 }
784 ssize_t result = -1;
785 ssize_t codecDataOffset = 0;
786 if (mCrypto) {
787 int32_t heapSeqNum = getHeapSeqNum(memory);
788 hardware::drm::V1_0::SharedBuffer src{(uint32_t)heapSeqNum, offset, size};
789 hardware::drm::V1_0::DestinationBuffer dst;
790 if (secure) {
791 dst.type = DrmBufferType::NATIVE_HANDLE;
792 dst.secureMemory = hardware::hidl_handle(block->handle());
793 } else {
794 dst.type = DrmBufferType::SHARED_MEMORY;
795 IMemoryToSharedBuffer(
796 mDecryptDestination, mHeapSeqNum, &dst.nonsecureMemory);
797 }
798 result = mCrypto->decrypt(
799 key, iv, mode, pattern, src, 0, subSamples, numSubSamples,
800 dst, errorDetailMsg);
801 if (result < 0) {
802 ALOGI("[%s] attachEncryptedBuffer: decrypt failed: result = %zd", mName, result);
803 return result;
804 }
805 } else {
806 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
807 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
808 hidl_vec<SubSample> hidlSubSamples;
809 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
810
811 hardware::cas::native::V1_0::SharedBuffer src{*memory, offset, size};
812 hardware::cas::native::V1_0::DestinationBuffer dst;
813 if (secure) {
814 dst.type = BufferType::NATIVE_HANDLE;
815 dst.secureMemory = hardware::hidl_handle(block->handle());
816 } else {
817 dst.type = BufferType::SHARED_MEMORY;
818 dst.nonsecureMemory = src;
819 }
820
821 CasStatus status = CasStatus::OK;
822 hidl_string detailedError;
823 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
824
825 if (key != nullptr) {
826 sctrl = (ScramblingControl)key[0];
827 // Adjust for the PES offset
828 codecDataOffset = key[2] | (key[3] << 8);
829 }
830
831 auto returnVoid = mDescrambler->descramble(
832 sctrl,
833 hidlSubSamples,
834 src,
835 0,
836 dst,
837 0,
838 [&status, &result, &detailedError] (
839 CasStatus _status, uint32_t _bytesWritten,
840 const hidl_string& _detailedError) {
841 status = _status;
842 result = (ssize_t)_bytesWritten;
843 detailedError = _detailedError;
844 });
845 if (errorDetailMsg) {
846 errorDetailMsg->setTo(detailedError.c_str(), detailedError.size());
847 }
848 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
849 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
850 mName, returnVoid.description().c_str(), status, result);
851 return UNKNOWN_ERROR;
852 }
853
854 if (result < codecDataOffset) {
855 ALOGD("[%s] invalid codec data offset: %zd, result %zd",
856 mName, codecDataOffset, result);
857 return BAD_VALUE;
858 }
859 }
860 if (!secure) {
861 C2WriteView view = block->map().get();
862 if (view.error() != C2_OK) {
863 ALOGI("[%s] attachEncryptedBuffer: block map error: %d (non-secure)",
864 mName, view.error());
865 return UNKNOWN_ERROR;
866 }
867 if (view.size() < result) {
868 ALOGI("[%s] attachEncryptedBuffer: block size too small: size=%u result=%zd "
869 "(non-secure)",
870 mName, view.size(), result);
871 return UNKNOWN_ERROR;
872 }
873 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
874 }
875 std::shared_ptr<C2Buffer> c2Buffer{C2Buffer::CreateLinearBuffer(
876 block->share(codecDataOffset, result - codecDataOffset, C2Fence{}))};
877 if (!buffer->copy(c2Buffer)) {
878 ALOGI("[%s] attachEncryptedBuffer: buffer copy failed", mName);
879 return -ENOSYS;
880 }
881 return OK;
882 }
883
queueInputBuffer(const sp<MediaCodecBuffer> & buffer)884 status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
885 QueueGuard guard(mSync);
886 if (!guard.isRunning()) {
887 ALOGD("[%s] No more buffers should be queued at current state.", mName);
888 return -ENOSYS;
889 }
890 return queueInputBufferInternal(buffer);
891 }
892
queueSecureInputBuffer(const sp<MediaCodecBuffer> & buffer,bool secure,const uint8_t * key,const uint8_t * iv,CryptoPlugin::Mode mode,CryptoPlugin::Pattern pattern,const CryptoPlugin::SubSample * subSamples,size_t numSubSamples,AString * errorDetailMsg)893 status_t CCodecBufferChannel::queueSecureInputBuffer(
894 const sp<MediaCodecBuffer> &buffer, bool secure, const uint8_t *key,
895 const uint8_t *iv, CryptoPlugin::Mode mode, CryptoPlugin::Pattern pattern,
896 const CryptoPlugin::SubSample *subSamples, size_t numSubSamples,
897 AString *errorDetailMsg) {
898 QueueGuard guard(mSync);
899 if (!guard.isRunning()) {
900 ALOGD("[%s] No more buffers should be queued at current state.", mName);
901 return -ENOSYS;
902 }
903
904 if (!hasCryptoOrDescrambler()) {
905 return -ENOSYS;
906 }
907 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get());
908
909 std::shared_ptr<C2LinearBlock> block;
910 size_t allocSize = buffer->size();
911 size_t bufferSize = 0;
912 c2_status_t blockRes = C2_OK;
913 bool copied = false;
914 ScopedTrace trace(ATRACE_TAG, android::base::StringPrintf(
915 "CCodecBufferChannel::decrypt(%s)", mName).c_str());
916 if (mSendEncryptedInfoBuffer) {
917 static const C2MemoryUsage kDefaultReadWriteUsage{
918 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
919 constexpr int kAllocGranule0 = 1024 * 64;
920 constexpr int kAllocGranule1 = 1024 * 1024;
921 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
922 // round up encrypted sizes to limit fragmentation and encourage buffer reuse
923 if (allocSize <= kAllocGranule1) {
924 bufferSize = align(allocSize, kAllocGranule0);
925 } else {
926 bufferSize = align(allocSize, kAllocGranule1);
927 }
928 blockRes = pool->fetchLinearBlock(
929 bufferSize, kDefaultReadWriteUsage, &block);
930
931 if (blockRes == C2_OK) {
932 C2WriteView view = block->map().get();
933 if (view.error() == C2_OK && view.size() == bufferSize) {
934 copied = true;
935 // TODO: only copy clear sections
936 memcpy(view.data(), buffer->data(), allocSize);
937 }
938 }
939 }
940
941 if (!copied) {
942 block.reset();
943 }
944
945 ssize_t result = -1;
946 ssize_t codecDataOffset = 0;
947 if (numSubSamples == 1
948 && subSamples[0].mNumBytesOfClearData == 0
949 && subSamples[0].mNumBytesOfEncryptedData == 0) {
950 // We don't need to go through crypto or descrambler if the input is empty.
951 result = 0;
952 } else if (mCrypto != nullptr) {
953 hardware::drm::V1_0::DestinationBuffer destination;
954 if (secure) {
955 destination.type = DrmBufferType::NATIVE_HANDLE;
956 destination.secureMemory = hidl_handle(encryptedBuffer->handle());
957 } else {
958 destination.type = DrmBufferType::SHARED_MEMORY;
959 IMemoryToSharedBuffer(
960 mDecryptDestination, mHeapSeqNum, &destination.nonsecureMemory);
961 }
962 hardware::drm::V1_0::SharedBuffer source;
963 encryptedBuffer->fillSourceBuffer(&source);
964 result = mCrypto->decrypt(
965 key, iv, mode, pattern, source, buffer->offset(),
966 subSamples, numSubSamples, destination, errorDetailMsg);
967 if (result < 0) {
968 ALOGI("[%s] decrypt failed: result=%zd", mName, result);
969 return result;
970 }
971 if (destination.type == DrmBufferType::SHARED_MEMORY) {
972 encryptedBuffer->copyDecryptedContent(mDecryptDestination, result);
973 }
974 } else {
975 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
976 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
977 hidl_vec<SubSample> hidlSubSamples;
978 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
979
980 hardware::cas::native::V1_0::SharedBuffer srcBuffer;
981 encryptedBuffer->fillSourceBuffer(&srcBuffer);
982
983 DestinationBuffer dstBuffer;
984 if (secure) {
985 dstBuffer.type = BufferType::NATIVE_HANDLE;
986 dstBuffer.secureMemory = hidl_handle(encryptedBuffer->handle());
987 } else {
988 dstBuffer.type = BufferType::SHARED_MEMORY;
989 dstBuffer.nonsecureMemory = srcBuffer;
990 }
991
992 CasStatus status = CasStatus::OK;
993 hidl_string detailedError;
994 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
995
996 if (key != nullptr) {
997 sctrl = (ScramblingControl)key[0];
998 // Adjust for the PES offset
999 codecDataOffset = key[2] | (key[3] << 8);
1000 }
1001
1002 auto returnVoid = mDescrambler->descramble(
1003 sctrl,
1004 hidlSubSamples,
1005 srcBuffer,
1006 0,
1007 dstBuffer,
1008 0,
1009 [&status, &result, &detailedError] (
1010 CasStatus _status, uint32_t _bytesWritten,
1011 const hidl_string& _detailedError) {
1012 status = _status;
1013 result = (ssize_t)_bytesWritten;
1014 detailedError = _detailedError;
1015 });
1016
1017 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
1018 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
1019 mName, returnVoid.description().c_str(), status, result);
1020 return UNKNOWN_ERROR;
1021 }
1022
1023 if (result < codecDataOffset) {
1024 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
1025 return BAD_VALUE;
1026 }
1027
1028 ALOGV("[%s] descramble succeeded, %zd bytes", mName, result);
1029
1030 if (dstBuffer.type == BufferType::SHARED_MEMORY) {
1031 encryptedBuffer->copyDecryptedContentFromMemory(result);
1032 }
1033 }
1034
1035 buffer->setRange(codecDataOffset, result - codecDataOffset);
1036
1037 return queueInputBufferInternal(buffer, block, bufferSize);
1038 }
1039
queueSecureInputBuffers(const sp<MediaCodecBuffer> & buffer,bool secure,AString * errorDetailMsg)1040 status_t CCodecBufferChannel::queueSecureInputBuffers(
1041 const sp<MediaCodecBuffer> &buffer,
1042 bool secure,
1043 AString *errorDetailMsg) {
1044 QueueGuard guard(mSync);
1045 if (!guard.isRunning()) {
1046 ALOGD("[%s] No more buffers should be queued at current state.", mName);
1047 return -ENOSYS;
1048 }
1049
1050 if (!hasCryptoOrDescrambler()) {
1051 ALOGE("queueSecureInputBuffers requires a Crypto/descrambler Object");
1052 return -ENOSYS;
1053 }
1054 sp<RefBase> obj;
1055 CHECK(buffer->meta()->findObject("cryptoInfos", &obj));
1056 sp<CryptoInfosWrapper> cryptoInfos{(CryptoInfosWrapper *)obj.get()};
1057 CHECK(buffer->meta()->findObject("accessUnitInfo", &obj));
1058 sp<BufferInfosWrapper> bufferInfos{(BufferInfosWrapper *)obj.get()};
1059 if (secure || mCrypto == nullptr) {
1060 if (cryptoInfos->value.size() != 1) {
1061 ALOGE("Cannot decrypt multiple access units on native handles");
1062 return -ENOSYS;
1063 }
1064 std::unique_ptr<CodecCryptoInfo> info = std::move(cryptoInfos->value[0]);
1065 if (info == nullptr) {
1066 ALOGE("Cannot decrypt, CryptoInfos are null");
1067 return -ENOSYS;
1068 }
1069 return queueSecureInputBuffer(
1070 buffer,
1071 secure,
1072 info->mKey,
1073 info->mIv,
1074 info->mMode,
1075 info->mPattern,
1076 info->mSubSamples,
1077 info->mNumSubSamples,
1078 errorDetailMsg);
1079 }
1080 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get());
1081
1082 std::shared_ptr<C2LinearBlock> block;
1083 size_t allocSize = buffer->size();
1084 size_t bufferSize = 0;
1085 c2_status_t blockRes = C2_OK;
1086 bool copied = false;
1087 ScopedTrace trace(ATRACE_TAG, android::base::StringPrintf(
1088 "CCodecBufferChannel::decrypt(%s)", mName).c_str());
1089 if (mSendEncryptedInfoBuffer) {
1090 static const C2MemoryUsage kDefaultReadWriteUsage{
1091 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
1092 constexpr int kAllocGranule0 = 1024 * 64;
1093 constexpr int kAllocGranule1 = 1024 * 1024;
1094 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
1095 // round up encrypted sizes to limit fragmentation and encourage buffer reuse
1096 if (allocSize <= kAllocGranule1) {
1097 bufferSize = align(allocSize, kAllocGranule0);
1098 } else {
1099 bufferSize = align(allocSize, kAllocGranule1);
1100 }
1101 blockRes = pool->fetchLinearBlock(
1102 bufferSize, kDefaultReadWriteUsage, &block);
1103
1104 if (blockRes == C2_OK) {
1105 C2WriteView view = block->map().get();
1106 if (view.error() == C2_OK && view.size() == bufferSize) {
1107 copied = true;
1108 // TODO: only copy clear sections
1109 memcpy(view.data(), buffer->data(), allocSize);
1110 }
1111 }
1112 }
1113
1114 if (!copied) {
1115 block.reset();
1116 }
1117 // size of cryptoInfo and accessUnitInfo should be the same?
1118 ssize_t result = -1;
1119 size_t srcOffset = 0;
1120 size_t outBufferSize = 0;
1121 uint32_t cryptoInfoIdx = 0;
1122 {
1123 // scoped this block to enable destruction of mappedBlock
1124 std::unique_ptr<EncryptedLinearBlockBuffer::MappedBlock> mappedBlock = nullptr;
1125 hardware::drm::V1_0::DestinationBuffer destination;
1126 destination.type = DrmBufferType::SHARED_MEMORY;
1127 IMemoryToSharedBuffer(
1128 mDecryptDestination, mHeapSeqNum, &destination.nonsecureMemory);
1129 encryptedBuffer->getMappedBlock(&mappedBlock);
1130 hardware::drm::V1_0::SharedBuffer source;
1131 encryptedBuffer->fillSourceBuffer(&source);
1132 srcOffset = source.offset;
1133 for (int i = 0 ; i < bufferInfos->value.size(); i++) {
1134 if (bufferInfos->value[i].mSize > 0) {
1135 std::unique_ptr<CodecCryptoInfo> info =
1136 std::move(cryptoInfos->value[cryptoInfoIdx++]);
1137 if (info->mNumSubSamples == 1
1138 && info->mSubSamples[0].mNumBytesOfClearData == 0
1139 && info->mSubSamples[0].mNumBytesOfEncryptedData == 0) {
1140 // no data so we only populate the bufferInfo
1141 result = 0;
1142 } else {
1143 source.offset = srcOffset;
1144 source.size = bufferInfos->value[i].mSize;
1145 result = mCrypto->decrypt(
1146 (uint8_t*)info->mKey,
1147 (uint8_t*)info->mIv,
1148 info->mMode,
1149 info->mPattern,
1150 source,
1151 buffer->offset(),
1152 info->mSubSamples,
1153 info->mNumSubSamples,
1154 destination,
1155 errorDetailMsg);
1156 srcOffset += bufferInfos->value[i].mSize;
1157 if (result < 0) {
1158 ALOGI("[%s] decrypt failed: result=%zd", mName, result);
1159 return result;
1160 }
1161 if (destination.type == DrmBufferType::SHARED_MEMORY && mappedBlock) {
1162 mappedBlock->copyDecryptedContent(mDecryptDestination, result);
1163 }
1164 bufferInfos->value[i].mSize = result;
1165 outBufferSize += result;
1166 }
1167 }
1168 }
1169 buffer->setRange(0, outBufferSize);
1170 }
1171 return queueInputBufferInternal(buffer, block, bufferSize);
1172 }
1173
feedInputBufferIfAvailable()1174 void CCodecBufferChannel::feedInputBufferIfAvailable() {
1175 QueueGuard guard(mSync);
1176 if (!guard.isRunning()) {
1177 ALOGV("[%s] We're not running --- no input buffer reported", mName);
1178 return;
1179 }
1180 feedInputBufferIfAvailableInternal();
1181 }
1182
feedInputBufferIfAvailableInternal()1183 void CCodecBufferChannel::feedInputBufferIfAvailableInternal() {
1184 if (mInputMetEos) {
1185 return;
1186 }
1187 int64_t numOutputSlots = 0;
1188 bool outputFull = [this, &numOutputSlots]() {
1189 Mutexed<Output>::Locked output(mOutput);
1190 if (!output->buffers) {
1191 ALOGV("[%s] feedInputBufferIfAvailableInternal: "
1192 "return because output buffers are null", mName);
1193 return true;
1194 }
1195 numOutputSlots = int64_t(output->numSlots);
1196 if (output->buffers->hasPending() ||
1197 (!output->bounded && output->buffers->numActiveSlots() >= output->numSlots)) {
1198 ALOGV("[%s] feedInputBufferIfAvailableInternal: "
1199 "return because there are no room for more output buffers", mName);
1200 return true;
1201 }
1202 return false;
1203 }();
1204 if (android::media::codec::provider_->input_surface_throttle()) {
1205 Mutexed<InputSurface>::Locked inputSurface(mInputSurface);
1206 if (inputSurface->surface) {
1207 if (inputSurface->numProcessingBuffersBalance <= numOutputSlots) {
1208 ++inputSurface->numProcessingBuffersBalance;
1209 ALOGV("[%s] feedInputBufferIfAvailableInternal: numProcessingBuffersBalance = %lld",
1210 mName, static_cast<long long>(inputSurface->numProcessingBuffersBalance));
1211 inputSurface->surface->onInputBufferEmptied();
1212 }
1213 }
1214 }
1215 if (outputFull) {
1216 return;
1217 }
1218 size_t numActiveSlots = 0;
1219 while (!mPipelineWatcher.lock()->pipelineFull()) {
1220 sp<MediaCodecBuffer> inBuffer;
1221 size_t index;
1222 {
1223 Mutexed<Input>::Locked input(mInput);
1224 numActiveSlots = input->buffers->numActiveSlots();
1225 if (numActiveSlots >= input->numSlots) {
1226 break;
1227 }
1228 if (!input->buffers->requestNewBuffer(&index, &inBuffer)) {
1229 ALOGV("[%s] no new buffer available", mName);
1230 break;
1231 }
1232 }
1233 ALOGV("[%s] new input index = %zu [%p]", mName, index, inBuffer.get());
1234 mCallback->onInputBufferAvailable(index, inBuffer);
1235 }
1236 ALOGV("[%s] # active slots after feedInputBufferIfAvailable = %zu", mName, numActiveSlots);
1237 }
1238
renderOutputBuffer(const sp<MediaCodecBuffer> & buffer,int64_t timestampNs)1239 status_t CCodecBufferChannel::renderOutputBuffer(
1240 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) {
1241 ALOGV("[%s] renderOutputBuffer: %p", mName, buffer.get());
1242 std::shared_ptr<C2Buffer> c2Buffer;
1243 bool released = false;
1244 {
1245 Mutexed<Output>::Locked output(mOutput);
1246 if (output->buffers) {
1247 released = output->buffers->releaseBuffer(buffer, &c2Buffer);
1248 }
1249 }
1250 // NOTE: some apps try to releaseOutputBuffer() with timestamp and/or render
1251 // set to true.
1252 sendOutputBuffers();
1253 // input buffer feeding may have been gated by pending output buffers
1254 feedInputBufferIfAvailable();
1255 if (!c2Buffer) {
1256 if (released) {
1257 std::call_once(mRenderWarningFlag, [this] {
1258 ALOGW("[%s] The app is calling releaseOutputBuffer() with "
1259 "timestamp or render=true with non-video buffers. Apps should "
1260 "call releaseOutputBuffer() with render=false for those.",
1261 mName);
1262 });
1263 }
1264 return INVALID_OPERATION;
1265 }
1266
1267 #if 0
1268 const std::vector<std::shared_ptr<const C2Info>> infoParams = c2Buffer->info();
1269 ALOGV("[%s] queuing gfx buffer with %zu infos", mName, infoParams.size());
1270 for (const std::shared_ptr<const C2Info> &info : infoParams) {
1271 AString res;
1272 for (size_t ix = 0; ix + 3 < info->size(); ix += 4) {
1273 if (ix) res.append(", ");
1274 res.append(*((int32_t*)info.get() + (ix / 4)));
1275 }
1276 ALOGV(" [%s]", res.c_str());
1277 }
1278 #endif
1279 std::shared_ptr<const C2StreamRotationInfo::output> rotation =
1280 std::static_pointer_cast<const C2StreamRotationInfo::output>(
1281 c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE));
1282 bool flip = rotation && (rotation->flip & 1);
1283 uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3;
1284
1285 {
1286 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1287 if (output->surface == nullptr) {
1288 ALOGI("[%s] cannot render buffer without surface", mName);
1289 return OK;
1290 }
1291 int64_t frameIndex;
1292 buffer->meta()->findInt64("frameIndex", &frameIndex);
1293 if (output->rotation.count(frameIndex) != 0) {
1294 auto it = output->rotation.find(frameIndex);
1295 quarters = (it->second / 90) & 3;
1296 output->rotation.erase(it);
1297 }
1298 }
1299
1300 uint32_t transform = 0;
1301 switch (quarters) {
1302 case 0: // no rotation
1303 transform = flip ? HAL_TRANSFORM_FLIP_H : 0;
1304 break;
1305 case 1: // 90 degrees counter-clockwise
1306 transform = flip ? (HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_ROT_90)
1307 : HAL_TRANSFORM_ROT_270;
1308 break;
1309 case 2: // 180 degrees
1310 transform = flip ? HAL_TRANSFORM_FLIP_V : HAL_TRANSFORM_ROT_180;
1311 break;
1312 case 3: // 90 degrees clockwise
1313 transform = flip ? (HAL_TRANSFORM_FLIP_H | HAL_TRANSFORM_ROT_90)
1314 : HAL_TRANSFORM_ROT_90;
1315 break;
1316 }
1317
1318 std::shared_ptr<const C2StreamSurfaceScalingInfo::output> surfaceScaling =
1319 std::static_pointer_cast<const C2StreamSurfaceScalingInfo::output>(
1320 c2Buffer->getInfo(C2StreamSurfaceScalingInfo::output::PARAM_TYPE));
1321 uint32_t videoScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW;
1322 if (surfaceScaling) {
1323 videoScalingMode = surfaceScaling->value;
1324 }
1325
1326 // Use dataspace from format as it has the default aspects already applied
1327 android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
1328 (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace);
1329
1330 // HDR static info
1331 std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo =
1332 std::static_pointer_cast<const C2StreamHdrStaticInfo::output>(
1333 c2Buffer->getInfo(C2StreamHdrStaticInfo::output::PARAM_TYPE));
1334
1335 // HDR10 plus info
1336 std::shared_ptr<const C2StreamHdr10PlusInfo::output> hdr10PlusInfo =
1337 std::static_pointer_cast<const C2StreamHdr10PlusInfo::output>(
1338 c2Buffer->getInfo(C2StreamHdr10PlusInfo::output::PARAM_TYPE));
1339 if (hdr10PlusInfo && hdr10PlusInfo->flexCount() == 0) {
1340 hdr10PlusInfo.reset();
1341 }
1342
1343 // HDR dynamic info
1344 std::shared_ptr<const C2StreamHdrDynamicMetadataInfo::output> hdrDynamicInfo =
1345 std::static_pointer_cast<const C2StreamHdrDynamicMetadataInfo::output>(
1346 c2Buffer->getInfo(C2StreamHdrDynamicMetadataInfo::output::PARAM_TYPE));
1347 // TODO: make this sticky & enable unset
1348 if (hdrDynamicInfo && hdrDynamicInfo->flexCount() == 0) {
1349 hdrDynamicInfo.reset();
1350 }
1351
1352 if (hdr10PlusInfo) {
1353 // C2StreamHdr10PlusInfo is deprecated; components should use
1354 // C2StreamHdrDynamicMetadataInfo
1355 // TODO: #metric
1356 if (hdrDynamicInfo) {
1357 // It is unexpected that C2StreamHdr10PlusInfo and
1358 // C2StreamHdrDynamicMetadataInfo is both present.
1359 // C2StreamHdrDynamicMetadataInfo takes priority.
1360 // TODO: #metric
1361 } else {
1362 std::shared_ptr<C2StreamHdrDynamicMetadataInfo::output> info =
1363 C2StreamHdrDynamicMetadataInfo::output::AllocShared(
1364 hdr10PlusInfo->flexCount(),
1365 0u,
1366 C2Config::HDR_DYNAMIC_METADATA_TYPE_SMPTE_2094_40);
1367 memcpy(info->m.data, hdr10PlusInfo->m.value, hdr10PlusInfo->flexCount());
1368 hdrDynamicInfo = info;
1369 }
1370 }
1371
1372 std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks();
1373 if (blocks.size() != 1u) {
1374 ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size());
1375 return UNKNOWN_ERROR;
1376 }
1377 const C2ConstGraphicBlock &block = blocks.front();
1378 C2Fence c2fence = block.fence();
1379 sp<Fence> fence = Fence::NO_FENCE;
1380 // TODO: it's not sufficient to just check isHW() and then construct android::fence from it.
1381 // Once C2Fence::type() is added, check the exact C2Fence type
1382 if (c2fence.isHW()) {
1383 int fenceFd = c2fence.fd();
1384 fence = sp<Fence>::make(fenceFd);
1385 if (!fence) {
1386 ALOGE("[%s] Failed to allocate a fence", mName);
1387 close(fenceFd);
1388 return NO_MEMORY;
1389 }
1390 }
1391
1392 // TODO: revisit this after C2Fence implementation.
1393 IGraphicBufferProducer::QueueBufferInput qbi(
1394 timestampNs,
1395 false, // droppable
1396 dataSpace,
1397 Rect(blocks.front().crop().left,
1398 blocks.front().crop().top,
1399 blocks.front().crop().right(),
1400 blocks.front().crop().bottom()),
1401 videoScalingMode,
1402 transform,
1403 fence, 0);
1404 if (hdrStaticInfo || hdrDynamicInfo) {
1405 HdrMetadata hdr;
1406 if (hdrStaticInfo) {
1407 // If mastering max and min luminance fields are 0, do not use them.
1408 // It indicates the value may not be present in the stream.
1409 if (hdrStaticInfo->mastering.maxLuminance > 0.0f &&
1410 hdrStaticInfo->mastering.minLuminance > 0.0f) {
1411 struct android_smpte2086_metadata smpte2086_meta = {
1412 .displayPrimaryRed = {
1413 hdrStaticInfo->mastering.red.x, hdrStaticInfo->mastering.red.y
1414 },
1415 .displayPrimaryGreen = {
1416 hdrStaticInfo->mastering.green.x, hdrStaticInfo->mastering.green.y
1417 },
1418 .displayPrimaryBlue = {
1419 hdrStaticInfo->mastering.blue.x, hdrStaticInfo->mastering.blue.y
1420 },
1421 .whitePoint = {
1422 hdrStaticInfo->mastering.white.x, hdrStaticInfo->mastering.white.y
1423 },
1424 .maxLuminance = hdrStaticInfo->mastering.maxLuminance,
1425 .minLuminance = hdrStaticInfo->mastering.minLuminance,
1426 };
1427 hdr.validTypes |= HdrMetadata::SMPTE2086;
1428 hdr.smpte2086 = smpte2086_meta;
1429 }
1430 // If the content light level fields are 0, do not use them, it
1431 // indicates the value may not be present in the stream.
1432 if (hdrStaticInfo->maxCll > 0.0f && hdrStaticInfo->maxFall > 0.0f) {
1433 struct android_cta861_3_metadata cta861_meta = {
1434 .maxContentLightLevel = hdrStaticInfo->maxCll,
1435 .maxFrameAverageLightLevel = hdrStaticInfo->maxFall,
1436 };
1437 hdr.validTypes |= HdrMetadata::CTA861_3;
1438 hdr.cta8613 = cta861_meta;
1439 }
1440
1441 // does not have valid info
1442 if (!(hdr.validTypes & (HdrMetadata::SMPTE2086 | HdrMetadata::CTA861_3))) {
1443 hdrStaticInfo.reset();
1444 }
1445 }
1446 if (hdrDynamicInfo
1447 && hdrDynamicInfo->m.type_ == C2Config::HDR_DYNAMIC_METADATA_TYPE_SMPTE_2094_40) {
1448 hdr.validTypes |= HdrMetadata::HDR10PLUS;
1449 hdr.hdr10plus.assign(
1450 hdrDynamicInfo->m.data,
1451 hdrDynamicInfo->m.data + hdrDynamicInfo->flexCount());
1452 }
1453 qbi.setHdrMetadata(hdr);
1454 }
1455 SetMetadataToGralloc4Handle(dataSpace, hdrStaticInfo, hdrDynamicInfo, block.handle());
1456
1457 qbi.setSurfaceDamage(Region::INVALID_REGION); // we don't have dirty regions
1458 qbi.getFrameTimestamps = true; // we need to know when a frame is rendered
1459 IGraphicBufferProducer::QueueBufferOutput qbo;
1460 status_t result = std::atomic_load(&mComponent)->queueToOutputSurface(block, qbi, &qbo);
1461 if (result != OK) {
1462 ALOGI("[%s] queueBuffer failed: %d", mName, result);
1463 if (result == NO_INIT) {
1464 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1465 }
1466 return result;
1467 }
1468
1469 if(android::base::GetBoolProperty("debug.stagefright.fps", false)) {
1470 ALOGD("[%s] queue buffer successful", mName);
1471 } else {
1472 ALOGV("[%s] queue buffer successful", mName);
1473 }
1474
1475 int64_t mediaTimeUs = 0;
1476 (void)buffer->meta()->findInt64("timeUs", &mediaTimeUs);
1477 if (mAreRenderMetricsEnabled && mIsSurfaceToDisplay) {
1478 trackReleasedFrame(qbo, mediaTimeUs, timestampNs);
1479 processRenderedFrames(qbo.frameTimestamps);
1480 } else {
1481 // When the surface is an intermediate surface, onFrameRendered is triggered immediately
1482 // when the frame is queued to the non-display surface
1483 mCCodecCallback->onOutputFramesRendered(mediaTimeUs, timestampNs);
1484 }
1485
1486 return OK;
1487 }
1488
initializeFrameTrackingFor(ANativeWindow * window)1489 void CCodecBufferChannel::initializeFrameTrackingFor(ANativeWindow * window) {
1490 mTrackedFrames.clear();
1491
1492 int isSurfaceToDisplay = 0;
1493 window->query(window, NATIVE_WINDOW_QUEUES_TO_WINDOW_COMPOSER, &isSurfaceToDisplay);
1494 mIsSurfaceToDisplay = isSurfaceToDisplay == 1;
1495 // No frame tracking is needed if we're not sending frames to the display
1496 if (!mIsSurfaceToDisplay) {
1497 // Return early so we don't call into SurfaceFlinger (requiring permissions)
1498 return;
1499 }
1500
1501 int hasPresentFenceTimes = 0;
1502 window->query(window, NATIVE_WINDOW_FRAME_TIMESTAMPS_SUPPORTS_PRESENT, &hasPresentFenceTimes);
1503 mHasPresentFenceTimes = hasPresentFenceTimes == 1;
1504 if (!mHasPresentFenceTimes) {
1505 ALOGI("Using latch times for frame rendered signals - present fences not supported");
1506 }
1507 }
1508
trackReleasedFrame(const IGraphicBufferProducer::QueueBufferOutput & qbo,int64_t mediaTimeUs,int64_t desiredRenderTimeNs)1509 void CCodecBufferChannel::trackReleasedFrame(const IGraphicBufferProducer::QueueBufferOutput& qbo,
1510 int64_t mediaTimeUs, int64_t desiredRenderTimeNs) {
1511 // If the render time is earlier than now, then we're suggesting it should be rendered ASAP,
1512 // so track the frame as if the desired render time is now.
1513 int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
1514 if (desiredRenderTimeNs < nowNs) {
1515 desiredRenderTimeNs = nowNs;
1516 }
1517
1518 // If the render time is more than a second from now, then pretend the frame is supposed to be
1519 // rendered immediately, because that's what SurfaceFlinger heuristics will do. This is a tight
1520 // coupling, but is really the only way to optimize away unnecessary present fence checks in
1521 // processRenderedFrames.
1522 if (desiredRenderTimeNs > nowNs + 1*1000*1000*1000LL) {
1523 desiredRenderTimeNs = nowNs;
1524 }
1525
1526 // We've just queued a frame to the surface, so keep track of it and later check to see if it is
1527 // actually rendered.
1528 TrackedFrame frame;
1529 frame.number = qbo.nextFrameNumber - 1;
1530 frame.mediaTimeUs = mediaTimeUs;
1531 frame.desiredRenderTimeNs = desiredRenderTimeNs;
1532 frame.latchTime = -1;
1533 frame.presentFence = nullptr;
1534 mTrackedFrames.push_back(frame);
1535 }
1536
processRenderedFrames(const FrameEventHistoryDelta & deltas)1537 void CCodecBufferChannel::processRenderedFrames(const FrameEventHistoryDelta& deltas) {
1538 // Grab the latch times and present fences from the frame event deltas
1539 for (const auto& delta : deltas) {
1540 for (auto& frame : mTrackedFrames) {
1541 if (delta.getFrameNumber() == frame.number) {
1542 delta.getLatchTime(&frame.latchTime);
1543 delta.getDisplayPresentFence(&frame.presentFence);
1544 }
1545 }
1546 }
1547
1548 // Scan all frames and check to see if the frames that SHOULD have been rendered by now, have,
1549 // in fact, been rendered.
1550 int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
1551 while (!mTrackedFrames.empty()) {
1552 TrackedFrame & frame = mTrackedFrames.front();
1553 // Frames that should have been rendered at least 100ms in the past are checked
1554 if (frame.desiredRenderTimeNs > nowNs - 100*1000*1000LL) {
1555 break;
1556 }
1557
1558 // If we don't have a render time by now, then consider the frame as dropped
1559 int64_t renderTimeNs = getRenderTimeNs(frame);
1560 if (renderTimeNs != -1) {
1561 mCCodecCallback->onOutputFramesRendered(frame.mediaTimeUs, renderTimeNs);
1562 }
1563 mTrackedFrames.pop_front();
1564 }
1565 }
1566
getRenderTimeNs(const TrackedFrame & frame)1567 int64_t CCodecBufferChannel::getRenderTimeNs(const TrackedFrame& frame) {
1568 // If the device doesn't have accurate present fence times, then use the latch time as a proxy
1569 if (!mHasPresentFenceTimes) {
1570 if (frame.latchTime == -1) {
1571 ALOGD("no latch time for frame %d", (int) frame.number);
1572 return -1;
1573 }
1574 return frame.latchTime;
1575 }
1576
1577 if (frame.presentFence == nullptr) {
1578 ALOGW("no present fence for frame %d", (int) frame.number);
1579 return -1;
1580 }
1581
1582 nsecs_t actualRenderTimeNs = frame.presentFence->getSignalTime();
1583
1584 if (actualRenderTimeNs == Fence::SIGNAL_TIME_INVALID) {
1585 ALOGW("invalid signal time for frame %d", (int) frame.number);
1586 return -1;
1587 }
1588
1589 if (actualRenderTimeNs == Fence::SIGNAL_TIME_PENDING) {
1590 ALOGD("present fence has not fired for frame %d", (int) frame.number);
1591 return -1;
1592 }
1593
1594 return actualRenderTimeNs;
1595 }
1596
pollForRenderedBuffers()1597 void CCodecBufferChannel::pollForRenderedBuffers() {
1598 FrameEventHistoryDelta delta;
1599 std::atomic_load(&mComponent)->pollForRenderedFrames(&delta);
1600 processRenderedFrames(delta);
1601 }
1602
onBufferReleasedFromOutputSurface(uint32_t generation)1603 void CCodecBufferChannel::onBufferReleasedFromOutputSurface(uint32_t generation) {
1604 // Note: Since this is called asynchronously from IProducerListener not
1605 // knowing the internal state of CCodec/CCodecBufferChannel,
1606 // prevent mComponent from being destroyed by holding the shared reference
1607 // during this interface being executed.
1608 std::shared_ptr<Codec2Client::Component> comp = std::atomic_load(&mComponent);
1609 if (comp) {
1610 SurfaceCallbackHandler::GetInstance().post(
1611 SurfaceCallbackHandler::ON_BUFFER_RELEASED, comp, generation);
1612 }
1613 }
1614
onBufferAttachedToOutputSurface(uint32_t generation)1615 void CCodecBufferChannel::onBufferAttachedToOutputSurface(uint32_t generation) {
1616 // Note: Since this is called asynchronously from IProducerListener not
1617 // knowing the internal state of CCodec/CCodecBufferChannel,
1618 // prevent mComponent from being destroyed by holding the shared reference
1619 // during this interface being executed.
1620 std::shared_ptr<Codec2Client::Component> comp = std::atomic_load(&mComponent);
1621 if (comp) {
1622 SurfaceCallbackHandler::GetInstance().post(
1623 SurfaceCallbackHandler::ON_BUFFER_ATTACHED, comp, generation);
1624 }
1625 }
1626
discardBuffer(const sp<MediaCodecBuffer> & buffer)1627 status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
1628 ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
1629 bool released = false;
1630 {
1631 Mutexed<Input>::Locked input(mInput);
1632 if (input->buffers && input->buffers->releaseBuffer(buffer, nullptr, true)) {
1633 released = true;
1634 }
1635 }
1636 {
1637 Mutexed<Output>::Locked output(mOutput);
1638 if (output->buffers && output->buffers->releaseBuffer(buffer, nullptr)) {
1639 released = true;
1640 }
1641 }
1642 if (released) {
1643 sendOutputBuffers();
1644 feedInputBufferIfAvailable();
1645 } else {
1646 ALOGD("[%s] MediaCodec discarded an unknown buffer", mName);
1647 }
1648 return OK;
1649 }
1650
getInputBufferArray(Vector<sp<MediaCodecBuffer>> * array)1651 void CCodecBufferChannel::getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
1652 array->clear();
1653 Mutexed<Input>::Locked input(mInput);
1654
1655 if (!input->buffers) {
1656 ALOGE("getInputBufferArray: No Input Buffers allocated");
1657 return;
1658 }
1659 if (!input->buffers->isArrayMode()) {
1660 input->buffers = input->buffers->toArrayMode(input->numSlots);
1661 }
1662
1663 input->buffers->getArray(array);
1664 }
1665
getOutputBufferArray(Vector<sp<MediaCodecBuffer>> * array)1666 void CCodecBufferChannel::getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
1667 array->clear();
1668 Mutexed<Output>::Locked output(mOutput);
1669 if (!output->buffers) {
1670 ALOGE("getOutputBufferArray: No Output Buffers allocated");
1671 return;
1672 }
1673 if (!output->buffers->isArrayMode()) {
1674 output->buffers = output->buffers->toArrayMode(output->numSlots);
1675 }
1676
1677 output->buffers->getArray(array);
1678 }
1679
start(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat,bool buffersBoundToCodec)1680 status_t CCodecBufferChannel::start(
1681 const sp<AMessage> &inputFormat,
1682 const sp<AMessage> &outputFormat,
1683 bool buffersBoundToCodec) {
1684 C2StreamBufferTypeSetting::input iStreamFormat(0u);
1685 C2StreamBufferTypeSetting::output oStreamFormat(0u);
1686 C2ComponentKindSetting kind;
1687 C2PortReorderBufferDepthTuning::output reorderDepth;
1688 C2PortReorderKeySetting::output reorderKey;
1689 C2PortActualDelayTuning::input inputDelay(0);
1690 C2PortActualDelayTuning::output outputDelay(0);
1691 C2ActualPipelineDelayTuning pipelineDelay(0);
1692 C2SecureModeTuning secureMode(C2Config::SM_UNPROTECTED);
1693
1694 c2_status_t err = std::atomic_load(&mComponent)->query(
1695 {
1696 &iStreamFormat,
1697 &oStreamFormat,
1698 &kind,
1699 &reorderDepth,
1700 &reorderKey,
1701 &inputDelay,
1702 &pipelineDelay,
1703 &outputDelay,
1704 &secureMode,
1705 },
1706 {},
1707 C2_DONT_BLOCK,
1708 nullptr);
1709 if (err == C2_BAD_INDEX) {
1710 if (!iStreamFormat || !oStreamFormat || !kind) {
1711 return UNKNOWN_ERROR;
1712 }
1713 } else if (err != C2_OK) {
1714 return UNKNOWN_ERROR;
1715 }
1716
1717 uint32_t inputDelayValue = inputDelay ? inputDelay.value : 0;
1718 uint32_t pipelineDelayValue = pipelineDelay ? pipelineDelay.value : 0;
1719 uint32_t outputDelayValue = outputDelay ? outputDelay.value : 0;
1720
1721 size_t numInputSlots = inputDelayValue + pipelineDelayValue + kSmoothnessFactor;
1722 size_t numOutputSlots = outputDelayValue + kSmoothnessFactor;
1723
1724 // TODO: get this from input format
1725 bool secure = std::atomic_load(&mComponent)->getName().find(".secure") != std::string::npos;
1726
1727 // secure mode is a static parameter (shall not change in the executing state)
1728 mSendEncryptedInfoBuffer = secureMode.value == C2Config::SM_READ_PROTECTED_WITH_ENCRYPTED;
1729
1730 std::shared_ptr<C2AllocatorStore> allocatorStore = GetCodec2PlatformAllocatorStore();
1731 int poolMask = GetCodec2PoolMask();
1732 C2PlatformAllocatorStore::id_t preferredLinearId = GetPreferredLinearAllocatorId(poolMask);
1733
1734 if (inputFormat != nullptr) {
1735 bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);
1736 bool audioEncoder = !graphic && (kind.value == C2Component::KIND_ENCODER);
1737 C2Config::api_feature_t apiFeatures = C2Config::api_feature_t(
1738 API_REFLECTION |
1739 API_VALUES |
1740 API_CURRENT_VALUES |
1741 API_DEPENDENCY |
1742 API_SAME_INPUT_BUFFER);
1743 C2StreamAudioFrameSizeInfo::input encoderFrameSize(0u);
1744 C2StreamSampleRateInfo::input sampleRate(0u);
1745 C2StreamChannelCountInfo::input channelCount(0u);
1746 C2StreamPcmEncodingInfo::input pcmEncoding(0u);
1747 std::shared_ptr<C2BlockPool> pool;
1748 {
1749 Mutexed<BlockPools>::Locked pools(mBlockPools);
1750
1751 // set default allocator ID.
1752 pools->inputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
1753 : preferredLinearId;
1754
1755 // query C2PortAllocatorsTuning::input from component. If an allocator ID is obtained
1756 // from component, create the input block pool with given ID. Otherwise, use default IDs.
1757 std::vector<std::unique_ptr<C2Param>> params;
1758 C2ApiFeaturesSetting featuresSetting{apiFeatures};
1759 std::vector<C2Param *> stackParams({&featuresSetting});
1760 if (audioEncoder) {
1761 stackParams.push_back(&encoderFrameSize);
1762 stackParams.push_back(&sampleRate);
1763 stackParams.push_back(&channelCount);
1764 stackParams.push_back(&pcmEncoding);
1765 } else {
1766 encoderFrameSize.invalidate();
1767 sampleRate.invalidate();
1768 channelCount.invalidate();
1769 pcmEncoding.invalidate();
1770 }
1771 err = std::atomic_load(&mComponent)->query(stackParams,
1772 { C2PortAllocatorsTuning::input::PARAM_TYPE },
1773 C2_DONT_BLOCK,
1774 ¶ms);
1775 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1776 ALOGD("[%s] Query input allocators returned %zu params => %s (%u)",
1777 mName, params.size(), asString(err), err);
1778 } else if (params.size() == 1) {
1779 C2PortAllocatorsTuning::input *inputAllocators =
1780 C2PortAllocatorsTuning::input::From(params[0].get());
1781 if (inputAllocators && inputAllocators->flexCount() > 0) {
1782 std::shared_ptr<C2Allocator> allocator;
1783 // verify allocator IDs and resolve default allocator
1784 allocatorStore->fetchAllocator(inputAllocators->m.values[0], &allocator);
1785 if (allocator) {
1786 pools->inputAllocatorId = allocator->getId();
1787 } else {
1788 ALOGD("[%s] component requested invalid input allocator ID %u",
1789 mName, inputAllocators->m.values[0]);
1790 }
1791 }
1792 }
1793 if (featuresSetting) {
1794 apiFeatures = featuresSetting.value;
1795 }
1796
1797 // TODO: use C2Component wrapper to associate this pool with ourselves
1798 if ((poolMask >> pools->inputAllocatorId) & 1) {
1799 err = CreateCodec2BlockPool(pools->inputAllocatorId, nullptr, &pool);
1800 ALOGD("[%s] Created input block pool with allocatorID %u => poolID %llu - %s (%d)",
1801 mName, pools->inputAllocatorId,
1802 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1803 asString(err), err);
1804 } else {
1805 err = C2_NOT_FOUND;
1806 }
1807 if (err != C2_OK) {
1808 C2BlockPool::local_id_t inputPoolId =
1809 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1810 err = GetCodec2BlockPool(inputPoolId, nullptr, &pool);
1811 ALOGD("[%s] Using basic input block pool with poolID %llu => got %llu - %s (%d)",
1812 mName, (unsigned long long)inputPoolId,
1813 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1814 asString(err), err);
1815 if (err != C2_OK) {
1816 return NO_MEMORY;
1817 }
1818 }
1819 pools->inputPool = pool;
1820 }
1821
1822 bool forceArrayMode = false;
1823 Mutexed<Input>::Locked input(mInput);
1824 input->inputDelay = inputDelayValue;
1825 input->pipelineDelay = pipelineDelayValue;
1826 input->numSlots = numInputSlots;
1827 input->extraBuffers.flush();
1828 input->numExtraSlots = 0u;
1829 input->lastFlushIndex = mFrameIndex.load(std::memory_order_relaxed);
1830 if (audioEncoder && encoderFrameSize && sampleRate && channelCount) {
1831 input->frameReassembler.init(
1832 pool,
1833 {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE},
1834 encoderFrameSize.value,
1835 sampleRate.value,
1836 channelCount.value,
1837 pcmEncoding ? pcmEncoding.value : C2Config::PCM_16);
1838 }
1839 if (!buffersBoundToCodec) {
1840 inputFormat->setInt32(KEY_NUM_SLOTS, numInputSlots);
1841 }
1842 bool conforming = (apiFeatures & API_SAME_INPUT_BUFFER);
1843 // For encrypted content, framework decrypts source buffer (ashmem) into
1844 // C2Buffers. Thus non-conforming codecs can process these.
1845 if (!buffersBoundToCodec
1846 && !input->frameReassembler
1847 && (hasCryptoOrDescrambler() || conforming)) {
1848 input->buffers.reset(new SlotInputBuffers(mName));
1849 } else if (graphic) {
1850 if (mHasInputSurface) {
1851 input->buffers.reset(new DummyInputBuffers(mName));
1852 } else if (mMetaMode == MODE_ANW) {
1853 input->buffers.reset(new GraphicMetadataInputBuffers(mName));
1854 // This is to ensure buffers do not get released prematurely.
1855 // TODO: handle this without going into array mode
1856 forceArrayMode = true;
1857 } else {
1858 input->buffers.reset(new GraphicInputBuffers(mName));
1859 }
1860 } else {
1861 if (hasCryptoOrDescrambler()) {
1862 int32_t capacity = kLinearBufferSize;
1863 (void)inputFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
1864 if ((size_t)capacity > kMaxLinearBufferSize) {
1865 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
1866 capacity = kMaxLinearBufferSize;
1867 }
1868 if (mDealer == nullptr) {
1869 mDealer = new MemoryDealer(
1870 align(capacity, MemoryDealer::getAllocationAlignment())
1871 * (numInputSlots + 1),
1872 "EncryptedLinearInputBuffers");
1873 mDecryptDestination = mDealer->allocate((size_t)capacity);
1874 }
1875 if (mCrypto != nullptr && mHeapSeqNum < 0) {
1876 sp<HidlMemory> heap = fromHeap(mDealer->getMemoryHeap());
1877 mHeapSeqNum = mCrypto->setHeap(heap);
1878 } else {
1879 mHeapSeqNum = -1;
1880 }
1881 input->buffers.reset(new EncryptedLinearInputBuffers(
1882 secure, mDealer, mCrypto, mHeapSeqNum, (size_t)capacity,
1883 numInputSlots, mName));
1884 forceArrayMode = true;
1885 } else {
1886 input->buffers.reset(new LinearInputBuffers(mName));
1887 }
1888 }
1889 input->buffers->setFormat(inputFormat);
1890
1891 if (err == C2_OK) {
1892 input->buffers->setPool(pool);
1893 } else {
1894 // TODO: error
1895 }
1896
1897 if (forceArrayMode) {
1898 input->buffers = input->buffers->toArrayMode(numInputSlots);
1899 }
1900 }
1901
1902 if (outputFormat != nullptr) {
1903 sp<IGraphicBufferProducer> outputSurface;
1904 uint32_t outputGeneration;
1905 int maxDequeueCount = 0;
1906 {
1907 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1908 maxDequeueCount = output->maxDequeueBuffers = numOutputSlots +
1909 reorderDepth.value + mRenderingDepth;
1910 outputSurface = output->surface ?
1911 output->surface->getIGraphicBufferProducer() : nullptr;
1912 if (outputSurface) {
1913 (void)SurfaceCallbackHandler::GetInstance();
1914 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1915 }
1916 outputGeneration = output->generation;
1917 }
1918
1919 bool graphic = (oStreamFormat.value == C2BufferData::GRAPHIC);
1920 C2BlockPool::local_id_t outputPoolId_;
1921 C2BlockPool::local_id_t prevOutputPoolId;
1922
1923 {
1924 Mutexed<BlockPools>::Locked pools(mBlockPools);
1925
1926 prevOutputPoolId = pools->outputPoolId;
1927
1928 // set default allocator ID.
1929 pools->outputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
1930 : preferredLinearId;
1931
1932 // query C2PortAllocatorsTuning::output from component, or use default allocator if
1933 // unsuccessful.
1934 std::vector<std::unique_ptr<C2Param>> params;
1935 err = std::atomic_load(&mComponent)->query({ },
1936 { C2PortAllocatorsTuning::output::PARAM_TYPE },
1937 C2_DONT_BLOCK,
1938 ¶ms);
1939 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1940 ALOGD("[%s] Query output allocators returned %zu params => %s (%u)",
1941 mName, params.size(), asString(err), err);
1942 } else if (err == C2_OK && params.size() == 1) {
1943 C2PortAllocatorsTuning::output *outputAllocators =
1944 C2PortAllocatorsTuning::output::From(params[0].get());
1945 if (outputAllocators && outputAllocators->flexCount() > 0) {
1946 std::shared_ptr<C2Allocator> allocator;
1947 // verify allocator IDs and resolve default allocator
1948 allocatorStore->fetchAllocator(outputAllocators->m.values[0], &allocator);
1949 if (allocator) {
1950 pools->outputAllocatorId = allocator->getId();
1951 } else {
1952 ALOGD("[%s] component requested invalid output allocator ID %u",
1953 mName, outputAllocators->m.values[0]);
1954 }
1955 }
1956 }
1957
1958 // use bufferqueue if outputting to a surface.
1959 // query C2PortSurfaceAllocatorTuning::output from component, or use default allocator
1960 // if unsuccessful.
1961 if (outputSurface) {
1962 params.clear();
1963 err = std::atomic_load(&mComponent)->query({ },
1964 { C2PortSurfaceAllocatorTuning::output::PARAM_TYPE },
1965 C2_DONT_BLOCK,
1966 ¶ms);
1967 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1968 ALOGD("[%s] Query output surface allocator returned %zu params => %s (%u)",
1969 mName, params.size(), asString(err), err);
1970 } else if (err == C2_OK && params.size() == 1) {
1971 C2PortSurfaceAllocatorTuning::output *surfaceAllocator =
1972 C2PortSurfaceAllocatorTuning::output::From(params[0].get());
1973 if (surfaceAllocator) {
1974 std::shared_ptr<C2Allocator> allocator;
1975 // verify allocator IDs and resolve default allocator
1976 allocatorStore->fetchAllocator(surfaceAllocator->value, &allocator);
1977 if (allocator) {
1978 pools->outputAllocatorId = allocator->getId();
1979 } else {
1980 ALOGD("[%s] component requested invalid surface output allocator ID %u",
1981 mName, surfaceAllocator->value);
1982 err = C2_BAD_VALUE;
1983 }
1984 }
1985 }
1986 if (pools->outputAllocatorId == C2PlatformAllocatorStore::GRALLOC
1987 && err != C2_OK
1988 && ((poolMask >> C2PlatformAllocatorStore::BUFFERQUEUE) & 1)) {
1989 pools->outputAllocatorId = C2PlatformAllocatorStore::BUFFERQUEUE;
1990 }
1991 }
1992
1993 if ((poolMask >> pools->outputAllocatorId) & 1) {
1994 err = std::atomic_load(&mComponent)->createBlockPool(
1995 pools->outputAllocatorId, &pools->outputPoolId, &pools->outputPoolIntf);
1996 ALOGI("[%s] Created output block pool with allocatorID %u => poolID %llu - %s",
1997 mName, pools->outputAllocatorId,
1998 (unsigned long long)pools->outputPoolId,
1999 asString(err));
2000 } else {
2001 err = C2_NOT_FOUND;
2002 }
2003 if (err != C2_OK) {
2004 // use basic pool instead
2005 pools->outputPoolId =
2006 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
2007 }
2008
2009 // Configure output block pool ID as parameter C2PortBlockPoolsTuning::output to
2010 // component.
2011 std::unique_ptr<C2PortBlockPoolsTuning::output> poolIdsTuning =
2012 C2PortBlockPoolsTuning::output::AllocUnique({ pools->outputPoolId });
2013
2014 std::vector<std::unique_ptr<C2SettingResult>> failures;
2015 err = std::atomic_load(&mComponent)->config(
2016 { poolIdsTuning.get() }, C2_MAY_BLOCK, &failures);
2017 ALOGD("[%s] Configured output block pool ids %llu => %s",
2018 mName, (unsigned long long)poolIdsTuning->m.values[0], asString(err));
2019 outputPoolId_ = pools->outputPoolId;
2020 }
2021
2022 if (prevOutputPoolId != C2BlockPool::BASIC_LINEAR
2023 && prevOutputPoolId != C2BlockPool::BASIC_GRAPHIC) {
2024 c2_status_t err = std::atomic_load(&mComponent)->destroyBlockPool(prevOutputPoolId);
2025 if (err != C2_OK) {
2026 ALOGW("Failed to clean up previous block pool %llu - %s (%d)\n",
2027 (unsigned long long) prevOutputPoolId, asString(err), err);
2028 }
2029 }
2030
2031 Mutexed<Output>::Locked output(mOutput);
2032 output->outputDelay = outputDelayValue;
2033 output->numSlots = numOutputSlots;
2034 output->bounded = bool(outputSurface);
2035 if (graphic) {
2036 if (outputSurface || !buffersBoundToCodec) {
2037 output->buffers.reset(new GraphicOutputBuffers(mName));
2038 } else {
2039 output->buffers.reset(new RawGraphicOutputBuffers(mName));
2040 }
2041 } else {
2042 output->buffers.reset(new LinearOutputBuffers(mName));
2043 }
2044 output->buffers->setFormat(outputFormat);
2045
2046 output->buffers->clearStash();
2047 if (reorderDepth) {
2048 output->buffers->setReorderDepth(reorderDepth.value);
2049 }
2050 if (reorderKey) {
2051 output->buffers->setReorderKey(reorderKey.value);
2052 }
2053
2054 // Try to set output surface to created block pool if given.
2055 if (outputSurface) {
2056 std::atomic_load(&mComponent)->setOutputSurface(
2057 outputPoolId_,
2058 outputSurface,
2059 outputGeneration,
2060 maxDequeueCount);
2061 } else {
2062 // configure CPU read consumer usage
2063 C2StreamUsageTuning::output outputUsage{0u, C2MemoryUsage::CPU_READ};
2064 std::vector<std::unique_ptr<C2SettingResult>> failures;
2065 err = std::atomic_load(&mComponent)->config({ &outputUsage }, C2_MAY_BLOCK, &failures);
2066 // do not print error message for now as most components may not yet
2067 // support this setting
2068 ALOGD_IF(err != C2_BAD_INDEX, "[%s] Configured output usage [%#llx]",
2069 mName, (long long)outputUsage.value);
2070 }
2071
2072 if (oStreamFormat.value == C2BufferData::LINEAR) {
2073 if (buffersBoundToCodec) {
2074 // WORKAROUND: if we're using early CSD workaround we convert to
2075 // array mode, to appease apps assuming the output
2076 // buffers to be of the same size.
2077 output->buffers = output->buffers->toArrayMode(numOutputSlots);
2078 }
2079
2080 int32_t channelCount;
2081 int32_t sampleRate;
2082 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
2083 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
2084 int32_t delay = 0;
2085 int32_t padding = 0;;
2086 if (!outputFormat->findInt32("encoder-delay", &delay)) {
2087 delay = 0;
2088 }
2089 if (!outputFormat->findInt32("encoder-padding", &padding)) {
2090 padding = 0;
2091 }
2092 if (delay || padding) {
2093 // We need write access to the buffers, so turn them into array mode.
2094 // TODO: b/321930152 - define SkipCutOutputBuffers that takes output from
2095 // component, runs it through SkipCutBuffer and allocate local buffer to be
2096 // used by fwk. Make initSkipCutBuffer() return OutputBuffers similar to
2097 // toArrayMode().
2098 if (!output->buffers->isArrayMode()) {
2099 output->buffers = output->buffers->toArrayMode(numOutputSlots);
2100 }
2101 output->buffers->initSkipCutBuffer(delay, padding, sampleRate, channelCount);
2102 }
2103 }
2104 }
2105
2106 int32_t tunneled = 0;
2107 if (!outputFormat->findInt32("android._tunneled", &tunneled)) {
2108 tunneled = 0;
2109 }
2110 mTunneled = (tunneled != 0);
2111 }
2112
2113 // Set up pipeline control. This has to be done after mInputBuffers and
2114 // mOutputBuffers are initialized to make sure that lingering callbacks
2115 // about buffers from the previous generation do not interfere with the
2116 // newly initialized pipeline capacity.
2117
2118 if (inputFormat || outputFormat) {
2119 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
2120 watcher->inputDelay(inputDelayValue)
2121 .pipelineDelay(pipelineDelayValue)
2122 .outputDelay(outputDelayValue)
2123 .smoothnessFactor(kSmoothnessFactor)
2124 .tunneled(mTunneled);
2125 watcher->flush();
2126 }
2127
2128 mInputMetEos = false;
2129 mSync.start();
2130 return OK;
2131 }
2132
prepareInitialInputBuffers(std::map<size_t,sp<MediaCodecBuffer>> * clientInputBuffers,bool retry)2133 status_t CCodecBufferChannel::prepareInitialInputBuffers(
2134 std::map<size_t, sp<MediaCodecBuffer>> *clientInputBuffers, bool retry) {
2135 if (mHasInputSurface) {
2136 return OK;
2137 }
2138
2139 size_t numInputSlots = mInput.lock()->numSlots;
2140 int retryCount = 1;
2141 for (; clientInputBuffers->empty() && retryCount >= 0; retryCount--) {
2142 {
2143 Mutexed<Input>::Locked input(mInput);
2144 while (clientInputBuffers->size() < numInputSlots) {
2145 size_t index;
2146 sp<MediaCodecBuffer> buffer;
2147 if (!input->buffers->requestNewBuffer(&index, &buffer)) {
2148 break;
2149 }
2150 clientInputBuffers->emplace(index, buffer);
2151 }
2152 }
2153 if (!retry || (retryCount <= 0)) {
2154 break;
2155 }
2156 if (clientInputBuffers->empty()) {
2157 // wait: buffer may be in transit from component.
2158 std::this_thread::sleep_for(std::chrono::milliseconds(4));
2159 }
2160 }
2161 if (clientInputBuffers->empty()) {
2162 ALOGW("[%s] start: cannot allocate memory at all", mName);
2163 return NO_MEMORY;
2164 } else if (clientInputBuffers->size() < numInputSlots) {
2165 ALOGD("[%s] start: cannot allocate memory for all slots, "
2166 "only %zu buffers allocated",
2167 mName, clientInputBuffers->size());
2168 } else {
2169 ALOGV("[%s] %zu initial input buffers available",
2170 mName, clientInputBuffers->size());
2171 }
2172 return OK;
2173 }
2174
requestInitialInputBuffers(std::map<size_t,sp<MediaCodecBuffer>> && clientInputBuffers)2175 status_t CCodecBufferChannel::requestInitialInputBuffers(
2176 std::map<size_t, sp<MediaCodecBuffer>> &&clientInputBuffers) {
2177 std::optional<QueueGuard> guard;
2178 if (android::media::codec::provider_->codec_buffer_state_cleanup()) {
2179 guard.emplace(mSync);
2180 if (!guard->isRunning()) {
2181 ALOGD("[%s] skip requestInitialInputBuffers when not running", mName);
2182 return OK;
2183 }
2184 }
2185 C2StreamBufferTypeSetting::output oStreamFormat(0u);
2186 C2PrependHeaderModeSetting prepend(PREPEND_HEADER_TO_NONE);
2187 c2_status_t err = std::atomic_load(&mComponent)->query(
2188 { &oStreamFormat, &prepend }, {}, C2_DONT_BLOCK, nullptr);
2189 if (err != C2_OK && err != C2_BAD_INDEX) {
2190 return UNKNOWN_ERROR;
2191 }
2192
2193 std::list<std::unique_ptr<C2Work>> flushedConfigs;
2194 mFlushedConfigs.lock()->swap(flushedConfigs);
2195 if (!flushedConfigs.empty()) {
2196 {
2197 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
2198 PipelineWatcher::Clock::time_point now = PipelineWatcher::Clock::now();
2199 for (const std::unique_ptr<C2Work> &work : flushedConfigs) {
2200 watcher->onWorkQueued(
2201 work->input.ordinal.frameIndex.peeku(),
2202 std::vector(work->input.buffers),
2203 now);
2204 }
2205 }
2206 err = std::atomic_load(&mComponent)->queue(&flushedConfigs);
2207 if (err != C2_OK) {
2208 ALOGW("[%s] Error while queueing a flushed config", mName);
2209 return UNKNOWN_ERROR;
2210 }
2211 }
2212 if (oStreamFormat.value == C2BufferData::LINEAR &&
2213 (!prepend || prepend.value == PREPEND_HEADER_TO_NONE) &&
2214 !clientInputBuffers.empty()) {
2215 size_t minIndex = clientInputBuffers.begin()->first;
2216 sp<MediaCodecBuffer> minBuffer = clientInputBuffers.begin()->second;
2217 for (const auto &[index, buffer] : clientInputBuffers) {
2218 if (minBuffer->capacity() > buffer->capacity()) {
2219 minIndex = index;
2220 minBuffer = buffer;
2221 }
2222 }
2223 // WORKAROUND: Some apps expect CSD available without queueing
2224 // any input. Queue an empty buffer to get the CSD.
2225 minBuffer->setRange(0, 0);
2226 minBuffer->meta()->clear();
2227 minBuffer->meta()->setInt64("timeUs", 0);
2228 if (queueInputBufferInternal(minBuffer) != OK) {
2229 ALOGW("[%s] Error while queueing an empty buffer to get CSD",
2230 mName);
2231 return UNKNOWN_ERROR;
2232 }
2233 clientInputBuffers.erase(minIndex);
2234 }
2235
2236 for (const auto &[index, buffer] : clientInputBuffers) {
2237 mCallback->onInputBufferAvailable(index, buffer);
2238 }
2239
2240 return OK;
2241 }
2242
stop()2243 void CCodecBufferChannel::stop() {
2244 mSync.stop();
2245 mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed);
2246 mInfoBuffers.clear();
2247 }
2248
stopUseOutputSurface(bool pushBlankBuffer)2249 void CCodecBufferChannel::stopUseOutputSurface(bool pushBlankBuffer) {
2250 sp<Surface> surface = mOutputSurface.lock()->surface;
2251 if (surface) {
2252 C2BlockPool::local_id_t outputPoolId;
2253 {
2254 Mutexed<BlockPools>::Locked pools(mBlockPools);
2255 outputPoolId = pools->outputPoolId;
2256 }
2257 std::shared_ptr<Codec2Client::Component> comp = std::atomic_load(&mComponent);
2258 if (comp) comp->stopUsingOutputSurface(outputPoolId);
2259
2260 if (pushBlankBuffer) {
2261 sp<ANativeWindow> anw = static_cast<ANativeWindow *>(surface.get());
2262 if (anw) {
2263 pushBlankBuffersToNativeWindow(anw.get());
2264 }
2265 }
2266 }
2267 }
2268
reset()2269 void CCodecBufferChannel::reset() {
2270 stop();
2271 mPipelineWatcher.lock()->flush();
2272 {
2273 mHasInputSurface = false;
2274 Mutexed<InputSurface>::Locked inputSurface(mInputSurface);
2275 inputSurface->surface.reset();
2276 }
2277 {
2278 Mutexed<Input>::Locked input(mInput);
2279 input->buffers.reset(new DummyInputBuffers(""));
2280 input->extraBuffers.flush();
2281 }
2282 {
2283 Mutexed<Output>::Locked output(mOutput);
2284 output->buffers.reset();
2285 }
2286 // reset the frames that are being tracked for onFrameRendered callbacks
2287 mTrackedFrames.clear();
2288 }
2289
release()2290 void CCodecBufferChannel::release() {
2291 mInfoBuffers.clear();
2292 std::shared_ptr<Codec2Client::Component> nullComp;
2293 std::atomic_store(&mComponent, nullComp);
2294 mInputAllocator.reset();
2295 mOutputSurface.lock()->surface.clear();
2296 {
2297 Mutexed<BlockPools>::Locked blockPools{mBlockPools};
2298 blockPools->inputPool.reset();
2299 blockPools->outputPoolIntf.reset();
2300 }
2301 setCrypto(nullptr);
2302 setDescrambler(nullptr);
2303 }
2304
flush(const std::list<std::unique_ptr<C2Work>> & flushedWork)2305 void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) {
2306 ALOGV("[%s] flush", mName);
2307 std::list<std::unique_ptr<C2Work>> configs;
2308 mInput.lock()->lastFlushIndex = mFrameIndex.load(std::memory_order_relaxed);
2309 {
2310 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
2311 for (const std::unique_ptr<C2Work> &work : flushedWork) {
2312 uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
2313 if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) {
2314 watcher->onWorkDone(frameIndex);
2315 continue;
2316 }
2317 if (work->input.buffers.empty()
2318 || work->input.buffers.front() == nullptr
2319 || work->input.buffers.front()->data().linearBlocks().empty()) {
2320 ALOGD("[%s] no linear codec config data found", mName);
2321 watcher->onWorkDone(frameIndex);
2322 continue;
2323 }
2324 std::unique_ptr<C2Work> copy(new C2Work);
2325 copy->input.flags = C2FrameData::flags_t(
2326 work->input.flags | C2FrameData::FLAG_DROP_FRAME);
2327 copy->input.ordinal = work->input.ordinal;
2328 copy->input.ordinal.frameIndex = mFrameIndex++;
2329 for (size_t i = 0; i < work->input.buffers.size(); ++i) {
2330 copy->input.buffers.push_back(watcher->onInputBufferReleased(frameIndex, i));
2331 }
2332 for (const std::unique_ptr<C2Param> ¶m : work->input.configUpdate) {
2333 copy->input.configUpdate.push_back(C2Param::Copy(*param));
2334 }
2335 copy->input.infoBuffers.insert(
2336 copy->input.infoBuffers.begin(),
2337 work->input.infoBuffers.begin(),
2338 work->input.infoBuffers.end());
2339 copy->worklets.emplace_back(new C2Worklet);
2340 configs.push_back(std::move(copy));
2341 watcher->onWorkDone(frameIndex);
2342 ALOGV("[%s] stashed flushed codec config data", mName);
2343 }
2344 }
2345 mFlushedConfigs.lock()->swap(configs);
2346 {
2347 Mutexed<Input>::Locked input(mInput);
2348 input->buffers->flush();
2349 input->extraBuffers.flush();
2350 }
2351 {
2352 Mutexed<Output>::Locked output(mOutput);
2353 if (output->buffers) {
2354 output->buffers->flush(flushedWork);
2355 output->buffers->flushStash();
2356 }
2357 }
2358 mInfoBuffers.clear();
2359 }
2360
onWorkDone(std::unique_ptr<C2Work> work,const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat,const C2StreamInitDataInfo::output * initData)2361 void CCodecBufferChannel::onWorkDone(
2362 std::unique_ptr<C2Work> work,
2363 const sp<AMessage> &inputFormat,
2364 const sp<AMessage> &outputFormat,
2365 const C2StreamInitDataInfo::output *initData) {
2366 if (handleWork(std::move(work), inputFormat, outputFormat, initData)) {
2367 feedInputBufferIfAvailable();
2368 }
2369 }
2370
onInputBufferDone(uint64_t frameIndex,size_t arrayIndex)2371 void CCodecBufferChannel::onInputBufferDone(
2372 uint64_t frameIndex, size_t arrayIndex) {
2373 std::shared_ptr<C2Buffer> buffer =
2374 mPipelineWatcher.lock()->onInputBufferReleased(frameIndex, arrayIndex);
2375 bool newInputSlotAvailable = false;
2376 {
2377 Mutexed<Input>::Locked input(mInput);
2378 if (input->lastFlushIndex >= frameIndex) {
2379 ALOGD("[%s] Ignoring stale input buffer done callback: "
2380 "last flush index = %lld, frameIndex = %lld",
2381 mName, input->lastFlushIndex.peekll(), (long long)frameIndex);
2382 } else {
2383 newInputSlotAvailable = input->buffers->expireComponentBuffer(buffer);
2384 if (!newInputSlotAvailable) {
2385 (void)input->extraBuffers.expireComponentBuffer(buffer);
2386 }
2387 }
2388 }
2389 if (newInputSlotAvailable) {
2390 feedInputBufferIfAvailable();
2391 }
2392 }
2393
handleWork(std::unique_ptr<C2Work> work,const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat,const C2StreamInitDataInfo::output * initData)2394 bool CCodecBufferChannel::handleWork(
2395 std::unique_ptr<C2Work> work,
2396 const sp<AMessage> &inputFormat,
2397 const sp<AMessage> &outputFormat,
2398 const C2StreamInitDataInfo::output *initData) {
2399 {
2400 Mutexed<Output>::Locked output(mOutput);
2401 if (!output->buffers) {
2402 return false;
2403 }
2404 }
2405
2406 // Whether the output buffer should be reported to the client or not.
2407 bool notifyClient = false;
2408
2409 if (work->result == C2_OK){
2410 notifyClient = true;
2411 } else if (work->result == C2_NOT_FOUND) {
2412 ALOGD("[%s] flushed work; ignored.", mName);
2413 } else {
2414 // C2_OK and C2_NOT_FOUND are the only results that we accept for processing
2415 // the config update.
2416 ALOGD("[%s] work failed to complete: %d", mName, work->result);
2417 mCCodecCallback->onError(work->result, ACTION_CODE_FATAL);
2418 return false;
2419 }
2420
2421 if ((work->input.ordinal.frameIndex -
2422 mFirstValidFrameIndex.load()).peek() < 0) {
2423 // Discard frames from previous generation.
2424 ALOGD("[%s] Discard frames from previous generation.", mName);
2425 notifyClient = false;
2426 }
2427
2428 if (!mHasInputSurface && (work->worklets.size() != 1u
2429 || !work->worklets.front()
2430 || !(work->worklets.front()->output.flags &
2431 C2FrameData::FLAG_INCOMPLETE))) {
2432 mPipelineWatcher.lock()->onWorkDone(
2433 work->input.ordinal.frameIndex.peeku());
2434 }
2435
2436 // NOTE: MediaCodec usage supposedly have only one worklet
2437 if (work->worklets.size() != 1u) {
2438 ALOGI("[%s] onWorkDone: incorrect number of worklets: %zu",
2439 mName, work->worklets.size());
2440 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
2441 return false;
2442 }
2443
2444 const std::unique_ptr<C2Worklet> &worklet = work->worklets.front();
2445
2446 std::shared_ptr<C2Buffer> buffer;
2447 // NOTE: MediaCodec usage supposedly have only one output stream.
2448 if (worklet->output.buffers.size() > 1u) {
2449 ALOGI("[%s] onWorkDone: incorrect number of output buffers: %zu",
2450 mName, worklet->output.buffers.size());
2451 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
2452 return false;
2453 } else if (worklet->output.buffers.size() == 1u) {
2454 buffer = worklet->output.buffers[0];
2455 if (!buffer) {
2456 ALOGD("[%s] onWorkDone: nullptr found in buffers; ignored.", mName);
2457 }
2458 }
2459
2460 std::optional<uint32_t> newInputDelay, newPipelineDelay, newOutputDelay, newReorderDepth;
2461 std::optional<C2Config::ordinal_key_t> newReorderKey;
2462 bool needMaxDequeueBufferCountUpdate = false;
2463 while (!worklet->output.configUpdate.empty()) {
2464 std::unique_ptr<C2Param> param;
2465 worklet->output.configUpdate.back().swap(param);
2466 worklet->output.configUpdate.pop_back();
2467 switch (param->coreIndex().coreIndex()) {
2468 case C2PortReorderBufferDepthTuning::CORE_INDEX: {
2469 C2PortReorderBufferDepthTuning::output reorderDepth;
2470 if (reorderDepth.updateFrom(*param)) {
2471 ALOGV("[%s] onWorkDone: updated reorder depth to %u",
2472 mName, reorderDepth.value);
2473 newReorderDepth = reorderDepth.value;
2474 needMaxDequeueBufferCountUpdate = true;
2475 } else {
2476 ALOGD("[%s] onWorkDone: failed to read reorder depth",
2477 mName);
2478 }
2479 break;
2480 }
2481 case C2PortReorderKeySetting::CORE_INDEX: {
2482 C2PortReorderKeySetting::output reorderKey;
2483 if (reorderKey.updateFrom(*param)) {
2484 newReorderKey = reorderKey.value;
2485 ALOGV("[%s] onWorkDone: updated reorder key to %u",
2486 mName, reorderKey.value);
2487 } else {
2488 ALOGD("[%s] onWorkDone: failed to read reorder key", mName);
2489 }
2490 break;
2491 }
2492 case C2PortActualDelayTuning::CORE_INDEX: {
2493 if (param->isGlobal()) {
2494 C2ActualPipelineDelayTuning pipelineDelay;
2495 if (pipelineDelay.updateFrom(*param)) {
2496 ALOGV("[%s] onWorkDone: updating pipeline delay %u",
2497 mName, pipelineDelay.value);
2498 newPipelineDelay = pipelineDelay.value;
2499 (void)mPipelineWatcher.lock()->pipelineDelay(
2500 pipelineDelay.value);
2501 }
2502 }
2503 if (param->forInput()) {
2504 C2PortActualDelayTuning::input inputDelay;
2505 if (inputDelay.updateFrom(*param)) {
2506 ALOGV("[%s] onWorkDone: updating input delay %u",
2507 mName, inputDelay.value);
2508 newInputDelay = inputDelay.value;
2509 (void)mPipelineWatcher.lock()->inputDelay(
2510 inputDelay.value);
2511 }
2512 }
2513 if (param->forOutput()) {
2514 C2PortActualDelayTuning::output outputDelay;
2515 if (outputDelay.updateFrom(*param)) {
2516 ALOGV("[%s] onWorkDone: updating output delay %u",
2517 mName, outputDelay.value);
2518 (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value);
2519 newOutputDelay = outputDelay.value;
2520 needMaxDequeueBufferCountUpdate = true;
2521
2522 }
2523 }
2524 break;
2525 }
2526 case C2PortTunnelSystemTime::CORE_INDEX: {
2527 C2PortTunnelSystemTime::output frameRenderTime;
2528 if (frameRenderTime.updateFrom(*param)) {
2529 ALOGV("[%s] onWorkDone: frame rendered (sys:%lld ns, media:%lld us)",
2530 mName, (long long)frameRenderTime.value,
2531 (long long)worklet->output.ordinal.timestamp.peekll());
2532 mCCodecCallback->onOutputFramesRendered(
2533 worklet->output.ordinal.timestamp.peek(), frameRenderTime.value);
2534 }
2535 break;
2536 }
2537 case C2StreamTunnelHoldRender::CORE_INDEX: {
2538 C2StreamTunnelHoldRender::output firstTunnelFrameHoldRender;
2539 if (!(worklet->output.flags & C2FrameData::FLAG_INCOMPLETE)) break;
2540 if (!firstTunnelFrameHoldRender.updateFrom(*param)) break;
2541 if (firstTunnelFrameHoldRender.value != C2_TRUE) break;
2542 ALOGV("[%s] onWorkDone: first tunnel frame ready", mName);
2543 mCCodecCallback->onFirstTunnelFrameReady();
2544 break;
2545 }
2546 default:
2547 ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
2548 mName, param->index());
2549 break;
2550 }
2551 }
2552 if (newInputDelay || newPipelineDelay) {
2553 Mutexed<Input>::Locked input(mInput);
2554 size_t newNumSlots =
2555 newInputDelay.value_or(input->inputDelay) +
2556 newPipelineDelay.value_or(input->pipelineDelay) +
2557 kSmoothnessFactor;
2558 input->inputDelay = newInputDelay.value_or(input->inputDelay);
2559 if (input->buffers->isArrayMode()) {
2560 if (input->numSlots >= newNumSlots) {
2561 input->numExtraSlots = 0;
2562 } else {
2563 input->numExtraSlots = newNumSlots - input->numSlots;
2564 }
2565 ALOGV("[%s] onWorkDone: updated number of extra slots to %zu (input array mode)",
2566 mName, input->numExtraSlots);
2567 } else {
2568 input->numSlots = newNumSlots;
2569 }
2570 if (inputFormat->contains(KEY_NUM_SLOTS)) {
2571 inputFormat->setInt32(KEY_NUM_SLOTS, input->numSlots);
2572 }
2573 }
2574 size_t numOutputSlots = 0;
2575 uint32_t reorderDepth = 0;
2576 bool outputBuffersChanged = false;
2577 if (newReorderKey || newReorderDepth || needMaxDequeueBufferCountUpdate) {
2578 Mutexed<Output>::Locked output(mOutput);
2579 if (!output->buffers) {
2580 return false;
2581 }
2582 numOutputSlots = output->numSlots;
2583 if (newReorderKey) {
2584 output->buffers->setReorderKey(newReorderKey.value());
2585 }
2586 if (newReorderDepth) {
2587 output->buffers->setReorderDepth(newReorderDepth.value());
2588 }
2589 reorderDepth = output->buffers->getReorderDepth();
2590 if (newOutputDelay) {
2591 output->outputDelay = newOutputDelay.value();
2592 numOutputSlots = newOutputDelay.value() + kSmoothnessFactor;
2593 if (output->numSlots < numOutputSlots) {
2594 output->numSlots = numOutputSlots;
2595 if (output->buffers->isArrayMode()) {
2596 OutputBuffersArray *array =
2597 (OutputBuffersArray *)output->buffers.get();
2598 ALOGV("[%s] onWorkDone: growing output buffer array to %zu",
2599 mName, numOutputSlots);
2600 array->grow(numOutputSlots);
2601 outputBuffersChanged = true;
2602 }
2603 }
2604 }
2605 numOutputSlots = output->numSlots;
2606 }
2607 if (outputBuffersChanged) {
2608 mCCodecCallback->onOutputBuffersChanged();
2609 }
2610 if (needMaxDequeueBufferCountUpdate) {
2611 int maxDequeueCount = 0;
2612 {
2613 Mutexed<OutputSurface>::Locked output(mOutputSurface);
2614 maxDequeueCount = output->maxDequeueBuffers =
2615 numOutputSlots + reorderDepth + mRenderingDepth;
2616 if (output->surface) {
2617 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
2618 }
2619 }
2620 if (maxDequeueCount > 0) {
2621 std::atomic_load(&mComponent)->setOutputSurfaceMaxDequeueCount(maxDequeueCount);
2622 }
2623 }
2624
2625 int32_t flags = 0;
2626 if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) {
2627 flags |= BUFFER_FLAG_END_OF_STREAM;
2628 ALOGV("[%s] onWorkDone: output EOS", mName);
2629 }
2630
2631 // WORKAROUND: adjust output timestamp based on client input timestamp and codec
2632 // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to
2633 // the codec input timestamp, but client output timestamp should (reported in timeUs)
2634 // shall correspond to the client input timesamp (in customOrdinal). By using the
2635 // delta between the two, this allows for some timestamp deviation - e.g. if one input
2636 // produces multiple output.
2637 c2_cntr64_t timestamp =
2638 worklet->output.ordinal.timestamp + work->input.ordinal.customOrdinal
2639 - work->input.ordinal.timestamp;
2640 if (mHasInputSurface) {
2641 // When using input surface we need to restore the original input timestamp.
2642 timestamp = work->input.ordinal.customOrdinal;
2643 }
2644 ScopedTrace trace(ATRACE_TAG, android::base::StringPrintf(
2645 "CCodecBufferChannel::onWorkDone(%s@ts=%lld)", mName, timestamp.peekll()).c_str());
2646 ALOGV("[%s] onWorkDone: input %lld, codec %lld => output %lld => %lld",
2647 mName,
2648 work->input.ordinal.customOrdinal.peekll(),
2649 work->input.ordinal.timestamp.peekll(),
2650 worklet->output.ordinal.timestamp.peekll(),
2651 timestamp.peekll());
2652
2653 // csd cannot be re-ordered and will always arrive first.
2654 if (initData != nullptr) {
2655 Mutexed<Output>::Locked output(mOutput);
2656 if (!output->buffers) {
2657 return false;
2658 }
2659 if (outputFormat) {
2660 output->buffers->updateSkipCutBuffer(outputFormat);
2661 output->buffers->setFormat(outputFormat);
2662 }
2663 if (!notifyClient) {
2664 return false;
2665 }
2666 size_t index;
2667 sp<MediaCodecBuffer> outBuffer;
2668 if (output->buffers->registerCsd(initData, &index, &outBuffer) == OK) {
2669 outBuffer->meta()->setInt64("timeUs", timestamp.peek());
2670 outBuffer->meta()->setInt32("flags", BUFFER_FLAG_CODEC_CONFIG);
2671 ALOGV("[%s] onWorkDone: csd index = %zu [%p]", mName, index, outBuffer.get());
2672
2673 // TRICKY: we want popped buffers reported in order, so sending
2674 // the callback while holding the lock here. This assumes that
2675 // onOutputBufferAvailable() does not block. onOutputBufferAvailable()
2676 // callbacks are always sent with the Output lock held.
2677 mCallback->onOutputBufferAvailable(index, outBuffer);
2678 } else {
2679 ALOGD("[%s] onWorkDone: unable to register csd", mName);
2680 output.unlock();
2681 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
2682 return false;
2683 }
2684 }
2685
2686 bool drop = false;
2687 if (worklet->output.flags & C2FrameData::FLAG_DROP_FRAME) {
2688 ALOGV("[%s] onWorkDone: drop buffer but keep metadata", mName);
2689 drop = true;
2690 }
2691
2692 // Workaround: if C2FrameData::FLAG_DROP_FRAME is not implemented in
2693 // HAL, the flag is then removed in the corresponding output buffer.
2694 if (work->input.flags & C2FrameData::FLAG_DROP_FRAME) {
2695 flags |= BUFFER_FLAG_DECODE_ONLY;
2696 }
2697
2698 if (notifyClient && !buffer && !flags) {
2699 if (mTunneled && drop && outputFormat) {
2700 if (mOutputFormat != outputFormat) {
2701 ALOGV("[%s] onWorkDone: Keep tunneled, drop frame with format change (%lld)",
2702 mName, work->input.ordinal.frameIndex.peekull());
2703 mOutputFormat = outputFormat;
2704 } else {
2705 ALOGV("[%s] onWorkDone: Not reporting output buffer without format change (%lld)",
2706 mName, work->input.ordinal.frameIndex.peekull());
2707 notifyClient = false;
2708 }
2709 } else {
2710 ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)",
2711 mName, work->input.ordinal.frameIndex.peekull());
2712 notifyClient = false;
2713 }
2714 }
2715
2716 if (buffer) {
2717 for (const std::shared_ptr<const C2Info> &info : buffer->info()) {
2718 // TODO: properly translate these to metadata
2719 switch (info->coreIndex().coreIndex()) {
2720 case C2StreamPictureTypeMaskInfo::CORE_INDEX:
2721 if (((C2StreamPictureTypeMaskInfo *)info.get())->value & C2Config::SYNC_FRAME) {
2722 flags |= BUFFER_FLAG_KEY_FRAME;
2723 }
2724 break;
2725 default:
2726 break;
2727 }
2728 }
2729 }
2730
2731 {
2732 Mutexed<Output>::Locked output(mOutput);
2733 if (!output->buffers) {
2734 return false;
2735 }
2736 output->buffers->pushToStash(
2737 buffer,
2738 notifyClient,
2739 timestamp.peek(),
2740 flags,
2741 outputFormat,
2742 worklet->output.ordinal);
2743 }
2744 sendOutputBuffers();
2745 return true;
2746 }
2747
sendOutputBuffers()2748 void CCodecBufferChannel::sendOutputBuffers() {
2749 OutputBuffers::BufferAction action;
2750 size_t index;
2751 sp<MediaCodecBuffer> outBuffer;
2752 std::shared_ptr<C2Buffer> c2Buffer;
2753
2754 constexpr int kMaxReallocTry = 5;
2755 int reallocTryNum = 0;
2756
2757 while (true) {
2758 Mutexed<Output>::Locked output(mOutput);
2759 if (!output->buffers) {
2760 return;
2761 }
2762 action = output->buffers->popFromStashAndRegister(
2763 &c2Buffer, &index, &outBuffer);
2764 if (action != OutputBuffers::REALLOCATE) {
2765 reallocTryNum = 0;
2766 }
2767 switch (action) {
2768 case OutputBuffers::SKIP:
2769 return;
2770 case OutputBuffers::NOTIFY_CLIENT:
2771 {
2772 // TRICKY: we want popped buffers reported in order, so sending
2773 // the callback while holding the lock here. This assumes that
2774 // onOutputBufferAvailable() does not block. onOutputBufferAvailable()
2775 // callbacks are always sent with the Output lock held.
2776 if (c2Buffer) {
2777 std::shared_ptr<const C2AccessUnitInfos::output> bufferMetadata =
2778 std::static_pointer_cast<const C2AccessUnitInfos::output>(
2779 c2Buffer->getInfo(C2AccessUnitInfos::output::PARAM_TYPE));
2780 if (bufferMetadata && bufferMetadata->flexCount() > 0) {
2781 uint32_t flag = 0;
2782 std::vector<AccessUnitInfo> accessUnitInfos;
2783 for (int nMeta = 0; nMeta < bufferMetadata->flexCount(); nMeta++) {
2784 const C2AccessUnitInfosStruct &bufferMetadataStruct =
2785 bufferMetadata->m.values[nMeta];
2786 flag = convertFlags(bufferMetadataStruct.flags, false);
2787 accessUnitInfos.emplace_back(flag,
2788 bufferMetadataStruct.size,
2789 bufferMetadataStruct.timestamp);
2790 }
2791 sp<WrapperObject<std::vector<AccessUnitInfo>>> obj{
2792 new WrapperObject<std::vector<AccessUnitInfo>>{accessUnitInfos}};
2793 outBuffer->meta()->setObject("accessUnitInfo", obj);
2794 }
2795 }
2796 mCallback->onOutputBufferAvailable(index, outBuffer);
2797 [[fallthrough]];
2798 }
2799 case OutputBuffers::DISCARD: {
2800 if (mHasInputSurface && android::media::codec::provider_->input_surface_throttle()) {
2801 Mutexed<InputSurface>::Locked inputSurface(mInputSurface);
2802 --inputSurface->numProcessingBuffersBalance;
2803 ALOGV("[%s] onWorkDone: numProcessingBuffersBalance = %lld",
2804 mName, static_cast<long long>(inputSurface->numProcessingBuffersBalance));
2805 }
2806 break;
2807 }
2808 case OutputBuffers::REALLOCATE:
2809 if (++reallocTryNum > kMaxReallocTry) {
2810 output.unlock();
2811 ALOGE("[%s] sendOutputBuffers: tried %d realloc and failed",
2812 mName, kMaxReallocTry);
2813 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
2814 return;
2815 }
2816 if (!output->buffers->isArrayMode()) {
2817 output->buffers =
2818 output->buffers->toArrayMode(output->numSlots);
2819 }
2820 static_cast<OutputBuffersArray*>(output->buffers.get())->
2821 realloc(c2Buffer);
2822 output.unlock();
2823 mCCodecCallback->onOutputBuffersChanged();
2824 break;
2825 case OutputBuffers::RETRY:
2826 ALOGV("[%s] sendOutputBuffers: unable to register output buffer",
2827 mName);
2828 return;
2829 default:
2830 LOG_ALWAYS_FATAL("[%s] sendOutputBuffers: "
2831 "corrupted BufferAction value (%d) "
2832 "returned from popFromStashAndRegister.",
2833 mName, int(action));
2834 return;
2835 }
2836 }
2837 }
2838
setSurface(const sp<Surface> & newSurface,uint32_t generation,bool pushBlankBuffer)2839 status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface,
2840 uint32_t generation, bool pushBlankBuffer) {
2841 sp<IGraphicBufferProducer> producer;
2842 int maxDequeueCount;
2843 sp<Surface> oldSurface;
2844 {
2845 Mutexed<OutputSurface>::Locked outputSurface(mOutputSurface);
2846 maxDequeueCount = outputSurface->maxDequeueBuffers;
2847 oldSurface = outputSurface->surface;
2848 }
2849 if (newSurface) {
2850 (void)SurfaceCallbackHandler::GetInstance();
2851 newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW);
2852 newSurface->setDequeueTimeout(kDequeueTimeoutNs);
2853 newSurface->setMaxDequeuedBufferCount(maxDequeueCount);
2854 producer = newSurface->getIGraphicBufferProducer();
2855 } else {
2856 ALOGE("[%s] setting output surface to null", mName);
2857 return INVALID_OPERATION;
2858 }
2859
2860 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf;
2861 C2BlockPool::local_id_t outputPoolId;
2862 {
2863 Mutexed<BlockPools>::Locked pools(mBlockPools);
2864 outputPoolId = pools->outputPoolId;
2865 outputPoolIntf = pools->outputPoolIntf;
2866 }
2867
2868 if (outputPoolIntf) {
2869 if (std::atomic_load(&mComponent)->setOutputSurface(
2870 outputPoolId,
2871 producer,
2872 generation,
2873 maxDequeueCount) != C2_OK) {
2874 ALOGI("[%s] setSurface: component setOutputSurface failed", mName);
2875 return INVALID_OPERATION;
2876 }
2877 }
2878
2879 {
2880 Mutexed<OutputSurface>::Locked output(mOutputSurface);
2881 output->surface = newSurface;
2882 output->generation = generation;
2883 initializeFrameTrackingFor(static_cast<ANativeWindow *>(newSurface.get()));
2884 }
2885
2886 if (oldSurface && pushBlankBuffer) {
2887 // When ReleaseSurface was set from MediaCodec,
2888 // pushing a blank buffer at the end might be necessary.
2889 sp<ANativeWindow> anw = static_cast<ANativeWindow *>(oldSurface.get());
2890 if (anw) {
2891 pushBlankBuffersToNativeWindow(anw.get());
2892 }
2893 }
2894
2895 return OK;
2896 }
2897
elapsed()2898 PipelineWatcher::Clock::duration CCodecBufferChannel::elapsed() {
2899 // Otherwise, component may have stalled work due to input starvation up to
2900 // the sum of the delay in the pipeline.
2901 // TODO(b/231253301): When client pushed EOS, the pipeline could have less
2902 // number of frames.
2903 size_t n = 0;
2904 size_t outputDelay = mOutput.lock()->outputDelay;
2905 {
2906 Mutexed<Input>::Locked input(mInput);
2907 n = input->inputDelay + input->pipelineDelay + outputDelay;
2908 }
2909 return mPipelineWatcher.lock()->elapsed(PipelineWatcher::Clock::now(), n);
2910 }
2911
setMetaMode(MetaMode mode)2912 void CCodecBufferChannel::setMetaMode(MetaMode mode) {
2913 mMetaMode = mode;
2914 }
2915
setCrypto(const sp<ICrypto> & crypto)2916 void CCodecBufferChannel::setCrypto(const sp<ICrypto> &crypto) {
2917 if (mCrypto != nullptr) {
2918 for (std::pair<wp<HidlMemory>, int32_t> entry : mHeapSeqNumMap) {
2919 mCrypto->unsetHeap(entry.second);
2920 }
2921 mHeapSeqNumMap.clear();
2922 if (mHeapSeqNum >= 0) {
2923 mCrypto->unsetHeap(mHeapSeqNum);
2924 mHeapSeqNum = -1;
2925 }
2926 }
2927 mCrypto = crypto;
2928 }
2929
setDescrambler(const sp<IDescrambler> & descrambler)2930 void CCodecBufferChannel::setDescrambler(const sp<IDescrambler> &descrambler) {
2931 mDescrambler = descrambler;
2932 }
2933
getBuffersPixelFormat(bool isEncoder)2934 uint32_t CCodecBufferChannel::getBuffersPixelFormat(bool isEncoder) {
2935 if (isEncoder) {
2936 return getInputBuffersPixelFormat();
2937 } else {
2938 return getOutputBuffersPixelFormat();
2939 }
2940 }
2941
getInputBuffersPixelFormat()2942 uint32_t CCodecBufferChannel::getInputBuffersPixelFormat() {
2943 Mutexed<Input>::Locked input(mInput);
2944 if (input->buffers == nullptr) {
2945 return PIXEL_FORMAT_UNKNOWN;
2946 }
2947 return input->buffers->getPixelFormatIfApplicable();
2948 }
2949
getOutputBuffersPixelFormat()2950 uint32_t CCodecBufferChannel::getOutputBuffersPixelFormat() {
2951 Mutexed<Output>::Locked output(mOutput);
2952 if (output->buffers == nullptr) {
2953 return PIXEL_FORMAT_UNKNOWN;
2954 }
2955 return output->buffers->getPixelFormatIfApplicable();
2956 }
2957
resetBuffersPixelFormat(bool isEncoder)2958 void CCodecBufferChannel::resetBuffersPixelFormat(bool isEncoder) {
2959 if (isEncoder) {
2960 Mutexed<Input>::Locked input(mInput);
2961 if (input->buffers == nullptr) {
2962 return;
2963 }
2964 input->buffers->resetPixelFormatIfApplicable();
2965 } else {
2966 Mutexed<Output>::Locked output(mOutput);
2967 if (output->buffers == nullptr) {
2968 return;
2969 }
2970 output->buffers->resetPixelFormatIfApplicable();
2971 }
2972 }
2973
setInfoBuffer(const std::shared_ptr<C2InfoBuffer> & buffer)2974 void CCodecBufferChannel::setInfoBuffer(const std::shared_ptr<C2InfoBuffer> &buffer) {
2975 if (!mHasInputSurface) {
2976 mInfoBuffers.push_back(buffer);
2977 } else {
2978 std::list<std::unique_ptr<C2Work>> items;
2979 std::unique_ptr<C2Work> work(new C2Work);
2980 work->input.infoBuffers.emplace_back(*buffer);
2981 work->worklets.emplace_back(new C2Worklet);
2982 items.push_back(std::move(work));
2983 }
2984 }
2985
toStatusT(c2_status_t c2s,c2_operation_t c2op)2986 status_t toStatusT(c2_status_t c2s, c2_operation_t c2op) {
2987 // C2_OK is always translated to OK.
2988 if (c2s == C2_OK) {
2989 return OK;
2990 }
2991
2992 // Operation-dependent translation
2993 // TODO: Add as necessary
2994 switch (c2op) {
2995 case C2_OPERATION_Component_start:
2996 switch (c2s) {
2997 case C2_NO_MEMORY:
2998 return NO_MEMORY;
2999 default:
3000 return UNKNOWN_ERROR;
3001 }
3002 default:
3003 break;
3004 }
3005
3006 // Backup operation-agnostic translation
3007 switch (c2s) {
3008 case C2_BAD_INDEX:
3009 return BAD_INDEX;
3010 case C2_BAD_VALUE:
3011 return BAD_VALUE;
3012 case C2_BLOCKING:
3013 return WOULD_BLOCK;
3014 case C2_DUPLICATE:
3015 return ALREADY_EXISTS;
3016 case C2_NO_INIT:
3017 return NO_INIT;
3018 case C2_NO_MEMORY:
3019 return NO_MEMORY;
3020 case C2_NOT_FOUND:
3021 return NAME_NOT_FOUND;
3022 case C2_TIMED_OUT:
3023 return TIMED_OUT;
3024 case C2_BAD_STATE:
3025 case C2_CANCELED:
3026 case C2_CANNOT_DO:
3027 case C2_CORRUPTED:
3028 case C2_OMITTED:
3029 case C2_REFUSED:
3030 return UNKNOWN_ERROR;
3031 default:
3032 return -static_cast<status_t>(c2s);
3033 }
3034 }
3035
3036 } // namespace android
3037