xref: /aosp_15_r20/frameworks/av/media/module/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp (revision ec779b8e0859a360c3d303172224686826e6e0e1) 
1 /*
2  * Copyright (C) 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 // #define LOG_NDEBUG 0
18 #define LOG_TAG "VideoTrackTranscoder"
19 
20 #include <android-base/logging.h>
21 #include <android-base/properties.h>
22 #include <media/NdkCommon.h>
23 #include <media/VideoTrackTranscoder.h>
24 #include <sys/prctl.h>
25 
26 using namespace AMediaFormatUtils;
27 
28 namespace android {
29 
30 // Check that the codec sample flags have the expected NDK meaning.
31 static_assert(SAMPLE_FLAG_CODEC_CONFIG == AMEDIACODEC_BUFFER_FLAG_CODEC_CONFIG,
32               "Sample flag mismatch: CODEC_CONFIG");
33 static_assert(SAMPLE_FLAG_END_OF_STREAM == AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM,
34               "Sample flag mismatch: END_OF_STREAM");
35 static_assert(SAMPLE_FLAG_PARTIAL_FRAME == AMEDIACODEC_BUFFER_FLAG_PARTIAL_FRAME,
36               "Sample flag mismatch: PARTIAL_FRAME");
37 
38 // Color format defined by surface. (See MediaCodecInfo.CodecCapabilities#COLOR_FormatSurface.)
39 static constexpr int32_t kColorFormatSurface = 0x7f000789;
40 // Default key frame interval in seconds.
41 static constexpr float kDefaultKeyFrameIntervalSeconds = 1.0f;
42 // Default codec operating rate.
43 static int32_t kDefaultCodecOperatingRate720P = base::GetIntProperty(
44         "debug.media.transcoding.codec_max_operating_rate_720P", /*default*/ 480);
45 static int32_t kDefaultCodecOperatingRate1080P = base::GetIntProperty(
46         "debug.media.transcoding.codec_max_operating_rate_1080P", /*default*/ 240);
47 // Default codec priority.
48 static constexpr int32_t kDefaultCodecPriority = 1;
49 // Default bitrate, in case source estimation fails.
50 static constexpr int32_t kDefaultBitrateMbps = 10 * 1000 * 1000;
51 // Default frame rate.
52 static constexpr int32_t kDefaultFrameRate = 30;
53 // Default codec complexity
54 static constexpr int32_t kDefaultCodecComplexity = 1;
55 
56 template <typename T>
push(T const & value,bool front)57 void VideoTrackTranscoder::BlockingQueue<T>::push(T const& value, bool front) {
58     {
59         std::scoped_lock lock(mMutex);
60         if (mAborted) {
61             return;
62         }
63 
64         if (front) {
65             mQueue.push_front(value);
66         } else {
67             mQueue.push_back(value);
68         }
69     }
70     mCondition.notify_one();
71 }
72 
73 template <typename T>
pop()74 T VideoTrackTranscoder::BlockingQueue<T>::pop() {
75     std::unique_lock lock(mMutex);
76     while (mQueue.empty()) {
77         mCondition.wait(lock);
78     }
79     T value = mQueue.front();
80     mQueue.pop_front();
81     return value;
82 }
83 
84 // Note: Do not call if another thread might waiting in pop.
85 template <typename T>
abort()86 void VideoTrackTranscoder::BlockingQueue<T>::abort() {
87     std::scoped_lock lock(mMutex);
88     mAborted = true;
89     mQueue.clear();
90 }
91 
92 // The CodecWrapper class is used to let AMediaCodec instances outlive the transcoder object itself
93 // by giving the codec a weak pointer to the transcoder. Codecs wrapped in this object are kept
94 // alive by the transcoder and the codec's outstanding buffers. Once the transcoder stops and all
95 // output buffers have been released by downstream components the codec will also be released.
96 class VideoTrackTranscoder::CodecWrapper {
97 public:
CodecWrapper(AMediaCodec * codec,const std::weak_ptr<VideoTrackTranscoder> & transcoder)98     CodecWrapper(AMediaCodec* codec, const std::weak_ptr<VideoTrackTranscoder>& transcoder)
99           : mCodec(codec), mTranscoder(transcoder), mCodecStarted(false) {}
~CodecWrapper()100     ~CodecWrapper() {
101         if (mCodecStarted) {
102             AMediaCodec_stop(mCodec);
103         }
104         AMediaCodec_delete(mCodec);
105     }
106 
getCodec()107     AMediaCodec* getCodec() { return mCodec; }
getTranscoder() const108     std::shared_ptr<VideoTrackTranscoder> getTranscoder() const { return mTranscoder.lock(); };
setStarted()109     void setStarted() { mCodecStarted = true; }
110 
111 private:
112     AMediaCodec* mCodec;
113     std::weak_ptr<VideoTrackTranscoder> mTranscoder;
114     bool mCodecStarted;
115 };
116 
117 // Dispatch responses to codec callbacks onto the message queue.
118 struct AsyncCodecCallbackDispatch {
onAsyncInputAvailableandroid::AsyncCodecCallbackDispatch119     static void onAsyncInputAvailable(AMediaCodec* codec, void* userdata, int32_t index) {
120         VideoTrackTranscoder::CodecWrapper* wrapper =
121                 static_cast<VideoTrackTranscoder::CodecWrapper*>(userdata);
122         if (auto transcoder = wrapper->getTranscoder()) {
123             if (codec == transcoder->mDecoder) {
124                 transcoder->mCodecMessageQueue.push(
125                         [transcoder, index] { transcoder->enqueueInputSample(index); });
126             }
127         }
128     }
129 
onAsyncOutputAvailableandroid::AsyncCodecCallbackDispatch130     static void onAsyncOutputAvailable(AMediaCodec* codec, void* userdata, int32_t index,
131                                        AMediaCodecBufferInfo* bufferInfoPtr) {
132         VideoTrackTranscoder::CodecWrapper* wrapper =
133                 static_cast<VideoTrackTranscoder::CodecWrapper*>(userdata);
134         AMediaCodecBufferInfo bufferInfo = *bufferInfoPtr;
135         if (auto transcoder = wrapper->getTranscoder()) {
136             transcoder->mCodecMessageQueue.push([transcoder, index, codec, bufferInfo] {
137                 if (codec == transcoder->mDecoder) {
138                     transcoder->transferBuffer(index, bufferInfo);
139                 } else if (codec == transcoder->mEncoder->getCodec()) {
140                     transcoder->dequeueOutputSample(index, bufferInfo);
141                 }
142             });
143         }
144     }
145 
onAsyncFormatChangedandroid::AsyncCodecCallbackDispatch146     static void onAsyncFormatChanged(AMediaCodec* codec, void* userdata, AMediaFormat* format) {
147         VideoTrackTranscoder::CodecWrapper* wrapper =
148                 static_cast<VideoTrackTranscoder::CodecWrapper*>(userdata);
149         if (auto transcoder = wrapper->getTranscoder()) {
150             const bool isDecoder = codec == transcoder->mDecoder;
151             const char* kCodecName = (isDecoder ? "Decoder" : "Encoder");
152             LOG(INFO) << kCodecName << " format changed: " << AMediaFormat_toString(format);
153             transcoder->mCodecMessageQueue.push([transcoder, format, isDecoder] {
154                 transcoder->updateTrackFormat(format, isDecoder);
155             });
156         }
157     }
158 
onAsyncErrorandroid::AsyncCodecCallbackDispatch159     static void onAsyncError(AMediaCodec* codec, void* userdata, media_status_t error,
160                              int32_t actionCode, const char* detail) {
161         LOG(ERROR) << "Error from codec " << codec << ", userdata " << userdata << ", error "
162                    << error << ", action " << actionCode << ", detail " << detail;
163         VideoTrackTranscoder::CodecWrapper* wrapper =
164                 static_cast<VideoTrackTranscoder::CodecWrapper*>(userdata);
165         if (auto transcoder = wrapper->getTranscoder()) {
166             transcoder->mCodecMessageQueue.push(
167                     [transcoder, error] { transcoder->mStatus = error; }, true);
168         }
169     }
170 };
171 
172 // static
create(const std::weak_ptr<MediaTrackTranscoderCallback> & transcoderCallback,pid_t pid,uid_t uid)173 std::shared_ptr<VideoTrackTranscoder> VideoTrackTranscoder::create(
174         const std::weak_ptr<MediaTrackTranscoderCallback>& transcoderCallback, pid_t pid,
175         uid_t uid) {
176     return std::shared_ptr<VideoTrackTranscoder>(
177             new VideoTrackTranscoder(transcoderCallback, pid, uid));
178 }
179 
~VideoTrackTranscoder()180 VideoTrackTranscoder::~VideoTrackTranscoder() {
181     if (mDecoder != nullptr) {
182         AMediaCodec_delete(mDecoder);
183     }
184 
185     if (mSurface != nullptr) {
186         ANativeWindow_release(mSurface);
187     }
188 }
189 
190 // Search the default operating rate based on resolution.
getDefaultOperatingRate(AMediaFormat * encoderFormat)191 static int32_t getDefaultOperatingRate(AMediaFormat* encoderFormat) {
192     int32_t width, height;
193     if (AMediaFormat_getInt32(encoderFormat, AMEDIAFORMAT_KEY_WIDTH, &width) && (width > 0) &&
194         AMediaFormat_getInt32(encoderFormat, AMEDIAFORMAT_KEY_HEIGHT, &height) && (height > 0)) {
195         if ((width == 1280 && height == 720) || (width == 720 && height == 1280)) {
196             return kDefaultCodecOperatingRate720P;
197         } else if ((width == 1920 && height == 1080) || (width == 1080 && height == 1920)) {
198             return kDefaultCodecOperatingRate1080P;
199         } else {
200             LOG(WARNING) << "Could not find default operating rate: " << width << " " << height;
201             // Don't set operating rate if the correct dimensions are not found.
202         }
203     } else {
204         LOG(ERROR) << "Failed to get default operating rate due to missing resolution";
205     }
206     return -1;
207 }
208 
209 // Creates and configures the codecs.
configureDestinationFormat(const std::shared_ptr<AMediaFormat> & destinationFormat)210 media_status_t VideoTrackTranscoder::configureDestinationFormat(
211         const std::shared_ptr<AMediaFormat>& destinationFormat) {
212     media_status_t status = AMEDIA_OK;
213 
214     if (destinationFormat == nullptr) {
215         LOG(ERROR) << "Destination format is null, use passthrough transcoder";
216         return AMEDIA_ERROR_INVALID_PARAMETER;
217     }
218 
219     AMediaFormat* encoderFormat = AMediaFormat_new();
220     if (!encoderFormat || AMediaFormat_copy(encoderFormat, destinationFormat.get()) != AMEDIA_OK) {
221         LOG(ERROR) << "Unable to copy destination format";
222         return AMEDIA_ERROR_INVALID_PARAMETER;
223     }
224 
225     if (!AMediaFormat_getInt32(encoderFormat, AMEDIAFORMAT_KEY_BIT_RATE, &mConfiguredBitrate)) {
226         status = mMediaSampleReader->getEstimatedBitrateForTrack(mTrackIndex, &mConfiguredBitrate);
227         if (status != AMEDIA_OK) {
228             LOG(ERROR) << "Unable to estimate bitrate. Using default " << kDefaultBitrateMbps;
229             mConfiguredBitrate = kDefaultBitrateMbps;
230         }
231 
232         LOG(INFO) << "Configuring bitrate " << mConfiguredBitrate;
233         AMediaFormat_setInt32(encoderFormat, AMEDIAFORMAT_KEY_BIT_RATE, mConfiguredBitrate);
234     }
235 
236     SetDefaultFormatValueFloat(AMEDIAFORMAT_KEY_I_FRAME_INTERVAL, encoderFormat,
237                                kDefaultKeyFrameIntervalSeconds);
238 
239     int32_t operatingRate = getDefaultOperatingRate(encoderFormat);
240 
241     if (operatingRate != -1) {
242         float tmpf;
243         int32_t tmpi;
244         if (!AMediaFormat_getFloat(encoderFormat, AMEDIAFORMAT_KEY_OPERATING_RATE, &tmpf) &&
245             !AMediaFormat_getInt32(encoderFormat, AMEDIAFORMAT_KEY_OPERATING_RATE, &tmpi)) {
246             AMediaFormat_setInt32(encoderFormat, AMEDIAFORMAT_KEY_OPERATING_RATE, operatingRate);
247         }
248     }
249 
250     SetDefaultFormatValueInt32(AMEDIAFORMAT_KEY_PRIORITY, encoderFormat, kDefaultCodecPriority);
251     SetDefaultFormatValueInt32(AMEDIAFORMAT_KEY_FRAME_RATE, encoderFormat, kDefaultFrameRate);
252     SetDefaultFormatValueInt32(AMEDIAFORMAT_KEY_COMPLEXITY, encoderFormat, kDefaultCodecComplexity);
253     AMediaFormat_setInt32(encoderFormat, AMEDIAFORMAT_KEY_COLOR_FORMAT, kColorFormatSurface);
254 
255     // Always encode without rotation. The rotation degree will be transferred directly to
256     // MediaSampleWriter track format, and MediaSampleWriter will call AMediaMuxer_setOrientationHint.
257     AMediaFormat_setInt32(encoderFormat, AMEDIAFORMAT_KEY_ROTATION, 0);
258 
259     // Request encoder to use background priorities by default.
260     SetDefaultFormatValueInt32(TBD_AMEDIACODEC_PARAMETER_KEY_BACKGROUND_MODE, encoderFormat,
261                                1 /* true */);
262 
263     mDestinationFormat = std::shared_ptr<AMediaFormat>(encoderFormat, &AMediaFormat_delete);
264 
265     // Create and configure the encoder.
266     const char* destinationMime = nullptr;
267     bool ok = AMediaFormat_getString(mDestinationFormat.get(), AMEDIAFORMAT_KEY_MIME,
268                                      &destinationMime);
269     if (!ok) {
270         LOG(ERROR) << "Destination MIME type is required for transcoding.";
271         return AMEDIA_ERROR_INVALID_PARAMETER;
272     }
273 
274 #define __TRANSCODING_MIN_API__ 31
275 
276     AMediaCodec* encoder;
277     if (__builtin_available(android __TRANSCODING_MIN_API__, *)) {
278         encoder = AMediaCodec_createEncoderByTypeForClient(destinationMime, mPid, mUid);
279     } else {
280         encoder = AMediaCodec_createEncoderByType(destinationMime);
281     }
282     if (encoder == nullptr) {
283         LOG(ERROR) << "Unable to create encoder for type " << destinationMime;
284         return AMEDIA_ERROR_UNSUPPORTED;
285     }
286     mEncoder = std::make_shared<CodecWrapper>(encoder, shared_from_this());
287 
288     LOG(INFO) << "Configuring encoder with: " << AMediaFormat_toString(mDestinationFormat.get());
289     status = AMediaCodec_configure(mEncoder->getCodec(), mDestinationFormat.get(),
290                                    NULL /* surface */, NULL /* crypto */,
291                                    AMEDIACODEC_CONFIGURE_FLAG_ENCODE);
292     if (status != AMEDIA_OK) {
293         LOG(ERROR) << "Unable to configure video encoder: " << status;
294         return status;
295     }
296 
297     status = AMediaCodec_createInputSurface(mEncoder->getCodec(), &mSurface);
298     if (status != AMEDIA_OK) {
299         LOG(ERROR) << "Unable to create an encoder input surface: %d" << status;
300         return status;
301     }
302 
303     // Create and configure the decoder.
304     const char* sourceMime = nullptr;
305     ok = AMediaFormat_getString(mSourceFormat.get(), AMEDIAFORMAT_KEY_MIME, &sourceMime);
306     if (!ok) {
307         LOG(ERROR) << "Source MIME type is required for transcoding.";
308         return AMEDIA_ERROR_INVALID_PARAMETER;
309     }
310 
311     if (__builtin_available(android __TRANSCODING_MIN_API__, *)) {
312         mDecoder = AMediaCodec_createDecoderByTypeForClient(sourceMime, mPid, mUid);
313     } else {
314         mDecoder = AMediaCodec_createDecoderByType(sourceMime);
315     }
316     if (mDecoder == nullptr) {
317         LOG(ERROR) << "Unable to create decoder for type " << sourceMime;
318         return AMEDIA_ERROR_UNSUPPORTED;
319     }
320 
321     auto decoderFormat = std::shared_ptr<AMediaFormat>(AMediaFormat_new(), &AMediaFormat_delete);
322     if (!decoderFormat ||
323         AMediaFormat_copy(decoderFormat.get(), mSourceFormat.get()) != AMEDIA_OK) {
324         LOG(ERROR) << "Unable to copy source format";
325         return AMEDIA_ERROR_INVALID_PARAMETER;
326     }
327 
328     // Request decoder to convert HDR content to SDR.
329     const bool sourceIsHdr = VideoIsHdr(mSourceFormat.get());
330     if (sourceIsHdr) {
331         AMediaFormat_setInt32(decoderFormat.get(),
332                               TBD_AMEDIACODEC_PARAMETER_KEY_COLOR_TRANSFER_REQUEST,
333                               COLOR_TRANSFER_SDR_VIDEO);
334     }
335 
336     // Prevent decoder from overwriting frames that the encoder has not yet consumed.
337     AMediaFormat_setInt32(decoderFormat.get(), TBD_AMEDIACODEC_PARAMETER_KEY_ALLOW_FRAME_DROP, 0);
338 
339     // Copy over configurations that apply to both encoder and decoder.
340     static const std::vector<EntryCopier> kEncoderEntriesToCopy{
341             ENTRY_COPIER2(AMEDIAFORMAT_KEY_OPERATING_RATE, Float, Int32),
342             ENTRY_COPIER(AMEDIAFORMAT_KEY_PRIORITY, Int32),
343             ENTRY_COPIER(TBD_AMEDIACODEC_PARAMETER_KEY_BACKGROUND_MODE, Int32),
344     };
345     CopyFormatEntries(mDestinationFormat.get(), decoderFormat.get(), kEncoderEntriesToCopy);
346 
347     LOG(INFO) << "Configuring decoder with: " << AMediaFormat_toString(decoderFormat.get());
348     status = AMediaCodec_configure(mDecoder, decoderFormat.get(), mSurface, NULL /* crypto */,
349                                    0 /* flags */);
350     if (status != AMEDIA_OK) {
351         LOG(ERROR) << "Unable to configure video decoder: " << status;
352         return status;
353     }
354 
355     if (sourceIsHdr) {
356         bool supported = false;
357         AMediaFormat* inputFormat = AMediaCodec_getInputFormat(mDecoder);
358 
359         if (inputFormat != nullptr) {
360             int32_t transferFunc;
361             supported = AMediaFormat_getInt32(inputFormat,
362                                               TBD_AMEDIACODEC_PARAMETER_KEY_COLOR_TRANSFER_REQUEST,
363                                               &transferFunc) &&
364                         transferFunc == COLOR_TRANSFER_SDR_VIDEO;
365             AMediaFormat_delete(inputFormat);
366         }
367 
368         if (!supported) {
369             LOG(ERROR) << "HDR to SDR conversion unsupported by the codec";
370             return AMEDIA_ERROR_UNSUPPORTED;
371         }
372     }
373 
374     // Configure codecs to run in async mode.
375     AMediaCodecOnAsyncNotifyCallback asyncCodecCallbacks = {
376             .onAsyncInputAvailable = AsyncCodecCallbackDispatch::onAsyncInputAvailable,
377             .onAsyncOutputAvailable = AsyncCodecCallbackDispatch::onAsyncOutputAvailable,
378             .onAsyncFormatChanged = AsyncCodecCallbackDispatch::onAsyncFormatChanged,
379             .onAsyncError = AsyncCodecCallbackDispatch::onAsyncError};
380 
381     // Note: The decoder does not need its own wrapper because its lifetime is tied to the
382     // transcoder. But the same callbacks are reused for decoder and encoder so we pass the encoder
383     // wrapper as userdata here but never read the codec from it in the callback.
384     status = AMediaCodec_setAsyncNotifyCallback(mDecoder, asyncCodecCallbacks, mEncoder.get());
385     if (status != AMEDIA_OK) {
386         LOG(ERROR) << "Unable to set decoder to async mode: " << status;
387         return status;
388     }
389 
390     status = AMediaCodec_setAsyncNotifyCallback(mEncoder->getCodec(), asyncCodecCallbacks,
391                                                 mEncoder.get());
392     if (status != AMEDIA_OK) {
393         LOG(ERROR) << "Unable to set encoder to async mode: " << status;
394         return status;
395     }
396 
397     return AMEDIA_OK;
398 }
399 
enqueueInputSample(int32_t bufferIndex)400 void VideoTrackTranscoder::enqueueInputSample(int32_t bufferIndex) {
401     media_status_t status = AMEDIA_OK;
402 
403     if (mEosFromSource) {
404         return;
405     }
406 
407     status = mMediaSampleReader->getSampleInfoForTrack(mTrackIndex, &mSampleInfo);
408     if (status != AMEDIA_OK && status != AMEDIA_ERROR_END_OF_STREAM) {
409         LOG(ERROR) << "Error getting next sample info: " << status;
410         mStatus = status;
411         return;
412     }
413     const bool endOfStream = (status == AMEDIA_ERROR_END_OF_STREAM);
414 
415     if (!endOfStream) {
416         size_t bufferSize = 0;
417         uint8_t* sourceBuffer = AMediaCodec_getInputBuffer(mDecoder, bufferIndex, &bufferSize);
418         if (sourceBuffer == nullptr) {
419             LOG(ERROR) << "Decoder returned a NULL input buffer.";
420             mStatus = AMEDIA_ERROR_UNKNOWN;
421             return;
422         } else if (bufferSize < mSampleInfo.size) {
423             LOG(ERROR) << "Decoder returned an input buffer that is smaller than the sample.";
424             mStatus = AMEDIA_ERROR_UNKNOWN;
425             return;
426         }
427 
428         status = mMediaSampleReader->readSampleDataForTrack(mTrackIndex, sourceBuffer,
429                                                             mSampleInfo.size);
430         if (status != AMEDIA_OK) {
431             LOG(ERROR) << "Unable to read next sample data. Aborting transcode.";
432             mStatus = status;
433             return;
434         }
435 
436         if (mSampleInfo.size) {
437             ++mInputFrameCount;
438         }
439     } else {
440         LOG(DEBUG) << "EOS from source.";
441         mEosFromSource = true;
442     }
443 
444     status = AMediaCodec_queueInputBuffer(mDecoder, bufferIndex, 0, mSampleInfo.size,
445                                           mSampleInfo.presentationTimeUs, mSampleInfo.flags);
446     if (status != AMEDIA_OK) {
447         LOG(ERROR) << "Unable to queue input buffer for decode: " << status;
448         mStatus = status;
449         return;
450     }
451 }
452 
transferBuffer(int32_t bufferIndex,AMediaCodecBufferInfo bufferInfo)453 void VideoTrackTranscoder::transferBuffer(int32_t bufferIndex, AMediaCodecBufferInfo bufferInfo) {
454     if (bufferIndex >= 0) {
455         bool needsRender = bufferInfo.size > 0;
456         AMediaCodec_releaseOutputBuffer(mDecoder, bufferIndex, needsRender);
457     }
458 
459     if (bufferInfo.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
460         LOG(DEBUG) << "EOS from decoder.";
461         // NOTE - b/360057459
462         // There is a synchronization problem between feeding the frame to the encoder input surface
463         // and signaling end of stream.
464         // Waiting before signaling end of stream so that input surface has time to feed remaining
465         // frames to the encoder.
466         std::this_thread::sleep_for(std::chrono::milliseconds(10));
467         media_status_t status = AMediaCodec_signalEndOfInputStream(mEncoder->getCodec());
468         if (status != AMEDIA_OK) {
469             LOG(ERROR) << "SignalEOS on encoder returned error: " << status;
470             mStatus = status;
471         }
472     }
473 }
474 
dequeueOutputSample(int32_t bufferIndex,AMediaCodecBufferInfo bufferInfo)475 void VideoTrackTranscoder::dequeueOutputSample(int32_t bufferIndex,
476                                                AMediaCodecBufferInfo bufferInfo) {
477     if (bufferIndex >= 0) {
478         size_t sampleSize = 0;
479         uint8_t* buffer =
480                 AMediaCodec_getOutputBuffer(mEncoder->getCodec(), bufferIndex, &sampleSize);
481 
482         MediaSample::OnSampleReleasedCallback bufferReleaseCallback =
483                 [encoder = mEncoder](MediaSample* sample) {
484                     AMediaCodec_releaseOutputBuffer(encoder->getCodec(), sample->bufferId,
485                                                     false /* render */);
486                 };
487 
488         std::shared_ptr<MediaSample> sample = MediaSample::createWithReleaseCallback(
489                 buffer, bufferInfo.offset, bufferIndex, bufferReleaseCallback);
490         sample->info.size = bufferInfo.size;
491         sample->info.flags = bufferInfo.flags;
492         sample->info.presentationTimeUs = bufferInfo.presentationTimeUs;
493 
494         if (bufferInfo.size > 0 && (bufferInfo.flags & SAMPLE_FLAG_CODEC_CONFIG) == 0) {
495             ++mOutputFrameCount;
496         }
497         onOutputSampleAvailable(sample);
498 
499         mLastSampleWasSync = sample->info.flags & SAMPLE_FLAG_SYNC_SAMPLE;
500     }
501 
502     if (bufferInfo.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) {
503         LOG(DEBUG) << "EOS from encoder.";
504         mEosFromEncoder = true;
505 
506         if (mInputFrameCount != mOutputFrameCount) {
507             LOG(WARNING) << "Input / Output frame count mismatch: " << mInputFrameCount << " vs "
508                          << mOutputFrameCount;
509             if (mInputFrameCount > 0 && mOutputFrameCount == 0) {
510                 LOG(ERROR) << "Encoder did not produce any output frames.";
511                 mStatus = AMEDIA_ERROR_UNKNOWN;
512             }
513         }
514     }
515 }
516 
updateTrackFormat(AMediaFormat * outputFormat,bool fromDecoder)517 void VideoTrackTranscoder::updateTrackFormat(AMediaFormat* outputFormat, bool fromDecoder) {
518     if (fromDecoder) {
519         static const std::vector<AMediaFormatUtils::EntryCopier> kValuesToCopy{
520                 ENTRY_COPIER(AMEDIAFORMAT_KEY_COLOR_RANGE, Int32),
521                 ENTRY_COPIER(AMEDIAFORMAT_KEY_COLOR_STANDARD, Int32),
522                 ENTRY_COPIER(AMEDIAFORMAT_KEY_COLOR_TRANSFER, Int32),
523         };
524         AMediaFormat* params = AMediaFormat_new();
525         if (params != nullptr) {
526             AMediaFormatUtils::CopyFormatEntries(outputFormat, params, kValuesToCopy);
527             if (AMediaCodec_setParameters(mEncoder->getCodec(), params) != AMEDIA_OK) {
528                 LOG(WARNING) << "Unable to update encoder with color information";
529             }
530             AMediaFormat_delete(params);
531         }
532         return;
533     }
534 
535     if (mActualOutputFormat != nullptr) {
536         LOG(WARNING) << "Ignoring duplicate format change.";
537         return;
538     }
539 
540     AMediaFormat* formatCopy = AMediaFormat_new();
541     if (!formatCopy || AMediaFormat_copy(formatCopy, outputFormat) != AMEDIA_OK) {
542         LOG(ERROR) << "Unable to copy outputFormat";
543         AMediaFormat_delete(formatCopy);
544         mStatus = AMEDIA_ERROR_INVALID_PARAMETER;
545         return;
546     }
547 
548     // Generate the actual track format for muxer based on the encoder output format,
549     // since many vital information comes in the encoder format (eg. CSD).
550     // Transfer necessary fields from the user-configured track format (derived from
551     // source track format and user transcoding request) where needed.
552 
553     // Transfer SAR settings:
554     // If mDestinationFormat has SAR set, it means the original source has SAR specified
555     // at container level. This is supposed to override any SAR settings in the bitstream,
556     // thus should always be transferred to the container of the transcoded file.
557     int32_t sarWidth, sarHeight;
558     if (AMediaFormat_getInt32(mSourceFormat.get(), AMEDIAFORMAT_KEY_SAR_WIDTH, &sarWidth) &&
559         (sarWidth > 0) &&
560         AMediaFormat_getInt32(mSourceFormat.get(), AMEDIAFORMAT_KEY_SAR_HEIGHT, &sarHeight) &&
561         (sarHeight > 0)) {
562         AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_SAR_WIDTH, sarWidth);
563         AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_SAR_HEIGHT, sarHeight);
564     }
565     // Transfer DAR settings.
566     int32_t displayWidth, displayHeight;
567     if (AMediaFormat_getInt32(mSourceFormat.get(), AMEDIAFORMAT_KEY_DISPLAY_WIDTH, &displayWidth) &&
568         (displayWidth > 0) &&
569         AMediaFormat_getInt32(mSourceFormat.get(), AMEDIAFORMAT_KEY_DISPLAY_HEIGHT,
570                               &displayHeight) &&
571         (displayHeight > 0)) {
572         AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_DISPLAY_WIDTH, displayWidth);
573         AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_DISPLAY_HEIGHT, displayHeight);
574     }
575 
576     // Transfer rotation settings.
577     // Note that muxer itself doesn't take rotation from the track format. It requires
578     // AMediaMuxer_setOrientationHint to set the rotation. Here we pass the rotation to
579     // MediaSampleWriter using the track format. MediaSampleWriter will then call
580     // AMediaMuxer_setOrientationHint as needed.
581     int32_t rotation;
582     if (AMediaFormat_getInt32(mSourceFormat.get(), AMEDIAFORMAT_KEY_ROTATION, &rotation) &&
583         (rotation != 0)) {
584         AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_ROTATION, rotation);
585     }
586 
587     // Transfer track duration.
588     // Preserve the source track duration by sending it to MediaSampleWriter.
589     int64_t durationUs;
590     if (AMediaFormat_getInt64(mSourceFormat.get(), AMEDIAFORMAT_KEY_DURATION, &durationUs) &&
591         durationUs > 0) {
592         AMediaFormat_setInt64(formatCopy, AMEDIAFORMAT_KEY_DURATION, durationUs);
593     }
594 
595     // TODO: transfer other fields as required.
596 
597     mActualOutputFormat = std::shared_ptr<AMediaFormat>(formatCopy, &AMediaFormat_delete);
598     LOG(INFO) << "Actual output format: " << AMediaFormat_toString(formatCopy);
599 
600     notifyTrackFormatAvailable();
601 }
602 
runTranscodeLoop(bool * stopped)603 media_status_t VideoTrackTranscoder::runTranscodeLoop(bool* stopped) {
604     prctl(PR_SET_NAME, (unsigned long)"VideTranscodTrd", 0, 0, 0);
605 
606     // Push start decoder and encoder as two messages, so that these are subject to the
607     // stop request as well. If the session is cancelled (or paused) immediately after start,
608     // we don't need to waste time start then stop the codecs.
609     mCodecMessageQueue.push([this] {
610         media_status_t status = AMediaCodec_start(mDecoder);
611         if (status != AMEDIA_OK) {
612             LOG(ERROR) << "Unable to start video decoder: " << status;
613             mStatus = status;
614         }
615     });
616 
617     mCodecMessageQueue.push([this] {
618         media_status_t status = AMediaCodec_start(mEncoder->getCodec());
619         if (status != AMEDIA_OK) {
620             LOG(ERROR) << "Unable to start video encoder: " << status;
621             mStatus = status;
622         }
623         mEncoder->setStarted();
624     });
625 
626     // Process codec events until EOS is reached, transcoding is stopped or an error occurs.
627     while (mStopRequest != STOP_NOW && !mEosFromEncoder && mStatus == AMEDIA_OK) {
628         std::function<void()> message = mCodecMessageQueue.pop();
629         message();
630 
631         if (mStopRequest == STOP_ON_SYNC && mLastSampleWasSync) {
632             break;
633         }
634     }
635 
636     mCodecMessageQueue.abort();
637     AMediaCodec_stop(mDecoder);
638 
639     // Signal if transcoding was stopped before it finished.
640     if (mStopRequest != NONE && !mEosFromEncoder && mStatus == AMEDIA_OK) {
641         *stopped = true;
642     }
643 
644     return mStatus;
645 }
646 
abortTranscodeLoop()647 void VideoTrackTranscoder::abortTranscodeLoop() {
648     if (mStopRequest == STOP_NOW) {
649         // Wake up transcoder thread.
650         mCodecMessageQueue.push([] {}, true /* front */);
651     }
652 }
653 
getOutputFormat() const654 std::shared_ptr<AMediaFormat> VideoTrackTranscoder::getOutputFormat() const {
655     return mActualOutputFormat;
656 }
657 
658 }  // namespace android
659