1*3f982cf4SFabien Sanglard // Copyright 2020 The Chromium Authors. All rights reserved. 2*3f982cf4SFabien Sanglard // Use of this source code is governed by a BSD-style license that can be 3*3f982cf4SFabien Sanglard // found in the LICENSE file. 4*3f982cf4SFabien Sanglard 5*3f982cf4SFabien Sanglard #ifndef CAST_STANDALONE_SENDER_SIMULATED_CAPTURER_H_ 6*3f982cf4SFabien Sanglard #define CAST_STANDALONE_SENDER_SIMULATED_CAPTURER_H_ 7*3f982cf4SFabien Sanglard 8*3f982cf4SFabien Sanglard #include <stdint.h> 9*3f982cf4SFabien Sanglard 10*3f982cf4SFabien Sanglard #include <string> 11*3f982cf4SFabien Sanglard #include <vector> 12*3f982cf4SFabien Sanglard 13*3f982cf4SFabien Sanglard #include "absl/types/optional.h" 14*3f982cf4SFabien Sanglard #include "cast/standalone_sender/ffmpeg_glue.h" 15*3f982cf4SFabien Sanglard #include "platform/api/time.h" 16*3f982cf4SFabien Sanglard #include "util/alarm.h" 17*3f982cf4SFabien Sanglard 18*3f982cf4SFabien Sanglard namespace openscreen { 19*3f982cf4SFabien Sanglard namespace cast { 20*3f982cf4SFabien Sanglard 21*3f982cf4SFabien Sanglard class Environment; 22*3f982cf4SFabien Sanglard 23*3f982cf4SFabien Sanglard // Simulates live media capture by demuxing, decoding, and emitting a stream of 24*3f982cf4SFabien Sanglard // frames from a file at normal (1X) speed. This is a base class containing 25*3f982cf4SFabien Sanglard // common functionality. Typical usage: Instantiate one SimulatedAudioCapturer 26*3f982cf4SFabien Sanglard // and one FileVideoStreamCapturer. 27*3f982cf4SFabien Sanglard class SimulatedCapturer { 28*3f982cf4SFabien Sanglard public: 29*3f982cf4SFabien Sanglard // Interface for receiving end-of-stream and fatal error notifications. 30*3f982cf4SFabien Sanglard class Observer { 31*3f982cf4SFabien Sanglard public: 32*3f982cf4SFabien Sanglard // Called once the end of the file has been reached and the |capturer| has 33*3f982cf4SFabien Sanglard // halted. 34*3f982cf4SFabien Sanglard virtual void OnEndOfFile(SimulatedCapturer* capturer) = 0; 35*3f982cf4SFabien Sanglard 36*3f982cf4SFabien Sanglard // Called if a non-recoverable error occurs and the |capturer| has halted. 37*3f982cf4SFabien Sanglard virtual void OnError(SimulatedCapturer* capturer, std::string message) = 0; 38*3f982cf4SFabien Sanglard 39*3f982cf4SFabien Sanglard protected: 40*3f982cf4SFabien Sanglard virtual ~Observer(); 41*3f982cf4SFabien Sanglard }; 42*3f982cf4SFabien Sanglard 43*3f982cf4SFabien Sanglard void SetPlaybackRate(double rate); 44*3f982cf4SFabien Sanglard 45*3f982cf4SFabien Sanglard protected: 46*3f982cf4SFabien Sanglard SimulatedCapturer(Environment* environment, 47*3f982cf4SFabien Sanglard const char* path, 48*3f982cf4SFabien Sanglard AVMediaType media_type, 49*3f982cf4SFabien Sanglard Clock::time_point start_time, 50*3f982cf4SFabien Sanglard Observer* observer); 51*3f982cf4SFabien Sanglard 52*3f982cf4SFabien Sanglard virtual ~SimulatedCapturer(); 53*3f982cf4SFabien Sanglard 54*3f982cf4SFabien Sanglard // Optionally overridden, to apply additional decoder context settings before 55*3f982cf4SFabien Sanglard // avcodec_open2() is called. 56*3f982cf4SFabien Sanglard virtual void SetAdditionalDecoderParameters(AVCodecContext* decoder_context); 57*3f982cf4SFabien Sanglard 58*3f982cf4SFabien Sanglard // Performs any additional processing on the decoded frame (e.g., audio 59*3f982cf4SFabien Sanglard // resampling), and returns any adjustments to the frame's capture time (e.g., 60*3f982cf4SFabien Sanglard // to account for any buffering). If a fatal error occurs, absl::nullopt is 61*3f982cf4SFabien Sanglard // returned. The default implementation does nothing. 62*3f982cf4SFabien Sanglard // 63*3f982cf4SFabien Sanglard // Mutating the |decoded_frame| is not allowed. If a subclass implementation 64*3f982cf4SFabien Sanglard // wants to deliver different data (e.g., resampled audio), it must stash the 65*3f982cf4SFabien Sanglard // data itself for the next DeliverDataToClient() call. 66*3f982cf4SFabien Sanglard virtual absl::optional<Clock::duration> ProcessDecodedFrame( 67*3f982cf4SFabien Sanglard const AVFrame& decoded_frame); 68*3f982cf4SFabien Sanglard 69*3f982cf4SFabien Sanglard // Delivers the decoded frame data to the client. 70*3f982cf4SFabien Sanglard virtual void DeliverDataToClient(const AVFrame& decoded_frame, 71*3f982cf4SFabien Sanglard Clock::time_point capture_time) = 0; 72*3f982cf4SFabien Sanglard 73*3f982cf4SFabien Sanglard // Called when any transient or fatal error occurs, generating an Error and 74*3f982cf4SFabien Sanglard // scheduling a task to notify the Observer of it soon. 75*3f982cf4SFabien Sanglard void OnError(const char* what, int av_errnum); 76*3f982cf4SFabien Sanglard 77*3f982cf4SFabien Sanglard // Converts the given FFMPEG tick count into an approximate Clock::duration. 78*3f982cf4SFabien Sanglard static Clock::duration ToApproximateClockDuration( 79*3f982cf4SFabien Sanglard int64_t ticks, 80*3f982cf4SFabien Sanglard const AVRational& time_base); 81*3f982cf4SFabien Sanglard 82*3f982cf4SFabien Sanglard private: 83*3f982cf4SFabien Sanglard // Reads the next frame from the file, sends it to the decoder, and schedules 84*3f982cf4SFabien Sanglard // a future ConsumeNextDecodedFrame() call to continue processing. 85*3f982cf4SFabien Sanglard void StartDecodingNextFrame(); 86*3f982cf4SFabien Sanglard 87*3f982cf4SFabien Sanglard // Receives the next decoded frame and schedules media delivery to the client, 88*3f982cf4SFabien Sanglard // and/or calls Observer::OnEndOfFile() if there are no more frames in the 89*3f982cf4SFabien Sanglard // file. 90*3f982cf4SFabien Sanglard void ConsumeNextDecodedFrame(); 91*3f982cf4SFabien Sanglard 92*3f982cf4SFabien Sanglard const AVFormatContextUniquePtr format_context_; 93*3f982cf4SFabien Sanglard const AVMediaType media_type_; // Audio or Video. 94*3f982cf4SFabien Sanglard const Clock::time_point start_time_; 95*3f982cf4SFabien Sanglard Observer* const observer_; 96*3f982cf4SFabien Sanglard const AVPacketUniquePtr packet_; // Decoder input buffer. 97*3f982cf4SFabien Sanglard const AVFrameUniquePtr decoded_frame_; // Decoder output frame. 98*3f982cf4SFabien Sanglard int stream_index_ = -1; // Selected stream from the file. 99*3f982cf4SFabien Sanglard AVCodecContextUniquePtr decoder_context_; 100*3f982cf4SFabien Sanglard 101*3f982cf4SFabien Sanglard // The last frame's stream timestamp. This is used to detect bad stream 102*3f982cf4SFabien Sanglard // timestamps in the file. 103*3f982cf4SFabien Sanglard absl::optional<Clock::duration> last_frame_timestamp_; 104*3f982cf4SFabien Sanglard 105*3f982cf4SFabien Sanglard // Used to schedule the next task to execute and when it should execute. There 106*3f982cf4SFabien Sanglard // is only ever one task scheduled/running at any time. 107*3f982cf4SFabien Sanglard Alarm next_task_; 108*3f982cf4SFabien Sanglard 109*3f982cf4SFabien Sanglard // Used to determine playback rate. Currently, we only support "playing" 110*3f982cf4SFabien Sanglard // at 1x speed, or "pausing" at 0x speed. 111*3f982cf4SFabien Sanglard bool playback_rate_is_non_zero_ = true; 112*3f982cf4SFabien Sanglard }; 113*3f982cf4SFabien Sanglard 114*3f982cf4SFabien Sanglard // Emits the primary audio stream from a file. 115*3f982cf4SFabien Sanglard class SimulatedAudioCapturer final : public SimulatedCapturer { 116*3f982cf4SFabien Sanglard public: 117*3f982cf4SFabien Sanglard class Client : public SimulatedCapturer::Observer { 118*3f982cf4SFabien Sanglard public: 119*3f982cf4SFabien Sanglard // Called to deliver more audio data as |interleaved_samples|, which 120*3f982cf4SFabien Sanglard // contains |num_samples| tuples (i.e., multiply by the number of channels 121*3f982cf4SFabien Sanglard // to determine the number of array elements). |capture_time| is used to 122*3f982cf4SFabien Sanglard // synchronize the play-out of the first audio sample with respect to video 123*3f982cf4SFabien Sanglard // frames. 124*3f982cf4SFabien Sanglard virtual void OnAudioData(const float* interleaved_samples, 125*3f982cf4SFabien Sanglard int num_samples, 126*3f982cf4SFabien Sanglard Clock::time_point capture_time) = 0; 127*3f982cf4SFabien Sanglard 128*3f982cf4SFabien Sanglard protected: 129*3f982cf4SFabien Sanglard ~Client() override; 130*3f982cf4SFabien Sanglard }; 131*3f982cf4SFabien Sanglard 132*3f982cf4SFabien Sanglard // Constructor: |num_channels| and |sample_rate| specify the required audio 133*3f982cf4SFabien Sanglard // format. If necessary, audio from the file will be resampled to match the 134*3f982cf4SFabien Sanglard // required format. 135*3f982cf4SFabien Sanglard SimulatedAudioCapturer(Environment* environment, 136*3f982cf4SFabien Sanglard const char* path, 137*3f982cf4SFabien Sanglard int num_channels, 138*3f982cf4SFabien Sanglard int sample_rate, 139*3f982cf4SFabien Sanglard Clock::time_point start_time, 140*3f982cf4SFabien Sanglard Client* client); 141*3f982cf4SFabien Sanglard 142*3f982cf4SFabien Sanglard ~SimulatedAudioCapturer() final; 143*3f982cf4SFabien Sanglard 144*3f982cf4SFabien Sanglard private: 145*3f982cf4SFabien Sanglard // Examines the audio format of the given |frame|, and ensures the resampler 146*3f982cf4SFabien Sanglard // is initialized to take that as input. 147*3f982cf4SFabien Sanglard bool EnsureResamplerIsInitializedFor(const AVFrame& frame); 148*3f982cf4SFabien Sanglard 149*3f982cf4SFabien Sanglard // Resamples the current |SimulatedCapturer::decoded_frame()| into the 150*3f982cf4SFabien Sanglard // required output format/channels/rate. The result is stored in 151*3f982cf4SFabien Sanglard // |resampled_audio_| for the next DeliverDataToClient() call. 152*3f982cf4SFabien Sanglard absl::optional<Clock::duration> ProcessDecodedFrame( 153*3f982cf4SFabien Sanglard const AVFrame& decoded_frame) final; 154*3f982cf4SFabien Sanglard 155*3f982cf4SFabien Sanglard // Called at the moment Client::OnAudioData() should be called to pass the 156*3f982cf4SFabien Sanglard // |resampled_audio_|. 157*3f982cf4SFabien Sanglard void DeliverDataToClient(const AVFrame& decoded_frame, 158*3f982cf4SFabien Sanglard Clock::time_point capture_time) final; 159*3f982cf4SFabien Sanglard 160*3f982cf4SFabien Sanglard const int num_channels_; // Output number of channels. 161*3f982cf4SFabien Sanglard const int sample_rate_; // Output sample rate. 162*3f982cf4SFabien Sanglard Client* const client_; 163*3f982cf4SFabien Sanglard 164*3f982cf4SFabien Sanglard const SwrContextUniquePtr resampler_; 165*3f982cf4SFabien Sanglard 166*3f982cf4SFabien Sanglard // Current resampler input audio parameters. 167*3f982cf4SFabien Sanglard AVSampleFormat input_sample_format_ = AV_SAMPLE_FMT_NONE; 168*3f982cf4SFabien Sanglard int input_sample_rate_; 169*3f982cf4SFabien Sanglard uint64_t input_channel_layout_; // Opaque value used by resampler library. 170*3f982cf4SFabien Sanglard 171*3f982cf4SFabien Sanglard std::vector<float> resampled_audio_; 172*3f982cf4SFabien Sanglard }; 173*3f982cf4SFabien Sanglard 174*3f982cf4SFabien Sanglard // Emits the primary video stream from a file. 175*3f982cf4SFabien Sanglard class SimulatedVideoCapturer final : public SimulatedCapturer { 176*3f982cf4SFabien Sanglard public: 177*3f982cf4SFabien Sanglard class Client : public SimulatedCapturer::Observer { 178*3f982cf4SFabien Sanglard public: 179*3f982cf4SFabien Sanglard // Called to deliver the next video |frame|, which is always in I420 format. 180*3f982cf4SFabien Sanglard // |capture_time| is used to synchronize the play-out of the video frame 181*3f982cf4SFabien Sanglard // with respect to the audio track. 182*3f982cf4SFabien Sanglard virtual void OnVideoFrame(const AVFrame& frame, 183*3f982cf4SFabien Sanglard Clock::time_point capture_time) = 0; 184*3f982cf4SFabien Sanglard 185*3f982cf4SFabien Sanglard protected: 186*3f982cf4SFabien Sanglard ~Client() override; 187*3f982cf4SFabien Sanglard }; 188*3f982cf4SFabien Sanglard 189*3f982cf4SFabien Sanglard SimulatedVideoCapturer(Environment* environment, 190*3f982cf4SFabien Sanglard const char* path, 191*3f982cf4SFabien Sanglard Clock::time_point start_time, 192*3f982cf4SFabien Sanglard Client* client); 193*3f982cf4SFabien Sanglard 194*3f982cf4SFabien Sanglard ~SimulatedVideoCapturer() final; 195*3f982cf4SFabien Sanglard 196*3f982cf4SFabien Sanglard private: 197*3f982cf4SFabien Sanglard Client* const client_; 198*3f982cf4SFabien Sanglard 199*3f982cf4SFabien Sanglard // Sets up the decoder to produce I420 format output. 200*3f982cf4SFabien Sanglard void SetAdditionalDecoderParameters(AVCodecContext* decoder_context) final; 201*3f982cf4SFabien Sanglard 202*3f982cf4SFabien Sanglard // Called at the moment Client::OnVideoFrame() should be called to provide the 203*3f982cf4SFabien Sanglard // next video frame. 204*3f982cf4SFabien Sanglard void DeliverDataToClient(const AVFrame& decoded_frame, 205*3f982cf4SFabien Sanglard Clock::time_point capture_time) final; 206*3f982cf4SFabien Sanglard }; 207*3f982cf4SFabien Sanglard 208*3f982cf4SFabien Sanglard } // namespace cast 209*3f982cf4SFabien Sanglard } // namespace openscreen 210*3f982cf4SFabien Sanglard 211*3f982cf4SFabien Sanglard #endif // CAST_STANDALONE_SENDER_SIMULATED_CAPTURER_H_ 212