xref: /aosp_15_r20/external/openscreen/cast/standalone_sender/simulated_capturer.h (revision 3f982cf4871df8771c9d4abe6e9a6f8d829b2736)
1*3f982cf4SFabien Sanglard // Copyright 2020 The Chromium Authors. All rights reserved.
2*3f982cf4SFabien Sanglard // Use of this source code is governed by a BSD-style license that can be
3*3f982cf4SFabien Sanglard // found in the LICENSE file.
4*3f982cf4SFabien Sanglard 
5*3f982cf4SFabien Sanglard #ifndef CAST_STANDALONE_SENDER_SIMULATED_CAPTURER_H_
6*3f982cf4SFabien Sanglard #define CAST_STANDALONE_SENDER_SIMULATED_CAPTURER_H_
7*3f982cf4SFabien Sanglard 
8*3f982cf4SFabien Sanglard #include <stdint.h>
9*3f982cf4SFabien Sanglard 
10*3f982cf4SFabien Sanglard #include <string>
11*3f982cf4SFabien Sanglard #include <vector>
12*3f982cf4SFabien Sanglard 
13*3f982cf4SFabien Sanglard #include "absl/types/optional.h"
14*3f982cf4SFabien Sanglard #include "cast/standalone_sender/ffmpeg_glue.h"
15*3f982cf4SFabien Sanglard #include "platform/api/time.h"
16*3f982cf4SFabien Sanglard #include "util/alarm.h"
17*3f982cf4SFabien Sanglard 
18*3f982cf4SFabien Sanglard namespace openscreen {
19*3f982cf4SFabien Sanglard namespace cast {
20*3f982cf4SFabien Sanglard 
21*3f982cf4SFabien Sanglard class Environment;
22*3f982cf4SFabien Sanglard 
23*3f982cf4SFabien Sanglard // Simulates live media capture by demuxing, decoding, and emitting a stream of
24*3f982cf4SFabien Sanglard // frames from a file at normal (1X) speed. This is a base class containing
25*3f982cf4SFabien Sanglard // common functionality. Typical usage: Instantiate one SimulatedAudioCapturer
26*3f982cf4SFabien Sanglard // and one FileVideoStreamCapturer.
27*3f982cf4SFabien Sanglard class SimulatedCapturer {
28*3f982cf4SFabien Sanglard  public:
29*3f982cf4SFabien Sanglard   // Interface for receiving end-of-stream and fatal error notifications.
30*3f982cf4SFabien Sanglard   class Observer {
31*3f982cf4SFabien Sanglard    public:
32*3f982cf4SFabien Sanglard     // Called once the end of the file has been reached and the |capturer| has
33*3f982cf4SFabien Sanglard     // halted.
34*3f982cf4SFabien Sanglard     virtual void OnEndOfFile(SimulatedCapturer* capturer) = 0;
35*3f982cf4SFabien Sanglard 
36*3f982cf4SFabien Sanglard     // Called if a non-recoverable error occurs and the |capturer| has halted.
37*3f982cf4SFabien Sanglard     virtual void OnError(SimulatedCapturer* capturer, std::string message) = 0;
38*3f982cf4SFabien Sanglard 
39*3f982cf4SFabien Sanglard    protected:
40*3f982cf4SFabien Sanglard     virtual ~Observer();
41*3f982cf4SFabien Sanglard   };
42*3f982cf4SFabien Sanglard 
43*3f982cf4SFabien Sanglard   void SetPlaybackRate(double rate);
44*3f982cf4SFabien Sanglard 
45*3f982cf4SFabien Sanglard  protected:
46*3f982cf4SFabien Sanglard   SimulatedCapturer(Environment* environment,
47*3f982cf4SFabien Sanglard                     const char* path,
48*3f982cf4SFabien Sanglard                     AVMediaType media_type,
49*3f982cf4SFabien Sanglard                     Clock::time_point start_time,
50*3f982cf4SFabien Sanglard                     Observer* observer);
51*3f982cf4SFabien Sanglard 
52*3f982cf4SFabien Sanglard   virtual ~SimulatedCapturer();
53*3f982cf4SFabien Sanglard 
54*3f982cf4SFabien Sanglard   // Optionally overridden, to apply additional decoder context settings before
55*3f982cf4SFabien Sanglard   // avcodec_open2() is called.
56*3f982cf4SFabien Sanglard   virtual void SetAdditionalDecoderParameters(AVCodecContext* decoder_context);
57*3f982cf4SFabien Sanglard 
58*3f982cf4SFabien Sanglard   // Performs any additional processing on the decoded frame (e.g., audio
59*3f982cf4SFabien Sanglard   // resampling), and returns any adjustments to the frame's capture time (e.g.,
60*3f982cf4SFabien Sanglard   // to account for any buffering). If a fatal error occurs, absl::nullopt is
61*3f982cf4SFabien Sanglard   // returned. The default implementation does nothing.
62*3f982cf4SFabien Sanglard   //
63*3f982cf4SFabien Sanglard   // Mutating the |decoded_frame| is not allowed. If a subclass implementation
64*3f982cf4SFabien Sanglard   // wants to deliver different data (e.g., resampled audio), it must stash the
65*3f982cf4SFabien Sanglard   // data itself for the next DeliverDataToClient() call.
66*3f982cf4SFabien Sanglard   virtual absl::optional<Clock::duration> ProcessDecodedFrame(
67*3f982cf4SFabien Sanglard       const AVFrame& decoded_frame);
68*3f982cf4SFabien Sanglard 
69*3f982cf4SFabien Sanglard   // Delivers the decoded frame data to the client.
70*3f982cf4SFabien Sanglard   virtual void DeliverDataToClient(const AVFrame& decoded_frame,
71*3f982cf4SFabien Sanglard                                    Clock::time_point capture_time) = 0;
72*3f982cf4SFabien Sanglard 
73*3f982cf4SFabien Sanglard   // Called when any transient or fatal error occurs, generating an Error and
74*3f982cf4SFabien Sanglard   // scheduling a task to notify the Observer of it soon.
75*3f982cf4SFabien Sanglard   void OnError(const char* what, int av_errnum);
76*3f982cf4SFabien Sanglard 
77*3f982cf4SFabien Sanglard   // Converts the given FFMPEG tick count into an approximate Clock::duration.
78*3f982cf4SFabien Sanglard   static Clock::duration ToApproximateClockDuration(
79*3f982cf4SFabien Sanglard       int64_t ticks,
80*3f982cf4SFabien Sanglard       const AVRational& time_base);
81*3f982cf4SFabien Sanglard 
82*3f982cf4SFabien Sanglard  private:
83*3f982cf4SFabien Sanglard   // Reads the next frame from the file, sends it to the decoder, and schedules
84*3f982cf4SFabien Sanglard   // a future ConsumeNextDecodedFrame() call to continue processing.
85*3f982cf4SFabien Sanglard   void StartDecodingNextFrame();
86*3f982cf4SFabien Sanglard 
87*3f982cf4SFabien Sanglard   // Receives the next decoded frame and schedules media delivery to the client,
88*3f982cf4SFabien Sanglard   // and/or calls Observer::OnEndOfFile() if there are no more frames in the
89*3f982cf4SFabien Sanglard   // file.
90*3f982cf4SFabien Sanglard   void ConsumeNextDecodedFrame();
91*3f982cf4SFabien Sanglard 
92*3f982cf4SFabien Sanglard   const AVFormatContextUniquePtr format_context_;
93*3f982cf4SFabien Sanglard   const AVMediaType media_type_;  // Audio or Video.
94*3f982cf4SFabien Sanglard   const Clock::time_point start_time_;
95*3f982cf4SFabien Sanglard   Observer* const observer_;
96*3f982cf4SFabien Sanglard   const AVPacketUniquePtr packet_;        // Decoder input buffer.
97*3f982cf4SFabien Sanglard   const AVFrameUniquePtr decoded_frame_;  // Decoder output frame.
98*3f982cf4SFabien Sanglard   int stream_index_ = -1;                 // Selected stream from the file.
99*3f982cf4SFabien Sanglard   AVCodecContextUniquePtr decoder_context_;
100*3f982cf4SFabien Sanglard 
101*3f982cf4SFabien Sanglard   // The last frame's stream timestamp. This is used to detect bad stream
102*3f982cf4SFabien Sanglard   // timestamps in the file.
103*3f982cf4SFabien Sanglard   absl::optional<Clock::duration> last_frame_timestamp_;
104*3f982cf4SFabien Sanglard 
105*3f982cf4SFabien Sanglard   // Used to schedule the next task to execute and when it should execute. There
106*3f982cf4SFabien Sanglard   // is only ever one task scheduled/running at any time.
107*3f982cf4SFabien Sanglard   Alarm next_task_;
108*3f982cf4SFabien Sanglard 
109*3f982cf4SFabien Sanglard   // Used to determine playback rate. Currently, we only support "playing"
110*3f982cf4SFabien Sanglard   // at 1x speed, or "pausing" at 0x speed.
111*3f982cf4SFabien Sanglard   bool playback_rate_is_non_zero_ = true;
112*3f982cf4SFabien Sanglard };
113*3f982cf4SFabien Sanglard 
114*3f982cf4SFabien Sanglard // Emits the primary audio stream from a file.
115*3f982cf4SFabien Sanglard class SimulatedAudioCapturer final : public SimulatedCapturer {
116*3f982cf4SFabien Sanglard  public:
117*3f982cf4SFabien Sanglard   class Client : public SimulatedCapturer::Observer {
118*3f982cf4SFabien Sanglard    public:
119*3f982cf4SFabien Sanglard     // Called to deliver more audio data as |interleaved_samples|, which
120*3f982cf4SFabien Sanglard     // contains |num_samples| tuples (i.e., multiply by the number of channels
121*3f982cf4SFabien Sanglard     // to determine the number of array elements). |capture_time| is used to
122*3f982cf4SFabien Sanglard     // synchronize the play-out of the first audio sample with respect to video
123*3f982cf4SFabien Sanglard     // frames.
124*3f982cf4SFabien Sanglard     virtual void OnAudioData(const float* interleaved_samples,
125*3f982cf4SFabien Sanglard                              int num_samples,
126*3f982cf4SFabien Sanglard                              Clock::time_point capture_time) = 0;
127*3f982cf4SFabien Sanglard 
128*3f982cf4SFabien Sanglard    protected:
129*3f982cf4SFabien Sanglard     ~Client() override;
130*3f982cf4SFabien Sanglard   };
131*3f982cf4SFabien Sanglard 
132*3f982cf4SFabien Sanglard   // Constructor: |num_channels| and |sample_rate| specify the required audio
133*3f982cf4SFabien Sanglard   // format. If necessary, audio from the file will be resampled to match the
134*3f982cf4SFabien Sanglard   // required format.
135*3f982cf4SFabien Sanglard   SimulatedAudioCapturer(Environment* environment,
136*3f982cf4SFabien Sanglard                          const char* path,
137*3f982cf4SFabien Sanglard                          int num_channels,
138*3f982cf4SFabien Sanglard                          int sample_rate,
139*3f982cf4SFabien Sanglard                          Clock::time_point start_time,
140*3f982cf4SFabien Sanglard                          Client* client);
141*3f982cf4SFabien Sanglard 
142*3f982cf4SFabien Sanglard   ~SimulatedAudioCapturer() final;
143*3f982cf4SFabien Sanglard 
144*3f982cf4SFabien Sanglard  private:
145*3f982cf4SFabien Sanglard   // Examines the audio format of the given |frame|, and ensures the resampler
146*3f982cf4SFabien Sanglard   // is initialized to take that as input.
147*3f982cf4SFabien Sanglard   bool EnsureResamplerIsInitializedFor(const AVFrame& frame);
148*3f982cf4SFabien Sanglard 
149*3f982cf4SFabien Sanglard   // Resamples the current |SimulatedCapturer::decoded_frame()| into the
150*3f982cf4SFabien Sanglard   // required output format/channels/rate. The result is stored in
151*3f982cf4SFabien Sanglard   // |resampled_audio_| for the next DeliverDataToClient() call.
152*3f982cf4SFabien Sanglard   absl::optional<Clock::duration> ProcessDecodedFrame(
153*3f982cf4SFabien Sanglard       const AVFrame& decoded_frame) final;
154*3f982cf4SFabien Sanglard 
155*3f982cf4SFabien Sanglard   // Called at the moment Client::OnAudioData() should be called to pass the
156*3f982cf4SFabien Sanglard   // |resampled_audio_|.
157*3f982cf4SFabien Sanglard   void DeliverDataToClient(const AVFrame& decoded_frame,
158*3f982cf4SFabien Sanglard                            Clock::time_point capture_time) final;
159*3f982cf4SFabien Sanglard 
160*3f982cf4SFabien Sanglard   const int num_channels_;  // Output number of channels.
161*3f982cf4SFabien Sanglard   const int sample_rate_;   // Output sample rate.
162*3f982cf4SFabien Sanglard   Client* const client_;
163*3f982cf4SFabien Sanglard 
164*3f982cf4SFabien Sanglard   const SwrContextUniquePtr resampler_;
165*3f982cf4SFabien Sanglard 
166*3f982cf4SFabien Sanglard   // Current resampler input audio parameters.
167*3f982cf4SFabien Sanglard   AVSampleFormat input_sample_format_ = AV_SAMPLE_FMT_NONE;
168*3f982cf4SFabien Sanglard   int input_sample_rate_;
169*3f982cf4SFabien Sanglard   uint64_t input_channel_layout_;  // Opaque value used by resampler library.
170*3f982cf4SFabien Sanglard 
171*3f982cf4SFabien Sanglard   std::vector<float> resampled_audio_;
172*3f982cf4SFabien Sanglard };
173*3f982cf4SFabien Sanglard 
174*3f982cf4SFabien Sanglard // Emits the primary video stream from a file.
175*3f982cf4SFabien Sanglard class SimulatedVideoCapturer final : public SimulatedCapturer {
176*3f982cf4SFabien Sanglard  public:
177*3f982cf4SFabien Sanglard   class Client : public SimulatedCapturer::Observer {
178*3f982cf4SFabien Sanglard    public:
179*3f982cf4SFabien Sanglard     // Called to deliver the next video |frame|, which is always in I420 format.
180*3f982cf4SFabien Sanglard     // |capture_time| is used to synchronize the play-out of the video frame
181*3f982cf4SFabien Sanglard     // with respect to the audio track.
182*3f982cf4SFabien Sanglard     virtual void OnVideoFrame(const AVFrame& frame,
183*3f982cf4SFabien Sanglard                               Clock::time_point capture_time) = 0;
184*3f982cf4SFabien Sanglard 
185*3f982cf4SFabien Sanglard    protected:
186*3f982cf4SFabien Sanglard     ~Client() override;
187*3f982cf4SFabien Sanglard   };
188*3f982cf4SFabien Sanglard 
189*3f982cf4SFabien Sanglard   SimulatedVideoCapturer(Environment* environment,
190*3f982cf4SFabien Sanglard                          const char* path,
191*3f982cf4SFabien Sanglard                          Clock::time_point start_time,
192*3f982cf4SFabien Sanglard                          Client* client);
193*3f982cf4SFabien Sanglard 
194*3f982cf4SFabien Sanglard   ~SimulatedVideoCapturer() final;
195*3f982cf4SFabien Sanglard 
196*3f982cf4SFabien Sanglard  private:
197*3f982cf4SFabien Sanglard   Client* const client_;
198*3f982cf4SFabien Sanglard 
199*3f982cf4SFabien Sanglard   // Sets up the decoder to produce I420 format output.
200*3f982cf4SFabien Sanglard   void SetAdditionalDecoderParameters(AVCodecContext* decoder_context) final;
201*3f982cf4SFabien Sanglard 
202*3f982cf4SFabien Sanglard   // Called at the moment Client::OnVideoFrame() should be called to provide the
203*3f982cf4SFabien Sanglard   // next video frame.
204*3f982cf4SFabien Sanglard   void DeliverDataToClient(const AVFrame& decoded_frame,
205*3f982cf4SFabien Sanglard                            Clock::time_point capture_time) final;
206*3f982cf4SFabien Sanglard };
207*3f982cf4SFabien Sanglard 
208*3f982cf4SFabien Sanglard }  // namespace cast
209*3f982cf4SFabien Sanglard }  // namespace openscreen
210*3f982cf4SFabien Sanglard 
211*3f982cf4SFabien Sanglard #endif  // CAST_STANDALONE_SENDER_SIMULATED_CAPTURER_H_
212