1 /* 2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_VIDEO_STREAM_BUFFER_CONTROLLER_H_ 12 #define VIDEO_VIDEO_STREAM_BUFFER_CONTROLLER_H_ 13 14 #include <memory> 15 16 #include "api/field_trials_view.h" 17 #include "api/task_queue/task_queue_base.h" 18 #include "api/video/encoded_frame.h" 19 #include "api/video/frame_buffer.h" 20 #include "modules/video_coding/include/video_coding_defines.h" 21 #include "modules/video_coding/timing/inter_frame_delay.h" 22 #include "modules/video_coding/timing/jitter_estimator.h" 23 #include "modules/video_coding/timing/timing.h" 24 #include "rtc_base/experiments/rtt_mult_experiment.h" 25 #include "system_wrappers/include/clock.h" 26 #include "video/decode_synchronizer.h" 27 #include "video/video_receive_stream_timeout_tracker.h" 28 29 namespace webrtc { 30 31 class FrameSchedulingReceiver { 32 public: 33 virtual ~FrameSchedulingReceiver() = default; 34 35 virtual void OnEncodedFrame(std::unique_ptr<EncodedFrame> frame) = 0; 36 virtual void OnDecodableFrameTimeout(TimeDelta wait_time) = 0; 37 }; 38 39 class VideoStreamBufferController { 40 public: 41 VideoStreamBufferController( 42 Clock* clock, 43 TaskQueueBase* worker_queue, 44 VCMTiming* timing, 45 VCMReceiveStatisticsCallback* stats_proxy, 46 FrameSchedulingReceiver* receiver, 47 TimeDelta max_wait_for_keyframe, 48 TimeDelta max_wait_for_frame, 49 std::unique_ptr<FrameDecodeScheduler> frame_decode_scheduler, 50 const FieldTrialsView& field_trials); 51 virtual ~VideoStreamBufferController() = default; 52 53 void Stop(); 54 void SetProtectionMode(VCMVideoProtection protection_mode); 55 void Clear(); 56 absl::optional<int64_t> InsertFrame(std::unique_ptr<EncodedFrame> frame); 57 void UpdateRtt(int64_t max_rtt_ms); 58 void SetMaxWaits(TimeDelta max_wait_for_keyframe, 59 TimeDelta max_wait_for_frame); 60 void StartNextDecode(bool keyframe_required); 61 int Size(); 62 63 private: 64 void OnFrameReady( 65 absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4> frames, 66 Timestamp render_time); 67 void OnTimeout(TimeDelta delay); 68 void FrameReadyForDecode(uint32_t rtp_timestamp, Timestamp render_time); 69 void UpdateDroppedFrames() RTC_RUN_ON(&worker_sequence_checker_); 70 void UpdateJitterDelay(); 71 void UpdateTimingFrameInfo(); 72 bool IsTooManyFramesQueued() const RTC_RUN_ON(&worker_sequence_checker_); 73 void ForceKeyFrameReleaseImmediately() RTC_RUN_ON(&worker_sequence_checker_); 74 void MaybeScheduleFrameForRelease() RTC_RUN_ON(&worker_sequence_checker_); 75 76 RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_sequence_checker_; 77 const FieldTrialsView& field_trials_; 78 const absl::optional<RttMultExperiment::Settings> rtt_mult_settings_ = 79 RttMultExperiment::GetRttMultValue(); 80 Clock* const clock_; 81 VCMReceiveStatisticsCallback* const stats_proxy_; 82 FrameSchedulingReceiver* const receiver_; 83 VCMTiming* const timing_; 84 const std::unique_ptr<FrameDecodeScheduler> frame_decode_scheduler_ 85 RTC_GUARDED_BY(&worker_sequence_checker_); 86 87 JitterEstimator jitter_estimator_ RTC_GUARDED_BY(&worker_sequence_checker_); 88 InterFrameDelay inter_frame_delay_ RTC_GUARDED_BY(&worker_sequence_checker_); 89 bool keyframe_required_ RTC_GUARDED_BY(&worker_sequence_checker_) = false; 90 std::unique_ptr<FrameBuffer> buffer_ 91 RTC_GUARDED_BY(&worker_sequence_checker_); 92 FrameDecodeTiming decode_timing_ RTC_GUARDED_BY(&worker_sequence_checker_); 93 VideoReceiveStreamTimeoutTracker timeout_tracker_ 94 RTC_GUARDED_BY(&worker_sequence_checker_); 95 int frames_dropped_before_last_new_frame_ 96 RTC_GUARDED_BY(&worker_sequence_checker_) = 0; 97 VCMVideoProtection protection_mode_ 98 RTC_GUARDED_BY(&worker_sequence_checker_) = kProtectionNack; 99 100 // This flag guards frames from queuing in front of the decoder. Without this 101 // guard, encoded frames will not wait for the decoder to finish decoding a 102 // frame and just queue up, meaning frames will not be dropped or 103 // fast-forwarded when the decoder is slow or hangs. 104 bool decoder_ready_for_new_frame_ RTC_GUARDED_BY(&worker_sequence_checker_) = 105 false; 106 107 // Maximum number of frames in the decode queue to allow pacing. If the 108 // queue grows beyond the max limit, pacing will be disabled and frames will 109 // be pushed to the decoder as soon as possible. This only has an effect 110 // when the low-latency rendering path is active, which is indicated by 111 // the frame's render time == 0. 112 FieldTrialParameter<unsigned> zero_playout_delay_max_decode_queue_size_; 113 114 ScopedTaskSafety worker_safety_; 115 }; 116 117 } // namespace webrtc 118 119 #endif // VIDEO_VIDEO_STREAM_BUFFER_CONTROLLER_H_ 120