1 /* 2 * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_VIDEO_CODING_TIMING_TIMING_H_ 12 #define MODULES_VIDEO_CODING_TIMING_TIMING_H_ 13 14 #include <memory> 15 16 #include "absl/types/optional.h" 17 #include "api/field_trials_view.h" 18 #include "api/units/time_delta.h" 19 #include "api/video/video_frame.h" 20 #include "api/video/video_timing.h" 21 #include "modules/video_coding/timing/codec_timer.h" 22 #include "modules/video_coding/timing/timestamp_extrapolator.h" 23 #include "rtc_base/experiments/field_trial_parser.h" 24 #include "rtc_base/synchronization/mutex.h" 25 #include "rtc_base/thread_annotations.h" 26 #include "system_wrappers/include/clock.h" 27 28 namespace webrtc { 29 30 class VCMTiming { 31 public: 32 static constexpr auto kDefaultRenderDelay = TimeDelta::Millis(10); 33 static constexpr auto kDelayMaxChangeMsPerS = 100; 34 35 VCMTiming(Clock* clock, const FieldTrialsView& field_trials); 36 virtual ~VCMTiming() = default; 37 38 // Resets the timing to the initial state. 39 void Reset(); 40 41 // Set the amount of time needed to render an image. Defaults to 10 ms. 42 void set_render_delay(TimeDelta render_delay); 43 44 // Set the minimum time the video must be delayed on the receiver to 45 // get the desired jitter buffer level. 46 void SetJitterDelay(TimeDelta required_delay); 47 48 // Set/get the minimum playout delay from capture to render. 49 TimeDelta min_playout_delay() const; 50 void set_min_playout_delay(TimeDelta min_playout_delay); 51 52 // Set/get the maximum playout delay from capture to render in ms. 53 void set_max_playout_delay(TimeDelta max_playout_delay); 54 55 // Increases or decreases the current delay to get closer to the target delay. 56 // Calculates how long it has been since the previous call to this function, 57 // and increases/decreases the delay in proportion to the time difference. 58 void UpdateCurrentDelay(uint32_t frame_timestamp); 59 60 // Increases or decreases the current delay to get closer to the target delay. 61 // Given the actual decode time in ms and the render time in ms for a frame, 62 // this function calculates how late the frame is and increases the delay 63 // accordingly. 64 void UpdateCurrentDelay(Timestamp render_time, Timestamp actual_decode_time); 65 66 // Stops the decoder timer, should be called when the decoder returns a frame 67 // or when the decoded frame callback is called. 68 void StopDecodeTimer(TimeDelta decode_time, Timestamp now); 69 70 // Used to report that a frame is passed to decoding. Updates the timestamp 71 // filter which is used to map between timestamps and receiver system time. 72 virtual void IncomingTimestamp(uint32_t rtp_timestamp, 73 Timestamp last_packet_time); 74 75 // Returns the receiver system time when the frame with timestamp 76 // `frame_timestamp` should be rendered, assuming that the system time 77 // currently is `now`. 78 virtual Timestamp RenderTime(uint32_t frame_timestamp, Timestamp now) const; 79 80 // Returns the maximum time in ms that we can wait for a frame to become 81 // complete before we must pass it to the decoder. render_time==0 indicates 82 // that the frames should be processed as quickly as possible, with possibly 83 // only a small delay added to make sure that the decoder is not overloaded. 84 // In this case, the parameter too_many_frames_queued is used to signal that 85 // the decode queue is full and that the frame should be decoded as soon as 86 // possible. 87 virtual TimeDelta MaxWaitingTime(Timestamp render_time, 88 Timestamp now, 89 bool too_many_frames_queued) const; 90 91 // Returns the current target delay which is required delay + decode time + 92 // render delay. 93 TimeDelta TargetVideoDelay() const; 94 95 // Return current timing information. Returns true if the first frame has been 96 // decoded, false otherwise. 97 struct VideoDelayTimings { 98 TimeDelta max_decode_duration; 99 TimeDelta current_delay; 100 TimeDelta target_delay; 101 TimeDelta jitter_buffer_delay; 102 TimeDelta min_playout_delay; 103 TimeDelta max_playout_delay; 104 TimeDelta render_delay; 105 size_t num_decoded_frames; 106 }; 107 VideoDelayTimings GetTimings() const; 108 109 void SetTimingFrameInfo(const TimingFrameInfo& info); 110 absl::optional<TimingFrameInfo> GetTimingFrameInfo(); 111 112 void SetMaxCompositionDelayInFrames( 113 absl::optional<int> max_composition_delay_in_frames); 114 115 VideoFrame::RenderParameters RenderParameters() const; 116 117 // Updates the last time a frame was scheduled for decoding. 118 void SetLastDecodeScheduledTimestamp(Timestamp last_decode_scheduled); 119 120 protected: 121 TimeDelta RequiredDecodeTime() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 122 Timestamp RenderTimeInternal(uint32_t frame_timestamp, Timestamp now) const 123 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 124 TimeDelta TargetDelayInternal() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 125 bool UseLowLatencyRendering() const RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 126 127 private: 128 mutable Mutex mutex_; 129 Clock* const clock_; 130 const std::unique_ptr<TimestampExtrapolator> ts_extrapolator_ 131 RTC_PT_GUARDED_BY(mutex_); 132 std::unique_ptr<CodecTimer> codec_timer_ RTC_GUARDED_BY(mutex_) 133 RTC_PT_GUARDED_BY(mutex_); 134 TimeDelta render_delay_ RTC_GUARDED_BY(mutex_); 135 // Best-effort playout delay range for frames from capture to render. 136 // The receiver tries to keep the delay between `min_playout_delay_ms_` 137 // and `max_playout_delay_ms_` taking the network jitter into account. 138 // A special case is where min_playout_delay_ms_ = max_playout_delay_ms_ = 0, 139 // in which case the receiver tries to play the frames as they arrive. 140 TimeDelta min_playout_delay_ RTC_GUARDED_BY(mutex_); 141 TimeDelta max_playout_delay_ RTC_GUARDED_BY(mutex_); 142 TimeDelta jitter_delay_ RTC_GUARDED_BY(mutex_); 143 TimeDelta current_delay_ RTC_GUARDED_BY(mutex_); 144 uint32_t prev_frame_timestamp_ RTC_GUARDED_BY(mutex_); 145 absl::optional<TimingFrameInfo> timing_frame_info_ RTC_GUARDED_BY(mutex_); 146 size_t num_decoded_frames_ RTC_GUARDED_BY(mutex_); 147 absl::optional<int> max_composition_delay_in_frames_ RTC_GUARDED_BY(mutex_); 148 // Set by the field trial WebRTC-ZeroPlayoutDelay. The parameter min_pacing 149 // determines the minimum delay between frames scheduled for decoding that is 150 // used when min playout delay=0 and max playout delay>=0. 151 FieldTrialParameter<TimeDelta> zero_playout_delay_min_pacing_ 152 RTC_GUARDED_BY(mutex_); 153 // Timestamp at which the last frame was scheduled to be sent to the decoder. 154 // Used only when the RTP header extension playout delay is set to min=0 ms 155 // which is indicated by a render time set to 0. 156 Timestamp last_decode_scheduled_ RTC_GUARDED_BY(mutex_); 157 }; 158 } // namespace webrtc 159 160 #endif // MODULES_VIDEO_CODING_TIMING_TIMING_H_ 161