1 /* 2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_DECODE_SYNCHRONIZER_H_ 12 #define VIDEO_DECODE_SYNCHRONIZER_H_ 13 14 #include <stdint.h> 15 16 #include <functional> 17 #include <memory> 18 #include <set> 19 #include <utility> 20 21 #include "absl/types/optional.h" 22 #include "api/metronome/metronome.h" 23 #include "api/sequence_checker.h" 24 #include "api/task_queue/task_queue_base.h" 25 #include "api/units/timestamp.h" 26 #include "rtc_base/checks.h" 27 #include "rtc_base/thread_annotations.h" 28 #include "video/frame_decode_scheduler.h" 29 #include "video/frame_decode_timing.h" 30 31 namespace webrtc { 32 33 // DecodeSynchronizer synchronizes the frame scheduling by coalescing decoding 34 // on the metronome. 35 // 36 // A video receive stream can use the DecodeSynchronizer by receiving a 37 // FrameDecodeScheduler instance with `CreateSynchronizedFrameScheduler()`. 38 // This instance implements FrameDecodeScheduler and can be used as a normal 39 // scheduler. This instance is owned by the receive stream, and is borrowed by 40 // the DecodeSynchronizer. The DecodeSynchronizer will stop borrowing the 41 // instance when `FrameDecodeScheduler::Stop()` is called, after which the 42 // scheduler may be destroyed by the receive stream. 43 // 44 // When a frame is scheduled for decode by a receive stream using the 45 // DecodeSynchronizer, it will instead be executed on the metronome during the 46 // tick interval where `max_decode_time` occurs. For example, if a frame is 47 // scheduled for decode in 50ms and the tick interval is 20ms, then the frame 48 // will be released for decoding in 2 ticks. See below for illustration, 49 // 50 // In the case where the decode time is in the past, or must occur before the 51 // next metronome tick then the frame will be released right away, allowing a 52 // delayed stream to catch up quickly. 53 // 54 // DecodeSynchronizer is single threaded - all method calls must run on the 55 // `worker_queue_`. 56 class DecodeSynchronizer { 57 public: 58 DecodeSynchronizer(Clock* clock, 59 Metronome* metronome, 60 TaskQueueBase* worker_queue); 61 ~DecodeSynchronizer(); 62 DecodeSynchronizer(const DecodeSynchronizer&) = delete; 63 DecodeSynchronizer& operator=(const DecodeSynchronizer&) = delete; 64 65 std::unique_ptr<FrameDecodeScheduler> CreateSynchronizedFrameScheduler(); 66 67 private: 68 class ScheduledFrame { 69 public: 70 ScheduledFrame(uint32_t rtp_timestamp, 71 FrameDecodeTiming::FrameSchedule schedule, 72 FrameDecodeScheduler::FrameReleaseCallback callback); 73 74 // Disallow copy since `callback` should only be moved. 75 ScheduledFrame(const ScheduledFrame&) = delete; 76 ScheduledFrame& operator=(const ScheduledFrame&) = delete; 77 ScheduledFrame(ScheduledFrame&&) = default; 78 ScheduledFrame& operator=(ScheduledFrame&&) = default; 79 80 // Executes `callback_`. 81 void RunFrameReleaseCallback() &&; 82 rtp_timestamp()83 uint32_t rtp_timestamp() const { return rtp_timestamp_; } 84 Timestamp LatestDecodeTime() const; 85 86 private: 87 uint32_t rtp_timestamp_; 88 FrameDecodeTiming::FrameSchedule schedule_; 89 FrameDecodeScheduler::FrameReleaseCallback callback_; 90 }; 91 92 class SynchronizedFrameDecodeScheduler : public FrameDecodeScheduler { 93 public: 94 explicit SynchronizedFrameDecodeScheduler(DecodeSynchronizer* sync); 95 ~SynchronizedFrameDecodeScheduler() override; 96 97 // Releases the outstanding frame for decoding. This invalidates 98 // `next_frame_`. There must be a frame scheduled. 99 ScheduledFrame ReleaseNextFrame(); 100 101 // Returns `next_frame_.schedule.max_decode_time`. There must be a frame 102 // scheduled when this is called. 103 Timestamp LatestDecodeTime(); 104 105 // FrameDecodeScheduler implementation. 106 absl::optional<uint32_t> ScheduledRtpTimestamp() override; 107 void ScheduleFrame(uint32_t rtp, 108 FrameDecodeTiming::FrameSchedule schedule, 109 FrameReleaseCallback cb) override; 110 void CancelOutstanding() override; 111 void Stop() override; 112 113 private: 114 DecodeSynchronizer* sync_; 115 absl::optional<ScheduledFrame> next_frame_; 116 bool stopped_ = false; 117 }; 118 119 void OnFrameScheduled(SynchronizedFrameDecodeScheduler* scheduler); 120 void RemoveFrameScheduler(SynchronizedFrameDecodeScheduler* scheduler); 121 122 void ScheduleNextTick(); 123 void OnTick(); 124 125 Clock* const clock_; 126 TaskQueueBase* const worker_queue_; 127 Metronome* const metronome_; 128 129 Timestamp expected_next_tick_ = Timestamp::PlusInfinity(); 130 std::set<SynchronizedFrameDecodeScheduler*> schedulers_ 131 RTC_GUARDED_BY(worker_queue_); 132 ScopedTaskSafetyDetached safety_; 133 }; 134 135 } // namespace webrtc 136 137 #endif // VIDEO_DECODE_SYNCHRONIZER_H_ 138