1 // Copyright 2019 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "cast/standalone_receiver/sdl_player_base.h"
6
7 #include <chrono>
8 #include <sstream>
9 #include <utility>
10
11 #include "absl/types/span.h"
12 #include "cast/standalone_receiver/avcodec_glue.h"
13 #include "cast/streaming/constants.h"
14 #include "cast/streaming/encoded_frame.h"
15 #include "util/big_endian.h"
16 #include "util/chrono_helpers.h"
17 #include "util/osp_logging.h"
18 #include "util/trace_logging.h"
19
20 namespace openscreen {
21 namespace cast {
22
SDLPlayerBase(ClockNowFunctionPtr now_function,TaskRunner * task_runner,Receiver * receiver,const std::string & codec_name,std::function<void ()> error_callback,const char * media_type)23 SDLPlayerBase::SDLPlayerBase(ClockNowFunctionPtr now_function,
24 TaskRunner* task_runner,
25 Receiver* receiver,
26 const std::string& codec_name,
27 std::function<void()> error_callback,
28 const char* media_type)
29 : now_(now_function),
30 receiver_(receiver),
31 error_callback_(std::move(error_callback)),
32 media_type_(media_type),
33 decoder_(codec_name),
34 decode_alarm_(now_, task_runner),
35 render_alarm_(now_, task_runner),
36 presentation_alarm_(now_, task_runner) {
37 OSP_DCHECK(receiver_);
38 OSP_DCHECK(media_type_);
39
40 decoder_.set_client(this);
41 receiver_->SetConsumer(this);
42 ResumeRendering();
43 }
44
~SDLPlayerBase()45 SDLPlayerBase::~SDLPlayerBase() {
46 receiver_->SetConsumer(nullptr);
47 decoder_.set_client(nullptr);
48 }
49
OnFatalError(std::string message)50 void SDLPlayerBase::OnFatalError(std::string message) {
51 state_ = kError;
52 error_status_ = Error(Error::Code::kUnknownError, std::move(message));
53
54 // Halt decoding and clear the rendering queue.
55 receiver_->SetConsumer(nullptr);
56 decoder_.set_client(nullptr);
57 decode_alarm_.Cancel();
58 frames_to_render_.clear();
59
60 // Resume rendering, to emit an error indication (e.g., "red splash" screen).
61 ResumeRendering();
62
63 if (error_callback_) {
64 const auto callback = std::move(error_callback_);
65 callback();
66 }
67 }
68
ResyncAndDeterminePresentationTime(const EncodedFrame & frame)69 Clock::time_point SDLPlayerBase::ResyncAndDeterminePresentationTime(
70 const EncodedFrame& frame) {
71 constexpr auto kMaxPlayoutDrift = milliseconds(100);
72 const auto media_time_since_last_sync =
73 (frame.rtp_timestamp - last_sync_rtp_timestamp_)
74 .ToDuration<Clock::duration>(receiver_->rtp_timebase());
75 Clock::time_point presentation_time =
76 last_sync_reference_time_ + media_time_since_last_sync;
77 const auto drift = to_milliseconds(frame.reference_time - presentation_time);
78 if (drift > kMaxPlayoutDrift || drift < -kMaxPlayoutDrift) {
79 // Only log if not the very first frame.
80 OSP_LOG_IF(INFO, frame.frame_id != FrameId::first())
81 << "Playout drift (" << drift.count() << " ms) exceeded threshold ("
82 << kMaxPlayoutDrift.count() << " ms) for " << media_type_
83 << ". Re-synchronizing...";
84 // This is the "big-stick" way to re-synchronize. If the amount of drift
85 // is small, a production-worthy player should "nudge" things gradually
86 // back into sync over several frames.
87 last_sync_rtp_timestamp_ = frame.rtp_timestamp;
88 last_sync_reference_time_ = frame.reference_time;
89 presentation_time = frame.reference_time;
90 }
91 return presentation_time;
92 }
93
OnFramesReady(int buffer_size)94 void SDLPlayerBase::OnFramesReady(int buffer_size) {
95 TRACE_DEFAULT_SCOPED(TraceCategory::kStandaloneReceiver);
96 // Do not consume anything if there are too many frames in the pipeline
97 // already.
98 if (static_cast<int>(frames_to_render_.size()) > kMaxFramesInPipeline) {
99 return;
100 }
101
102 // Consume the next frame.
103 const Clock::time_point start_time = now_();
104 buffer_.Resize(buffer_size);
105 EncodedFrame frame = receiver_->ConsumeNextFrame(buffer_.GetSpan());
106
107 // Create the tracking state for the frame in the player pipeline.
108 OSP_DCHECK_EQ(frames_to_render_.count(frame.frame_id), 0);
109 PendingFrame& pending_frame = frames_to_render_[frame.frame_id];
110 pending_frame.start_time = start_time;
111
112 pending_frame.presentation_time = ResyncAndDeterminePresentationTime(frame);
113
114 // Start decoding the frame. This call may synchronously call back into the
115 // AVCodecDecoder::Client methods in this class.
116 decoder_.Decode(frame.frame_id, buffer_);
117 }
118
OnFrameDecoded(FrameId frame_id,const AVFrame & frame)119 void SDLPlayerBase::OnFrameDecoded(FrameId frame_id, const AVFrame& frame) {
120 TRACE_DEFAULT_SCOPED(TraceCategory::kStandaloneReceiver);
121 const auto it = frames_to_render_.find(frame_id);
122 if (it == frames_to_render_.end()) {
123 return;
124 }
125 OSP_DCHECK(!it->second.decoded_frame);
126 // av_clone_frame() does a shallow copy here, incrementing a ref-count on the
127 // memory backing the frame.
128 it->second.decoded_frame = AVFrameUniquePtr(av_frame_clone(&frame));
129 ResumeRendering();
130 }
131
OnDecodeError(FrameId frame_id,std::string message)132 void SDLPlayerBase::OnDecodeError(FrameId frame_id, std::string message) {
133 const auto it = frames_to_render_.find(frame_id);
134 if (it != frames_to_render_.end()) {
135 frames_to_render_.erase(it);
136 }
137 OSP_LOG_WARN << "Requesting " << media_type_
138 << " key frame because of error decoding" << frame_id << ": "
139 << message;
140 receiver_->RequestKeyFrame();
141 ResumeDecoding();
142 }
143
RenderAndSchedulePresentation()144 void SDLPlayerBase::RenderAndSchedulePresentation() {
145 TRACE_DEFAULT_SCOPED(TraceCategory::kStandaloneReceiver);
146 // If something has already been scheduled to present at an exact time point,
147 // don't render anything new yet.
148 if (state_ == kScheduledToPresent) {
149 return;
150 }
151
152 // If no frames are available, just re-render the currently-presented frame
153 // (or the error screen).
154 auto it =
155 (state_ == kError) ? frames_to_render_.end() : frames_to_render_.begin();
156 if (it == frames_to_render_.end() || !it->second.decoded_frame) {
157 if (RenderWhileIdle(state_ == kPresented ? ¤t_frame_ : nullptr)) {
158 // Schedule presentation to happen after a rather lengthy interval, to
159 // minimize redraw/etc. resource usage while doing "idle mode" play-out.
160 // The interval here, is "lengthy" from the program's perspective, but
161 // reasonably "snappy" from the user's perspective.
162 constexpr auto kIdlePresentInterval = milliseconds(250);
163 presentation_alarm_.ScheduleFromNow(
164 [this] {
165 Present();
166 ResumeRendering();
167 },
168 kIdlePresentInterval);
169 }
170 return;
171 }
172
173 // Skip late frames, to render the first not-late frame. If all decoded frames
174 // are late, skip-forward to the least-late frame.
175 const Clock::time_point now = now_();
176 while (it->second.presentation_time < now) {
177 const auto next_it = std::next(it);
178 if (next_it == frames_to_render_.end() || !next_it->second.decoded_frame) {
179 break;
180 }
181 frames_to_render_.erase(it); // Drop the late frame.
182 it = next_it;
183 }
184
185 // Remove the frame from the queue, making it the |current_frame_|. Then,
186 // render it and, if successful, schedule its presentation.
187 current_frame_ = std::move(it->second);
188 frames_to_render_.erase(it);
189 const ErrorOr<Clock::time_point> presentation_time =
190 RenderNextFrame(current_frame_);
191 if (!presentation_time) {
192 OnFatalError(presentation_time.error().message());
193 return;
194 }
195 state_ = kScheduledToPresent;
196 presentation_alarm_.Schedule(
197 [this] {
198 Present();
199 if (state_ == kScheduledToPresent) {
200 state_ = kPresented;
201 }
202 ResumeRendering();
203 },
204 presentation_time.value());
205
206 // Resume consuming/decoding frames, since some of the prior OnFramesReady()
207 // calls may have been ignored to leave things in the Receiver's queue.
208 ResumeDecoding();
209
210 // Compute how long it took to decode/render this frame, and notify the
211 // Receiver of the recent-average per-frame processing time. This is used by
212 // the Receiver to determine when to drop late frames.
213 const Clock::duration measured_processing_time =
214 now_() - current_frame_.start_time;
215 constexpr int kCumulativeAveragePoints = 8;
216 recent_processing_time_ =
217 ((kCumulativeAveragePoints - 1) * recent_processing_time_ +
218 1 * measured_processing_time) /
219 kCumulativeAveragePoints;
220 receiver_->SetPlayerProcessingTime(recent_processing_time_);
221 }
222
ResumeDecoding()223 void SDLPlayerBase::ResumeDecoding() {
224 decode_alarm_.Schedule(
225 [this] {
226 const int buffer_size = receiver_->AdvanceToNextFrame();
227 if (buffer_size != Receiver::kNoFramesReady) {
228 OnFramesReady(buffer_size);
229 }
230 },
231 Alarm::kImmediately);
232 }
233
ResumeRendering()234 void SDLPlayerBase::ResumeRendering() {
235 render_alarm_.Schedule([this] { RenderAndSchedulePresentation(); },
236 Alarm::kImmediately);
237 }
238
239 // static
240 constexpr int SDLPlayerBase::kMaxFramesInPipeline;
241
242 SDLPlayerBase::PresentableFrame::PresentableFrame() = default;
243 SDLPlayerBase::PresentableFrame::~PresentableFrame() = default;
244 SDLPlayerBase::PresentableFrame::PresentableFrame(PresentableFrame&&) noexcept =
245 default;
246 SDLPlayerBase::PresentableFrame& SDLPlayerBase::PresentableFrame::operator=(
247 PresentableFrame&&) noexcept = default;
248
249 SDLPlayerBase::PendingFrame::PendingFrame() = default;
250 SDLPlayerBase::PendingFrame::~PendingFrame() = default;
251 SDLPlayerBase::PendingFrame::PendingFrame(PendingFrame&&) noexcept = default;
252 SDLPlayerBase::PendingFrame& SDLPlayerBase::PendingFrame::operator=(
253 PendingFrame&&) noexcept = default;
254
255 } // namespace cast
256 } // namespace openscreen
257