1*3f982cf4SFabien Sanglard // Copyright 2020 The Chromium Authors. All rights reserved.
2*3f982cf4SFabien Sanglard // Use of this source code is governed by a BSD-style license that can be
3*3f982cf4SFabien Sanglard // found in the LICENSE file.
4*3f982cf4SFabien Sanglard
5*3f982cf4SFabien Sanglard #include "cast/streaming/sender.h"
6*3f982cf4SFabien Sanglard
7*3f982cf4SFabien Sanglard #include <algorithm>
8*3f982cf4SFabien Sanglard #include <chrono>
9*3f982cf4SFabien Sanglard #include <ratio>
10*3f982cf4SFabien Sanglard
11*3f982cf4SFabien Sanglard #include "cast/streaming/session_config.h"
12*3f982cf4SFabien Sanglard #include "util/chrono_helpers.h"
13*3f982cf4SFabien Sanglard #include "util/osp_logging.h"
14*3f982cf4SFabien Sanglard #include "util/std_util.h"
15*3f982cf4SFabien Sanglard #include "util/trace_logging.h"
16*3f982cf4SFabien Sanglard
17*3f982cf4SFabien Sanglard namespace openscreen {
18*3f982cf4SFabien Sanglard namespace cast {
19*3f982cf4SFabien Sanglard
20*3f982cf4SFabien Sanglard using openscreen::operator<<; // For std::chrono::duration logging.
21*3f982cf4SFabien Sanglard
Sender(Environment * environment,SenderPacketRouter * packet_router,SessionConfig config,RtpPayloadType rtp_payload_type)22*3f982cf4SFabien Sanglard Sender::Sender(Environment* environment,
23*3f982cf4SFabien Sanglard SenderPacketRouter* packet_router,
24*3f982cf4SFabien Sanglard SessionConfig config,
25*3f982cf4SFabien Sanglard RtpPayloadType rtp_payload_type)
26*3f982cf4SFabien Sanglard : config_(config),
27*3f982cf4SFabien Sanglard packet_router_(packet_router),
28*3f982cf4SFabien Sanglard rtcp_session_(config.sender_ssrc,
29*3f982cf4SFabien Sanglard config.receiver_ssrc,
30*3f982cf4SFabien Sanglard environment->now()),
31*3f982cf4SFabien Sanglard rtcp_parser_(&rtcp_session_, this),
32*3f982cf4SFabien Sanglard sender_report_builder_(&rtcp_session_),
33*3f982cf4SFabien Sanglard rtp_packetizer_(rtp_payload_type,
34*3f982cf4SFabien Sanglard config.sender_ssrc,
35*3f982cf4SFabien Sanglard packet_router_->max_packet_size()),
36*3f982cf4SFabien Sanglard rtp_timebase_(config.rtp_timebase),
37*3f982cf4SFabien Sanglard crypto_(config.aes_secret_key, config.aes_iv_mask),
38*3f982cf4SFabien Sanglard target_playout_delay_(config.target_playout_delay) {
39*3f982cf4SFabien Sanglard OSP_DCHECK(packet_router_);
40*3f982cf4SFabien Sanglard OSP_DCHECK_NE(rtcp_session_.sender_ssrc(), rtcp_session_.receiver_ssrc());
41*3f982cf4SFabien Sanglard OSP_DCHECK_GT(rtp_timebase_, 0);
42*3f982cf4SFabien Sanglard OSP_DCHECK(target_playout_delay_ > milliseconds::zero());
43*3f982cf4SFabien Sanglard
44*3f982cf4SFabien Sanglard pending_sender_report_.reference_time = SenderPacketRouter::kNever;
45*3f982cf4SFabien Sanglard
46*3f982cf4SFabien Sanglard packet_router_->OnSenderCreated(rtcp_session_.receiver_ssrc(), this);
47*3f982cf4SFabien Sanglard }
48*3f982cf4SFabien Sanglard
~Sender()49*3f982cf4SFabien Sanglard Sender::~Sender() {
50*3f982cf4SFabien Sanglard packet_router_->OnSenderDestroyed(rtcp_session_.receiver_ssrc());
51*3f982cf4SFabien Sanglard }
52*3f982cf4SFabien Sanglard
SetObserver(Sender::Observer * observer)53*3f982cf4SFabien Sanglard void Sender::SetObserver(Sender::Observer* observer) {
54*3f982cf4SFabien Sanglard observer_ = observer;
55*3f982cf4SFabien Sanglard }
56*3f982cf4SFabien Sanglard
GetInFlightFrameCount() const57*3f982cf4SFabien Sanglard int Sender::GetInFlightFrameCount() const {
58*3f982cf4SFabien Sanglard return num_frames_in_flight_;
59*3f982cf4SFabien Sanglard }
60*3f982cf4SFabien Sanglard
GetInFlightMediaDuration(RtpTimeTicks next_frame_rtp_timestamp) const61*3f982cf4SFabien Sanglard Clock::duration Sender::GetInFlightMediaDuration(
62*3f982cf4SFabien Sanglard RtpTimeTicks next_frame_rtp_timestamp) const {
63*3f982cf4SFabien Sanglard if (num_frames_in_flight_ == 0) {
64*3f982cf4SFabien Sanglard return Clock::duration::zero(); // No frames are currently in-flight.
65*3f982cf4SFabien Sanglard }
66*3f982cf4SFabien Sanglard
67*3f982cf4SFabien Sanglard const PendingFrameSlot& oldest_slot = *get_slot_for(checkpoint_frame_id_ + 1);
68*3f982cf4SFabien Sanglard // Note: The oldest slot's frame cannot have been canceled because the
69*3f982cf4SFabien Sanglard // protocol does not allow ACK'ing this particular frame without also moving
70*3f982cf4SFabien Sanglard // the checkpoint forward. See "CST2 feedback" discussion in rtp_defines.h.
71*3f982cf4SFabien Sanglard OSP_DCHECK(oldest_slot.is_active_for_frame(checkpoint_frame_id_ + 1));
72*3f982cf4SFabien Sanglard
73*3f982cf4SFabien Sanglard return (next_frame_rtp_timestamp - oldest_slot.frame->rtp_timestamp)
74*3f982cf4SFabien Sanglard .ToDuration<Clock::duration>(rtp_timebase_);
75*3f982cf4SFabien Sanglard }
76*3f982cf4SFabien Sanglard
GetMaxInFlightMediaDuration() const77*3f982cf4SFabien Sanglard Clock::duration Sender::GetMaxInFlightMediaDuration() const {
78*3f982cf4SFabien Sanglard // Assumption: The total amount of allowed in-flight media should equal the
79*3f982cf4SFabien Sanglard // half of the playout delay window, plus the amount of time it takes to
80*3f982cf4SFabien Sanglard // receive an ACK from the Receiver.
81*3f982cf4SFabien Sanglard //
82*3f982cf4SFabien Sanglard // Why half of the playout delay window? It's assumed here that capture and
83*3f982cf4SFabien Sanglard // media encoding, which occur before EnqueueFrame() is called, are executing
84*3f982cf4SFabien Sanglard // within the first half of the playout delay window. This leaves the second
85*3f982cf4SFabien Sanglard // half for executing all network transmits/re-transmits, plus decoding and
86*3f982cf4SFabien Sanglard // play-out at the Receiver.
87*3f982cf4SFabien Sanglard return (target_playout_delay_ / 2) + (round_trip_time_ / 2);
88*3f982cf4SFabien Sanglard }
89*3f982cf4SFabien Sanglard
NeedsKeyFrame() const90*3f982cf4SFabien Sanglard bool Sender::NeedsKeyFrame() const {
91*3f982cf4SFabien Sanglard return last_enqueued_key_frame_id_ <= picture_lost_at_frame_id_;
92*3f982cf4SFabien Sanglard }
93*3f982cf4SFabien Sanglard
GetNextFrameId() const94*3f982cf4SFabien Sanglard FrameId Sender::GetNextFrameId() const {
95*3f982cf4SFabien Sanglard return last_enqueued_frame_id_ + 1;
96*3f982cf4SFabien Sanglard }
97*3f982cf4SFabien Sanglard
EnqueueFrame(const EncodedFrame & frame)98*3f982cf4SFabien Sanglard Sender::EnqueueFrameResult Sender::EnqueueFrame(const EncodedFrame& frame) {
99*3f982cf4SFabien Sanglard // Assume the fields of the |frame| have all been set correctly, with
100*3f982cf4SFabien Sanglard // monotonically increasing timestamps and a valid pointer to the data.
101*3f982cf4SFabien Sanglard OSP_DCHECK_EQ(frame.frame_id, GetNextFrameId());
102*3f982cf4SFabien Sanglard OSP_DCHECK_GE(frame.referenced_frame_id, FrameId::first());
103*3f982cf4SFabien Sanglard if (frame.frame_id != FrameId::first()) {
104*3f982cf4SFabien Sanglard OSP_DCHECK_GT(frame.rtp_timestamp, pending_sender_report_.rtp_timestamp);
105*3f982cf4SFabien Sanglard OSP_DCHECK_GT(frame.reference_time, pending_sender_report_.reference_time);
106*3f982cf4SFabien Sanglard }
107*3f982cf4SFabien Sanglard OSP_DCHECK(frame.data.data());
108*3f982cf4SFabien Sanglard
109*3f982cf4SFabien Sanglard // Check whether enqueuing the frame would exceed the design limit for the
110*3f982cf4SFabien Sanglard // span of FrameIds. Even if |num_frames_in_flight_| is less than
111*3f982cf4SFabien Sanglard // kMaxUnackedFrames, it's the span of FrameIds that is restricted.
112*3f982cf4SFabien Sanglard if ((frame.frame_id - checkpoint_frame_id_) > kMaxUnackedFrames) {
113*3f982cf4SFabien Sanglard return REACHED_ID_SPAN_LIMIT;
114*3f982cf4SFabien Sanglard }
115*3f982cf4SFabien Sanglard
116*3f982cf4SFabien Sanglard // Check whether enqueuing the frame would exceed the current maximum media
117*3f982cf4SFabien Sanglard // duration limit.
118*3f982cf4SFabien Sanglard if (GetInFlightMediaDuration(frame.rtp_timestamp) >
119*3f982cf4SFabien Sanglard GetMaxInFlightMediaDuration()) {
120*3f982cf4SFabien Sanglard return MAX_DURATION_IN_FLIGHT;
121*3f982cf4SFabien Sanglard }
122*3f982cf4SFabien Sanglard
123*3f982cf4SFabien Sanglard // Encrypt the frame and initialize the slot tracking its sending.
124*3f982cf4SFabien Sanglard PendingFrameSlot* const slot = get_slot_for(frame.frame_id);
125*3f982cf4SFabien Sanglard OSP_DCHECK(!slot->frame);
126*3f982cf4SFabien Sanglard slot->frame = crypto_.Encrypt(frame);
127*3f982cf4SFabien Sanglard const int packet_count = rtp_packetizer_.ComputeNumberOfPackets(*slot->frame);
128*3f982cf4SFabien Sanglard if (packet_count <= 0) {
129*3f982cf4SFabien Sanglard slot->frame.reset();
130*3f982cf4SFabien Sanglard return PAYLOAD_TOO_LARGE;
131*3f982cf4SFabien Sanglard }
132*3f982cf4SFabien Sanglard slot->send_flags.Resize(packet_count, YetAnotherBitVector::SET);
133*3f982cf4SFabien Sanglard slot->packet_sent_times.assign(packet_count, SenderPacketRouter::kNever);
134*3f982cf4SFabien Sanglard
135*3f982cf4SFabien Sanglard // Officially record the "enqueue."
136*3f982cf4SFabien Sanglard ++num_frames_in_flight_;
137*3f982cf4SFabien Sanglard last_enqueued_frame_id_ = slot->frame->frame_id;
138*3f982cf4SFabien Sanglard OSP_DCHECK_LE(num_frames_in_flight_,
139*3f982cf4SFabien Sanglard last_enqueued_frame_id_ - checkpoint_frame_id_);
140*3f982cf4SFabien Sanglard if (slot->frame->dependency == EncodedFrame::KEY_FRAME) {
141*3f982cf4SFabien Sanglard last_enqueued_key_frame_id_ = slot->frame->frame_id;
142*3f982cf4SFabien Sanglard }
143*3f982cf4SFabien Sanglard
144*3f982cf4SFabien Sanglard // Update the target playout delay, if necessary.
145*3f982cf4SFabien Sanglard if (slot->frame->new_playout_delay > milliseconds::zero()) {
146*3f982cf4SFabien Sanglard target_playout_delay_ = slot->frame->new_playout_delay;
147*3f982cf4SFabien Sanglard playout_delay_change_at_frame_id_ = slot->frame->frame_id;
148*3f982cf4SFabien Sanglard }
149*3f982cf4SFabien Sanglard
150*3f982cf4SFabien Sanglard // Update the lip-sync information for the next Sender Report.
151*3f982cf4SFabien Sanglard pending_sender_report_.reference_time = slot->frame->reference_time;
152*3f982cf4SFabien Sanglard pending_sender_report_.rtp_timestamp = slot->frame->rtp_timestamp;
153*3f982cf4SFabien Sanglard
154*3f982cf4SFabien Sanglard // If the round trip time hasn't been computed yet, immediately send a RTCP
155*3f982cf4SFabien Sanglard // packet (i.e., before the RTP packets are sent). The RTCP packet will
156*3f982cf4SFabien Sanglard // provide a Sender Report which contains the required lip-sync information
157*3f982cf4SFabien Sanglard // the Receiver needs for timing the media playout.
158*3f982cf4SFabien Sanglard //
159*3f982cf4SFabien Sanglard // Detail: Working backwards, if the round trip time is not known, then this
160*3f982cf4SFabien Sanglard // Sender has never processed a Receiver Report. Thus, the Receiver has never
161*3f982cf4SFabien Sanglard // provided a Receiver Report, which it can only do after having processed a
162*3f982cf4SFabien Sanglard // Sender Report from this Sender. Thus, this Sender really needs to send
163*3f982cf4SFabien Sanglard // that, right now!
164*3f982cf4SFabien Sanglard if (round_trip_time_ == Clock::duration::zero()) {
165*3f982cf4SFabien Sanglard packet_router_->RequestRtcpSend(rtcp_session_.receiver_ssrc());
166*3f982cf4SFabien Sanglard }
167*3f982cf4SFabien Sanglard
168*3f982cf4SFabien Sanglard // Re-activate RTP sending if it was suspended.
169*3f982cf4SFabien Sanglard packet_router_->RequestRtpSend(rtcp_session_.receiver_ssrc());
170*3f982cf4SFabien Sanglard
171*3f982cf4SFabien Sanglard return OK;
172*3f982cf4SFabien Sanglard }
173*3f982cf4SFabien Sanglard
CancelInFlightData()174*3f982cf4SFabien Sanglard void Sender::CancelInFlightData() {
175*3f982cf4SFabien Sanglard while (checkpoint_frame_id_ <= last_enqueued_frame_id_) {
176*3f982cf4SFabien Sanglard ++checkpoint_frame_id_;
177*3f982cf4SFabien Sanglard CancelPendingFrame(checkpoint_frame_id_);
178*3f982cf4SFabien Sanglard }
179*3f982cf4SFabien Sanglard }
180*3f982cf4SFabien Sanglard
OnReceivedRtcpPacket(Clock::time_point arrival_time,absl::Span<const uint8_t> packet)181*3f982cf4SFabien Sanglard void Sender::OnReceivedRtcpPacket(Clock::time_point arrival_time,
182*3f982cf4SFabien Sanglard absl::Span<const uint8_t> packet) {
183*3f982cf4SFabien Sanglard rtcp_packet_arrival_time_ = arrival_time;
184*3f982cf4SFabien Sanglard // This call to Parse() invoke zero or more of the OnReceiverXYZ() methods in
185*3f982cf4SFabien Sanglard // the current call stack:
186*3f982cf4SFabien Sanglard if (rtcp_parser_.Parse(packet, last_enqueued_frame_id_)) {
187*3f982cf4SFabien Sanglard packet_router_->OnRtcpReceived(arrival_time, round_trip_time_);
188*3f982cf4SFabien Sanglard }
189*3f982cf4SFabien Sanglard }
190*3f982cf4SFabien Sanglard
GetRtcpPacketForImmediateSend(Clock::time_point send_time,absl::Span<uint8_t> buffer)191*3f982cf4SFabien Sanglard absl::Span<uint8_t> Sender::GetRtcpPacketForImmediateSend(
192*3f982cf4SFabien Sanglard Clock::time_point send_time,
193*3f982cf4SFabien Sanglard absl::Span<uint8_t> buffer) {
194*3f982cf4SFabien Sanglard if (pending_sender_report_.reference_time == SenderPacketRouter::kNever) {
195*3f982cf4SFabien Sanglard // Cannot send a report if one is not available (i.e., a frame has never
196*3f982cf4SFabien Sanglard // been enqueued).
197*3f982cf4SFabien Sanglard return buffer.subspan(0, 0);
198*3f982cf4SFabien Sanglard }
199*3f982cf4SFabien Sanglard
200*3f982cf4SFabien Sanglard // The Sender Report to be sent is a snapshot of the "pending Sender Report,"
201*3f982cf4SFabien Sanglard // but with its timestamp fields modified. First, the reference time is set to
202*3f982cf4SFabien Sanglard // the RTCP packet's send time. Then, the corresponding RTP timestamp is
203*3f982cf4SFabien Sanglard // translated to match (for lip-sync).
204*3f982cf4SFabien Sanglard RtcpSenderReport sender_report = pending_sender_report_;
205*3f982cf4SFabien Sanglard sender_report.reference_time = send_time;
206*3f982cf4SFabien Sanglard sender_report.rtp_timestamp += RtpTimeDelta::FromDuration(
207*3f982cf4SFabien Sanglard sender_report.reference_time - pending_sender_report_.reference_time,
208*3f982cf4SFabien Sanglard rtp_timebase_);
209*3f982cf4SFabien Sanglard
210*3f982cf4SFabien Sanglard return sender_report_builder_.BuildPacket(sender_report, buffer).first;
211*3f982cf4SFabien Sanglard }
212*3f982cf4SFabien Sanglard
GetRtpPacketForImmediateSend(Clock::time_point send_time,absl::Span<uint8_t> buffer)213*3f982cf4SFabien Sanglard absl::Span<uint8_t> Sender::GetRtpPacketForImmediateSend(
214*3f982cf4SFabien Sanglard Clock::time_point send_time,
215*3f982cf4SFabien Sanglard absl::Span<uint8_t> buffer) {
216*3f982cf4SFabien Sanglard ChosenPacket chosen = ChooseNextRtpPacketNeedingSend();
217*3f982cf4SFabien Sanglard
218*3f982cf4SFabien Sanglard // If no packets need sending (i.e., all packets have been sent at least once
219*3f982cf4SFabien Sanglard // and do not need to be re-sent yet), check whether a Kickstart packet should
220*3f982cf4SFabien Sanglard // be sent. It's possible that there has been complete packet loss of some
221*3f982cf4SFabien Sanglard // frames, and the Receiver may not be aware of the existence of the latest
222*3f982cf4SFabien Sanglard // frame(s). Kickstarting is the only way the Receiver can discover the newer
223*3f982cf4SFabien Sanglard // frames it doesn't know about.
224*3f982cf4SFabien Sanglard if (!chosen) {
225*3f982cf4SFabien Sanglard const ChosenPacketAndWhen kickstart = ChooseKickstartPacket();
226*3f982cf4SFabien Sanglard if (kickstart.when > send_time) {
227*3f982cf4SFabien Sanglard // Nothing to send, so return "empty" signal to the packet router. The
228*3f982cf4SFabien Sanglard // packet router will suspend RTP sending until this Sender explicitly
229*3f982cf4SFabien Sanglard // resumes it.
230*3f982cf4SFabien Sanglard return buffer.subspan(0, 0);
231*3f982cf4SFabien Sanglard }
232*3f982cf4SFabien Sanglard chosen = kickstart;
233*3f982cf4SFabien Sanglard OSP_DCHECK(chosen);
234*3f982cf4SFabien Sanglard }
235*3f982cf4SFabien Sanglard
236*3f982cf4SFabien Sanglard const absl::Span<uint8_t> result = rtp_packetizer_.GeneratePacket(
237*3f982cf4SFabien Sanglard *chosen.slot->frame, chosen.packet_id, buffer);
238*3f982cf4SFabien Sanglard chosen.slot->send_flags.Clear(chosen.packet_id);
239*3f982cf4SFabien Sanglard chosen.slot->packet_sent_times[chosen.packet_id] = send_time;
240*3f982cf4SFabien Sanglard
241*3f982cf4SFabien Sanglard ++pending_sender_report_.send_packet_count;
242*3f982cf4SFabien Sanglard // According to RFC3550, the octet count does not include the RTP header. The
243*3f982cf4SFabien Sanglard // following is just a good approximation, however, because the header size
244*3f982cf4SFabien Sanglard // will very infrequently be 4 bytes greater (see
245*3f982cf4SFabien Sanglard // RtpPacketizer::kAdaptiveLatencyHeaderSize). No known Cast Streaming
246*3f982cf4SFabien Sanglard // Receiver implementations use this for anything, and so this should be fine.
247*3f982cf4SFabien Sanglard const int approximate_octet_count =
248*3f982cf4SFabien Sanglard static_cast<int>(result.size()) - RtpPacketizer::kBaseRtpHeaderSize;
249*3f982cf4SFabien Sanglard OSP_DCHECK_GE(approximate_octet_count, 0);
250*3f982cf4SFabien Sanglard pending_sender_report_.send_octet_count += approximate_octet_count;
251*3f982cf4SFabien Sanglard
252*3f982cf4SFabien Sanglard return result;
253*3f982cf4SFabien Sanglard }
254*3f982cf4SFabien Sanglard
GetRtpResumeTime()255*3f982cf4SFabien Sanglard Clock::time_point Sender::GetRtpResumeTime() {
256*3f982cf4SFabien Sanglard if (ChooseNextRtpPacketNeedingSend()) {
257*3f982cf4SFabien Sanglard return Alarm::kImmediately;
258*3f982cf4SFabien Sanglard }
259*3f982cf4SFabien Sanglard return ChooseKickstartPacket().when;
260*3f982cf4SFabien Sanglard }
261*3f982cf4SFabien Sanglard
OnReceiverReferenceTimeAdvanced(Clock::time_point reference_time)262*3f982cf4SFabien Sanglard void Sender::OnReceiverReferenceTimeAdvanced(Clock::time_point reference_time) {
263*3f982cf4SFabien Sanglard // Not used.
264*3f982cf4SFabien Sanglard }
265*3f982cf4SFabien Sanglard
OnReceiverReport(const RtcpReportBlock & receiver_report)266*3f982cf4SFabien Sanglard void Sender::OnReceiverReport(const RtcpReportBlock& receiver_report) {
267*3f982cf4SFabien Sanglard OSP_DCHECK_NE(rtcp_packet_arrival_time_, SenderPacketRouter::kNever);
268*3f982cf4SFabien Sanglard
269*3f982cf4SFabien Sanglard const Clock::duration total_delay =
270*3f982cf4SFabien Sanglard rtcp_packet_arrival_time_ -
271*3f982cf4SFabien Sanglard sender_report_builder_.GetRecentReportTime(
272*3f982cf4SFabien Sanglard receiver_report.last_status_report_id, rtcp_packet_arrival_time_);
273*3f982cf4SFabien Sanglard const auto non_network_delay =
274*3f982cf4SFabien Sanglard Clock::to_duration(receiver_report.delay_since_last_report);
275*3f982cf4SFabien Sanglard
276*3f982cf4SFabien Sanglard // Round trip time measurement: This is the time elapsed since the Sender
277*3f982cf4SFabien Sanglard // Report was sent, minus the time the Receiver did other stuff before sending
278*3f982cf4SFabien Sanglard // the Receiver Report back.
279*3f982cf4SFabien Sanglard //
280*3f982cf4SFabien Sanglard // If the round trip time seems to be less than or equal to zero, assume clock
281*3f982cf4SFabien Sanglard // imprecision by one or both peers caused a bad value to be calculated. The
282*3f982cf4SFabien Sanglard // true value is likely very close to zero (i.e., this is ideal network
283*3f982cf4SFabien Sanglard // behavior); and so just represent this as 75 µs, an optimistic
284*3f982cf4SFabien Sanglard // wired-Ethernet LAN ping time.
285*3f982cf4SFabien Sanglard constexpr auto kNearZeroRoundTripTime = Clock::to_duration(microseconds(75));
286*3f982cf4SFabien Sanglard static_assert(kNearZeroRoundTripTime > Clock::duration::zero(),
287*3f982cf4SFabien Sanglard "More precision in Clock::duration needed!");
288*3f982cf4SFabien Sanglard const Clock::duration measurement =
289*3f982cf4SFabien Sanglard std::max(total_delay - non_network_delay, kNearZeroRoundTripTime);
290*3f982cf4SFabien Sanglard
291*3f982cf4SFabien Sanglard // Validate the measurement by using the current target playout delay as a
292*3f982cf4SFabien Sanglard // "reasonable upper-bound." It's certainly possible that the actual network
293*3f982cf4SFabien Sanglard // round-trip time could exceed the target playout delay, but that would mean
294*3f982cf4SFabien Sanglard // the current network performance is totally inadequate for streaming anyway.
295*3f982cf4SFabien Sanglard if (measurement > target_playout_delay_) {
296*3f982cf4SFabien Sanglard OSP_LOG_WARN << "Invalidating a round-trip time measurement ("
297*3f982cf4SFabien Sanglard << measurement
298*3f982cf4SFabien Sanglard << ") since it exceeds the current target playout delay ("
299*3f982cf4SFabien Sanglard << target_playout_delay_ << ").";
300*3f982cf4SFabien Sanglard return;
301*3f982cf4SFabien Sanglard }
302*3f982cf4SFabien Sanglard
303*3f982cf4SFabien Sanglard // Measurements will typically have high variance. Use a simple smoothing
304*3f982cf4SFabien Sanglard // filter to track a short-term average that changes less drastically.
305*3f982cf4SFabien Sanglard if (round_trip_time_ == Clock::duration::zero()) {
306*3f982cf4SFabien Sanglard round_trip_time_ = measurement;
307*3f982cf4SFabien Sanglard } else {
308*3f982cf4SFabien Sanglard // Arbitrary constant, to provide 1/8 weight to the new measurement, and 7/8
309*3f982cf4SFabien Sanglard // weight to the old estimate, which seems to work well for de-noising the
310*3f982cf4SFabien Sanglard // estimate.
311*3f982cf4SFabien Sanglard constexpr int kInertia = 7;
312*3f982cf4SFabien Sanglard round_trip_time_ =
313*3f982cf4SFabien Sanglard (kInertia * round_trip_time_ + measurement) / (kInertia + 1);
314*3f982cf4SFabien Sanglard }
315*3f982cf4SFabien Sanglard TRACE_SCOPED(TraceCategory::kSender, "UpdatedRTT");
316*3f982cf4SFabien Sanglard }
317*3f982cf4SFabien Sanglard
OnReceiverIndicatesPictureLoss()318*3f982cf4SFabien Sanglard void Sender::OnReceiverIndicatesPictureLoss() {
319*3f982cf4SFabien Sanglard TRACE_DEFAULT_SCOPED(TraceCategory::kSender);
320*3f982cf4SFabien Sanglard // The Receiver will continue the PLI notifications until it has received a
321*3f982cf4SFabien Sanglard // key frame. Thus, if a key frame is already in-flight, don't make a state
322*3f982cf4SFabien Sanglard // change that would cause this Sender to force another expensive key frame.
323*3f982cf4SFabien Sanglard if (checkpoint_frame_id_ < last_enqueued_key_frame_id_) {
324*3f982cf4SFabien Sanglard return;
325*3f982cf4SFabien Sanglard }
326*3f982cf4SFabien Sanglard
327*3f982cf4SFabien Sanglard picture_lost_at_frame_id_ = checkpoint_frame_id_;
328*3f982cf4SFabien Sanglard
329*3f982cf4SFabien Sanglard if (observer_) {
330*3f982cf4SFabien Sanglard observer_->OnPictureLost();
331*3f982cf4SFabien Sanglard }
332*3f982cf4SFabien Sanglard
333*3f982cf4SFabien Sanglard // Note: It may seem that all pending frames should be canceled until
334*3f982cf4SFabien Sanglard // EnqueueFrame() is called with a key frame. However:
335*3f982cf4SFabien Sanglard //
336*3f982cf4SFabien Sanglard // 1. The Receiver should still be the main authority on what frames/packets
337*3f982cf4SFabien Sanglard // are being ACK'ed and NACK'ed.
338*3f982cf4SFabien Sanglard //
339*3f982cf4SFabien Sanglard // 2. It may be desirable for the Receiver to be "limping along" in the
340*3f982cf4SFabien Sanglard // meantime. For example, video may be corrupted but mostly watchable,
341*3f982cf4SFabien Sanglard // and so it's best for the Sender to continue sending the non-key frames
342*3f982cf4SFabien Sanglard // until the Receiver indicates otherwise.
343*3f982cf4SFabien Sanglard }
344*3f982cf4SFabien Sanglard
OnReceiverCheckpoint(FrameId frame_id,milliseconds playout_delay)345*3f982cf4SFabien Sanglard void Sender::OnReceiverCheckpoint(FrameId frame_id,
346*3f982cf4SFabien Sanglard milliseconds playout_delay) {
347*3f982cf4SFabien Sanglard TRACE_DEFAULT_SCOPED(TraceCategory::kSender);
348*3f982cf4SFabien Sanglard if (frame_id > last_enqueued_frame_id_) {
349*3f982cf4SFabien Sanglard OSP_LOG_ERROR
350*3f982cf4SFabien Sanglard << "Ignoring checkpoint for " << latest_expected_frame_id_
351*3f982cf4SFabien Sanglard << " because this Sender could not have sent any frames after "
352*3f982cf4SFabien Sanglard << last_enqueued_frame_id_ << '.';
353*3f982cf4SFabien Sanglard return;
354*3f982cf4SFabien Sanglard }
355*3f982cf4SFabien Sanglard // CompoundRtcpParser should guarantee this:
356*3f982cf4SFabien Sanglard OSP_DCHECK(playout_delay >= milliseconds::zero());
357*3f982cf4SFabien Sanglard
358*3f982cf4SFabien Sanglard while (checkpoint_frame_id_ < frame_id) {
359*3f982cf4SFabien Sanglard ++checkpoint_frame_id_;
360*3f982cf4SFabien Sanglard CancelPendingFrame(checkpoint_frame_id_);
361*3f982cf4SFabien Sanglard }
362*3f982cf4SFabien Sanglard latest_expected_frame_id_ = std::max(latest_expected_frame_id_, frame_id);
363*3f982cf4SFabien Sanglard
364*3f982cf4SFabien Sanglard if (playout_delay != target_playout_delay_ &&
365*3f982cf4SFabien Sanglard frame_id >= playout_delay_change_at_frame_id_) {
366*3f982cf4SFabien Sanglard OSP_LOG_WARN << "Sender's target playout delay (" << target_playout_delay_
367*3f982cf4SFabien Sanglard << ") disagrees with the Receiver's (" << playout_delay << ")";
368*3f982cf4SFabien Sanglard }
369*3f982cf4SFabien Sanglard }
370*3f982cf4SFabien Sanglard
OnReceiverHasFrames(std::vector<FrameId> acks)371*3f982cf4SFabien Sanglard void Sender::OnReceiverHasFrames(std::vector<FrameId> acks) {
372*3f982cf4SFabien Sanglard OSP_DCHECK(!acks.empty() && AreElementsSortedAndUnique(acks));
373*3f982cf4SFabien Sanglard
374*3f982cf4SFabien Sanglard if (acks.back() > last_enqueued_frame_id_) {
375*3f982cf4SFabien Sanglard OSP_LOG_ERROR << "Ignoring individual frame ACKs: ACKing frame "
376*3f982cf4SFabien Sanglard << latest_expected_frame_id_
377*3f982cf4SFabien Sanglard << " is invalid because this Sender could not have sent any "
378*3f982cf4SFabien Sanglard "frames after "
379*3f982cf4SFabien Sanglard << last_enqueued_frame_id_ << '.';
380*3f982cf4SFabien Sanglard return;
381*3f982cf4SFabien Sanglard }
382*3f982cf4SFabien Sanglard
383*3f982cf4SFabien Sanglard for (FrameId id : acks) {
384*3f982cf4SFabien Sanglard CancelPendingFrame(id);
385*3f982cf4SFabien Sanglard }
386*3f982cf4SFabien Sanglard latest_expected_frame_id_ = std::max(latest_expected_frame_id_, acks.back());
387*3f982cf4SFabien Sanglard }
388*3f982cf4SFabien Sanglard
OnReceiverIsMissingPackets(std::vector<PacketNack> nacks)389*3f982cf4SFabien Sanglard void Sender::OnReceiverIsMissingPackets(std::vector<PacketNack> nacks) {
390*3f982cf4SFabien Sanglard OSP_DCHECK(!nacks.empty() && AreElementsSortedAndUnique(nacks));
391*3f982cf4SFabien Sanglard OSP_DCHECK_NE(rtcp_packet_arrival_time_, SenderPacketRouter::kNever);
392*3f982cf4SFabien Sanglard
393*3f982cf4SFabien Sanglard // This is a point-in-time threshold that indicates whether each NACK will
394*3f982cf4SFabien Sanglard // trigger a packet retransmit. The threshold is based on the network round
395*3f982cf4SFabien Sanglard // trip time because a Receiver's NACK may have been issued while the needed
396*3f982cf4SFabien Sanglard // packet was in-flight from the Sender. In such cases, the Receiver's NACK is
397*3f982cf4SFabien Sanglard // likely stale and this Sender should not redundantly re-transmit the packet
398*3f982cf4SFabien Sanglard // again.
399*3f982cf4SFabien Sanglard const Clock::time_point too_recent_a_send_time =
400*3f982cf4SFabien Sanglard rtcp_packet_arrival_time_ - round_trip_time_;
401*3f982cf4SFabien Sanglard
402*3f982cf4SFabien Sanglard // Iterate over all the NACKs...
403*3f982cf4SFabien Sanglard bool need_to_send = false;
404*3f982cf4SFabien Sanglard for (auto nack_it = nacks.begin(); nack_it != nacks.end();) {
405*3f982cf4SFabien Sanglard // Find the slot associated with the NACK's frame ID.
406*3f982cf4SFabien Sanglard const FrameId frame_id = nack_it->frame_id;
407*3f982cf4SFabien Sanglard PendingFrameSlot* slot = nullptr;
408*3f982cf4SFabien Sanglard if (frame_id <= last_enqueued_frame_id_) {
409*3f982cf4SFabien Sanglard PendingFrameSlot* const candidate_slot = get_slot_for(frame_id);
410*3f982cf4SFabien Sanglard if (candidate_slot->is_active_for_frame(frame_id)) {
411*3f982cf4SFabien Sanglard slot = candidate_slot;
412*3f982cf4SFabien Sanglard }
413*3f982cf4SFabien Sanglard }
414*3f982cf4SFabien Sanglard
415*3f982cf4SFabien Sanglard // If no slot was found (i.e., the NACK is invalid) for the frame, skip-over
416*3f982cf4SFabien Sanglard // all other NACKs for the same frame. While it seems to be a bug that the
417*3f982cf4SFabien Sanglard // Receiver would attempt to NACK a frame that does not yet exist, this can
418*3f982cf4SFabien Sanglard // happen in rare cases where RTCP packets arrive out-of-order (i.e., the
419*3f982cf4SFabien Sanglard // network shuffled them).
420*3f982cf4SFabien Sanglard if (!slot) {
421*3f982cf4SFabien Sanglard TRACE_SCOPED(TraceCategory::kSender, "MissingNackSlot");
422*3f982cf4SFabien Sanglard for (++nack_it; nack_it != nacks.end() && nack_it->frame_id == frame_id;
423*3f982cf4SFabien Sanglard ++nack_it) {
424*3f982cf4SFabien Sanglard }
425*3f982cf4SFabien Sanglard continue;
426*3f982cf4SFabien Sanglard }
427*3f982cf4SFabien Sanglard
428*3f982cf4SFabien Sanglard latest_expected_frame_id_ = std::max(latest_expected_frame_id_, frame_id);
429*3f982cf4SFabien Sanglard
430*3f982cf4SFabien Sanglard const auto HandleIndividualNack = [&](FramePacketId packet_id) {
431*3f982cf4SFabien Sanglard if (slot->packet_sent_times[packet_id] <= too_recent_a_send_time) {
432*3f982cf4SFabien Sanglard slot->send_flags.Set(packet_id);
433*3f982cf4SFabien Sanglard need_to_send = true;
434*3f982cf4SFabien Sanglard }
435*3f982cf4SFabien Sanglard };
436*3f982cf4SFabien Sanglard const FramePacketId range_end = slot->packet_sent_times.size();
437*3f982cf4SFabien Sanglard if (nack_it->packet_id == kAllPacketsLost) {
438*3f982cf4SFabien Sanglard for (FramePacketId packet_id = 0; packet_id < range_end; ++packet_id) {
439*3f982cf4SFabien Sanglard HandleIndividualNack(packet_id);
440*3f982cf4SFabien Sanglard }
441*3f982cf4SFabien Sanglard ++nack_it;
442*3f982cf4SFabien Sanglard } else {
443*3f982cf4SFabien Sanglard do {
444*3f982cf4SFabien Sanglard if (nack_it->packet_id < range_end) {
445*3f982cf4SFabien Sanglard HandleIndividualNack(nack_it->packet_id);
446*3f982cf4SFabien Sanglard } else {
447*3f982cf4SFabien Sanglard OSP_LOG_WARN
448*3f982cf4SFabien Sanglard << "Ignoring NACK for packet that doesn't exist in frame "
449*3f982cf4SFabien Sanglard << frame_id << ": " << static_cast<int>(nack_it->packet_id);
450*3f982cf4SFabien Sanglard }
451*3f982cf4SFabien Sanglard ++nack_it;
452*3f982cf4SFabien Sanglard } while (nack_it != nacks.end() && nack_it->frame_id == frame_id);
453*3f982cf4SFabien Sanglard }
454*3f982cf4SFabien Sanglard }
455*3f982cf4SFabien Sanglard
456*3f982cf4SFabien Sanglard if (need_to_send) {
457*3f982cf4SFabien Sanglard packet_router_->RequestRtpSend(rtcp_session_.receiver_ssrc());
458*3f982cf4SFabien Sanglard }
459*3f982cf4SFabien Sanglard }
460*3f982cf4SFabien Sanglard
ChooseNextRtpPacketNeedingSend()461*3f982cf4SFabien Sanglard Sender::ChosenPacket Sender::ChooseNextRtpPacketNeedingSend() {
462*3f982cf4SFabien Sanglard // Find the oldest packet needing to be sent (or re-sent).
463*3f982cf4SFabien Sanglard for (FrameId frame_id = checkpoint_frame_id_ + 1;
464*3f982cf4SFabien Sanglard frame_id <= last_enqueued_frame_id_; ++frame_id) {
465*3f982cf4SFabien Sanglard PendingFrameSlot* const slot = get_slot_for(frame_id);
466*3f982cf4SFabien Sanglard if (!slot->is_active_for_frame(frame_id)) {
467*3f982cf4SFabien Sanglard continue; // Frame was canceled. None of its packets need to be sent.
468*3f982cf4SFabien Sanglard }
469*3f982cf4SFabien Sanglard const FramePacketId packet_id = slot->send_flags.FindFirstSet();
470*3f982cf4SFabien Sanglard if (packet_id < slot->send_flags.size()) {
471*3f982cf4SFabien Sanglard return {slot, packet_id};
472*3f982cf4SFabien Sanglard }
473*3f982cf4SFabien Sanglard }
474*3f982cf4SFabien Sanglard
475*3f982cf4SFabien Sanglard return {}; // Nothing needs to be sent.
476*3f982cf4SFabien Sanglard }
477*3f982cf4SFabien Sanglard
ChooseKickstartPacket()478*3f982cf4SFabien Sanglard Sender::ChosenPacketAndWhen Sender::ChooseKickstartPacket() {
479*3f982cf4SFabien Sanglard if (latest_expected_frame_id_ >= last_enqueued_frame_id_) {
480*3f982cf4SFabien Sanglard // Since the Receiver must know about all of the frames currently queued, no
481*3f982cf4SFabien Sanglard // Kickstart packet is necessary.
482*3f982cf4SFabien Sanglard return {};
483*3f982cf4SFabien Sanglard }
484*3f982cf4SFabien Sanglard
485*3f982cf4SFabien Sanglard // The Kickstart packet is always in the last-enqueued frame, so that the
486*3f982cf4SFabien Sanglard // Receiver will know about every frame the Sender has. However, which packet
487*3f982cf4SFabien Sanglard // should be chosen? Any would do, since all packets contain the frame's total
488*3f982cf4SFabien Sanglard // packet count. For historical reasons, all sender implementations have
489*3f982cf4SFabien Sanglard // always just sent the last packet; and so that tradition is continued here.
490*3f982cf4SFabien Sanglard ChosenPacketAndWhen chosen;
491*3f982cf4SFabien Sanglard chosen.slot = get_slot_for(last_enqueued_frame_id_);
492*3f982cf4SFabien Sanglard // Note: This frame cannot have been canceled since
493*3f982cf4SFabien Sanglard // |latest_expected_frame_id_| hasn't yet reached this point.
494*3f982cf4SFabien Sanglard OSP_DCHECK(chosen.slot->is_active_for_frame(last_enqueued_frame_id_));
495*3f982cf4SFabien Sanglard chosen.packet_id = chosen.slot->send_flags.size() - 1;
496*3f982cf4SFabien Sanglard
497*3f982cf4SFabien Sanglard const Clock::time_point time_last_sent =
498*3f982cf4SFabien Sanglard chosen.slot->packet_sent_times[chosen.packet_id];
499*3f982cf4SFabien Sanglard // Sanity-check: This method should not be called to choose a packet while
500*3f982cf4SFabien Sanglard // there are still unsent packets.
501*3f982cf4SFabien Sanglard OSP_DCHECK_NE(time_last_sent, SenderPacketRouter::kNever);
502*3f982cf4SFabien Sanglard
503*3f982cf4SFabien Sanglard // The desired Kickstart interval is a fraction of the total
504*3f982cf4SFabien Sanglard // |target_playout_delay_|. The reason for the specific ratio here is based on
505*3f982cf4SFabien Sanglard // lost knowledge (from legacy implementations); but it makes sense (i.e., to
506*3f982cf4SFabien Sanglard // be a good "network citizen") to be less aggressive for larger playout delay
507*3f982cf4SFabien Sanglard // windows, and more aggressive for shorter ones to avoid too-late packet
508*3f982cf4SFabien Sanglard // arrivals.
509*3f982cf4SFabien Sanglard using kWaitFraction = std::ratio<1, 20>;
510*3f982cf4SFabien Sanglard const Clock::duration desired_kickstart_interval =
511*3f982cf4SFabien Sanglard Clock::to_duration(target_playout_delay_) * kWaitFraction::num /
512*3f982cf4SFabien Sanglard kWaitFraction::den;
513*3f982cf4SFabien Sanglard // The actual interval used is increased, if current network performance
514*3f982cf4SFabien Sanglard // warrants waiting longer. Don't send a Kickstart packet until no NACKs
515*3f982cf4SFabien Sanglard // have been received for two network round-trip periods.
516*3f982cf4SFabien Sanglard constexpr int kLowerBoundRoundTrips = 2;
517*3f982cf4SFabien Sanglard const Clock::duration kickstart_interval = std::max(
518*3f982cf4SFabien Sanglard desired_kickstart_interval, round_trip_time_ * kLowerBoundRoundTrips);
519*3f982cf4SFabien Sanglard chosen.when = time_last_sent + kickstart_interval;
520*3f982cf4SFabien Sanglard
521*3f982cf4SFabien Sanglard return chosen;
522*3f982cf4SFabien Sanglard }
523*3f982cf4SFabien Sanglard
CancelPendingFrame(FrameId frame_id)524*3f982cf4SFabien Sanglard void Sender::CancelPendingFrame(FrameId frame_id) {
525*3f982cf4SFabien Sanglard PendingFrameSlot* const slot = get_slot_for(frame_id);
526*3f982cf4SFabien Sanglard if (!slot->is_active_for_frame(frame_id)) {
527*3f982cf4SFabien Sanglard return; // Frame was already canceled.
528*3f982cf4SFabien Sanglard }
529*3f982cf4SFabien Sanglard
530*3f982cf4SFabien Sanglard packet_router_->OnPayloadReceived(
531*3f982cf4SFabien Sanglard slot->frame->data.size(), rtcp_packet_arrival_time_, round_trip_time_);
532*3f982cf4SFabien Sanglard
533*3f982cf4SFabien Sanglard slot->frame.reset();
534*3f982cf4SFabien Sanglard OSP_DCHECK_GT(num_frames_in_flight_, 0);
535*3f982cf4SFabien Sanglard --num_frames_in_flight_;
536*3f982cf4SFabien Sanglard if (observer_) {
537*3f982cf4SFabien Sanglard observer_->OnFrameCanceled(frame_id);
538*3f982cf4SFabien Sanglard }
539*3f982cf4SFabien Sanglard }
540*3f982cf4SFabien Sanglard
OnFrameCanceled(FrameId frame_id)541*3f982cf4SFabien Sanglard void Sender::Observer::OnFrameCanceled(FrameId frame_id) {}
OnPictureLost()542*3f982cf4SFabien Sanglard void Sender::Observer::OnPictureLost() {}
543*3f982cf4SFabien Sanglard Sender::Observer::~Observer() = default;
544*3f982cf4SFabien Sanglard
545*3f982cf4SFabien Sanglard Sender::PendingFrameSlot::PendingFrameSlot() = default;
546*3f982cf4SFabien Sanglard Sender::PendingFrameSlot::~PendingFrameSlot() = default;
547*3f982cf4SFabien Sanglard
548*3f982cf4SFabien Sanglard } // namespace cast
549*3f982cf4SFabien Sanglard } // namespace openscreen
550