xref: /aosp_15_r20/external/openscreen/cast/streaming/sender.cc (revision 3f982cf4871df8771c9d4abe6e9a6f8d829b2736)
1*3f982cf4SFabien Sanglard // Copyright 2020 The Chromium Authors. All rights reserved.
2*3f982cf4SFabien Sanglard // Use of this source code is governed by a BSD-style license that can be
3*3f982cf4SFabien Sanglard // found in the LICENSE file.
4*3f982cf4SFabien Sanglard 
5*3f982cf4SFabien Sanglard #include "cast/streaming/sender.h"
6*3f982cf4SFabien Sanglard 
7*3f982cf4SFabien Sanglard #include <algorithm>
8*3f982cf4SFabien Sanglard #include <chrono>
9*3f982cf4SFabien Sanglard #include <ratio>
10*3f982cf4SFabien Sanglard 
11*3f982cf4SFabien Sanglard #include "cast/streaming/session_config.h"
12*3f982cf4SFabien Sanglard #include "util/chrono_helpers.h"
13*3f982cf4SFabien Sanglard #include "util/osp_logging.h"
14*3f982cf4SFabien Sanglard #include "util/std_util.h"
15*3f982cf4SFabien Sanglard #include "util/trace_logging.h"
16*3f982cf4SFabien Sanglard 
17*3f982cf4SFabien Sanglard namespace openscreen {
18*3f982cf4SFabien Sanglard namespace cast {
19*3f982cf4SFabien Sanglard 
20*3f982cf4SFabien Sanglard using openscreen::operator<<;  // For std::chrono::duration logging.
21*3f982cf4SFabien Sanglard 
Sender(Environment * environment,SenderPacketRouter * packet_router,SessionConfig config,RtpPayloadType rtp_payload_type)22*3f982cf4SFabien Sanglard Sender::Sender(Environment* environment,
23*3f982cf4SFabien Sanglard                SenderPacketRouter* packet_router,
24*3f982cf4SFabien Sanglard                SessionConfig config,
25*3f982cf4SFabien Sanglard                RtpPayloadType rtp_payload_type)
26*3f982cf4SFabien Sanglard     : config_(config),
27*3f982cf4SFabien Sanglard       packet_router_(packet_router),
28*3f982cf4SFabien Sanglard       rtcp_session_(config.sender_ssrc,
29*3f982cf4SFabien Sanglard                     config.receiver_ssrc,
30*3f982cf4SFabien Sanglard                     environment->now()),
31*3f982cf4SFabien Sanglard       rtcp_parser_(&rtcp_session_, this),
32*3f982cf4SFabien Sanglard       sender_report_builder_(&rtcp_session_),
33*3f982cf4SFabien Sanglard       rtp_packetizer_(rtp_payload_type,
34*3f982cf4SFabien Sanglard                       config.sender_ssrc,
35*3f982cf4SFabien Sanglard                       packet_router_->max_packet_size()),
36*3f982cf4SFabien Sanglard       rtp_timebase_(config.rtp_timebase),
37*3f982cf4SFabien Sanglard       crypto_(config.aes_secret_key, config.aes_iv_mask),
38*3f982cf4SFabien Sanglard       target_playout_delay_(config.target_playout_delay) {
39*3f982cf4SFabien Sanglard   OSP_DCHECK(packet_router_);
40*3f982cf4SFabien Sanglard   OSP_DCHECK_NE(rtcp_session_.sender_ssrc(), rtcp_session_.receiver_ssrc());
41*3f982cf4SFabien Sanglard   OSP_DCHECK_GT(rtp_timebase_, 0);
42*3f982cf4SFabien Sanglard   OSP_DCHECK(target_playout_delay_ > milliseconds::zero());
43*3f982cf4SFabien Sanglard 
44*3f982cf4SFabien Sanglard   pending_sender_report_.reference_time = SenderPacketRouter::kNever;
45*3f982cf4SFabien Sanglard 
46*3f982cf4SFabien Sanglard   packet_router_->OnSenderCreated(rtcp_session_.receiver_ssrc(), this);
47*3f982cf4SFabien Sanglard }
48*3f982cf4SFabien Sanglard 
~Sender()49*3f982cf4SFabien Sanglard Sender::~Sender() {
50*3f982cf4SFabien Sanglard   packet_router_->OnSenderDestroyed(rtcp_session_.receiver_ssrc());
51*3f982cf4SFabien Sanglard }
52*3f982cf4SFabien Sanglard 
SetObserver(Sender::Observer * observer)53*3f982cf4SFabien Sanglard void Sender::SetObserver(Sender::Observer* observer) {
54*3f982cf4SFabien Sanglard   observer_ = observer;
55*3f982cf4SFabien Sanglard }
56*3f982cf4SFabien Sanglard 
GetInFlightFrameCount() const57*3f982cf4SFabien Sanglard int Sender::GetInFlightFrameCount() const {
58*3f982cf4SFabien Sanglard   return num_frames_in_flight_;
59*3f982cf4SFabien Sanglard }
60*3f982cf4SFabien Sanglard 
GetInFlightMediaDuration(RtpTimeTicks next_frame_rtp_timestamp) const61*3f982cf4SFabien Sanglard Clock::duration Sender::GetInFlightMediaDuration(
62*3f982cf4SFabien Sanglard     RtpTimeTicks next_frame_rtp_timestamp) const {
63*3f982cf4SFabien Sanglard   if (num_frames_in_flight_ == 0) {
64*3f982cf4SFabien Sanglard     return Clock::duration::zero();  // No frames are currently in-flight.
65*3f982cf4SFabien Sanglard   }
66*3f982cf4SFabien Sanglard 
67*3f982cf4SFabien Sanglard   const PendingFrameSlot& oldest_slot = *get_slot_for(checkpoint_frame_id_ + 1);
68*3f982cf4SFabien Sanglard   // Note: The oldest slot's frame cannot have been canceled because the
69*3f982cf4SFabien Sanglard   // protocol does not allow ACK'ing this particular frame without also moving
70*3f982cf4SFabien Sanglard   // the checkpoint forward. See "CST2 feedback" discussion in rtp_defines.h.
71*3f982cf4SFabien Sanglard   OSP_DCHECK(oldest_slot.is_active_for_frame(checkpoint_frame_id_ + 1));
72*3f982cf4SFabien Sanglard 
73*3f982cf4SFabien Sanglard   return (next_frame_rtp_timestamp - oldest_slot.frame->rtp_timestamp)
74*3f982cf4SFabien Sanglard       .ToDuration<Clock::duration>(rtp_timebase_);
75*3f982cf4SFabien Sanglard }
76*3f982cf4SFabien Sanglard 
GetMaxInFlightMediaDuration() const77*3f982cf4SFabien Sanglard Clock::duration Sender::GetMaxInFlightMediaDuration() const {
78*3f982cf4SFabien Sanglard   // Assumption: The total amount of allowed in-flight media should equal the
79*3f982cf4SFabien Sanglard   // half of the playout delay window, plus the amount of time it takes to
80*3f982cf4SFabien Sanglard   // receive an ACK from the Receiver.
81*3f982cf4SFabien Sanglard   //
82*3f982cf4SFabien Sanglard   // Why half of the playout delay window? It's assumed here that capture and
83*3f982cf4SFabien Sanglard   // media encoding, which occur before EnqueueFrame() is called, are executing
84*3f982cf4SFabien Sanglard   // within the first half of the playout delay window. This leaves the second
85*3f982cf4SFabien Sanglard   // half for executing all network transmits/re-transmits, plus decoding and
86*3f982cf4SFabien Sanglard   // play-out at the Receiver.
87*3f982cf4SFabien Sanglard   return (target_playout_delay_ / 2) + (round_trip_time_ / 2);
88*3f982cf4SFabien Sanglard }
89*3f982cf4SFabien Sanglard 
NeedsKeyFrame() const90*3f982cf4SFabien Sanglard bool Sender::NeedsKeyFrame() const {
91*3f982cf4SFabien Sanglard   return last_enqueued_key_frame_id_ <= picture_lost_at_frame_id_;
92*3f982cf4SFabien Sanglard }
93*3f982cf4SFabien Sanglard 
GetNextFrameId() const94*3f982cf4SFabien Sanglard FrameId Sender::GetNextFrameId() const {
95*3f982cf4SFabien Sanglard   return last_enqueued_frame_id_ + 1;
96*3f982cf4SFabien Sanglard }
97*3f982cf4SFabien Sanglard 
EnqueueFrame(const EncodedFrame & frame)98*3f982cf4SFabien Sanglard Sender::EnqueueFrameResult Sender::EnqueueFrame(const EncodedFrame& frame) {
99*3f982cf4SFabien Sanglard   // Assume the fields of the |frame| have all been set correctly, with
100*3f982cf4SFabien Sanglard   // monotonically increasing timestamps and a valid pointer to the data.
101*3f982cf4SFabien Sanglard   OSP_DCHECK_EQ(frame.frame_id, GetNextFrameId());
102*3f982cf4SFabien Sanglard   OSP_DCHECK_GE(frame.referenced_frame_id, FrameId::first());
103*3f982cf4SFabien Sanglard   if (frame.frame_id != FrameId::first()) {
104*3f982cf4SFabien Sanglard     OSP_DCHECK_GT(frame.rtp_timestamp, pending_sender_report_.rtp_timestamp);
105*3f982cf4SFabien Sanglard     OSP_DCHECK_GT(frame.reference_time, pending_sender_report_.reference_time);
106*3f982cf4SFabien Sanglard   }
107*3f982cf4SFabien Sanglard   OSP_DCHECK(frame.data.data());
108*3f982cf4SFabien Sanglard 
109*3f982cf4SFabien Sanglard   // Check whether enqueuing the frame would exceed the design limit for the
110*3f982cf4SFabien Sanglard   // span of FrameIds. Even if |num_frames_in_flight_| is less than
111*3f982cf4SFabien Sanglard   // kMaxUnackedFrames, it's the span of FrameIds that is restricted.
112*3f982cf4SFabien Sanglard   if ((frame.frame_id - checkpoint_frame_id_) > kMaxUnackedFrames) {
113*3f982cf4SFabien Sanglard     return REACHED_ID_SPAN_LIMIT;
114*3f982cf4SFabien Sanglard   }
115*3f982cf4SFabien Sanglard 
116*3f982cf4SFabien Sanglard   // Check whether enqueuing the frame would exceed the current maximum media
117*3f982cf4SFabien Sanglard   // duration limit.
118*3f982cf4SFabien Sanglard   if (GetInFlightMediaDuration(frame.rtp_timestamp) >
119*3f982cf4SFabien Sanglard       GetMaxInFlightMediaDuration()) {
120*3f982cf4SFabien Sanglard     return MAX_DURATION_IN_FLIGHT;
121*3f982cf4SFabien Sanglard   }
122*3f982cf4SFabien Sanglard 
123*3f982cf4SFabien Sanglard   // Encrypt the frame and initialize the slot tracking its sending.
124*3f982cf4SFabien Sanglard   PendingFrameSlot* const slot = get_slot_for(frame.frame_id);
125*3f982cf4SFabien Sanglard   OSP_DCHECK(!slot->frame);
126*3f982cf4SFabien Sanglard   slot->frame = crypto_.Encrypt(frame);
127*3f982cf4SFabien Sanglard   const int packet_count = rtp_packetizer_.ComputeNumberOfPackets(*slot->frame);
128*3f982cf4SFabien Sanglard   if (packet_count <= 0) {
129*3f982cf4SFabien Sanglard     slot->frame.reset();
130*3f982cf4SFabien Sanglard     return PAYLOAD_TOO_LARGE;
131*3f982cf4SFabien Sanglard   }
132*3f982cf4SFabien Sanglard   slot->send_flags.Resize(packet_count, YetAnotherBitVector::SET);
133*3f982cf4SFabien Sanglard   slot->packet_sent_times.assign(packet_count, SenderPacketRouter::kNever);
134*3f982cf4SFabien Sanglard 
135*3f982cf4SFabien Sanglard   // Officially record the "enqueue."
136*3f982cf4SFabien Sanglard   ++num_frames_in_flight_;
137*3f982cf4SFabien Sanglard   last_enqueued_frame_id_ = slot->frame->frame_id;
138*3f982cf4SFabien Sanglard   OSP_DCHECK_LE(num_frames_in_flight_,
139*3f982cf4SFabien Sanglard                 last_enqueued_frame_id_ - checkpoint_frame_id_);
140*3f982cf4SFabien Sanglard   if (slot->frame->dependency == EncodedFrame::KEY_FRAME) {
141*3f982cf4SFabien Sanglard     last_enqueued_key_frame_id_ = slot->frame->frame_id;
142*3f982cf4SFabien Sanglard   }
143*3f982cf4SFabien Sanglard 
144*3f982cf4SFabien Sanglard   // Update the target playout delay, if necessary.
145*3f982cf4SFabien Sanglard   if (slot->frame->new_playout_delay > milliseconds::zero()) {
146*3f982cf4SFabien Sanglard     target_playout_delay_ = slot->frame->new_playout_delay;
147*3f982cf4SFabien Sanglard     playout_delay_change_at_frame_id_ = slot->frame->frame_id;
148*3f982cf4SFabien Sanglard   }
149*3f982cf4SFabien Sanglard 
150*3f982cf4SFabien Sanglard   // Update the lip-sync information for the next Sender Report.
151*3f982cf4SFabien Sanglard   pending_sender_report_.reference_time = slot->frame->reference_time;
152*3f982cf4SFabien Sanglard   pending_sender_report_.rtp_timestamp = slot->frame->rtp_timestamp;
153*3f982cf4SFabien Sanglard 
154*3f982cf4SFabien Sanglard   // If the round trip time hasn't been computed yet, immediately send a RTCP
155*3f982cf4SFabien Sanglard   // packet (i.e., before the RTP packets are sent). The RTCP packet will
156*3f982cf4SFabien Sanglard   // provide a Sender Report which contains the required lip-sync information
157*3f982cf4SFabien Sanglard   // the Receiver needs for timing the media playout.
158*3f982cf4SFabien Sanglard   //
159*3f982cf4SFabien Sanglard   // Detail: Working backwards, if the round trip time is not known, then this
160*3f982cf4SFabien Sanglard   // Sender has never processed a Receiver Report. Thus, the Receiver has never
161*3f982cf4SFabien Sanglard   // provided a Receiver Report, which it can only do after having processed a
162*3f982cf4SFabien Sanglard   // Sender Report from this Sender. Thus, this Sender really needs to send
163*3f982cf4SFabien Sanglard   // that, right now!
164*3f982cf4SFabien Sanglard   if (round_trip_time_ == Clock::duration::zero()) {
165*3f982cf4SFabien Sanglard     packet_router_->RequestRtcpSend(rtcp_session_.receiver_ssrc());
166*3f982cf4SFabien Sanglard   }
167*3f982cf4SFabien Sanglard 
168*3f982cf4SFabien Sanglard   // Re-activate RTP sending if it was suspended.
169*3f982cf4SFabien Sanglard   packet_router_->RequestRtpSend(rtcp_session_.receiver_ssrc());
170*3f982cf4SFabien Sanglard 
171*3f982cf4SFabien Sanglard   return OK;
172*3f982cf4SFabien Sanglard }
173*3f982cf4SFabien Sanglard 
CancelInFlightData()174*3f982cf4SFabien Sanglard void Sender::CancelInFlightData() {
175*3f982cf4SFabien Sanglard   while (checkpoint_frame_id_ <= last_enqueued_frame_id_) {
176*3f982cf4SFabien Sanglard     ++checkpoint_frame_id_;
177*3f982cf4SFabien Sanglard     CancelPendingFrame(checkpoint_frame_id_);
178*3f982cf4SFabien Sanglard   }
179*3f982cf4SFabien Sanglard }
180*3f982cf4SFabien Sanglard 
OnReceivedRtcpPacket(Clock::time_point arrival_time,absl::Span<const uint8_t> packet)181*3f982cf4SFabien Sanglard void Sender::OnReceivedRtcpPacket(Clock::time_point arrival_time,
182*3f982cf4SFabien Sanglard                                   absl::Span<const uint8_t> packet) {
183*3f982cf4SFabien Sanglard   rtcp_packet_arrival_time_ = arrival_time;
184*3f982cf4SFabien Sanglard   // This call to Parse() invoke zero or more of the OnReceiverXYZ() methods in
185*3f982cf4SFabien Sanglard   // the current call stack:
186*3f982cf4SFabien Sanglard   if (rtcp_parser_.Parse(packet, last_enqueued_frame_id_)) {
187*3f982cf4SFabien Sanglard     packet_router_->OnRtcpReceived(arrival_time, round_trip_time_);
188*3f982cf4SFabien Sanglard   }
189*3f982cf4SFabien Sanglard }
190*3f982cf4SFabien Sanglard 
GetRtcpPacketForImmediateSend(Clock::time_point send_time,absl::Span<uint8_t> buffer)191*3f982cf4SFabien Sanglard absl::Span<uint8_t> Sender::GetRtcpPacketForImmediateSend(
192*3f982cf4SFabien Sanglard     Clock::time_point send_time,
193*3f982cf4SFabien Sanglard     absl::Span<uint8_t> buffer) {
194*3f982cf4SFabien Sanglard   if (pending_sender_report_.reference_time == SenderPacketRouter::kNever) {
195*3f982cf4SFabien Sanglard     // Cannot send a report if one is not available (i.e., a frame has never
196*3f982cf4SFabien Sanglard     // been enqueued).
197*3f982cf4SFabien Sanglard     return buffer.subspan(0, 0);
198*3f982cf4SFabien Sanglard   }
199*3f982cf4SFabien Sanglard 
200*3f982cf4SFabien Sanglard   // The Sender Report to be sent is a snapshot of the "pending Sender Report,"
201*3f982cf4SFabien Sanglard   // but with its timestamp fields modified. First, the reference time is set to
202*3f982cf4SFabien Sanglard   // the RTCP packet's send time. Then, the corresponding RTP timestamp is
203*3f982cf4SFabien Sanglard   // translated to match (for lip-sync).
204*3f982cf4SFabien Sanglard   RtcpSenderReport sender_report = pending_sender_report_;
205*3f982cf4SFabien Sanglard   sender_report.reference_time = send_time;
206*3f982cf4SFabien Sanglard   sender_report.rtp_timestamp += RtpTimeDelta::FromDuration(
207*3f982cf4SFabien Sanglard       sender_report.reference_time - pending_sender_report_.reference_time,
208*3f982cf4SFabien Sanglard       rtp_timebase_);
209*3f982cf4SFabien Sanglard 
210*3f982cf4SFabien Sanglard   return sender_report_builder_.BuildPacket(sender_report, buffer).first;
211*3f982cf4SFabien Sanglard }
212*3f982cf4SFabien Sanglard 
GetRtpPacketForImmediateSend(Clock::time_point send_time,absl::Span<uint8_t> buffer)213*3f982cf4SFabien Sanglard absl::Span<uint8_t> Sender::GetRtpPacketForImmediateSend(
214*3f982cf4SFabien Sanglard     Clock::time_point send_time,
215*3f982cf4SFabien Sanglard     absl::Span<uint8_t> buffer) {
216*3f982cf4SFabien Sanglard   ChosenPacket chosen = ChooseNextRtpPacketNeedingSend();
217*3f982cf4SFabien Sanglard 
218*3f982cf4SFabien Sanglard   // If no packets need sending (i.e., all packets have been sent at least once
219*3f982cf4SFabien Sanglard   // and do not need to be re-sent yet), check whether a Kickstart packet should
220*3f982cf4SFabien Sanglard   // be sent. It's possible that there has been complete packet loss of some
221*3f982cf4SFabien Sanglard   // frames, and the Receiver may not be aware of the existence of the latest
222*3f982cf4SFabien Sanglard   // frame(s). Kickstarting is the only way the Receiver can discover the newer
223*3f982cf4SFabien Sanglard   // frames it doesn't know about.
224*3f982cf4SFabien Sanglard   if (!chosen) {
225*3f982cf4SFabien Sanglard     const ChosenPacketAndWhen kickstart = ChooseKickstartPacket();
226*3f982cf4SFabien Sanglard     if (kickstart.when > send_time) {
227*3f982cf4SFabien Sanglard       // Nothing to send, so return "empty" signal to the packet router. The
228*3f982cf4SFabien Sanglard       // packet router will suspend RTP sending until this Sender explicitly
229*3f982cf4SFabien Sanglard       // resumes it.
230*3f982cf4SFabien Sanglard       return buffer.subspan(0, 0);
231*3f982cf4SFabien Sanglard     }
232*3f982cf4SFabien Sanglard     chosen = kickstart;
233*3f982cf4SFabien Sanglard     OSP_DCHECK(chosen);
234*3f982cf4SFabien Sanglard   }
235*3f982cf4SFabien Sanglard 
236*3f982cf4SFabien Sanglard   const absl::Span<uint8_t> result = rtp_packetizer_.GeneratePacket(
237*3f982cf4SFabien Sanglard       *chosen.slot->frame, chosen.packet_id, buffer);
238*3f982cf4SFabien Sanglard   chosen.slot->send_flags.Clear(chosen.packet_id);
239*3f982cf4SFabien Sanglard   chosen.slot->packet_sent_times[chosen.packet_id] = send_time;
240*3f982cf4SFabien Sanglard 
241*3f982cf4SFabien Sanglard   ++pending_sender_report_.send_packet_count;
242*3f982cf4SFabien Sanglard   // According to RFC3550, the octet count does not include the RTP header. The
243*3f982cf4SFabien Sanglard   // following is just a good approximation, however, because the header size
244*3f982cf4SFabien Sanglard   // will very infrequently be 4 bytes greater (see
245*3f982cf4SFabien Sanglard   // RtpPacketizer::kAdaptiveLatencyHeaderSize). No known Cast Streaming
246*3f982cf4SFabien Sanglard   // Receiver implementations use this for anything, and so this should be fine.
247*3f982cf4SFabien Sanglard   const int approximate_octet_count =
248*3f982cf4SFabien Sanglard       static_cast<int>(result.size()) - RtpPacketizer::kBaseRtpHeaderSize;
249*3f982cf4SFabien Sanglard   OSP_DCHECK_GE(approximate_octet_count, 0);
250*3f982cf4SFabien Sanglard   pending_sender_report_.send_octet_count += approximate_octet_count;
251*3f982cf4SFabien Sanglard 
252*3f982cf4SFabien Sanglard   return result;
253*3f982cf4SFabien Sanglard }
254*3f982cf4SFabien Sanglard 
GetRtpResumeTime()255*3f982cf4SFabien Sanglard Clock::time_point Sender::GetRtpResumeTime() {
256*3f982cf4SFabien Sanglard   if (ChooseNextRtpPacketNeedingSend()) {
257*3f982cf4SFabien Sanglard     return Alarm::kImmediately;
258*3f982cf4SFabien Sanglard   }
259*3f982cf4SFabien Sanglard   return ChooseKickstartPacket().when;
260*3f982cf4SFabien Sanglard }
261*3f982cf4SFabien Sanglard 
OnReceiverReferenceTimeAdvanced(Clock::time_point reference_time)262*3f982cf4SFabien Sanglard void Sender::OnReceiverReferenceTimeAdvanced(Clock::time_point reference_time) {
263*3f982cf4SFabien Sanglard   // Not used.
264*3f982cf4SFabien Sanglard }
265*3f982cf4SFabien Sanglard 
OnReceiverReport(const RtcpReportBlock & receiver_report)266*3f982cf4SFabien Sanglard void Sender::OnReceiverReport(const RtcpReportBlock& receiver_report) {
267*3f982cf4SFabien Sanglard   OSP_DCHECK_NE(rtcp_packet_arrival_time_, SenderPacketRouter::kNever);
268*3f982cf4SFabien Sanglard 
269*3f982cf4SFabien Sanglard   const Clock::duration total_delay =
270*3f982cf4SFabien Sanglard       rtcp_packet_arrival_time_ -
271*3f982cf4SFabien Sanglard       sender_report_builder_.GetRecentReportTime(
272*3f982cf4SFabien Sanglard           receiver_report.last_status_report_id, rtcp_packet_arrival_time_);
273*3f982cf4SFabien Sanglard   const auto non_network_delay =
274*3f982cf4SFabien Sanglard       Clock::to_duration(receiver_report.delay_since_last_report);
275*3f982cf4SFabien Sanglard 
276*3f982cf4SFabien Sanglard   // Round trip time measurement: This is the time elapsed since the Sender
277*3f982cf4SFabien Sanglard   // Report was sent, minus the time the Receiver did other stuff before sending
278*3f982cf4SFabien Sanglard   // the Receiver Report back.
279*3f982cf4SFabien Sanglard   //
280*3f982cf4SFabien Sanglard   // If the round trip time seems to be less than or equal to zero, assume clock
281*3f982cf4SFabien Sanglard   // imprecision by one or both peers caused a bad value to be calculated. The
282*3f982cf4SFabien Sanglard   // true value is likely very close to zero (i.e., this is ideal network
283*3f982cf4SFabien Sanglard   // behavior); and so just represent this as 75 µs, an optimistic
284*3f982cf4SFabien Sanglard   // wired-Ethernet LAN ping time.
285*3f982cf4SFabien Sanglard   constexpr auto kNearZeroRoundTripTime = Clock::to_duration(microseconds(75));
286*3f982cf4SFabien Sanglard   static_assert(kNearZeroRoundTripTime > Clock::duration::zero(),
287*3f982cf4SFabien Sanglard                 "More precision in Clock::duration needed!");
288*3f982cf4SFabien Sanglard   const Clock::duration measurement =
289*3f982cf4SFabien Sanglard       std::max(total_delay - non_network_delay, kNearZeroRoundTripTime);
290*3f982cf4SFabien Sanglard 
291*3f982cf4SFabien Sanglard   // Validate the measurement by using the current target playout delay as a
292*3f982cf4SFabien Sanglard   // "reasonable upper-bound." It's certainly possible that the actual network
293*3f982cf4SFabien Sanglard   // round-trip time could exceed the target playout delay, but that would mean
294*3f982cf4SFabien Sanglard   // the current network performance is totally inadequate for streaming anyway.
295*3f982cf4SFabien Sanglard   if (measurement > target_playout_delay_) {
296*3f982cf4SFabien Sanglard     OSP_LOG_WARN << "Invalidating a round-trip time measurement ("
297*3f982cf4SFabien Sanglard                  << measurement
298*3f982cf4SFabien Sanglard                  << ") since it exceeds the current target playout delay ("
299*3f982cf4SFabien Sanglard                  << target_playout_delay_ << ").";
300*3f982cf4SFabien Sanglard     return;
301*3f982cf4SFabien Sanglard   }
302*3f982cf4SFabien Sanglard 
303*3f982cf4SFabien Sanglard   // Measurements will typically have high variance. Use a simple smoothing
304*3f982cf4SFabien Sanglard   // filter to track a short-term average that changes less drastically.
305*3f982cf4SFabien Sanglard   if (round_trip_time_ == Clock::duration::zero()) {
306*3f982cf4SFabien Sanglard     round_trip_time_ = measurement;
307*3f982cf4SFabien Sanglard   } else {
308*3f982cf4SFabien Sanglard     // Arbitrary constant, to provide 1/8 weight to the new measurement, and 7/8
309*3f982cf4SFabien Sanglard     // weight to the old estimate, which seems to work well for de-noising the
310*3f982cf4SFabien Sanglard     // estimate.
311*3f982cf4SFabien Sanglard     constexpr int kInertia = 7;
312*3f982cf4SFabien Sanglard     round_trip_time_ =
313*3f982cf4SFabien Sanglard         (kInertia * round_trip_time_ + measurement) / (kInertia + 1);
314*3f982cf4SFabien Sanglard   }
315*3f982cf4SFabien Sanglard   TRACE_SCOPED(TraceCategory::kSender, "UpdatedRTT");
316*3f982cf4SFabien Sanglard }
317*3f982cf4SFabien Sanglard 
OnReceiverIndicatesPictureLoss()318*3f982cf4SFabien Sanglard void Sender::OnReceiverIndicatesPictureLoss() {
319*3f982cf4SFabien Sanglard   TRACE_DEFAULT_SCOPED(TraceCategory::kSender);
320*3f982cf4SFabien Sanglard   // The Receiver will continue the PLI notifications until it has received a
321*3f982cf4SFabien Sanglard   // key frame. Thus, if a key frame is already in-flight, don't make a state
322*3f982cf4SFabien Sanglard   // change that would cause this Sender to force another expensive key frame.
323*3f982cf4SFabien Sanglard   if (checkpoint_frame_id_ < last_enqueued_key_frame_id_) {
324*3f982cf4SFabien Sanglard     return;
325*3f982cf4SFabien Sanglard   }
326*3f982cf4SFabien Sanglard 
327*3f982cf4SFabien Sanglard   picture_lost_at_frame_id_ = checkpoint_frame_id_;
328*3f982cf4SFabien Sanglard 
329*3f982cf4SFabien Sanglard   if (observer_) {
330*3f982cf4SFabien Sanglard     observer_->OnPictureLost();
331*3f982cf4SFabien Sanglard   }
332*3f982cf4SFabien Sanglard 
333*3f982cf4SFabien Sanglard   // Note: It may seem that all pending frames should be canceled until
334*3f982cf4SFabien Sanglard   // EnqueueFrame() is called with a key frame. However:
335*3f982cf4SFabien Sanglard   //
336*3f982cf4SFabien Sanglard   //   1. The Receiver should still be the main authority on what frames/packets
337*3f982cf4SFabien Sanglard   //      are being ACK'ed and NACK'ed.
338*3f982cf4SFabien Sanglard   //
339*3f982cf4SFabien Sanglard   //   2. It may be desirable for the Receiver to be "limping along" in the
340*3f982cf4SFabien Sanglard   //      meantime. For example, video may be corrupted but mostly watchable,
341*3f982cf4SFabien Sanglard   //      and so it's best for the Sender to continue sending the non-key frames
342*3f982cf4SFabien Sanglard   //      until the Receiver indicates otherwise.
343*3f982cf4SFabien Sanglard }
344*3f982cf4SFabien Sanglard 
OnReceiverCheckpoint(FrameId frame_id,milliseconds playout_delay)345*3f982cf4SFabien Sanglard void Sender::OnReceiverCheckpoint(FrameId frame_id,
346*3f982cf4SFabien Sanglard                                   milliseconds playout_delay) {
347*3f982cf4SFabien Sanglard   TRACE_DEFAULT_SCOPED(TraceCategory::kSender);
348*3f982cf4SFabien Sanglard   if (frame_id > last_enqueued_frame_id_) {
349*3f982cf4SFabien Sanglard     OSP_LOG_ERROR
350*3f982cf4SFabien Sanglard         << "Ignoring checkpoint for " << latest_expected_frame_id_
351*3f982cf4SFabien Sanglard         << " because this Sender could not have sent any frames after "
352*3f982cf4SFabien Sanglard         << last_enqueued_frame_id_ << '.';
353*3f982cf4SFabien Sanglard     return;
354*3f982cf4SFabien Sanglard   }
355*3f982cf4SFabien Sanglard   // CompoundRtcpParser should guarantee this:
356*3f982cf4SFabien Sanglard   OSP_DCHECK(playout_delay >= milliseconds::zero());
357*3f982cf4SFabien Sanglard 
358*3f982cf4SFabien Sanglard   while (checkpoint_frame_id_ < frame_id) {
359*3f982cf4SFabien Sanglard     ++checkpoint_frame_id_;
360*3f982cf4SFabien Sanglard     CancelPendingFrame(checkpoint_frame_id_);
361*3f982cf4SFabien Sanglard   }
362*3f982cf4SFabien Sanglard   latest_expected_frame_id_ = std::max(latest_expected_frame_id_, frame_id);
363*3f982cf4SFabien Sanglard 
364*3f982cf4SFabien Sanglard   if (playout_delay != target_playout_delay_ &&
365*3f982cf4SFabien Sanglard       frame_id >= playout_delay_change_at_frame_id_) {
366*3f982cf4SFabien Sanglard     OSP_LOG_WARN << "Sender's target playout delay (" << target_playout_delay_
367*3f982cf4SFabien Sanglard                  << ") disagrees with the Receiver's (" << playout_delay << ")";
368*3f982cf4SFabien Sanglard   }
369*3f982cf4SFabien Sanglard }
370*3f982cf4SFabien Sanglard 
OnReceiverHasFrames(std::vector<FrameId> acks)371*3f982cf4SFabien Sanglard void Sender::OnReceiverHasFrames(std::vector<FrameId> acks) {
372*3f982cf4SFabien Sanglard   OSP_DCHECK(!acks.empty() && AreElementsSortedAndUnique(acks));
373*3f982cf4SFabien Sanglard 
374*3f982cf4SFabien Sanglard   if (acks.back() > last_enqueued_frame_id_) {
375*3f982cf4SFabien Sanglard     OSP_LOG_ERROR << "Ignoring individual frame ACKs: ACKing frame "
376*3f982cf4SFabien Sanglard                   << latest_expected_frame_id_
377*3f982cf4SFabien Sanglard                   << " is invalid because this Sender could not have sent any "
378*3f982cf4SFabien Sanglard                      "frames after "
379*3f982cf4SFabien Sanglard                   << last_enqueued_frame_id_ << '.';
380*3f982cf4SFabien Sanglard     return;
381*3f982cf4SFabien Sanglard   }
382*3f982cf4SFabien Sanglard 
383*3f982cf4SFabien Sanglard   for (FrameId id : acks) {
384*3f982cf4SFabien Sanglard     CancelPendingFrame(id);
385*3f982cf4SFabien Sanglard   }
386*3f982cf4SFabien Sanglard   latest_expected_frame_id_ = std::max(latest_expected_frame_id_, acks.back());
387*3f982cf4SFabien Sanglard }
388*3f982cf4SFabien Sanglard 
OnReceiverIsMissingPackets(std::vector<PacketNack> nacks)389*3f982cf4SFabien Sanglard void Sender::OnReceiverIsMissingPackets(std::vector<PacketNack> nacks) {
390*3f982cf4SFabien Sanglard   OSP_DCHECK(!nacks.empty() && AreElementsSortedAndUnique(nacks));
391*3f982cf4SFabien Sanglard   OSP_DCHECK_NE(rtcp_packet_arrival_time_, SenderPacketRouter::kNever);
392*3f982cf4SFabien Sanglard 
393*3f982cf4SFabien Sanglard   // This is a point-in-time threshold that indicates whether each NACK will
394*3f982cf4SFabien Sanglard   // trigger a packet retransmit. The threshold is based on the network round
395*3f982cf4SFabien Sanglard   // trip time because a Receiver's NACK may have been issued while the needed
396*3f982cf4SFabien Sanglard   // packet was in-flight from the Sender. In such cases, the Receiver's NACK is
397*3f982cf4SFabien Sanglard   // likely stale and this Sender should not redundantly re-transmit the packet
398*3f982cf4SFabien Sanglard   // again.
399*3f982cf4SFabien Sanglard   const Clock::time_point too_recent_a_send_time =
400*3f982cf4SFabien Sanglard       rtcp_packet_arrival_time_ - round_trip_time_;
401*3f982cf4SFabien Sanglard 
402*3f982cf4SFabien Sanglard   // Iterate over all the NACKs...
403*3f982cf4SFabien Sanglard   bool need_to_send = false;
404*3f982cf4SFabien Sanglard   for (auto nack_it = nacks.begin(); nack_it != nacks.end();) {
405*3f982cf4SFabien Sanglard     // Find the slot associated with the NACK's frame ID.
406*3f982cf4SFabien Sanglard     const FrameId frame_id = nack_it->frame_id;
407*3f982cf4SFabien Sanglard     PendingFrameSlot* slot = nullptr;
408*3f982cf4SFabien Sanglard     if (frame_id <= last_enqueued_frame_id_) {
409*3f982cf4SFabien Sanglard       PendingFrameSlot* const candidate_slot = get_slot_for(frame_id);
410*3f982cf4SFabien Sanglard       if (candidate_slot->is_active_for_frame(frame_id)) {
411*3f982cf4SFabien Sanglard         slot = candidate_slot;
412*3f982cf4SFabien Sanglard       }
413*3f982cf4SFabien Sanglard     }
414*3f982cf4SFabien Sanglard 
415*3f982cf4SFabien Sanglard     // If no slot was found (i.e., the NACK is invalid) for the frame, skip-over
416*3f982cf4SFabien Sanglard     // all other NACKs for the same frame. While it seems to be a bug that the
417*3f982cf4SFabien Sanglard     // Receiver would attempt to NACK a frame that does not yet exist, this can
418*3f982cf4SFabien Sanglard     // happen in rare cases where RTCP packets arrive out-of-order (i.e., the
419*3f982cf4SFabien Sanglard     // network shuffled them).
420*3f982cf4SFabien Sanglard     if (!slot) {
421*3f982cf4SFabien Sanglard       TRACE_SCOPED(TraceCategory::kSender, "MissingNackSlot");
422*3f982cf4SFabien Sanglard       for (++nack_it; nack_it != nacks.end() && nack_it->frame_id == frame_id;
423*3f982cf4SFabien Sanglard            ++nack_it) {
424*3f982cf4SFabien Sanglard       }
425*3f982cf4SFabien Sanglard       continue;
426*3f982cf4SFabien Sanglard     }
427*3f982cf4SFabien Sanglard 
428*3f982cf4SFabien Sanglard     latest_expected_frame_id_ = std::max(latest_expected_frame_id_, frame_id);
429*3f982cf4SFabien Sanglard 
430*3f982cf4SFabien Sanglard     const auto HandleIndividualNack = [&](FramePacketId packet_id) {
431*3f982cf4SFabien Sanglard       if (slot->packet_sent_times[packet_id] <= too_recent_a_send_time) {
432*3f982cf4SFabien Sanglard         slot->send_flags.Set(packet_id);
433*3f982cf4SFabien Sanglard         need_to_send = true;
434*3f982cf4SFabien Sanglard       }
435*3f982cf4SFabien Sanglard     };
436*3f982cf4SFabien Sanglard     const FramePacketId range_end = slot->packet_sent_times.size();
437*3f982cf4SFabien Sanglard     if (nack_it->packet_id == kAllPacketsLost) {
438*3f982cf4SFabien Sanglard       for (FramePacketId packet_id = 0; packet_id < range_end; ++packet_id) {
439*3f982cf4SFabien Sanglard         HandleIndividualNack(packet_id);
440*3f982cf4SFabien Sanglard       }
441*3f982cf4SFabien Sanglard       ++nack_it;
442*3f982cf4SFabien Sanglard     } else {
443*3f982cf4SFabien Sanglard       do {
444*3f982cf4SFabien Sanglard         if (nack_it->packet_id < range_end) {
445*3f982cf4SFabien Sanglard           HandleIndividualNack(nack_it->packet_id);
446*3f982cf4SFabien Sanglard         } else {
447*3f982cf4SFabien Sanglard           OSP_LOG_WARN
448*3f982cf4SFabien Sanglard               << "Ignoring NACK for packet that doesn't exist in frame "
449*3f982cf4SFabien Sanglard               << frame_id << ": " << static_cast<int>(nack_it->packet_id);
450*3f982cf4SFabien Sanglard         }
451*3f982cf4SFabien Sanglard         ++nack_it;
452*3f982cf4SFabien Sanglard       } while (nack_it != nacks.end() && nack_it->frame_id == frame_id);
453*3f982cf4SFabien Sanglard     }
454*3f982cf4SFabien Sanglard   }
455*3f982cf4SFabien Sanglard 
456*3f982cf4SFabien Sanglard   if (need_to_send) {
457*3f982cf4SFabien Sanglard     packet_router_->RequestRtpSend(rtcp_session_.receiver_ssrc());
458*3f982cf4SFabien Sanglard   }
459*3f982cf4SFabien Sanglard }
460*3f982cf4SFabien Sanglard 
ChooseNextRtpPacketNeedingSend()461*3f982cf4SFabien Sanglard Sender::ChosenPacket Sender::ChooseNextRtpPacketNeedingSend() {
462*3f982cf4SFabien Sanglard   // Find the oldest packet needing to be sent (or re-sent).
463*3f982cf4SFabien Sanglard   for (FrameId frame_id = checkpoint_frame_id_ + 1;
464*3f982cf4SFabien Sanglard        frame_id <= last_enqueued_frame_id_; ++frame_id) {
465*3f982cf4SFabien Sanglard     PendingFrameSlot* const slot = get_slot_for(frame_id);
466*3f982cf4SFabien Sanglard     if (!slot->is_active_for_frame(frame_id)) {
467*3f982cf4SFabien Sanglard       continue;  // Frame was canceled. None of its packets need to be sent.
468*3f982cf4SFabien Sanglard     }
469*3f982cf4SFabien Sanglard     const FramePacketId packet_id = slot->send_flags.FindFirstSet();
470*3f982cf4SFabien Sanglard     if (packet_id < slot->send_flags.size()) {
471*3f982cf4SFabien Sanglard       return {slot, packet_id};
472*3f982cf4SFabien Sanglard     }
473*3f982cf4SFabien Sanglard   }
474*3f982cf4SFabien Sanglard 
475*3f982cf4SFabien Sanglard   return {};  // Nothing needs to be sent.
476*3f982cf4SFabien Sanglard }
477*3f982cf4SFabien Sanglard 
ChooseKickstartPacket()478*3f982cf4SFabien Sanglard Sender::ChosenPacketAndWhen Sender::ChooseKickstartPacket() {
479*3f982cf4SFabien Sanglard   if (latest_expected_frame_id_ >= last_enqueued_frame_id_) {
480*3f982cf4SFabien Sanglard     // Since the Receiver must know about all of the frames currently queued, no
481*3f982cf4SFabien Sanglard     // Kickstart packet is necessary.
482*3f982cf4SFabien Sanglard     return {};
483*3f982cf4SFabien Sanglard   }
484*3f982cf4SFabien Sanglard 
485*3f982cf4SFabien Sanglard   // The Kickstart packet is always in the last-enqueued frame, so that the
486*3f982cf4SFabien Sanglard   // Receiver will know about every frame the Sender has. However, which packet
487*3f982cf4SFabien Sanglard   // should be chosen? Any would do, since all packets contain the frame's total
488*3f982cf4SFabien Sanglard   // packet count. For historical reasons, all sender implementations have
489*3f982cf4SFabien Sanglard   // always just sent the last packet; and so that tradition is continued here.
490*3f982cf4SFabien Sanglard   ChosenPacketAndWhen chosen;
491*3f982cf4SFabien Sanglard   chosen.slot = get_slot_for(last_enqueued_frame_id_);
492*3f982cf4SFabien Sanglard   // Note: This frame cannot have been canceled since
493*3f982cf4SFabien Sanglard   // |latest_expected_frame_id_| hasn't yet reached this point.
494*3f982cf4SFabien Sanglard   OSP_DCHECK(chosen.slot->is_active_for_frame(last_enqueued_frame_id_));
495*3f982cf4SFabien Sanglard   chosen.packet_id = chosen.slot->send_flags.size() - 1;
496*3f982cf4SFabien Sanglard 
497*3f982cf4SFabien Sanglard   const Clock::time_point time_last_sent =
498*3f982cf4SFabien Sanglard       chosen.slot->packet_sent_times[chosen.packet_id];
499*3f982cf4SFabien Sanglard   // Sanity-check: This method should not be called to choose a packet while
500*3f982cf4SFabien Sanglard   // there are still unsent packets.
501*3f982cf4SFabien Sanglard   OSP_DCHECK_NE(time_last_sent, SenderPacketRouter::kNever);
502*3f982cf4SFabien Sanglard 
503*3f982cf4SFabien Sanglard   // The desired Kickstart interval is a fraction of the total
504*3f982cf4SFabien Sanglard   // |target_playout_delay_|. The reason for the specific ratio here is based on
505*3f982cf4SFabien Sanglard   // lost knowledge (from legacy implementations); but it makes sense (i.e., to
506*3f982cf4SFabien Sanglard   // be a good "network citizen") to be less aggressive for larger playout delay
507*3f982cf4SFabien Sanglard   // windows, and more aggressive for shorter ones to avoid too-late packet
508*3f982cf4SFabien Sanglard   // arrivals.
509*3f982cf4SFabien Sanglard   using kWaitFraction = std::ratio<1, 20>;
510*3f982cf4SFabien Sanglard   const Clock::duration desired_kickstart_interval =
511*3f982cf4SFabien Sanglard       Clock::to_duration(target_playout_delay_) * kWaitFraction::num /
512*3f982cf4SFabien Sanglard       kWaitFraction::den;
513*3f982cf4SFabien Sanglard   // The actual interval used is increased, if current network performance
514*3f982cf4SFabien Sanglard   // warrants waiting longer. Don't send a Kickstart packet until no NACKs
515*3f982cf4SFabien Sanglard   // have been received for two network round-trip periods.
516*3f982cf4SFabien Sanglard   constexpr int kLowerBoundRoundTrips = 2;
517*3f982cf4SFabien Sanglard   const Clock::duration kickstart_interval = std::max(
518*3f982cf4SFabien Sanglard       desired_kickstart_interval, round_trip_time_ * kLowerBoundRoundTrips);
519*3f982cf4SFabien Sanglard   chosen.when = time_last_sent + kickstart_interval;
520*3f982cf4SFabien Sanglard 
521*3f982cf4SFabien Sanglard   return chosen;
522*3f982cf4SFabien Sanglard }
523*3f982cf4SFabien Sanglard 
CancelPendingFrame(FrameId frame_id)524*3f982cf4SFabien Sanglard void Sender::CancelPendingFrame(FrameId frame_id) {
525*3f982cf4SFabien Sanglard   PendingFrameSlot* const slot = get_slot_for(frame_id);
526*3f982cf4SFabien Sanglard   if (!slot->is_active_for_frame(frame_id)) {
527*3f982cf4SFabien Sanglard     return;  // Frame was already canceled.
528*3f982cf4SFabien Sanglard   }
529*3f982cf4SFabien Sanglard 
530*3f982cf4SFabien Sanglard   packet_router_->OnPayloadReceived(
531*3f982cf4SFabien Sanglard       slot->frame->data.size(), rtcp_packet_arrival_time_, round_trip_time_);
532*3f982cf4SFabien Sanglard 
533*3f982cf4SFabien Sanglard   slot->frame.reset();
534*3f982cf4SFabien Sanglard   OSP_DCHECK_GT(num_frames_in_flight_, 0);
535*3f982cf4SFabien Sanglard   --num_frames_in_flight_;
536*3f982cf4SFabien Sanglard   if (observer_) {
537*3f982cf4SFabien Sanglard     observer_->OnFrameCanceled(frame_id);
538*3f982cf4SFabien Sanglard   }
539*3f982cf4SFabien Sanglard }
540*3f982cf4SFabien Sanglard 
OnFrameCanceled(FrameId frame_id)541*3f982cf4SFabien Sanglard void Sender::Observer::OnFrameCanceled(FrameId frame_id) {}
OnPictureLost()542*3f982cf4SFabien Sanglard void Sender::Observer::OnPictureLost() {}
543*3f982cf4SFabien Sanglard Sender::Observer::~Observer() = default;
544*3f982cf4SFabien Sanglard 
545*3f982cf4SFabien Sanglard Sender::PendingFrameSlot::PendingFrameSlot() = default;
546*3f982cf4SFabien Sanglard Sender::PendingFrameSlot::~PendingFrameSlot() = default;
547*3f982cf4SFabien Sanglard 
548*3f982cf4SFabien Sanglard }  // namespace cast
549*3f982cf4SFabien Sanglard }  // namespace openscreen
550