xref: /aosp_15_r20/external/webrtc/video/video_send_stream_impl.cc (revision d9f758449e529ab9291ac668be2861e7a55c2422)
1*d9f75844SAndroid Build Coastguard Worker /*
2*d9f75844SAndroid Build Coastguard Worker  *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
3*d9f75844SAndroid Build Coastguard Worker  *
4*d9f75844SAndroid Build Coastguard Worker  *  Use of this source code is governed by a BSD-style license
5*d9f75844SAndroid Build Coastguard Worker  *  that can be found in the LICENSE file in the root of the source
6*d9f75844SAndroid Build Coastguard Worker  *  tree. An additional intellectual property rights grant can be found
7*d9f75844SAndroid Build Coastguard Worker  *  in the file PATENTS.  All contributing project authors may
8*d9f75844SAndroid Build Coastguard Worker  *  be found in the AUTHORS file in the root of the source tree.
9*d9f75844SAndroid Build Coastguard Worker  */
10*d9f75844SAndroid Build Coastguard Worker #include "video/video_send_stream_impl.h"
11*d9f75844SAndroid Build Coastguard Worker 
12*d9f75844SAndroid Build Coastguard Worker #include <stdio.h>
13*d9f75844SAndroid Build Coastguard Worker 
14*d9f75844SAndroid Build Coastguard Worker #include <algorithm>
15*d9f75844SAndroid Build Coastguard Worker #include <cstdint>
16*d9f75844SAndroid Build Coastguard Worker #include <string>
17*d9f75844SAndroid Build Coastguard Worker #include <utility>
18*d9f75844SAndroid Build Coastguard Worker 
19*d9f75844SAndroid Build Coastguard Worker #include "absl/algorithm/container.h"
20*d9f75844SAndroid Build Coastguard Worker #include "api/crypto/crypto_options.h"
21*d9f75844SAndroid Build Coastguard Worker #include "api/rtp_parameters.h"
22*d9f75844SAndroid Build Coastguard Worker #include "api/scoped_refptr.h"
23*d9f75844SAndroid Build Coastguard Worker #include "api/sequence_checker.h"
24*d9f75844SAndroid Build Coastguard Worker #include "api/task_queue/pending_task_safety_flag.h"
25*d9f75844SAndroid Build Coastguard Worker #include "api/task_queue/task_queue_base.h"
26*d9f75844SAndroid Build Coastguard Worker #include "api/video_codecs/video_codec.h"
27*d9f75844SAndroid Build Coastguard Worker #include "call/rtp_transport_controller_send_interface.h"
28*d9f75844SAndroid Build Coastguard Worker #include "call/video_send_stream.h"
29*d9f75844SAndroid Build Coastguard Worker #include "modules/pacing/pacing_controller.h"
30*d9f75844SAndroid Build Coastguard Worker #include "rtc_base/checks.h"
31*d9f75844SAndroid Build Coastguard Worker #include "rtc_base/experiments/alr_experiment.h"
32*d9f75844SAndroid Build Coastguard Worker #include "rtc_base/experiments/field_trial_parser.h"
33*d9f75844SAndroid Build Coastguard Worker #include "rtc_base/experiments/min_video_bitrate_experiment.h"
34*d9f75844SAndroid Build Coastguard Worker #include "rtc_base/experiments/rate_control_settings.h"
35*d9f75844SAndroid Build Coastguard Worker #include "rtc_base/logging.h"
36*d9f75844SAndroid Build Coastguard Worker #include "rtc_base/numerics/safe_conversions.h"
37*d9f75844SAndroid Build Coastguard Worker #include "rtc_base/trace_event.h"
38*d9f75844SAndroid Build Coastguard Worker #include "system_wrappers/include/clock.h"
39*d9f75844SAndroid Build Coastguard Worker #include "system_wrappers/include/field_trial.h"
40*d9f75844SAndroid Build Coastguard Worker 
41*d9f75844SAndroid Build Coastguard Worker namespace webrtc {
42*d9f75844SAndroid Build Coastguard Worker namespace internal {
43*d9f75844SAndroid Build Coastguard Worker namespace {
44*d9f75844SAndroid Build Coastguard Worker 
45*d9f75844SAndroid Build Coastguard Worker // Max positive size difference to treat allocations as "similar".
46*d9f75844SAndroid Build Coastguard Worker static constexpr int kMaxVbaSizeDifferencePercent = 10;
47*d9f75844SAndroid Build Coastguard Worker // Max time we will throttle similar video bitrate allocations.
48*d9f75844SAndroid Build Coastguard Worker static constexpr int64_t kMaxVbaThrottleTimeMs = 500;
49*d9f75844SAndroid Build Coastguard Worker 
50*d9f75844SAndroid Build Coastguard Worker constexpr TimeDelta kEncoderTimeOut = TimeDelta::Seconds(2);
51*d9f75844SAndroid Build Coastguard Worker 
52*d9f75844SAndroid Build Coastguard Worker constexpr double kVideoHysteresis = 1.2;
53*d9f75844SAndroid Build Coastguard Worker constexpr double kScreenshareHysteresis = 1.35;
54*d9f75844SAndroid Build Coastguard Worker 
55*d9f75844SAndroid Build Coastguard Worker // When send-side BWE is used a stricter 1.1x pacing factor is used, rather than
56*d9f75844SAndroid Build Coastguard Worker // the 2.5x which is used with receive-side BWE. Provides a more careful
57*d9f75844SAndroid Build Coastguard Worker // bandwidth rampup with less risk of overshoots causing adverse effects like
58*d9f75844SAndroid Build Coastguard Worker // packet loss. Not used for receive side BWE, since there we lack the probing
59*d9f75844SAndroid Build Coastguard Worker // feature and so may result in too slow initial rampup.
60*d9f75844SAndroid Build Coastguard Worker static constexpr double kStrictPacingMultiplier = 1.1;
61*d9f75844SAndroid Build Coastguard Worker 
TransportSeqNumExtensionConfigured(const VideoSendStream::Config & config)62*d9f75844SAndroid Build Coastguard Worker bool TransportSeqNumExtensionConfigured(const VideoSendStream::Config& config) {
63*d9f75844SAndroid Build Coastguard Worker   const std::vector<RtpExtension>& extensions = config.rtp.extensions;
64*d9f75844SAndroid Build Coastguard Worker   return absl::c_any_of(extensions, [](const RtpExtension& ext) {
65*d9f75844SAndroid Build Coastguard Worker     return ext.uri == RtpExtension::kTransportSequenceNumberUri;
66*d9f75844SAndroid Build Coastguard Worker   });
67*d9f75844SAndroid Build Coastguard Worker }
68*d9f75844SAndroid Build Coastguard Worker 
69*d9f75844SAndroid Build Coastguard Worker // Calculate max padding bitrate for a multi layer codec.
CalculateMaxPadBitrateBps(const std::vector<VideoStream> & streams,bool is_svc,VideoEncoderConfig::ContentType content_type,int min_transmit_bitrate_bps,bool pad_to_min_bitrate,bool alr_probing)70*d9f75844SAndroid Build Coastguard Worker int CalculateMaxPadBitrateBps(const std::vector<VideoStream>& streams,
71*d9f75844SAndroid Build Coastguard Worker                               bool is_svc,
72*d9f75844SAndroid Build Coastguard Worker                               VideoEncoderConfig::ContentType content_type,
73*d9f75844SAndroid Build Coastguard Worker                               int min_transmit_bitrate_bps,
74*d9f75844SAndroid Build Coastguard Worker                               bool pad_to_min_bitrate,
75*d9f75844SAndroid Build Coastguard Worker                               bool alr_probing) {
76*d9f75844SAndroid Build Coastguard Worker   int pad_up_to_bitrate_bps = 0;
77*d9f75844SAndroid Build Coastguard Worker 
78*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK(!is_svc || streams.size() <= 1) << "Only one stream is allowed in "
79*d9f75844SAndroid Build Coastguard Worker                                                 "SVC mode.";
80*d9f75844SAndroid Build Coastguard Worker 
81*d9f75844SAndroid Build Coastguard Worker   // Filter out only the active streams;
82*d9f75844SAndroid Build Coastguard Worker   std::vector<VideoStream> active_streams;
83*d9f75844SAndroid Build Coastguard Worker   for (const VideoStream& stream : streams) {
84*d9f75844SAndroid Build Coastguard Worker     if (stream.active)
85*d9f75844SAndroid Build Coastguard Worker       active_streams.emplace_back(stream);
86*d9f75844SAndroid Build Coastguard Worker   }
87*d9f75844SAndroid Build Coastguard Worker 
88*d9f75844SAndroid Build Coastguard Worker   if (active_streams.size() > 1 || (!active_streams.empty() && is_svc)) {
89*d9f75844SAndroid Build Coastguard Worker     // Simulcast or SVC is used.
90*d9f75844SAndroid Build Coastguard Worker     // if SVC is used, stream bitrates should already encode svc bitrates:
91*d9f75844SAndroid Build Coastguard Worker     // min_bitrate = min bitrate of a lowest svc layer.
92*d9f75844SAndroid Build Coastguard Worker     // target_bitrate = sum of target bitrates of lower layers + min bitrate
93*d9f75844SAndroid Build Coastguard Worker     // of the last one (as used in the calculations below).
94*d9f75844SAndroid Build Coastguard Worker     // max_bitrate = sum of all active layers' max_bitrate.
95*d9f75844SAndroid Build Coastguard Worker     if (alr_probing) {
96*d9f75844SAndroid Build Coastguard Worker       // With alr probing, just pad to the min bitrate of the lowest stream,
97*d9f75844SAndroid Build Coastguard Worker       // probing will handle the rest of the rampup.
98*d9f75844SAndroid Build Coastguard Worker       pad_up_to_bitrate_bps = active_streams[0].min_bitrate_bps;
99*d9f75844SAndroid Build Coastguard Worker     } else {
100*d9f75844SAndroid Build Coastguard Worker       // Without alr probing, pad up to start bitrate of the
101*d9f75844SAndroid Build Coastguard Worker       // highest active stream.
102*d9f75844SAndroid Build Coastguard Worker       const double hysteresis_factor =
103*d9f75844SAndroid Build Coastguard Worker           content_type == VideoEncoderConfig::ContentType::kScreen
104*d9f75844SAndroid Build Coastguard Worker               ? kScreenshareHysteresis
105*d9f75844SAndroid Build Coastguard Worker               : kVideoHysteresis;
106*d9f75844SAndroid Build Coastguard Worker       if (is_svc) {
107*d9f75844SAndroid Build Coastguard Worker         // For SVC, since there is only one "stream", the padding bitrate
108*d9f75844SAndroid Build Coastguard Worker         // needed to enable the top spatial layer is stored in the
109*d9f75844SAndroid Build Coastguard Worker         // `target_bitrate_bps` field.
110*d9f75844SAndroid Build Coastguard Worker         // TODO(sprang): This behavior needs to die.
111*d9f75844SAndroid Build Coastguard Worker         pad_up_to_bitrate_bps = static_cast<int>(
112*d9f75844SAndroid Build Coastguard Worker             hysteresis_factor * active_streams[0].target_bitrate_bps + 0.5);
113*d9f75844SAndroid Build Coastguard Worker       } else {
114*d9f75844SAndroid Build Coastguard Worker         const size_t top_active_stream_idx = active_streams.size() - 1;
115*d9f75844SAndroid Build Coastguard Worker         pad_up_to_bitrate_bps = std::min(
116*d9f75844SAndroid Build Coastguard Worker             static_cast<int>(
117*d9f75844SAndroid Build Coastguard Worker                 hysteresis_factor *
118*d9f75844SAndroid Build Coastguard Worker                     active_streams[top_active_stream_idx].min_bitrate_bps +
119*d9f75844SAndroid Build Coastguard Worker                 0.5),
120*d9f75844SAndroid Build Coastguard Worker             active_streams[top_active_stream_idx].target_bitrate_bps);
121*d9f75844SAndroid Build Coastguard Worker 
122*d9f75844SAndroid Build Coastguard Worker         // Add target_bitrate_bps of the lower active streams.
123*d9f75844SAndroid Build Coastguard Worker         for (size_t i = 0; i < top_active_stream_idx; ++i) {
124*d9f75844SAndroid Build Coastguard Worker           pad_up_to_bitrate_bps += active_streams[i].target_bitrate_bps;
125*d9f75844SAndroid Build Coastguard Worker         }
126*d9f75844SAndroid Build Coastguard Worker       }
127*d9f75844SAndroid Build Coastguard Worker     }
128*d9f75844SAndroid Build Coastguard Worker   } else if (!active_streams.empty() && pad_to_min_bitrate) {
129*d9f75844SAndroid Build Coastguard Worker     pad_up_to_bitrate_bps = active_streams[0].min_bitrate_bps;
130*d9f75844SAndroid Build Coastguard Worker   }
131*d9f75844SAndroid Build Coastguard Worker 
132*d9f75844SAndroid Build Coastguard Worker   pad_up_to_bitrate_bps =
133*d9f75844SAndroid Build Coastguard Worker       std::max(pad_up_to_bitrate_bps, min_transmit_bitrate_bps);
134*d9f75844SAndroid Build Coastguard Worker 
135*d9f75844SAndroid Build Coastguard Worker   return pad_up_to_bitrate_bps;
136*d9f75844SAndroid Build Coastguard Worker }
137*d9f75844SAndroid Build Coastguard Worker 
GetAlrSettings(VideoEncoderConfig::ContentType content_type)138*d9f75844SAndroid Build Coastguard Worker absl::optional<AlrExperimentSettings> GetAlrSettings(
139*d9f75844SAndroid Build Coastguard Worker     VideoEncoderConfig::ContentType content_type) {
140*d9f75844SAndroid Build Coastguard Worker   if (content_type == VideoEncoderConfig::ContentType::kScreen) {
141*d9f75844SAndroid Build Coastguard Worker     return AlrExperimentSettings::CreateFromFieldTrial(
142*d9f75844SAndroid Build Coastguard Worker         AlrExperimentSettings::kScreenshareProbingBweExperimentName);
143*d9f75844SAndroid Build Coastguard Worker   }
144*d9f75844SAndroid Build Coastguard Worker   return AlrExperimentSettings::CreateFromFieldTrial(
145*d9f75844SAndroid Build Coastguard Worker       AlrExperimentSettings::kStrictPacingAndProbingExperimentName);
146*d9f75844SAndroid Build Coastguard Worker }
147*d9f75844SAndroid Build Coastguard Worker 
SameStreamsEnabled(const VideoBitrateAllocation & lhs,const VideoBitrateAllocation & rhs)148*d9f75844SAndroid Build Coastguard Worker bool SameStreamsEnabled(const VideoBitrateAllocation& lhs,
149*d9f75844SAndroid Build Coastguard Worker                         const VideoBitrateAllocation& rhs) {
150*d9f75844SAndroid Build Coastguard Worker   for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
151*d9f75844SAndroid Build Coastguard Worker     for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
152*d9f75844SAndroid Build Coastguard Worker       if (lhs.HasBitrate(si, ti) != rhs.HasBitrate(si, ti)) {
153*d9f75844SAndroid Build Coastguard Worker         return false;
154*d9f75844SAndroid Build Coastguard Worker       }
155*d9f75844SAndroid Build Coastguard Worker     }
156*d9f75844SAndroid Build Coastguard Worker   }
157*d9f75844SAndroid Build Coastguard Worker   return true;
158*d9f75844SAndroid Build Coastguard Worker }
159*d9f75844SAndroid Build Coastguard Worker 
160*d9f75844SAndroid Build Coastguard Worker // Returns an optional that has value iff TransportSeqNumExtensionConfigured
161*d9f75844SAndroid Build Coastguard Worker // is `true` for the given video send stream config.
GetConfiguredPacingFactor(const VideoSendStream::Config & config,VideoEncoderConfig::ContentType content_type,const PacingConfig & default_pacing_config)162*d9f75844SAndroid Build Coastguard Worker absl::optional<float> GetConfiguredPacingFactor(
163*d9f75844SAndroid Build Coastguard Worker     const VideoSendStream::Config& config,
164*d9f75844SAndroid Build Coastguard Worker     VideoEncoderConfig::ContentType content_type,
165*d9f75844SAndroid Build Coastguard Worker     const PacingConfig& default_pacing_config) {
166*d9f75844SAndroid Build Coastguard Worker   if (!TransportSeqNumExtensionConfigured(config))
167*d9f75844SAndroid Build Coastguard Worker     return absl::nullopt;
168*d9f75844SAndroid Build Coastguard Worker 
169*d9f75844SAndroid Build Coastguard Worker   absl::optional<AlrExperimentSettings> alr_settings =
170*d9f75844SAndroid Build Coastguard Worker       GetAlrSettings(content_type);
171*d9f75844SAndroid Build Coastguard Worker   if (alr_settings)
172*d9f75844SAndroid Build Coastguard Worker     return alr_settings->pacing_factor;
173*d9f75844SAndroid Build Coastguard Worker 
174*d9f75844SAndroid Build Coastguard Worker   RateControlSettings rate_control_settings =
175*d9f75844SAndroid Build Coastguard Worker       RateControlSettings::ParseFromFieldTrials();
176*d9f75844SAndroid Build Coastguard Worker   return rate_control_settings.GetPacingFactor().value_or(
177*d9f75844SAndroid Build Coastguard Worker       default_pacing_config.pacing_factor);
178*d9f75844SAndroid Build Coastguard Worker }
179*d9f75844SAndroid Build Coastguard Worker 
GetInitialEncoderMaxBitrate(int initial_encoder_max_bitrate)180*d9f75844SAndroid Build Coastguard Worker uint32_t GetInitialEncoderMaxBitrate(int initial_encoder_max_bitrate) {
181*d9f75844SAndroid Build Coastguard Worker   if (initial_encoder_max_bitrate > 0)
182*d9f75844SAndroid Build Coastguard Worker     return rtc::dchecked_cast<uint32_t>(initial_encoder_max_bitrate);
183*d9f75844SAndroid Build Coastguard Worker 
184*d9f75844SAndroid Build Coastguard Worker   // TODO(srte): Make sure max bitrate is not set to negative values. We don't
185*d9f75844SAndroid Build Coastguard Worker   // have any way to handle unset values in downstream code, such as the
186*d9f75844SAndroid Build Coastguard Worker   // bitrate allocator. Previously -1 was implicitly casted to UINT32_MAX, a
187*d9f75844SAndroid Build Coastguard Worker   // behaviour that is not safe. Converting to 10 Mbps should be safe for
188*d9f75844SAndroid Build Coastguard Worker   // reasonable use cases as it allows adding the max of multiple streams
189*d9f75844SAndroid Build Coastguard Worker   // without wrappping around.
190*d9f75844SAndroid Build Coastguard Worker   const int kFallbackMaxBitrateBps = 10000000;
191*d9f75844SAndroid Build Coastguard Worker   RTC_DLOG(LS_ERROR) << "ERROR: Initial encoder max bitrate = "
192*d9f75844SAndroid Build Coastguard Worker                      << initial_encoder_max_bitrate << " which is <= 0!";
193*d9f75844SAndroid Build Coastguard Worker   RTC_DLOG(LS_INFO) << "Using default encoder max bitrate = 10 Mbps";
194*d9f75844SAndroid Build Coastguard Worker   return kFallbackMaxBitrateBps;
195*d9f75844SAndroid Build Coastguard Worker }
196*d9f75844SAndroid Build Coastguard Worker 
197*d9f75844SAndroid Build Coastguard Worker }  // namespace
198*d9f75844SAndroid Build Coastguard Worker 
PacingConfig(const FieldTrialsView & field_trials)199*d9f75844SAndroid Build Coastguard Worker PacingConfig::PacingConfig(const FieldTrialsView& field_trials)
200*d9f75844SAndroid Build Coastguard Worker     : pacing_factor("factor", kStrictPacingMultiplier),
201*d9f75844SAndroid Build Coastguard Worker       max_pacing_delay("max_delay", PacingController::kMaxExpectedQueueLength) {
202*d9f75844SAndroid Build Coastguard Worker   ParseFieldTrial({&pacing_factor, &max_pacing_delay},
203*d9f75844SAndroid Build Coastguard Worker                   field_trials.Lookup("WebRTC-Video-Pacing"));
204*d9f75844SAndroid Build Coastguard Worker }
205*d9f75844SAndroid Build Coastguard Worker PacingConfig::PacingConfig(const PacingConfig&) = default;
206*d9f75844SAndroid Build Coastguard Worker PacingConfig::~PacingConfig() = default;
207*d9f75844SAndroid Build Coastguard Worker 
VideoSendStreamImpl(Clock * clock,SendStatisticsProxy * stats_proxy,RtpTransportControllerSendInterface * transport,BitrateAllocatorInterface * bitrate_allocator,VideoStreamEncoderInterface * video_stream_encoder,const VideoSendStream::Config * config,int initial_encoder_max_bitrate,double initial_encoder_bitrate_priority,VideoEncoderConfig::ContentType content_type,RtpVideoSenderInterface * rtp_video_sender,const FieldTrialsView & field_trials)208*d9f75844SAndroid Build Coastguard Worker VideoSendStreamImpl::VideoSendStreamImpl(
209*d9f75844SAndroid Build Coastguard Worker     Clock* clock,
210*d9f75844SAndroid Build Coastguard Worker     SendStatisticsProxy* stats_proxy,
211*d9f75844SAndroid Build Coastguard Worker     RtpTransportControllerSendInterface* transport,
212*d9f75844SAndroid Build Coastguard Worker     BitrateAllocatorInterface* bitrate_allocator,
213*d9f75844SAndroid Build Coastguard Worker     VideoStreamEncoderInterface* video_stream_encoder,
214*d9f75844SAndroid Build Coastguard Worker     const VideoSendStream::Config* config,
215*d9f75844SAndroid Build Coastguard Worker     int initial_encoder_max_bitrate,
216*d9f75844SAndroid Build Coastguard Worker     double initial_encoder_bitrate_priority,
217*d9f75844SAndroid Build Coastguard Worker     VideoEncoderConfig::ContentType content_type,
218*d9f75844SAndroid Build Coastguard Worker     RtpVideoSenderInterface* rtp_video_sender,
219*d9f75844SAndroid Build Coastguard Worker     const FieldTrialsView& field_trials)
220*d9f75844SAndroid Build Coastguard Worker     : clock_(clock),
221*d9f75844SAndroid Build Coastguard Worker       has_alr_probing_(config->periodic_alr_bandwidth_probing ||
222*d9f75844SAndroid Build Coastguard Worker                        GetAlrSettings(content_type)),
223*d9f75844SAndroid Build Coastguard Worker       pacing_config_(PacingConfig(field_trials)),
224*d9f75844SAndroid Build Coastguard Worker       stats_proxy_(stats_proxy),
225*d9f75844SAndroid Build Coastguard Worker       config_(config),
226*d9f75844SAndroid Build Coastguard Worker       rtp_transport_queue_(transport->GetWorkerQueue()),
227*d9f75844SAndroid Build Coastguard Worker       timed_out_(false),
228*d9f75844SAndroid Build Coastguard Worker       transport_(transport),
229*d9f75844SAndroid Build Coastguard Worker       bitrate_allocator_(bitrate_allocator),
230*d9f75844SAndroid Build Coastguard Worker       disable_padding_(true),
231*d9f75844SAndroid Build Coastguard Worker       max_padding_bitrate_(0),
232*d9f75844SAndroid Build Coastguard Worker       encoder_min_bitrate_bps_(0),
233*d9f75844SAndroid Build Coastguard Worker       encoder_max_bitrate_bps_(
234*d9f75844SAndroid Build Coastguard Worker           GetInitialEncoderMaxBitrate(initial_encoder_max_bitrate)),
235*d9f75844SAndroid Build Coastguard Worker       encoder_target_rate_bps_(0),
236*d9f75844SAndroid Build Coastguard Worker       encoder_bitrate_priority_(initial_encoder_bitrate_priority),
237*d9f75844SAndroid Build Coastguard Worker       video_stream_encoder_(video_stream_encoder),
238*d9f75844SAndroid Build Coastguard Worker       bandwidth_observer_(transport->GetBandwidthObserver()),
239*d9f75844SAndroid Build Coastguard Worker       rtp_video_sender_(rtp_video_sender),
240*d9f75844SAndroid Build Coastguard Worker       configured_pacing_factor_(
241*d9f75844SAndroid Build Coastguard Worker           GetConfiguredPacingFactor(*config_, content_type, pacing_config_)) {
242*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK_GE(config_->rtp.payload_type, 0);
243*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK_LE(config_->rtp.payload_type, 127);
244*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK(!config_->rtp.ssrcs.empty());
245*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK(transport_);
246*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK_NE(initial_encoder_max_bitrate, 0);
247*d9f75844SAndroid Build Coastguard Worker   RTC_LOG(LS_INFO) << "VideoSendStreamImpl: " << config_->ToString();
248*d9f75844SAndroid Build Coastguard Worker 
249*d9f75844SAndroid Build Coastguard Worker   RTC_CHECK(AlrExperimentSettings::MaxOneFieldTrialEnabled());
250*d9f75844SAndroid Build Coastguard Worker 
251*d9f75844SAndroid Build Coastguard Worker   // Only request rotation at the source when we positively know that the remote
252*d9f75844SAndroid Build Coastguard Worker   // side doesn't support the rotation extension. This allows us to prepare the
253*d9f75844SAndroid Build Coastguard Worker   // encoder in the expectation that rotation is supported - which is the common
254*d9f75844SAndroid Build Coastguard Worker   // case.
255*d9f75844SAndroid Build Coastguard Worker   bool rotation_applied = absl::c_none_of(
256*d9f75844SAndroid Build Coastguard Worker       config_->rtp.extensions, [](const RtpExtension& extension) {
257*d9f75844SAndroid Build Coastguard Worker         return extension.uri == RtpExtension::kVideoRotationUri;
258*d9f75844SAndroid Build Coastguard Worker       });
259*d9f75844SAndroid Build Coastguard Worker 
260*d9f75844SAndroid Build Coastguard Worker   video_stream_encoder_->SetSink(this, rotation_applied);
261*d9f75844SAndroid Build Coastguard Worker 
262*d9f75844SAndroid Build Coastguard Worker   absl::optional<bool> enable_alr_bw_probing;
263*d9f75844SAndroid Build Coastguard Worker 
264*d9f75844SAndroid Build Coastguard Worker   // If send-side BWE is enabled, check if we should apply updated probing and
265*d9f75844SAndroid Build Coastguard Worker   // pacing settings.
266*d9f75844SAndroid Build Coastguard Worker   if (configured_pacing_factor_) {
267*d9f75844SAndroid Build Coastguard Worker     absl::optional<AlrExperimentSettings> alr_settings =
268*d9f75844SAndroid Build Coastguard Worker         GetAlrSettings(content_type);
269*d9f75844SAndroid Build Coastguard Worker     int queue_time_limit_ms;
270*d9f75844SAndroid Build Coastguard Worker     if (alr_settings) {
271*d9f75844SAndroid Build Coastguard Worker       enable_alr_bw_probing = true;
272*d9f75844SAndroid Build Coastguard Worker       queue_time_limit_ms = alr_settings->max_paced_queue_time;
273*d9f75844SAndroid Build Coastguard Worker     } else {
274*d9f75844SAndroid Build Coastguard Worker       RateControlSettings rate_control_settings =
275*d9f75844SAndroid Build Coastguard Worker           RateControlSettings::ParseFromFieldTrials();
276*d9f75844SAndroid Build Coastguard Worker       enable_alr_bw_probing = rate_control_settings.UseAlrProbing();
277*d9f75844SAndroid Build Coastguard Worker       queue_time_limit_ms = pacing_config_.max_pacing_delay.Get().ms();
278*d9f75844SAndroid Build Coastguard Worker     }
279*d9f75844SAndroid Build Coastguard Worker 
280*d9f75844SAndroid Build Coastguard Worker     transport->SetQueueTimeLimit(queue_time_limit_ms);
281*d9f75844SAndroid Build Coastguard Worker   }
282*d9f75844SAndroid Build Coastguard Worker 
283*d9f75844SAndroid Build Coastguard Worker   if (config_->periodic_alr_bandwidth_probing) {
284*d9f75844SAndroid Build Coastguard Worker     enable_alr_bw_probing = config_->periodic_alr_bandwidth_probing;
285*d9f75844SAndroid Build Coastguard Worker   }
286*d9f75844SAndroid Build Coastguard Worker 
287*d9f75844SAndroid Build Coastguard Worker   if (enable_alr_bw_probing) {
288*d9f75844SAndroid Build Coastguard Worker     transport->EnablePeriodicAlrProbing(*enable_alr_bw_probing);
289*d9f75844SAndroid Build Coastguard Worker   }
290*d9f75844SAndroid Build Coastguard Worker 
291*d9f75844SAndroid Build Coastguard Worker   rtp_transport_queue_->RunOrPost(SafeTask(transport_queue_safety_, [this] {
292*d9f75844SAndroid Build Coastguard Worker     if (configured_pacing_factor_)
293*d9f75844SAndroid Build Coastguard Worker       transport_->SetPacingFactor(*configured_pacing_factor_);
294*d9f75844SAndroid Build Coastguard Worker 
295*d9f75844SAndroid Build Coastguard Worker     video_stream_encoder_->SetStartBitrate(
296*d9f75844SAndroid Build Coastguard Worker         bitrate_allocator_->GetStartBitrate(this));
297*d9f75844SAndroid Build Coastguard Worker   }));
298*d9f75844SAndroid Build Coastguard Worker }
299*d9f75844SAndroid Build Coastguard Worker 
~VideoSendStreamImpl()300*d9f75844SAndroid Build Coastguard Worker VideoSendStreamImpl::~VideoSendStreamImpl() {
301*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK_RUN_ON(&thread_checker_);
302*d9f75844SAndroid Build Coastguard Worker   RTC_LOG(LS_INFO) << "~VideoSendStreamImpl: " << config_->ToString();
303*d9f75844SAndroid Build Coastguard Worker   // TODO(webrtc:14502): Change `transport_queue_safety_` to be of type
304*d9f75844SAndroid Build Coastguard Worker   // ScopedTaskSafety if experiment WebRTC-SendPacketsOnWorkerThread succeed.
305*d9f75844SAndroid Build Coastguard Worker   if (rtp_transport_queue_->IsCurrent()) {
306*d9f75844SAndroid Build Coastguard Worker     transport_queue_safety_->SetNotAlive();
307*d9f75844SAndroid Build Coastguard Worker   }
308*d9f75844SAndroid Build Coastguard Worker }
309*d9f75844SAndroid Build Coastguard Worker 
DeliverRtcp(const uint8_t * packet,size_t length)310*d9f75844SAndroid Build Coastguard Worker void VideoSendStreamImpl::DeliverRtcp(const uint8_t* packet, size_t length) {
311*d9f75844SAndroid Build Coastguard Worker   // Runs on a worker thread.
312*d9f75844SAndroid Build Coastguard Worker   rtp_video_sender_->DeliverRtcp(packet, length);
313*d9f75844SAndroid Build Coastguard Worker }
314*d9f75844SAndroid Build Coastguard Worker 
StartPerRtpStream(const std::vector<bool> active_layers)315*d9f75844SAndroid Build Coastguard Worker void VideoSendStreamImpl::StartPerRtpStream(
316*d9f75844SAndroid Build Coastguard Worker     const std::vector<bool> active_layers) {
317*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK_RUN_ON(rtp_transport_queue_);
318*d9f75844SAndroid Build Coastguard Worker   bool previously_active = rtp_video_sender_->IsActive();
319*d9f75844SAndroid Build Coastguard Worker   rtp_video_sender_->SetActiveModules(active_layers);
320*d9f75844SAndroid Build Coastguard Worker   if (!rtp_video_sender_->IsActive() && previously_active) {
321*d9f75844SAndroid Build Coastguard Worker     StopVideoSendStream();
322*d9f75844SAndroid Build Coastguard Worker   } else if (rtp_video_sender_->IsActive() && !previously_active) {
323*d9f75844SAndroid Build Coastguard Worker     StartupVideoSendStream();
324*d9f75844SAndroid Build Coastguard Worker   }
325*d9f75844SAndroid Build Coastguard Worker }
326*d9f75844SAndroid Build Coastguard Worker 
StartupVideoSendStream()327*d9f75844SAndroid Build Coastguard Worker void VideoSendStreamImpl::StartupVideoSendStream() {
328*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK_RUN_ON(rtp_transport_queue_);
329*d9f75844SAndroid Build Coastguard Worker   transport_queue_safety_->SetAlive();
330*d9f75844SAndroid Build Coastguard Worker 
331*d9f75844SAndroid Build Coastguard Worker   bitrate_allocator_->AddObserver(this, GetAllocationConfig());
332*d9f75844SAndroid Build Coastguard Worker   // Start monitoring encoder activity.
333*d9f75844SAndroid Build Coastguard Worker   {
334*d9f75844SAndroid Build Coastguard Worker     RTC_DCHECK(!check_encoder_activity_task_.Running());
335*d9f75844SAndroid Build Coastguard Worker 
336*d9f75844SAndroid Build Coastguard Worker     activity_ = false;
337*d9f75844SAndroid Build Coastguard Worker     timed_out_ = false;
338*d9f75844SAndroid Build Coastguard Worker     check_encoder_activity_task_ = RepeatingTaskHandle::DelayedStart(
339*d9f75844SAndroid Build Coastguard Worker         rtp_transport_queue_->TaskQueueForDelayedTasks(), kEncoderTimeOut,
340*d9f75844SAndroid Build Coastguard Worker         [this] {
341*d9f75844SAndroid Build Coastguard Worker           RTC_DCHECK_RUN_ON(rtp_transport_queue_);
342*d9f75844SAndroid Build Coastguard Worker           if (!activity_) {
343*d9f75844SAndroid Build Coastguard Worker             if (!timed_out_) {
344*d9f75844SAndroid Build Coastguard Worker               SignalEncoderTimedOut();
345*d9f75844SAndroid Build Coastguard Worker             }
346*d9f75844SAndroid Build Coastguard Worker             timed_out_ = true;
347*d9f75844SAndroid Build Coastguard Worker             disable_padding_ = true;
348*d9f75844SAndroid Build Coastguard Worker           } else if (timed_out_) {
349*d9f75844SAndroid Build Coastguard Worker             SignalEncoderActive();
350*d9f75844SAndroid Build Coastguard Worker             timed_out_ = false;
351*d9f75844SAndroid Build Coastguard Worker           }
352*d9f75844SAndroid Build Coastguard Worker           activity_ = false;
353*d9f75844SAndroid Build Coastguard Worker           return kEncoderTimeOut;
354*d9f75844SAndroid Build Coastguard Worker         });
355*d9f75844SAndroid Build Coastguard Worker   }
356*d9f75844SAndroid Build Coastguard Worker 
357*d9f75844SAndroid Build Coastguard Worker   video_stream_encoder_->SendKeyFrame();
358*d9f75844SAndroid Build Coastguard Worker }
359*d9f75844SAndroid Build Coastguard Worker 
Stop()360*d9f75844SAndroid Build Coastguard Worker void VideoSendStreamImpl::Stop() {
361*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK_RUN_ON(rtp_transport_queue_);
362*d9f75844SAndroid Build Coastguard Worker   RTC_LOG(LS_INFO) << "VideoSendStreamImpl::Stop";
363*d9f75844SAndroid Build Coastguard Worker   if (!rtp_video_sender_->IsActive())
364*d9f75844SAndroid Build Coastguard Worker     return;
365*d9f75844SAndroid Build Coastguard Worker 
366*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK(transport_queue_safety_->alive());
367*d9f75844SAndroid Build Coastguard Worker   TRACE_EVENT_INSTANT0("webrtc", "VideoSendStream::Stop");
368*d9f75844SAndroid Build Coastguard Worker   rtp_video_sender_->Stop();
369*d9f75844SAndroid Build Coastguard Worker   StopVideoSendStream();
370*d9f75844SAndroid Build Coastguard Worker }
371*d9f75844SAndroid Build Coastguard Worker 
StopVideoSendStream()372*d9f75844SAndroid Build Coastguard Worker void VideoSendStreamImpl::StopVideoSendStream() {
373*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK_RUN_ON(rtp_transport_queue_);
374*d9f75844SAndroid Build Coastguard Worker   bitrate_allocator_->RemoveObserver(this);
375*d9f75844SAndroid Build Coastguard Worker   check_encoder_activity_task_.Stop();
376*d9f75844SAndroid Build Coastguard Worker   video_stream_encoder_->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
377*d9f75844SAndroid Build Coastguard Worker                                           DataRate::Zero(), 0, 0, 0);
378*d9f75844SAndroid Build Coastguard Worker   stats_proxy_->OnSetEncoderTargetRate(0);
379*d9f75844SAndroid Build Coastguard Worker   transport_queue_safety_->SetNotAlive();
380*d9f75844SAndroid Build Coastguard Worker }
381*d9f75844SAndroid Build Coastguard Worker 
SignalEncoderTimedOut()382*d9f75844SAndroid Build Coastguard Worker void VideoSendStreamImpl::SignalEncoderTimedOut() {
383*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK_RUN_ON(rtp_transport_queue_);
384*d9f75844SAndroid Build Coastguard Worker   // If the encoder has not produced anything the last kEncoderTimeOut and it
385*d9f75844SAndroid Build Coastguard Worker   // is supposed to, deregister as BitrateAllocatorObserver. This can happen
386*d9f75844SAndroid Build Coastguard Worker   // if a camera stops producing frames.
387*d9f75844SAndroid Build Coastguard Worker   if (encoder_target_rate_bps_ > 0) {
388*d9f75844SAndroid Build Coastguard Worker     RTC_LOG(LS_INFO) << "SignalEncoderTimedOut, Encoder timed out.";
389*d9f75844SAndroid Build Coastguard Worker     bitrate_allocator_->RemoveObserver(this);
390*d9f75844SAndroid Build Coastguard Worker   }
391*d9f75844SAndroid Build Coastguard Worker }
392*d9f75844SAndroid Build Coastguard Worker 
OnBitrateAllocationUpdated(const VideoBitrateAllocation & allocation)393*d9f75844SAndroid Build Coastguard Worker void VideoSendStreamImpl::OnBitrateAllocationUpdated(
394*d9f75844SAndroid Build Coastguard Worker     const VideoBitrateAllocation& allocation) {
395*d9f75844SAndroid Build Coastguard Worker   // OnBitrateAllocationUpdated is invoked from  the encoder task queue or
396*d9f75844SAndroid Build Coastguard Worker   // the rtp_transport_queue_.
397*d9f75844SAndroid Build Coastguard Worker   auto task = [=] {
398*d9f75844SAndroid Build Coastguard Worker     RTC_DCHECK_RUN_ON(rtp_transport_queue_);
399*d9f75844SAndroid Build Coastguard Worker     if (encoder_target_rate_bps_ == 0) {
400*d9f75844SAndroid Build Coastguard Worker       return;
401*d9f75844SAndroid Build Coastguard Worker     }
402*d9f75844SAndroid Build Coastguard Worker     int64_t now_ms = clock_->TimeInMilliseconds();
403*d9f75844SAndroid Build Coastguard Worker     if (video_bitrate_allocation_context_) {
404*d9f75844SAndroid Build Coastguard Worker       // If new allocation is within kMaxVbaSizeDifferencePercent larger
405*d9f75844SAndroid Build Coastguard Worker       // than the previously sent allocation and the same streams are still
406*d9f75844SAndroid Build Coastguard Worker       // enabled, it is considered "similar". We do not want send similar
407*d9f75844SAndroid Build Coastguard Worker       // allocations more once per kMaxVbaThrottleTimeMs.
408*d9f75844SAndroid Build Coastguard Worker       const VideoBitrateAllocation& last =
409*d9f75844SAndroid Build Coastguard Worker           video_bitrate_allocation_context_->last_sent_allocation;
410*d9f75844SAndroid Build Coastguard Worker       const bool is_similar =
411*d9f75844SAndroid Build Coastguard Worker           allocation.get_sum_bps() >= last.get_sum_bps() &&
412*d9f75844SAndroid Build Coastguard Worker           allocation.get_sum_bps() <
413*d9f75844SAndroid Build Coastguard Worker               (last.get_sum_bps() * (100 + kMaxVbaSizeDifferencePercent)) /
414*d9f75844SAndroid Build Coastguard Worker                   100 &&
415*d9f75844SAndroid Build Coastguard Worker           SameStreamsEnabled(allocation, last);
416*d9f75844SAndroid Build Coastguard Worker       if (is_similar &&
417*d9f75844SAndroid Build Coastguard Worker           (now_ms - video_bitrate_allocation_context_->last_send_time_ms) <
418*d9f75844SAndroid Build Coastguard Worker               kMaxVbaThrottleTimeMs) {
419*d9f75844SAndroid Build Coastguard Worker         // This allocation is too similar, cache it and return.
420*d9f75844SAndroid Build Coastguard Worker         video_bitrate_allocation_context_->throttled_allocation = allocation;
421*d9f75844SAndroid Build Coastguard Worker         return;
422*d9f75844SAndroid Build Coastguard Worker       }
423*d9f75844SAndroid Build Coastguard Worker     } else {
424*d9f75844SAndroid Build Coastguard Worker       video_bitrate_allocation_context_.emplace();
425*d9f75844SAndroid Build Coastguard Worker     }
426*d9f75844SAndroid Build Coastguard Worker 
427*d9f75844SAndroid Build Coastguard Worker     video_bitrate_allocation_context_->last_sent_allocation = allocation;
428*d9f75844SAndroid Build Coastguard Worker     video_bitrate_allocation_context_->throttled_allocation.reset();
429*d9f75844SAndroid Build Coastguard Worker     video_bitrate_allocation_context_->last_send_time_ms = now_ms;
430*d9f75844SAndroid Build Coastguard Worker 
431*d9f75844SAndroid Build Coastguard Worker     // Send bitrate allocation metadata only if encoder is not paused.
432*d9f75844SAndroid Build Coastguard Worker     rtp_video_sender_->OnBitrateAllocationUpdated(allocation);
433*d9f75844SAndroid Build Coastguard Worker   };
434*d9f75844SAndroid Build Coastguard Worker   if (!rtp_transport_queue_->IsCurrent()) {
435*d9f75844SAndroid Build Coastguard Worker     rtp_transport_queue_->TaskQueueForPost()->PostTask(
436*d9f75844SAndroid Build Coastguard Worker         SafeTask(transport_queue_safety_, std::move(task)));
437*d9f75844SAndroid Build Coastguard Worker   } else {
438*d9f75844SAndroid Build Coastguard Worker     task();
439*d9f75844SAndroid Build Coastguard Worker   }
440*d9f75844SAndroid Build Coastguard Worker }
441*d9f75844SAndroid Build Coastguard Worker 
OnVideoLayersAllocationUpdated(VideoLayersAllocation allocation)442*d9f75844SAndroid Build Coastguard Worker void VideoSendStreamImpl::OnVideoLayersAllocationUpdated(
443*d9f75844SAndroid Build Coastguard Worker     VideoLayersAllocation allocation) {
444*d9f75844SAndroid Build Coastguard Worker   // OnVideoLayersAllocationUpdated is handled on the encoder task queue in
445*d9f75844SAndroid Build Coastguard Worker   // order to not race with OnEncodedImage callbacks.
446*d9f75844SAndroid Build Coastguard Worker   rtp_video_sender_->OnVideoLayersAllocationUpdated(allocation);
447*d9f75844SAndroid Build Coastguard Worker }
448*d9f75844SAndroid Build Coastguard Worker 
SignalEncoderActive()449*d9f75844SAndroid Build Coastguard Worker void VideoSendStreamImpl::SignalEncoderActive() {
450*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK_RUN_ON(rtp_transport_queue_);
451*d9f75844SAndroid Build Coastguard Worker   if (rtp_video_sender_->IsActive()) {
452*d9f75844SAndroid Build Coastguard Worker     RTC_LOG(LS_INFO) << "SignalEncoderActive, Encoder is active.";
453*d9f75844SAndroid Build Coastguard Worker     bitrate_allocator_->AddObserver(this, GetAllocationConfig());
454*d9f75844SAndroid Build Coastguard Worker   }
455*d9f75844SAndroid Build Coastguard Worker }
456*d9f75844SAndroid Build Coastguard Worker 
GetAllocationConfig() const457*d9f75844SAndroid Build Coastguard Worker MediaStreamAllocationConfig VideoSendStreamImpl::GetAllocationConfig() const {
458*d9f75844SAndroid Build Coastguard Worker   return MediaStreamAllocationConfig{
459*d9f75844SAndroid Build Coastguard Worker       static_cast<uint32_t>(encoder_min_bitrate_bps_),
460*d9f75844SAndroid Build Coastguard Worker       encoder_max_bitrate_bps_,
461*d9f75844SAndroid Build Coastguard Worker       static_cast<uint32_t>(disable_padding_ ? 0 : max_padding_bitrate_),
462*d9f75844SAndroid Build Coastguard Worker       /* priority_bitrate */ 0,
463*d9f75844SAndroid Build Coastguard Worker       !config_->suspend_below_min_bitrate,
464*d9f75844SAndroid Build Coastguard Worker       encoder_bitrate_priority_};
465*d9f75844SAndroid Build Coastguard Worker }
466*d9f75844SAndroid Build Coastguard Worker 
OnEncoderConfigurationChanged(std::vector<VideoStream> streams,bool is_svc,VideoEncoderConfig::ContentType content_type,int min_transmit_bitrate_bps)467*d9f75844SAndroid Build Coastguard Worker void VideoSendStreamImpl::OnEncoderConfigurationChanged(
468*d9f75844SAndroid Build Coastguard Worker     std::vector<VideoStream> streams,
469*d9f75844SAndroid Build Coastguard Worker     bool is_svc,
470*d9f75844SAndroid Build Coastguard Worker     VideoEncoderConfig::ContentType content_type,
471*d9f75844SAndroid Build Coastguard Worker     int min_transmit_bitrate_bps) {
472*d9f75844SAndroid Build Coastguard Worker   // Currently called on the encoder TQ
473*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK(!rtp_transport_queue_->IsCurrent());
474*d9f75844SAndroid Build Coastguard Worker   auto closure = [this, streams = std::move(streams), is_svc, content_type,
475*d9f75844SAndroid Build Coastguard Worker                   min_transmit_bitrate_bps]() mutable {
476*d9f75844SAndroid Build Coastguard Worker     RTC_DCHECK_GE(config_->rtp.ssrcs.size(), streams.size());
477*d9f75844SAndroid Build Coastguard Worker     TRACE_EVENT0("webrtc", "VideoSendStream::OnEncoderConfigurationChanged");
478*d9f75844SAndroid Build Coastguard Worker     RTC_DCHECK_RUN_ON(rtp_transport_queue_);
479*d9f75844SAndroid Build Coastguard Worker 
480*d9f75844SAndroid Build Coastguard Worker     const VideoCodecType codec_type =
481*d9f75844SAndroid Build Coastguard Worker         PayloadStringToCodecType(config_->rtp.payload_name);
482*d9f75844SAndroid Build Coastguard Worker 
483*d9f75844SAndroid Build Coastguard Worker     const absl::optional<DataRate> experimental_min_bitrate =
484*d9f75844SAndroid Build Coastguard Worker         GetExperimentalMinVideoBitrate(codec_type);
485*d9f75844SAndroid Build Coastguard Worker     encoder_min_bitrate_bps_ =
486*d9f75844SAndroid Build Coastguard Worker         experimental_min_bitrate
487*d9f75844SAndroid Build Coastguard Worker             ? experimental_min_bitrate->bps()
488*d9f75844SAndroid Build Coastguard Worker             : std::max(streams[0].min_bitrate_bps, kDefaultMinVideoBitrateBps);
489*d9f75844SAndroid Build Coastguard Worker 
490*d9f75844SAndroid Build Coastguard Worker     encoder_max_bitrate_bps_ = 0;
491*d9f75844SAndroid Build Coastguard Worker     double stream_bitrate_priority_sum = 0;
492*d9f75844SAndroid Build Coastguard Worker     for (const auto& stream : streams) {
493*d9f75844SAndroid Build Coastguard Worker       // We don't want to allocate more bitrate than needed to inactive streams.
494*d9f75844SAndroid Build Coastguard Worker       encoder_max_bitrate_bps_ += stream.active ? stream.max_bitrate_bps : 0;
495*d9f75844SAndroid Build Coastguard Worker       if (stream.bitrate_priority) {
496*d9f75844SAndroid Build Coastguard Worker         RTC_DCHECK_GT(*stream.bitrate_priority, 0);
497*d9f75844SAndroid Build Coastguard Worker         stream_bitrate_priority_sum += *stream.bitrate_priority;
498*d9f75844SAndroid Build Coastguard Worker       }
499*d9f75844SAndroid Build Coastguard Worker     }
500*d9f75844SAndroid Build Coastguard Worker     RTC_DCHECK_GT(stream_bitrate_priority_sum, 0);
501*d9f75844SAndroid Build Coastguard Worker     encoder_bitrate_priority_ = stream_bitrate_priority_sum;
502*d9f75844SAndroid Build Coastguard Worker     encoder_max_bitrate_bps_ =
503*d9f75844SAndroid Build Coastguard Worker         std::max(static_cast<uint32_t>(encoder_min_bitrate_bps_),
504*d9f75844SAndroid Build Coastguard Worker                  encoder_max_bitrate_bps_);
505*d9f75844SAndroid Build Coastguard Worker 
506*d9f75844SAndroid Build Coastguard Worker     // TODO(bugs.webrtc.org/10266): Query the VideoBitrateAllocator instead.
507*d9f75844SAndroid Build Coastguard Worker     max_padding_bitrate_ = CalculateMaxPadBitrateBps(
508*d9f75844SAndroid Build Coastguard Worker         streams, is_svc, content_type, min_transmit_bitrate_bps,
509*d9f75844SAndroid Build Coastguard Worker         config_->suspend_below_min_bitrate, has_alr_probing_);
510*d9f75844SAndroid Build Coastguard Worker 
511*d9f75844SAndroid Build Coastguard Worker     // Clear stats for disabled layers.
512*d9f75844SAndroid Build Coastguard Worker     for (size_t i = streams.size(); i < config_->rtp.ssrcs.size(); ++i) {
513*d9f75844SAndroid Build Coastguard Worker       stats_proxy_->OnInactiveSsrc(config_->rtp.ssrcs[i]);
514*d9f75844SAndroid Build Coastguard Worker     }
515*d9f75844SAndroid Build Coastguard Worker 
516*d9f75844SAndroid Build Coastguard Worker     const size_t num_temporal_layers =
517*d9f75844SAndroid Build Coastguard Worker         streams.back().num_temporal_layers.value_or(1);
518*d9f75844SAndroid Build Coastguard Worker 
519*d9f75844SAndroid Build Coastguard Worker     rtp_video_sender_->SetEncodingData(streams[0].width, streams[0].height,
520*d9f75844SAndroid Build Coastguard Worker                                        num_temporal_layers);
521*d9f75844SAndroid Build Coastguard Worker 
522*d9f75844SAndroid Build Coastguard Worker     if (rtp_video_sender_->IsActive()) {
523*d9f75844SAndroid Build Coastguard Worker       // The send stream is started already. Update the allocator with new
524*d9f75844SAndroid Build Coastguard Worker       // bitrate limits.
525*d9f75844SAndroid Build Coastguard Worker       bitrate_allocator_->AddObserver(this, GetAllocationConfig());
526*d9f75844SAndroid Build Coastguard Worker     }
527*d9f75844SAndroid Build Coastguard Worker   };
528*d9f75844SAndroid Build Coastguard Worker 
529*d9f75844SAndroid Build Coastguard Worker   rtp_transport_queue_->TaskQueueForPost()->PostTask(
530*d9f75844SAndroid Build Coastguard Worker       SafeTask(transport_queue_safety_, std::move(closure)));
531*d9f75844SAndroid Build Coastguard Worker }
532*d9f75844SAndroid Build Coastguard Worker 
OnEncodedImage(const EncodedImage & encoded_image,const CodecSpecificInfo * codec_specific_info)533*d9f75844SAndroid Build Coastguard Worker EncodedImageCallback::Result VideoSendStreamImpl::OnEncodedImage(
534*d9f75844SAndroid Build Coastguard Worker     const EncodedImage& encoded_image,
535*d9f75844SAndroid Build Coastguard Worker     const CodecSpecificInfo* codec_specific_info) {
536*d9f75844SAndroid Build Coastguard Worker   // Encoded is called on whatever thread the real encoder implementation run
537*d9f75844SAndroid Build Coastguard Worker   // on. In the case of hardware encoders, there might be several encoders
538*d9f75844SAndroid Build Coastguard Worker   // running in parallel on different threads.
539*d9f75844SAndroid Build Coastguard Worker 
540*d9f75844SAndroid Build Coastguard Worker   // Indicate that there still is activity going on.
541*d9f75844SAndroid Build Coastguard Worker   activity_ = true;
542*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK(!rtp_transport_queue_->IsCurrent());
543*d9f75844SAndroid Build Coastguard Worker 
544*d9f75844SAndroid Build Coastguard Worker   auto task_to_run_on_worker = [this]() {
545*d9f75844SAndroid Build Coastguard Worker     RTC_DCHECK_RUN_ON(rtp_transport_queue_);
546*d9f75844SAndroid Build Coastguard Worker     if (disable_padding_) {
547*d9f75844SAndroid Build Coastguard Worker       disable_padding_ = false;
548*d9f75844SAndroid Build Coastguard Worker       // To ensure that padding bitrate is propagated to the bitrate allocator.
549*d9f75844SAndroid Build Coastguard Worker       SignalEncoderActive();
550*d9f75844SAndroid Build Coastguard Worker     }
551*d9f75844SAndroid Build Coastguard Worker     // Check if there's a throttled VideoBitrateAllocation that we should try
552*d9f75844SAndroid Build Coastguard Worker     // sending.
553*d9f75844SAndroid Build Coastguard Worker     auto& context = video_bitrate_allocation_context_;
554*d9f75844SAndroid Build Coastguard Worker     if (context && context->throttled_allocation) {
555*d9f75844SAndroid Build Coastguard Worker       OnBitrateAllocationUpdated(*context->throttled_allocation);
556*d9f75844SAndroid Build Coastguard Worker     }
557*d9f75844SAndroid Build Coastguard Worker   };
558*d9f75844SAndroid Build Coastguard Worker   rtp_transport_queue_->TaskQueueForPost()->PostTask(
559*d9f75844SAndroid Build Coastguard Worker       SafeTask(transport_queue_safety_, std::move(task_to_run_on_worker)));
560*d9f75844SAndroid Build Coastguard Worker 
561*d9f75844SAndroid Build Coastguard Worker   return rtp_video_sender_->OnEncodedImage(encoded_image, codec_specific_info);
562*d9f75844SAndroid Build Coastguard Worker }
563*d9f75844SAndroid Build Coastguard Worker 
OnDroppedFrame(EncodedImageCallback::DropReason reason)564*d9f75844SAndroid Build Coastguard Worker void VideoSendStreamImpl::OnDroppedFrame(
565*d9f75844SAndroid Build Coastguard Worker     EncodedImageCallback::DropReason reason) {
566*d9f75844SAndroid Build Coastguard Worker   activity_ = true;
567*d9f75844SAndroid Build Coastguard Worker }
568*d9f75844SAndroid Build Coastguard Worker 
GetRtpStates() const569*d9f75844SAndroid Build Coastguard Worker std::map<uint32_t, RtpState> VideoSendStreamImpl::GetRtpStates() const {
570*d9f75844SAndroid Build Coastguard Worker   return rtp_video_sender_->GetRtpStates();
571*d9f75844SAndroid Build Coastguard Worker }
572*d9f75844SAndroid Build Coastguard Worker 
GetRtpPayloadStates() const573*d9f75844SAndroid Build Coastguard Worker std::map<uint32_t, RtpPayloadState> VideoSendStreamImpl::GetRtpPayloadStates()
574*d9f75844SAndroid Build Coastguard Worker     const {
575*d9f75844SAndroid Build Coastguard Worker   return rtp_video_sender_->GetRtpPayloadStates();
576*d9f75844SAndroid Build Coastguard Worker }
577*d9f75844SAndroid Build Coastguard Worker 
OnBitrateUpdated(BitrateAllocationUpdate update)578*d9f75844SAndroid Build Coastguard Worker uint32_t VideoSendStreamImpl::OnBitrateUpdated(BitrateAllocationUpdate update) {
579*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK_RUN_ON(rtp_transport_queue_);
580*d9f75844SAndroid Build Coastguard Worker   RTC_DCHECK(rtp_video_sender_->IsActive())
581*d9f75844SAndroid Build Coastguard Worker       << "VideoSendStream::Start has not been called.";
582*d9f75844SAndroid Build Coastguard Worker 
583*d9f75844SAndroid Build Coastguard Worker   // When the BWE algorithm doesn't pass a stable estimate, we'll use the
584*d9f75844SAndroid Build Coastguard Worker   // unstable one instead.
585*d9f75844SAndroid Build Coastguard Worker   if (update.stable_target_bitrate.IsZero()) {
586*d9f75844SAndroid Build Coastguard Worker     update.stable_target_bitrate = update.target_bitrate;
587*d9f75844SAndroid Build Coastguard Worker   }
588*d9f75844SAndroid Build Coastguard Worker 
589*d9f75844SAndroid Build Coastguard Worker   rtp_video_sender_->OnBitrateUpdated(update, stats_proxy_->GetSendFrameRate());
590*d9f75844SAndroid Build Coastguard Worker   encoder_target_rate_bps_ = rtp_video_sender_->GetPayloadBitrateBps();
591*d9f75844SAndroid Build Coastguard Worker   const uint32_t protection_bitrate_bps =
592*d9f75844SAndroid Build Coastguard Worker       rtp_video_sender_->GetProtectionBitrateBps();
593*d9f75844SAndroid Build Coastguard Worker   DataRate link_allocation = DataRate::Zero();
594*d9f75844SAndroid Build Coastguard Worker   if (encoder_target_rate_bps_ > protection_bitrate_bps) {
595*d9f75844SAndroid Build Coastguard Worker     link_allocation =
596*d9f75844SAndroid Build Coastguard Worker         DataRate::BitsPerSec(encoder_target_rate_bps_ - protection_bitrate_bps);
597*d9f75844SAndroid Build Coastguard Worker   }
598*d9f75844SAndroid Build Coastguard Worker   DataRate overhead =
599*d9f75844SAndroid Build Coastguard Worker       update.target_bitrate - DataRate::BitsPerSec(encoder_target_rate_bps_);
600*d9f75844SAndroid Build Coastguard Worker   DataRate encoder_stable_target_rate = update.stable_target_bitrate;
601*d9f75844SAndroid Build Coastguard Worker   if (encoder_stable_target_rate > overhead) {
602*d9f75844SAndroid Build Coastguard Worker     encoder_stable_target_rate = encoder_stable_target_rate - overhead;
603*d9f75844SAndroid Build Coastguard Worker   } else {
604*d9f75844SAndroid Build Coastguard Worker     encoder_stable_target_rate = DataRate::BitsPerSec(encoder_target_rate_bps_);
605*d9f75844SAndroid Build Coastguard Worker   }
606*d9f75844SAndroid Build Coastguard Worker 
607*d9f75844SAndroid Build Coastguard Worker   encoder_target_rate_bps_ =
608*d9f75844SAndroid Build Coastguard Worker       std::min(encoder_max_bitrate_bps_, encoder_target_rate_bps_);
609*d9f75844SAndroid Build Coastguard Worker 
610*d9f75844SAndroid Build Coastguard Worker   encoder_stable_target_rate =
611*d9f75844SAndroid Build Coastguard Worker       std::min(DataRate::BitsPerSec(encoder_max_bitrate_bps_),
612*d9f75844SAndroid Build Coastguard Worker                encoder_stable_target_rate);
613*d9f75844SAndroid Build Coastguard Worker 
614*d9f75844SAndroid Build Coastguard Worker   DataRate encoder_target_rate = DataRate::BitsPerSec(encoder_target_rate_bps_);
615*d9f75844SAndroid Build Coastguard Worker   link_allocation = std::max(encoder_target_rate, link_allocation);
616*d9f75844SAndroid Build Coastguard Worker   video_stream_encoder_->OnBitrateUpdated(
617*d9f75844SAndroid Build Coastguard Worker       encoder_target_rate, encoder_stable_target_rate, link_allocation,
618*d9f75844SAndroid Build Coastguard Worker       rtc::dchecked_cast<uint8_t>(update.packet_loss_ratio * 256),
619*d9f75844SAndroid Build Coastguard Worker       update.round_trip_time.ms(), update.cwnd_reduce_ratio);
620*d9f75844SAndroid Build Coastguard Worker   stats_proxy_->OnSetEncoderTargetRate(encoder_target_rate_bps_);
621*d9f75844SAndroid Build Coastguard Worker   return protection_bitrate_bps;
622*d9f75844SAndroid Build Coastguard Worker }
623*d9f75844SAndroid Build Coastguard Worker 
624*d9f75844SAndroid Build Coastguard Worker }  // namespace internal
625*d9f75844SAndroid Build Coastguard Worker }  // namespace webrtc
626