1 /* 2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef AUDIO_AUDIO_SEND_STREAM_H_ 12 #define AUDIO_AUDIO_SEND_STREAM_H_ 13 14 #include <memory> 15 #include <utility> 16 #include <vector> 17 18 #include "absl/functional/any_invocable.h" 19 #include "api/field_trials_view.h" 20 #include "api/sequence_checker.h" 21 #include "api/task_queue/task_queue_base.h" 22 #include "audio/audio_level.h" 23 #include "audio/channel_send.h" 24 #include "call/audio_send_stream.h" 25 #include "call/audio_state.h" 26 #include "call/bitrate_allocator.h" 27 #include "modules/rtp_rtcp/source/rtp_rtcp_interface.h" 28 #include "modules/utility/maybe_worker_thread.h" 29 #include "rtc_base/experiments/struct_parameters_parser.h" 30 #include "rtc_base/race_checker.h" 31 #include "rtc_base/synchronization/mutex.h" 32 #include "rtc_base/task_queue.h" 33 34 namespace webrtc { 35 class RtcEventLog; 36 class RtcpBandwidthObserver; 37 class RtcpRttStats; 38 class RtpTransportControllerSendInterface; 39 40 struct AudioAllocationConfig { 41 static constexpr char kKey[] = "WebRTC-Audio-Allocation"; 42 // Field Trial configured bitrates to use as overrides over default/user 43 // configured bitrate range when audio bitrate allocation is enabled. 44 absl::optional<DataRate> min_bitrate; 45 absl::optional<DataRate> max_bitrate; 46 DataRate priority_bitrate = DataRate::Zero(); 47 // By default the priority_bitrate is compensated for packet overhead. 48 // Use this flag to configure a raw value instead. 49 absl::optional<DataRate> priority_bitrate_raw; 50 absl::optional<double> bitrate_priority; 51 52 std::unique_ptr<StructParametersParser> Parser(); 53 explicit AudioAllocationConfig(const FieldTrialsView& field_trials); 54 }; 55 namespace internal { 56 class AudioState; 57 58 class AudioSendStream final : public webrtc::AudioSendStream, 59 public webrtc::BitrateAllocatorObserver { 60 public: 61 AudioSendStream(Clock* clock, 62 const webrtc::AudioSendStream::Config& config, 63 const rtc::scoped_refptr<webrtc::AudioState>& audio_state, 64 TaskQueueFactory* task_queue_factory, 65 RtpTransportControllerSendInterface* rtp_transport, 66 BitrateAllocatorInterface* bitrate_allocator, 67 RtcEventLog* event_log, 68 RtcpRttStats* rtcp_rtt_stats, 69 const absl::optional<RtpState>& suspended_rtp_state, 70 const FieldTrialsView& field_trials); 71 // For unit tests, which need to supply a mock ChannelSend. 72 AudioSendStream(Clock* clock, 73 const webrtc::AudioSendStream::Config& config, 74 const rtc::scoped_refptr<webrtc::AudioState>& audio_state, 75 TaskQueueFactory* task_queue_factory, 76 RtpTransportControllerSendInterface* rtp_transport, 77 BitrateAllocatorInterface* bitrate_allocator, 78 RtcEventLog* event_log, 79 const absl::optional<RtpState>& suspended_rtp_state, 80 std::unique_ptr<voe::ChannelSendInterface> channel_send, 81 const FieldTrialsView& field_trials); 82 83 AudioSendStream() = delete; 84 AudioSendStream(const AudioSendStream&) = delete; 85 AudioSendStream& operator=(const AudioSendStream&) = delete; 86 87 ~AudioSendStream() override; 88 89 // webrtc::AudioSendStream implementation. 90 const webrtc::AudioSendStream::Config& GetConfig() const override; 91 void Reconfigure(const webrtc::AudioSendStream::Config& config, 92 SetParametersCallback callback) override; 93 void Start() override; 94 void Stop() override; 95 void SendAudioData(std::unique_ptr<AudioFrame> audio_frame) override; 96 bool SendTelephoneEvent(int payload_type, 97 int payload_frequency, 98 int event, 99 int duration_ms) override; 100 void SetMuted(bool muted) override; 101 webrtc::AudioSendStream::Stats GetStats() const override; 102 webrtc::AudioSendStream::Stats GetStats( 103 bool has_remote_tracks) const override; 104 105 void DeliverRtcp(const uint8_t* packet, size_t length); 106 107 // Implements BitrateAllocatorObserver. 108 uint32_t OnBitrateUpdated(BitrateAllocationUpdate update) override; 109 110 void SetTransportOverhead(int transport_overhead_per_packet_bytes); 111 112 RtpState GetRtpState() const; 113 const voe::ChannelSendInterface* GetChannel() const; 114 115 // Returns combined per-packet overhead. 116 size_t TestOnlyGetPerPacketOverheadBytes() const 117 RTC_LOCKS_EXCLUDED(overhead_per_packet_lock_); 118 119 private: 120 class TimedTransport; 121 // Constraints including overhead. 122 struct TargetAudioBitrateConstraints { 123 DataRate min; 124 DataRate max; 125 }; 126 127 internal::AudioState* audio_state(); 128 const internal::AudioState* audio_state() const; 129 130 void StoreEncoderProperties(int sample_rate_hz, size_t num_channels) 131 RTC_RUN_ON(worker_thread_checker_); 132 133 void ConfigureStream(const Config& new_config, 134 bool first_time, 135 SetParametersCallback callback) 136 RTC_RUN_ON(worker_thread_checker_); 137 bool SetupSendCodec(const Config& new_config) 138 RTC_RUN_ON(worker_thread_checker_); 139 bool ReconfigureSendCodec(const Config& new_config) 140 RTC_RUN_ON(worker_thread_checker_); 141 void ReconfigureANA(const Config& new_config) 142 RTC_RUN_ON(worker_thread_checker_); 143 void ReconfigureCNG(const Config& new_config) 144 RTC_RUN_ON(worker_thread_checker_); 145 void ReconfigureBitrateObserver(const Config& new_config) 146 RTC_RUN_ON(worker_thread_checker_); 147 148 void ConfigureBitrateObserver() RTC_RUN_ON(worker_thread_checker_); 149 void RemoveBitrateObserver() RTC_RUN_ON(worker_thread_checker_); 150 151 // Returns bitrate constraints, maybe including overhead when enabled by 152 // field trial. 153 absl::optional<TargetAudioBitrateConstraints> GetMinMaxBitrateConstraints() 154 const RTC_RUN_ON(worker_thread_checker_); 155 156 // Sets per-packet overhead on encoded (for ANA) based on current known values 157 // of transport and packetization overheads. 158 void UpdateOverheadForEncoder() 159 RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_); 160 161 // Returns combined per-packet overhead. 162 size_t GetPerPacketOverheadBytes() const 163 RTC_EXCLUSIVE_LOCKS_REQUIRED(overhead_per_packet_lock_); 164 165 void RegisterCngPayloadType(int payload_type, int clockrate_hz) 166 RTC_RUN_ON(worker_thread_checker_); 167 168 void UpdateCachedTargetAudioBitrateConstraints() 169 RTC_RUN_ON(worker_thread_checker_); 170 171 Clock* clock_; 172 const FieldTrialsView& field_trials_; 173 174 SequenceChecker worker_thread_checker_; 175 rtc::RaceChecker audio_capture_race_checker_; 176 MaybeWorkerThread* rtp_transport_queue_; 177 178 const bool allocate_audio_without_feedback_; 179 const bool force_no_audio_feedback_ = allocate_audio_without_feedback_; 180 const bool enable_audio_alr_probing_; 181 const AudioAllocationConfig allocation_settings_; 182 183 webrtc::AudioSendStream::Config config_ 184 RTC_GUARDED_BY(worker_thread_checker_); 185 rtc::scoped_refptr<webrtc::AudioState> audio_state_; 186 const std::unique_ptr<voe::ChannelSendInterface> channel_send_; 187 RtcEventLog* const event_log_; 188 const bool use_legacy_overhead_calculation_; 189 190 int encoder_sample_rate_hz_ RTC_GUARDED_BY(worker_thread_checker_) = 0; 191 size_t encoder_num_channels_ RTC_GUARDED_BY(worker_thread_checker_) = 0; 192 bool sending_ RTC_GUARDED_BY(worker_thread_checker_) = false; 193 mutable Mutex audio_level_lock_; 194 // Keeps track of audio level, total audio energy and total samples duration. 195 // https://w3c.github.io/webrtc-stats/#dom-rtcaudiohandlerstats-totalaudioenergy 196 webrtc::voe::AudioLevel audio_level_ RTC_GUARDED_BY(audio_level_lock_); 197 198 BitrateAllocatorInterface* const bitrate_allocator_ 199 RTC_GUARDED_BY(rtp_transport_queue_); 200 // Constrains cached to be accessed from `rtp_transport_queue_`. 201 absl::optional<AudioSendStream::TargetAudioBitrateConstraints> 202 cached_constraints_ RTC_GUARDED_BY(rtp_transport_queue_) = absl::nullopt; 203 RtpTransportControllerSendInterface* const rtp_transport_; 204 205 RtpRtcpInterface* const rtp_rtcp_module_; 206 absl::optional<RtpState> const suspended_rtp_state_; 207 208 // RFC 5285: Each distinct extension MUST have a unique ID. The value 0 is 209 // reserved for padding and MUST NOT be used as a local identifier. 210 // So it should be safe to use 0 here to indicate "not configured". 211 struct ExtensionIds { 212 int audio_level = 0; 213 int abs_send_time = 0; 214 int abs_capture_time = 0; 215 int transport_sequence_number = 0; 216 int mid = 0; 217 int rid = 0; 218 int repaired_rid = 0; 219 }; 220 static ExtensionIds FindExtensionIds( 221 const std::vector<RtpExtension>& extensions); 222 static int TransportSeqNumId(const Config& config); 223 224 mutable Mutex overhead_per_packet_lock_; 225 size_t overhead_per_packet_ RTC_GUARDED_BY(overhead_per_packet_lock_) = 0; 226 227 // Current transport overhead (ICE, TURN, etc.) 228 size_t transport_overhead_per_packet_bytes_ 229 RTC_GUARDED_BY(overhead_per_packet_lock_) = 0; 230 231 bool registered_with_allocator_ RTC_GUARDED_BY(worker_thread_checker_) = 232 false; 233 size_t total_packet_overhead_bytes_ RTC_GUARDED_BY(worker_thread_checker_) = 234 0; 235 absl::optional<std::pair<TimeDelta, TimeDelta>> frame_length_range_ 236 RTC_GUARDED_BY(worker_thread_checker_); 237 }; 238 } // namespace internal 239 } // namespace webrtc 240 241 #endif // AUDIO_AUDIO_SEND_STREAM_H_ 242