1 /*
2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "audio/voip/audio_ingress.h"
12
13 #include <algorithm>
14 #include <utility>
15 #include <vector>
16
17 #include "api/audio_codecs/audio_format.h"
18 #include "audio/utility/audio_frame_operations.h"
19 #include "modules/audio_coding/include/audio_coding_module.h"
20 #include "modules/rtp_rtcp/source/byte_io.h"
21 #include "modules/rtp_rtcp/source/rtcp_packet/common_header.h"
22 #include "modules/rtp_rtcp/source/rtcp_packet/receiver_report.h"
23 #include "modules/rtp_rtcp/source/rtcp_packet/sender_report.h"
24 #include "rtc_base/logging.h"
25 #include "rtc_base/numerics/safe_minmax.h"
26
27 namespace webrtc {
28
29 namespace {
30
CreateAcmConfig(rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)31 AudioCodingModule::Config CreateAcmConfig(
32 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory) {
33 AudioCodingModule::Config acm_config;
34 acm_config.neteq_config.enable_muted_state = true;
35 acm_config.decoder_factory = decoder_factory;
36 return acm_config;
37 }
38
39 } // namespace
40
AudioIngress(RtpRtcpInterface * rtp_rtcp,Clock * clock,ReceiveStatistics * receive_statistics,rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)41 AudioIngress::AudioIngress(
42 RtpRtcpInterface* rtp_rtcp,
43 Clock* clock,
44 ReceiveStatistics* receive_statistics,
45 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory)
46 : playing_(false),
47 remote_ssrc_(0),
48 first_rtp_timestamp_(-1),
49 rtp_receive_statistics_(receive_statistics),
50 rtp_rtcp_(rtp_rtcp),
51 acm_receiver_(CreateAcmConfig(decoder_factory)),
52 ntp_estimator_(clock) {}
53
54 AudioIngress::~AudioIngress() = default;
55
GetAudioFrameWithInfo(int sampling_rate,AudioFrame * audio_frame)56 AudioMixer::Source::AudioFrameInfo AudioIngress::GetAudioFrameWithInfo(
57 int sampling_rate,
58 AudioFrame* audio_frame) {
59 audio_frame->sample_rate_hz_ = sampling_rate;
60
61 // Get 10ms raw PCM data from the ACM.
62 bool muted = false;
63 if (acm_receiver_.GetAudio(sampling_rate, audio_frame, &muted) == -1) {
64 RTC_DLOG(LS_ERROR) << "GetAudio() failed!";
65 // In all likelihood, the audio in this frame is garbage. We return an
66 // error so that the audio mixer module doesn't add it to the mix. As
67 // a result, it won't be played out and the actions skipped here are
68 // irrelevant.
69 return AudioMixer::Source::AudioFrameInfo::kError;
70 }
71
72 if (muted) {
73 AudioFrameOperations::Mute(audio_frame);
74 }
75
76 // Measure audio level.
77 constexpr double kAudioSampleDurationSeconds = 0.01;
78 output_audio_level_.ComputeLevel(*audio_frame, kAudioSampleDurationSeconds);
79
80 // If caller invoked StopPlay(), then mute the frame.
81 if (!playing_) {
82 AudioFrameOperations::Mute(audio_frame);
83 muted = true;
84 }
85
86 // Set first rtp timestamp with first audio frame with valid timestamp.
87 if (first_rtp_timestamp_ < 0 && audio_frame->timestamp_ != 0) {
88 first_rtp_timestamp_ = audio_frame->timestamp_;
89 }
90
91 if (first_rtp_timestamp_ >= 0) {
92 // Compute elapsed and NTP times.
93 int64_t unwrap_timestamp;
94 {
95 MutexLock lock(&lock_);
96 unwrap_timestamp =
97 timestamp_wrap_handler_.Unwrap(audio_frame->timestamp_);
98 audio_frame->ntp_time_ms_ =
99 ntp_estimator_.Estimate(audio_frame->timestamp_);
100 }
101 // For clock rate, default to the playout sampling rate if we haven't
102 // received any packets yet.
103 absl::optional<std::pair<int, SdpAudioFormat>> decoder =
104 acm_receiver_.LastDecoder();
105 int clock_rate = decoder ? decoder->second.clockrate_hz
106 : acm_receiver_.last_output_sample_rate_hz();
107 RTC_DCHECK_GT(clock_rate, 0);
108 audio_frame->elapsed_time_ms_ =
109 (unwrap_timestamp - first_rtp_timestamp_) / (clock_rate / 1000);
110 }
111
112 return muted ? AudioMixer::Source::AudioFrameInfo::kMuted
113 : AudioMixer::Source::AudioFrameInfo::kNormal;
114 }
115
StartPlay()116 bool AudioIngress::StartPlay() {
117 {
118 MutexLock lock(&lock_);
119 if (receive_codec_info_.empty()) {
120 RTC_DLOG(LS_WARNING) << "Receive codecs have not been set yet";
121 return false;
122 }
123 }
124 playing_ = true;
125 return true;
126 }
127
SetReceiveCodecs(const std::map<int,SdpAudioFormat> & codecs)128 void AudioIngress::SetReceiveCodecs(
129 const std::map<int, SdpAudioFormat>& codecs) {
130 {
131 MutexLock lock(&lock_);
132 for (const auto& kv : codecs) {
133 receive_codec_info_[kv.first] = kv.second.clockrate_hz;
134 }
135 }
136 acm_receiver_.SetCodecs(codecs);
137 }
138
ReceivedRTPPacket(rtc::ArrayView<const uint8_t> rtp_packet)139 void AudioIngress::ReceivedRTPPacket(rtc::ArrayView<const uint8_t> rtp_packet) {
140 RtpPacketReceived rtp_packet_received;
141 rtp_packet_received.Parse(rtp_packet.data(), rtp_packet.size());
142
143 // Set payload type's sampling rate before we feed it into ReceiveStatistics.
144 {
145 MutexLock lock(&lock_);
146 const auto& it =
147 receive_codec_info_.find(rtp_packet_received.PayloadType());
148 // If sampling rate info is not available in our received codec set, it
149 // would mean that remote media endpoint is sending incorrect payload id
150 // which can't be processed correctly especially on payload type id in
151 // dynamic range.
152 if (it == receive_codec_info_.end()) {
153 RTC_DLOG(LS_WARNING) << "Unexpected payload id received: "
154 << rtp_packet_received.PayloadType();
155 return;
156 }
157 rtp_packet_received.set_payload_type_frequency(it->second);
158 }
159
160 // Track current remote SSRC.
161 if (rtp_packet_received.Ssrc() != remote_ssrc_) {
162 rtp_rtcp_->SetRemoteSSRC(rtp_packet_received.Ssrc());
163 remote_ssrc_.store(rtp_packet_received.Ssrc());
164 }
165
166 rtp_receive_statistics_->OnRtpPacket(rtp_packet_received);
167
168 RTPHeader header;
169 rtp_packet_received.GetHeader(&header);
170
171 size_t packet_length = rtp_packet_received.size();
172 if (packet_length < header.headerLength ||
173 (packet_length - header.headerLength) < header.paddingLength) {
174 RTC_DLOG(LS_ERROR) << "Packet length(" << packet_length << ") header("
175 << header.headerLength << ") padding("
176 << header.paddingLength << ")";
177 return;
178 }
179
180 const uint8_t* payload = rtp_packet_received.data() + header.headerLength;
181 size_t payload_length = packet_length - header.headerLength;
182 size_t payload_data_length = payload_length - header.paddingLength;
183 auto data_view = rtc::ArrayView<const uint8_t>(payload, payload_data_length);
184
185 // Push the incoming payload (parsed and ready for decoding) into the ACM.
186 if (acm_receiver_.InsertPacket(header, data_view) != 0) {
187 RTC_DLOG(LS_ERROR) << "AudioIngress::ReceivedRTPPacket() unable to "
188 "push data to the ACM";
189 }
190 }
191
ReceivedRTCPPacket(rtc::ArrayView<const uint8_t> rtcp_packet)192 void AudioIngress::ReceivedRTCPPacket(
193 rtc::ArrayView<const uint8_t> rtcp_packet) {
194 rtcp::CommonHeader rtcp_header;
195 if (rtcp_header.Parse(rtcp_packet.data(), rtcp_packet.size()) &&
196 (rtcp_header.type() == rtcp::SenderReport::kPacketType ||
197 rtcp_header.type() == rtcp::ReceiverReport::kPacketType)) {
198 RTC_DCHECK_GE(rtcp_packet.size(), 8);
199
200 uint32_t sender_ssrc =
201 ByteReader<uint32_t>::ReadBigEndian(rtcp_packet.data() + 4);
202
203 // If we don't have remote ssrc at this point, it's likely that remote
204 // endpoint is receive-only or it could have restarted the media.
205 if (sender_ssrc != remote_ssrc_) {
206 rtp_rtcp_->SetRemoteSSRC(sender_ssrc);
207 remote_ssrc_.store(sender_ssrc);
208 }
209 }
210
211 // Deliver RTCP packet to RTP/RTCP module for parsing and processing.
212 rtp_rtcp_->IncomingRtcpPacket(rtcp_packet.data(), rtcp_packet.size());
213
214 int64_t rtt = 0;
215 if (rtp_rtcp_->RTT(remote_ssrc_, &rtt, nullptr, nullptr, nullptr) != 0) {
216 // Waiting for valid RTT.
217 return;
218 }
219
220 uint32_t ntp_secs = 0, ntp_frac = 0, rtp_timestamp = 0;
221 if (rtp_rtcp_->RemoteNTP(&ntp_secs, &ntp_frac, nullptr, nullptr,
222 &rtp_timestamp) != 0) {
223 // Waiting for RTCP.
224 return;
225 }
226
227 {
228 MutexLock lock(&lock_);
229 ntp_estimator_.UpdateRtcpTimestamp(
230 TimeDelta::Millis(rtt), NtpTime(ntp_secs, ntp_frac), rtp_timestamp);
231 }
232 }
233
GetChannelStatistics()234 ChannelStatistics AudioIngress::GetChannelStatistics() {
235 ChannelStatistics channel_stats;
236
237 // Get clockrate for current decoder ahead of jitter calculation.
238 uint32_t clockrate_hz = 0;
239 absl::optional<std::pair<int, SdpAudioFormat>> decoder =
240 acm_receiver_.LastDecoder();
241 if (decoder) {
242 clockrate_hz = decoder->second.clockrate_hz;
243 }
244
245 StreamStatistician* statistician =
246 rtp_receive_statistics_->GetStatistician(remote_ssrc_);
247 if (statistician) {
248 RtpReceiveStats stats = statistician->GetStats();
249 channel_stats.packets_lost = stats.packets_lost;
250 channel_stats.packets_received = stats.packet_counter.packets;
251 channel_stats.bytes_received = stats.packet_counter.payload_bytes;
252 channel_stats.remote_ssrc = remote_ssrc_;
253 if (clockrate_hz > 0) {
254 channel_stats.jitter = static_cast<double>(stats.jitter) / clockrate_hz;
255 }
256 }
257
258 // Get RTCP report using remote SSRC.
259 const std::vector<ReportBlockData>& report_data =
260 rtp_rtcp_->GetLatestReportBlockData();
261 for (const ReportBlockData& block_data : report_data) {
262 const RTCPReportBlock& rtcp_report = block_data.report_block();
263 if (rtp_rtcp_->SSRC() != rtcp_report.source_ssrc ||
264 remote_ssrc_ != rtcp_report.sender_ssrc) {
265 continue;
266 }
267 RemoteRtcpStatistics remote_stat;
268 remote_stat.packets_lost = rtcp_report.packets_lost;
269 remote_stat.fraction_lost =
270 static_cast<double>(rtcp_report.fraction_lost) / (1 << 8);
271 if (clockrate_hz > 0) {
272 remote_stat.jitter =
273 static_cast<double>(rtcp_report.jitter) / clockrate_hz;
274 }
275 if (block_data.has_rtt()) {
276 remote_stat.round_trip_time =
277 static_cast<double>(block_data.last_rtt_ms()) /
278 rtc::kNumMillisecsPerSec;
279 }
280 remote_stat.last_report_received_timestamp_ms =
281 block_data.report_block_timestamp_utc_us() /
282 rtc::kNumMicrosecsPerMillisec;
283 channel_stats.remote_rtcp = remote_stat;
284
285 // Receive only channel won't send any RTP packets.
286 if (!channel_stats.remote_ssrc.has_value()) {
287 channel_stats.remote_ssrc = remote_ssrc_;
288 }
289 break;
290 }
291
292 return channel_stats;
293 }
294
295 } // namespace webrtc
296