1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_coding/neteq/tools/neteq_test.h"
12
13 #include <iomanip>
14 #include <iostream>
15
16 #include "modules/audio_coding/neteq/default_neteq_factory.h"
17 #include "modules/rtp_rtcp/source/byte_io.h"
18 #include "system_wrappers/include/clock.h"
19
20 namespace webrtc {
21 namespace test {
22 namespace {
23
ActionToOperations(absl::optional<NetEqSimulator::Action> a)24 absl::optional<NetEq::Operation> ActionToOperations(
25 absl::optional<NetEqSimulator::Action> a) {
26 if (!a) {
27 return absl::nullopt;
28 }
29 switch (*a) {
30 case NetEqSimulator::Action::kAccelerate:
31 return absl::make_optional(NetEq::Operation::kAccelerate);
32 case NetEqSimulator::Action::kExpand:
33 return absl::make_optional(NetEq::Operation::kExpand);
34 case NetEqSimulator::Action::kNormal:
35 return absl::make_optional(NetEq::Operation::kNormal);
36 case NetEqSimulator::Action::kPreemptiveExpand:
37 return absl::make_optional(NetEq::Operation::kPreemptiveExpand);
38 }
39 }
40
CreateNetEq(const NetEq::Config & config,Clock * clock,const rtc::scoped_refptr<AudioDecoderFactory> & decoder_factory)41 std::unique_ptr<NetEq> CreateNetEq(
42 const NetEq::Config& config,
43 Clock* clock,
44 const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory) {
45 return DefaultNetEqFactory().CreateNetEq(config, decoder_factory, clock);
46 }
47
48 } // namespace
49
OnInsertPacketError(const NetEqInput::PacketData & packet)50 void DefaultNetEqTestErrorCallback::OnInsertPacketError(
51 const NetEqInput::PacketData& packet) {
52 std::cerr << "InsertPacket returned an error." << std::endl;
53 std::cerr << "Packet data: " << packet.ToString() << std::endl;
54 RTC_FATAL();
55 }
56
OnGetAudioError()57 void DefaultNetEqTestErrorCallback::OnGetAudioError() {
58 std::cerr << "GetAudio returned an error." << std::endl;
59 RTC_FATAL();
60 }
61
NetEqTest(const NetEq::Config & config,rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,const DecoderMap & codecs,std::unique_ptr<std::ofstream> text_log,NetEqFactory * neteq_factory,std::unique_ptr<NetEqInput> input,std::unique_ptr<AudioSink> output,Callbacks callbacks)62 NetEqTest::NetEqTest(const NetEq::Config& config,
63 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
64 const DecoderMap& codecs,
65 std::unique_ptr<std::ofstream> text_log,
66 NetEqFactory* neteq_factory,
67 std::unique_ptr<NetEqInput> input,
68 std::unique_ptr<AudioSink> output,
69 Callbacks callbacks)
70 : clock_(0),
71 neteq_(neteq_factory
72 ? neteq_factory->CreateNetEq(config, decoder_factory, &clock_)
73 : CreateNetEq(config, &clock_, decoder_factory)),
74 input_(std::move(input)),
75 output_(std::move(output)),
76 callbacks_(callbacks),
77 sample_rate_hz_(config.sample_rate_hz),
78 text_log_(std::move(text_log)) {
79 RTC_CHECK(!config.enable_muted_state)
80 << "The code does not handle enable_muted_state";
81 RegisterDecoders(codecs);
82 }
83
84 NetEqTest::~NetEqTest() = default;
85
Run()86 int64_t NetEqTest::Run() {
87 int64_t simulation_time = 0;
88 SimulationStepResult step_result;
89 do {
90 step_result = RunToNextGetAudio();
91 simulation_time += step_result.simulation_step_ms;
92 } while (!step_result.is_simulation_finished);
93 if (callbacks_.simulation_ended_callback) {
94 callbacks_.simulation_ended_callback->SimulationEnded(simulation_time);
95 }
96 return simulation_time;
97 }
98
RunToNextGetAudio()99 NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() {
100 SimulationStepResult result;
101 const int64_t start_time_ms = *input_->NextEventTime();
102 int64_t time_now_ms = start_time_ms;
103 current_state_.packet_iat_ms.clear();
104
105 while (!input_->ended()) {
106 // Advance time to next event.
107 RTC_DCHECK(input_->NextEventTime());
108 clock_.AdvanceTimeMilliseconds(*input_->NextEventTime() - time_now_ms);
109 time_now_ms = *input_->NextEventTime();
110 // Check if it is time to insert packet.
111 if (input_->NextPacketTime() && time_now_ms >= *input_->NextPacketTime()) {
112 std::unique_ptr<NetEqInput::PacketData> packet_data = input_->PopPacket();
113 RTC_CHECK(packet_data);
114 const size_t payload_data_length =
115 packet_data->payload.size() - packet_data->header.paddingLength;
116 if (payload_data_length != 0) {
117 int error = neteq_->InsertPacket(
118 packet_data->header,
119 rtc::ArrayView<const uint8_t>(packet_data->payload));
120 if (error != NetEq::kOK && callbacks_.error_callback) {
121 callbacks_.error_callback->OnInsertPacketError(*packet_data);
122 }
123 if (callbacks_.post_insert_packet) {
124 callbacks_.post_insert_packet->AfterInsertPacket(*packet_data,
125 neteq_.get());
126 }
127 } else {
128 neteq_->InsertEmptyPacket(packet_data->header);
129 }
130 if (last_packet_time_ms_) {
131 current_state_.packet_iat_ms.push_back(time_now_ms -
132 *last_packet_time_ms_);
133 }
134 if (text_log_) {
135 const auto ops_state = neteq_->GetOperationsAndState();
136 const auto delta_wallclock =
137 last_packet_time_ms_ ? (time_now_ms - *last_packet_time_ms_) : -1;
138 const auto delta_timestamp =
139 last_packet_timestamp_
140 ? (static_cast<int64_t>(packet_data->header.timestamp) -
141 *last_packet_timestamp_) *
142 1000 / sample_rate_hz_
143 : -1;
144 const auto packet_size_bytes =
145 packet_data->payload.size() == 12
146 ? ByteReader<uint32_t>::ReadLittleEndian(
147 &packet_data->payload[8])
148 : -1;
149 *text_log_ << "Packet - wallclock: " << std::setw(5) << time_now_ms
150 << ", delta wc: " << std::setw(4) << delta_wallclock
151 << ", seq_no: " << packet_data->header.sequenceNumber
152 << ", timestamp: " << std::setw(10)
153 << packet_data->header.timestamp
154 << ", delta ts: " << std::setw(4) << delta_timestamp
155 << ", size: " << std::setw(5) << packet_size_bytes
156 << ", frame size: " << std::setw(3)
157 << ops_state.current_frame_size_ms
158 << ", buffer size: " << std::setw(4)
159 << ops_state.current_buffer_size_ms << std::endl;
160 }
161 last_packet_time_ms_ = absl::make_optional<int>(time_now_ms);
162 last_packet_timestamp_ =
163 absl::make_optional<uint32_t>(packet_data->header.timestamp);
164 }
165
166 // Check if it is time to get output audio.
167 if (input_->NextOutputEventTime() &&
168 time_now_ms >= *input_->NextOutputEventTime()) {
169 if (callbacks_.get_audio_callback) {
170 callbacks_.get_audio_callback->BeforeGetAudio(neteq_.get());
171 }
172 AudioFrame out_frame;
173 bool muted;
174 int error = neteq_->GetAudio(&out_frame, &muted, nullptr,
175 ActionToOperations(next_action_));
176 next_action_ = absl::nullopt;
177 RTC_CHECK(!muted) << "The code does not handle enable_muted_state";
178 if (error != NetEq::kOK) {
179 if (callbacks_.error_callback) {
180 callbacks_.error_callback->OnGetAudioError();
181 }
182 } else {
183 sample_rate_hz_ = out_frame.sample_rate_hz_;
184 }
185 if (callbacks_.get_audio_callback) {
186 callbacks_.get_audio_callback->AfterGetAudio(time_now_ms, out_frame,
187 muted, neteq_.get());
188 }
189
190 if (output_) {
191 RTC_CHECK(output_->WriteArray(
192 out_frame.data(),
193 out_frame.samples_per_channel_ * out_frame.num_channels_));
194 }
195
196 input_->AdvanceOutputEvent();
197 result.simulation_step_ms =
198 input_->NextEventTime().value_or(time_now_ms) - start_time_ms;
199 const auto operations_state = neteq_->GetOperationsAndState();
200 current_state_.current_delay_ms = operations_state.current_buffer_size_ms;
201 current_state_.packet_size_ms = operations_state.current_frame_size_ms;
202 current_state_.next_packet_available =
203 operations_state.next_packet_available;
204 current_state_.packet_buffer_flushed =
205 operations_state.packet_buffer_flushes >
206 prev_ops_state_.packet_buffer_flushes;
207 // TODO(ivoc): Add more accurate reporting by tracking the origin of
208 // samples in the sync buffer.
209 result.action_times_ms[Action::kExpand] = 0;
210 result.action_times_ms[Action::kAccelerate] = 0;
211 result.action_times_ms[Action::kPreemptiveExpand] = 0;
212 result.action_times_ms[Action::kNormal] = 0;
213
214 if (out_frame.speech_type_ == AudioFrame::SpeechType::kPLC ||
215 out_frame.speech_type_ == AudioFrame::SpeechType::kPLCCNG) {
216 // Consider the whole frame to be the result of expansion.
217 result.action_times_ms[Action::kExpand] = 10;
218 } else if (operations_state.accelerate_samples -
219 prev_ops_state_.accelerate_samples >
220 0) {
221 // Consider the whole frame to be the result of acceleration.
222 result.action_times_ms[Action::kAccelerate] = 10;
223 } else if (operations_state.preemptive_samples -
224 prev_ops_state_.preemptive_samples >
225 0) {
226 // Consider the whole frame to be the result of preemptive expansion.
227 result.action_times_ms[Action::kPreemptiveExpand] = 10;
228 } else {
229 // Consider the whole frame to be the result of normal playout.
230 result.action_times_ms[Action::kNormal] = 10;
231 }
232 auto lifetime_stats = LifetimeStats();
233 if (text_log_) {
234 const bool plc =
235 (out_frame.speech_type_ == AudioFrame::SpeechType::kPLC) ||
236 (out_frame.speech_type_ == AudioFrame::SpeechType::kPLCCNG);
237 const bool cng = out_frame.speech_type_ == AudioFrame::SpeechType::kCNG;
238 const bool voice_concealed =
239 (lifetime_stats.concealed_samples -
240 lifetime_stats.silent_concealed_samples) >
241 (prev_lifetime_stats_.concealed_samples -
242 prev_lifetime_stats_.silent_concealed_samples);
243 *text_log_ << "GetAudio - wallclock: " << std::setw(5) << time_now_ms
244 << ", delta wc: " << std::setw(4)
245 << (input_->NextEventTime().value_or(time_now_ms) -
246 start_time_ms)
247 << ", CNG: " << cng << ", PLC: " << plc
248 << ", voice concealed: " << voice_concealed
249 << ", buffer size: " << std::setw(4)
250 << current_state_.current_delay_ms << std::endl;
251 if (lifetime_stats.packets_discarded >
252 prev_lifetime_stats_.packets_discarded) {
253 *text_log_ << "Discarded "
254 << (lifetime_stats.packets_discarded -
255 prev_lifetime_stats_.packets_discarded)
256 << " primary packets." << std::endl;
257 }
258 if (operations_state.packet_buffer_flushes >
259 prev_ops_state_.packet_buffer_flushes) {
260 *text_log_ << "Flushed packet buffer "
261 << (operations_state.packet_buffer_flushes -
262 prev_ops_state_.packet_buffer_flushes)
263 << " times." << std::endl;
264 }
265 }
266 prev_lifetime_stats_ = lifetime_stats;
267 const bool no_more_packets_to_decode =
268 !input_->NextPacketTime() && !operations_state.next_packet_available;
269 // End the simulation if the gap is too large. This indicates an issue
270 // with the event log file.
271 const bool simulation_step_too_large = result.simulation_step_ms > 1000;
272 if (simulation_step_too_large) {
273 // If we don't reset the step time, the large gap will be included in
274 // the simulation time, which can be a large distortion.
275 result.simulation_step_ms = 10;
276 }
277 result.is_simulation_finished = simulation_step_too_large ||
278 no_more_packets_to_decode ||
279 input_->ended();
280 prev_ops_state_ = operations_state;
281 return result;
282 }
283 }
284 result.simulation_step_ms =
285 input_->NextEventTime().value_or(time_now_ms) - start_time_ms;
286 result.is_simulation_finished = true;
287 return result;
288 }
289
SetNextAction(NetEqTest::Action next_operation)290 void NetEqTest::SetNextAction(NetEqTest::Action next_operation) {
291 next_action_ = absl::optional<Action>(next_operation);
292 }
293
GetNetEqState()294 NetEqTest::NetEqState NetEqTest::GetNetEqState() {
295 return current_state_;
296 }
297
SimulationStats()298 NetEqNetworkStatistics NetEqTest::SimulationStats() {
299 NetEqNetworkStatistics stats;
300 RTC_CHECK_EQ(neteq_->NetworkStatistics(&stats), 0);
301 return stats;
302 }
303
LifetimeStats() const304 NetEqLifetimeStatistics NetEqTest::LifetimeStats() const {
305 return neteq_->GetLifetimeStatistics();
306 }
307
StandardDecoderMap()308 NetEqTest::DecoderMap NetEqTest::StandardDecoderMap() {
309 DecoderMap codecs = {
310 {0, SdpAudioFormat("pcmu", 8000, 1)},
311 {8, SdpAudioFormat("pcma", 8000, 1)},
312 #ifdef WEBRTC_CODEC_ILBC
313 {102, SdpAudioFormat("ilbc", 8000, 1)},
314 #endif
315 #ifdef WEBRTC_CODEC_OPUS
316 {111, SdpAudioFormat("opus", 48000, 2)},
317 #endif
318 {93, SdpAudioFormat("l16", 8000, 1)},
319 {94, SdpAudioFormat("l16", 16000, 1)},
320 {95, SdpAudioFormat("l16", 32000, 1)},
321 {96, SdpAudioFormat("l16", 48000, 1)},
322 {9, SdpAudioFormat("g722", 8000, 1)},
323 {106, SdpAudioFormat("telephone-event", 8000, 1)},
324 {114, SdpAudioFormat("telephone-event", 16000, 1)},
325 {115, SdpAudioFormat("telephone-event", 32000, 1)},
326 {116, SdpAudioFormat("telephone-event", 48000, 1)},
327 {117, SdpAudioFormat("red", 8000, 1)},
328 {13, SdpAudioFormat("cn", 8000, 1)},
329 {98, SdpAudioFormat("cn", 16000, 1)},
330 {99, SdpAudioFormat("cn", 32000, 1)},
331 {100, SdpAudioFormat("cn", 48000, 1)}
332 };
333 return codecs;
334 }
335
RegisterDecoders(const DecoderMap & codecs)336 void NetEqTest::RegisterDecoders(const DecoderMap& codecs) {
337 for (const auto& c : codecs) {
338 RTC_CHECK(neteq_->RegisterPayloadType(c.first, c.second))
339 << "Cannot register " << c.second.name << " to payload type "
340 << c.first;
341 }
342 }
343
344 } // namespace test
345 } // namespace webrtc
346