1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/audio_mixer/audio_mixer_impl.h"
12
13 #include <string.h>
14
15 #include <cstdint>
16 #include <limits>
17 #include <memory>
18 #include <string>
19 #include <utility>
20 #include <vector>
21
22 #include "absl/types/optional.h"
23 #include "api/audio/audio_mixer.h"
24 #include "api/rtp_packet_info.h"
25 #include "api/rtp_packet_infos.h"
26 #include "api/units/timestamp.h"
27 #include "modules/audio_mixer/default_output_rate_calculator.h"
28 #include "rtc_base/checks.h"
29 #include "rtc_base/strings/string_builder.h"
30 #include "rtc_base/task_queue_for_test.h"
31 #include "test/gmock.h"
32 #include "test/gtest.h"
33
34 using ::testing::_;
35 using ::testing::Exactly;
36 using ::testing::Invoke;
37 using ::testing::Return;
38 using ::testing::UnorderedElementsAre;
39
40 namespace webrtc {
41
42 namespace {
43
44 constexpr int kDefaultSampleRateHz = 48000;
45
46 // Utility function that resets the frame member variables with
47 // sensible defaults.
ResetFrame(AudioFrame * frame)48 void ResetFrame(AudioFrame* frame) {
49 frame->sample_rate_hz_ = kDefaultSampleRateHz;
50 frame->num_channels_ = 1;
51
52 // Frame duration 10ms.
53 frame->samples_per_channel_ = kDefaultSampleRateHz / 100;
54 frame->vad_activity_ = AudioFrame::kVadActive;
55 frame->speech_type_ = AudioFrame::kNormalSpeech;
56 }
57
ProduceDebugText(int sample_rate_hz,int number_of_channels,int number_of_sources)58 std::string ProduceDebugText(int sample_rate_hz,
59 int number_of_channels,
60 int number_of_sources) {
61 rtc::StringBuilder ss;
62 ss << "Sample rate: " << sample_rate_hz << " ";
63 ss << "Number of channels: " << number_of_channels << " ";
64 ss << "Number of sources: " << number_of_sources;
65 return ss.Release();
66 }
67
68 AudioFrame frame_for_mixing;
69
70 } // namespace
71
72 class MockMixerAudioSource : public ::testing::NiceMock<AudioMixer::Source> {
73 public:
MockMixerAudioSource()74 MockMixerAudioSource()
75 : fake_audio_frame_info_(AudioMixer::Source::AudioFrameInfo::kNormal) {
76 ON_CALL(*this, GetAudioFrameWithInfo(_, _))
77 .WillByDefault(
78 Invoke(this, &MockMixerAudioSource::FakeAudioFrameWithInfo));
79 ON_CALL(*this, PreferredSampleRate())
80 .WillByDefault(Return(kDefaultSampleRateHz));
81 }
82
83 MOCK_METHOD(AudioFrameInfo,
84 GetAudioFrameWithInfo,
85 (int sample_rate_hz, AudioFrame* audio_frame),
86 (override));
87
88 MOCK_METHOD(int, PreferredSampleRate, (), (const, override));
89 MOCK_METHOD(int, Ssrc, (), (const, override));
90
fake_frame()91 AudioFrame* fake_frame() { return &fake_frame_; }
fake_info()92 AudioFrameInfo fake_info() { return fake_audio_frame_info_; }
set_fake_info(const AudioFrameInfo audio_frame_info)93 void set_fake_info(const AudioFrameInfo audio_frame_info) {
94 fake_audio_frame_info_ = audio_frame_info;
95 }
96
set_packet_infos(const RtpPacketInfos & packet_infos)97 void set_packet_infos(const RtpPacketInfos& packet_infos) {
98 packet_infos_ = packet_infos;
99 }
100
101 private:
FakeAudioFrameWithInfo(int sample_rate_hz,AudioFrame * audio_frame)102 AudioFrameInfo FakeAudioFrameWithInfo(int sample_rate_hz,
103 AudioFrame* audio_frame) {
104 audio_frame->CopyFrom(fake_frame_);
105 audio_frame->sample_rate_hz_ = sample_rate_hz;
106 audio_frame->samples_per_channel_ =
107 rtc::CheckedDivExact(sample_rate_hz, 100);
108 audio_frame->packet_infos_ = packet_infos_;
109 return fake_info();
110 }
111
112 AudioFrame fake_frame_;
113 AudioFrameInfo fake_audio_frame_info_;
114 RtpPacketInfos packet_infos_;
115 };
116
117 class CustomRateCalculator : public OutputRateCalculator {
118 public:
CustomRateCalculator(int rate)119 explicit CustomRateCalculator(int rate) : rate_(rate) {}
CalculateOutputRateFromRange(rtc::ArrayView<const int> preferred_rates)120 int CalculateOutputRateFromRange(
121 rtc::ArrayView<const int> preferred_rates) override {
122 return rate_;
123 }
124
125 private:
126 const int rate_;
127 };
128
129 // Creates participants from `frames` and `frame_info` and adds them
130 // to the mixer. Compares mixed status with `expected_status`
MixAndCompare(const std::vector<AudioFrame> & frames,const std::vector<AudioMixer::Source::AudioFrameInfo> & frame_info,const std::vector<bool> & expected_status)131 void MixAndCompare(
132 const std::vector<AudioFrame>& frames,
133 const std::vector<AudioMixer::Source::AudioFrameInfo>& frame_info,
134 const std::vector<bool>& expected_status) {
135 const size_t num_audio_sources = frames.size();
136 RTC_DCHECK(frames.size() == frame_info.size());
137 RTC_DCHECK(frame_info.size() == expected_status.size());
138
139 const auto mixer = AudioMixerImpl::Create();
140 std::vector<MockMixerAudioSource> participants(num_audio_sources);
141
142 for (size_t i = 0; i < num_audio_sources; ++i) {
143 participants[i].fake_frame()->CopyFrom(frames[i]);
144 participants[i].set_fake_info(frame_info[i]);
145 }
146
147 for (size_t i = 0; i < num_audio_sources; ++i) {
148 EXPECT_TRUE(mixer->AddSource(&participants[i]));
149 EXPECT_CALL(participants[i], GetAudioFrameWithInfo(kDefaultSampleRateHz, _))
150 .Times(Exactly(1));
151 }
152
153 mixer->Mix(1, &frame_for_mixing);
154
155 for (size_t i = 0; i < num_audio_sources; ++i) {
156 EXPECT_EQ(expected_status[i],
157 mixer->GetAudioSourceMixabilityStatusForTest(&participants[i]))
158 << "Mixed status of AudioSource #" << i << " wrong.";
159 }
160 }
161
MixMonoAtGivenNativeRate(int native_sample_rate,AudioFrame * mix_frame,rtc::scoped_refptr<AudioMixer> mixer,MockMixerAudioSource * audio_source)162 void MixMonoAtGivenNativeRate(int native_sample_rate,
163 AudioFrame* mix_frame,
164 rtc::scoped_refptr<AudioMixer> mixer,
165 MockMixerAudioSource* audio_source) {
166 ON_CALL(*audio_source, PreferredSampleRate())
167 .WillByDefault(Return(native_sample_rate));
168 audio_source->fake_frame()->sample_rate_hz_ = native_sample_rate;
169 audio_source->fake_frame()->samples_per_channel_ = native_sample_rate / 100;
170
171 mixer->Mix(1, mix_frame);
172 }
173
TEST(AudioMixer,LargestEnergyVadActiveMixed)174 TEST(AudioMixer, LargestEnergyVadActiveMixed) {
175 constexpr int kAudioSources =
176 AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 3;
177
178 const auto mixer = AudioMixerImpl::Create();
179
180 MockMixerAudioSource participants[kAudioSources];
181
182 for (int i = 0; i < kAudioSources; ++i) {
183 ResetFrame(participants[i].fake_frame());
184
185 // We set the 80-th sample value since the first 80 samples may be
186 // modified by a ramped-in window.
187 participants[i].fake_frame()->mutable_data()[80] = i;
188
189 EXPECT_TRUE(mixer->AddSource(&participants[i]));
190 EXPECT_CALL(participants[i], GetAudioFrameWithInfo(_, _)).Times(Exactly(1));
191 }
192
193 // Last participant gives audio frame with passive VAD, although it has the
194 // largest energy.
195 participants[kAudioSources - 1].fake_frame()->vad_activity_ =
196 AudioFrame::kVadPassive;
197
198 AudioFrame audio_frame;
199 mixer->Mix(1, // number of channels
200 &audio_frame);
201
202 for (int i = 0; i < kAudioSources; ++i) {
203 bool is_mixed =
204 mixer->GetAudioSourceMixabilityStatusForTest(&participants[i]);
205 if (i == kAudioSources - 1 ||
206 i < kAudioSources - 1 -
207 AudioMixerImpl::kDefaultNumberOfMixedAudioSources) {
208 EXPECT_FALSE(is_mixed)
209 << "Mixing status of AudioSource #" << i << " wrong.";
210 } else {
211 EXPECT_TRUE(is_mixed)
212 << "Mixing status of AudioSource #" << i << " wrong.";
213 }
214 }
215 }
216
TEST(AudioMixer,FrameNotModifiedForSingleParticipant)217 TEST(AudioMixer, FrameNotModifiedForSingleParticipant) {
218 const auto mixer = AudioMixerImpl::Create();
219
220 MockMixerAudioSource participant;
221
222 ResetFrame(participant.fake_frame());
223 const size_t n_samples = participant.fake_frame()->samples_per_channel_;
224
225 // Modify the frame so that it's not zero.
226 int16_t* fake_frame_data = participant.fake_frame()->mutable_data();
227 for (size_t j = 0; j < n_samples; ++j) {
228 fake_frame_data[j] = static_cast<int16_t>(j);
229 }
230
231 EXPECT_TRUE(mixer->AddSource(&participant));
232 EXPECT_CALL(participant, GetAudioFrameWithInfo(_, _)).Times(Exactly(2));
233
234 AudioFrame audio_frame;
235 // Two mix iteration to compare after the ramp-up step.
236 for (int i = 0; i < 2; ++i) {
237 mixer->Mix(1, // number of channels
238 &audio_frame);
239 }
240
241 EXPECT_EQ(0, memcmp(participant.fake_frame()->data(), audio_frame.data(),
242 n_samples));
243 }
244
TEST(AudioMixer,SourceAtNativeRateShouldNeverResample)245 TEST(AudioMixer, SourceAtNativeRateShouldNeverResample) {
246 const auto mixer = AudioMixerImpl::Create();
247
248 MockMixerAudioSource audio_source;
249 ResetFrame(audio_source.fake_frame());
250
251 mixer->AddSource(&audio_source);
252
253 for (auto frequency : {8000, 16000, 32000, 48000}) {
254 EXPECT_CALL(audio_source, GetAudioFrameWithInfo(frequency, _))
255 .Times(Exactly(1));
256
257 MixMonoAtGivenNativeRate(frequency, &frame_for_mixing, mixer,
258 &audio_source);
259 }
260 }
261
TEST(AudioMixer,MixerShouldMixAtNativeSourceRate)262 TEST(AudioMixer, MixerShouldMixAtNativeSourceRate) {
263 const auto mixer = AudioMixerImpl::Create();
264
265 MockMixerAudioSource audio_source;
266 ResetFrame(audio_source.fake_frame());
267
268 mixer->AddSource(&audio_source);
269
270 for (auto frequency : {8000, 16000, 32000, 48000}) {
271 MixMonoAtGivenNativeRate(frequency, &frame_for_mixing, mixer,
272 &audio_source);
273
274 EXPECT_EQ(frequency, frame_for_mixing.sample_rate_hz_);
275 }
276 }
277
TEST(AudioMixer,MixerShouldAlwaysMixAtNativeRate)278 TEST(AudioMixer, MixerShouldAlwaysMixAtNativeRate) {
279 const auto mixer = AudioMixerImpl::Create();
280
281 MockMixerAudioSource participant;
282 ResetFrame(participant.fake_frame());
283 mixer->AddSource(&participant);
284
285 const int needed_frequency = 44100;
286 ON_CALL(participant, PreferredSampleRate())
287 .WillByDefault(Return(needed_frequency));
288
289 // We expect mixing frequency to be native and >= needed_frequency.
290 const int expected_mix_frequency = 48000;
291 EXPECT_CALL(participant, GetAudioFrameWithInfo(expected_mix_frequency, _))
292 .Times(Exactly(1));
293 participant.fake_frame()->sample_rate_hz_ = expected_mix_frequency;
294 participant.fake_frame()->samples_per_channel_ = expected_mix_frequency / 100;
295
296 mixer->Mix(1, &frame_for_mixing);
297
298 EXPECT_EQ(48000, frame_for_mixing.sample_rate_hz_);
299 }
300
301 // Check that the mixing rate is always >= participants preferred rate.
TEST(AudioMixer,ShouldNotCauseQualityLossForMultipleSources)302 TEST(AudioMixer, ShouldNotCauseQualityLossForMultipleSources) {
303 const auto mixer = AudioMixerImpl::Create();
304
305 std::vector<MockMixerAudioSource> audio_sources(2);
306 const std::vector<int> source_sample_rates = {8000, 16000};
307 for (int i = 0; i < 2; ++i) {
308 auto& source = audio_sources[i];
309 ResetFrame(source.fake_frame());
310 mixer->AddSource(&source);
311 const auto sample_rate = source_sample_rates[i];
312 EXPECT_CALL(source, PreferredSampleRate()).WillOnce(Return(sample_rate));
313
314 EXPECT_CALL(source, GetAudioFrameWithInfo(::testing::Ge(sample_rate), _));
315 }
316 mixer->Mix(1, &frame_for_mixing);
317 }
318
TEST(AudioMixer,ParticipantNumberOfChannels)319 TEST(AudioMixer, ParticipantNumberOfChannels) {
320 const auto mixer = AudioMixerImpl::Create();
321
322 MockMixerAudioSource participant;
323 ResetFrame(participant.fake_frame());
324
325 EXPECT_TRUE(mixer->AddSource(&participant));
326 for (size_t number_of_channels : {1, 2}) {
327 EXPECT_CALL(participant, GetAudioFrameWithInfo(kDefaultSampleRateHz, _))
328 .Times(Exactly(1));
329 mixer->Mix(number_of_channels, &frame_for_mixing);
330 EXPECT_EQ(number_of_channels, frame_for_mixing.num_channels_);
331 }
332 }
333
334 // Maximal amount of participants are mixed one iteration, then
335 // another participant with higher energy is added.
TEST(AudioMixer,RampedOutSourcesShouldNotBeMarkedMixed)336 TEST(AudioMixer, RampedOutSourcesShouldNotBeMarkedMixed) {
337 constexpr int kAudioSources =
338 AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1;
339
340 const auto mixer = AudioMixerImpl::Create();
341 MockMixerAudioSource participants[kAudioSources];
342
343 for (int i = 0; i < kAudioSources; ++i) {
344 ResetFrame(participants[i].fake_frame());
345 // Set the participant audio energy to increase with the index
346 // `i`.
347 participants[i].fake_frame()->mutable_data()[0] = 100 * i;
348 }
349
350 // Add all participants but the loudest for mixing.
351 for (int i = 0; i < kAudioSources - 1; ++i) {
352 EXPECT_TRUE(mixer->AddSource(&participants[i]));
353 EXPECT_CALL(participants[i], GetAudioFrameWithInfo(kDefaultSampleRateHz, _))
354 .Times(Exactly(1));
355 }
356
357 // First mixer iteration
358 mixer->Mix(1, &frame_for_mixing);
359
360 // All participants but the loudest should have been mixed.
361 for (int i = 0; i < kAudioSources - 1; ++i) {
362 EXPECT_TRUE(mixer->GetAudioSourceMixabilityStatusForTest(&participants[i]))
363 << "Mixed status of AudioSource #" << i << " wrong.";
364 }
365
366 // Add new participant with higher energy.
367 EXPECT_TRUE(mixer->AddSource(&participants[kAudioSources - 1]));
368 for (int i = 0; i < kAudioSources; ++i) {
369 EXPECT_CALL(participants[i], GetAudioFrameWithInfo(kDefaultSampleRateHz, _))
370 .Times(Exactly(1));
371 }
372
373 mixer->Mix(1, &frame_for_mixing);
374
375 // The most quiet participant should not have been mixed.
376 EXPECT_FALSE(mixer->GetAudioSourceMixabilityStatusForTest(&participants[0]))
377 << "Mixed status of AudioSource #0 wrong.";
378
379 // The loudest participants should have been mixed.
380 for (int i = 1; i < kAudioSources; ++i) {
381 EXPECT_EQ(true,
382 mixer->GetAudioSourceMixabilityStatusForTest(&participants[i]))
383 << "Mixed status of AudioSource #" << i << " wrong.";
384 }
385 }
386
387 // This test checks that the initialization and participant addition
388 // can be done on a different thread.
TEST(AudioMixer,ConstructFromOtherThread)389 TEST(AudioMixer, ConstructFromOtherThread) {
390 TaskQueueForTest init_queue("init");
391 rtc::scoped_refptr<AudioMixer> mixer;
392 init_queue.SendTask([&mixer]() { mixer = AudioMixerImpl::Create(); });
393
394 MockMixerAudioSource participant;
395 EXPECT_CALL(participant, PreferredSampleRate())
396 .WillRepeatedly(Return(kDefaultSampleRateHz));
397
398 ResetFrame(participant.fake_frame());
399
400 TaskQueueForTest participant_queue("participant");
401 participant_queue.SendTask(
402 [&mixer, &participant]() { mixer->AddSource(&participant); });
403
404 EXPECT_CALL(participant, GetAudioFrameWithInfo(kDefaultSampleRateHz, _))
405 .Times(Exactly(1));
406
407 // Do one mixer iteration
408 mixer->Mix(1, &frame_for_mixing);
409 }
410
TEST(AudioMixer,MutedShouldMixAfterUnmuted)411 TEST(AudioMixer, MutedShouldMixAfterUnmuted) {
412 constexpr int kAudioSources =
413 AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1;
414
415 std::vector<AudioFrame> frames(kAudioSources);
416 for (auto& frame : frames) {
417 ResetFrame(&frame);
418 }
419
420 std::vector<AudioMixer::Source::AudioFrameInfo> frame_info(
421 kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal);
422 frame_info[0] = AudioMixer::Source::AudioFrameInfo::kMuted;
423 std::vector<bool> expected_status(kAudioSources, true);
424 expected_status[0] = false;
425
426 MixAndCompare(frames, frame_info, expected_status);
427 }
428
TEST(AudioMixer,PassiveShouldMixAfterNormal)429 TEST(AudioMixer, PassiveShouldMixAfterNormal) {
430 constexpr int kAudioSources =
431 AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1;
432
433 std::vector<AudioFrame> frames(kAudioSources);
434 for (auto& frame : frames) {
435 ResetFrame(&frame);
436 }
437
438 std::vector<AudioMixer::Source::AudioFrameInfo> frame_info(
439 kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal);
440 frames[0].vad_activity_ = AudioFrame::kVadPassive;
441 std::vector<bool> expected_status(kAudioSources, true);
442 expected_status[0] = false;
443
444 MixAndCompare(frames, frame_info, expected_status);
445 }
446
TEST(AudioMixer,ActiveShouldMixBeforeLoud)447 TEST(AudioMixer, ActiveShouldMixBeforeLoud) {
448 constexpr int kAudioSources =
449 AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1;
450
451 std::vector<AudioFrame> frames(kAudioSources);
452 for (auto& frame : frames) {
453 ResetFrame(&frame);
454 }
455
456 std::vector<AudioMixer::Source::AudioFrameInfo> frame_info(
457 kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal);
458 frames[0].vad_activity_ = AudioFrame::kVadPassive;
459 int16_t* frame_data = frames[0].mutable_data();
460 std::fill(frame_data, frame_data + kDefaultSampleRateHz / 100,
461 std::numeric_limits<int16_t>::max());
462 std::vector<bool> expected_status(kAudioSources, true);
463 expected_status[0] = false;
464
465 MixAndCompare(frames, frame_info, expected_status);
466 }
467
TEST(AudioMixer,ShouldMixUpToSpecifiedNumberOfSourcesToMix)468 TEST(AudioMixer, ShouldMixUpToSpecifiedNumberOfSourcesToMix) {
469 constexpr int kAudioSources = 5;
470 constexpr int kSourcesToMix = 2;
471
472 std::vector<AudioFrame> frames(kAudioSources);
473 for (auto& frame : frames) {
474 ResetFrame(&frame);
475 }
476
477 std::vector<AudioMixer::Source::AudioFrameInfo> frame_info(
478 kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal);
479 // Set up to kSourceToMix sources with kVadActive so that they're mixed.
480 const std::vector<AudioFrame::VADActivity> kVadActivities = {
481 AudioFrame::kVadUnknown, AudioFrame::kVadPassive, AudioFrame::kVadPassive,
482 AudioFrame::kVadActive, AudioFrame::kVadActive};
483 // Populate VAD and frame for all sources.
484 for (int i = 0; i < kAudioSources; i++) {
485 frames[i].vad_activity_ = kVadActivities[i];
486 }
487
488 std::vector<MockMixerAudioSource> participants(kAudioSources);
489 for (int i = 0; i < kAudioSources; ++i) {
490 participants[i].fake_frame()->CopyFrom(frames[i]);
491 participants[i].set_fake_info(frame_info[i]);
492 }
493
494 const auto mixer = AudioMixerImpl::Create(kSourcesToMix);
495 for (int i = 0; i < kAudioSources; ++i) {
496 EXPECT_TRUE(mixer->AddSource(&participants[i]));
497 EXPECT_CALL(participants[i], GetAudioFrameWithInfo(kDefaultSampleRateHz, _))
498 .Times(Exactly(1));
499 }
500
501 mixer->Mix(1, &frame_for_mixing);
502
503 std::vector<bool> expected_status = {false, false, false, true, true};
504 for (int i = 0; i < kAudioSources; ++i) {
505 EXPECT_EQ(expected_status[i],
506 mixer->GetAudioSourceMixabilityStatusForTest(&participants[i]))
507 << "Wrong mix status for source #" << i << " is wrong";
508 }
509 }
510
TEST(AudioMixer,UnmutedShouldMixBeforeLoud)511 TEST(AudioMixer, UnmutedShouldMixBeforeLoud) {
512 constexpr int kAudioSources =
513 AudioMixerImpl::kDefaultNumberOfMixedAudioSources + 1;
514
515 std::vector<AudioFrame> frames(kAudioSources);
516 for (auto& frame : frames) {
517 ResetFrame(&frame);
518 }
519
520 std::vector<AudioMixer::Source::AudioFrameInfo> frame_info(
521 kAudioSources, AudioMixer::Source::AudioFrameInfo::kNormal);
522 frame_info[0] = AudioMixer::Source::AudioFrameInfo::kMuted;
523 int16_t* frame_data = frames[0].mutable_data();
524 std::fill(frame_data, frame_data + kDefaultSampleRateHz / 100,
525 std::numeric_limits<int16_t>::max());
526 std::vector<bool> expected_status(kAudioSources, true);
527 expected_status[0] = false;
528
529 MixAndCompare(frames, frame_info, expected_status);
530 }
531
TEST(AudioMixer,MixingRateShouldBeDecidedByRateCalculator)532 TEST(AudioMixer, MixingRateShouldBeDecidedByRateCalculator) {
533 constexpr int kOutputRate = 22000;
534 const auto mixer =
535 AudioMixerImpl::Create(std::unique_ptr<OutputRateCalculator>(
536 new CustomRateCalculator(kOutputRate)),
537 true);
538 MockMixerAudioSource audio_source;
539 mixer->AddSource(&audio_source);
540 ResetFrame(audio_source.fake_frame());
541
542 EXPECT_CALL(audio_source, GetAudioFrameWithInfo(kOutputRate, _))
543 .Times(Exactly(1));
544
545 mixer->Mix(1, &frame_for_mixing);
546 }
547
TEST(AudioMixer,ZeroSourceRateShouldBeDecidedByRateCalculator)548 TEST(AudioMixer, ZeroSourceRateShouldBeDecidedByRateCalculator) {
549 constexpr int kOutputRate = 8000;
550 const auto mixer =
551 AudioMixerImpl::Create(std::unique_ptr<OutputRateCalculator>(
552 new CustomRateCalculator(kOutputRate)),
553 true);
554
555 mixer->Mix(1, &frame_for_mixing);
556
557 EXPECT_EQ(kOutputRate, frame_for_mixing.sample_rate_hz_);
558 }
559
TEST(AudioMixer,NoLimiterBasicApiCalls)560 TEST(AudioMixer, NoLimiterBasicApiCalls) {
561 const auto mixer = AudioMixerImpl::Create(
562 std::unique_ptr<OutputRateCalculator>(new DefaultOutputRateCalculator()),
563 false);
564 mixer->Mix(1, &frame_for_mixing);
565 }
566
TEST(AudioMixer,AnyRateIsPossibleWithNoLimiter)567 TEST(AudioMixer, AnyRateIsPossibleWithNoLimiter) {
568 // No APM limiter means no AudioProcessing::NativeRate restriction
569 // on mixing rate. The rate has to be divisible by 100 since we use
570 // 10 ms frames, though.
571 for (const auto rate : {8000, 20000, 24000, 32000, 44100}) {
572 for (const size_t number_of_channels : {1, 2}) {
573 for (const auto number_of_sources : {0, 1, 2, 3, 4}) {
574 SCOPED_TRACE(
575 ProduceDebugText(rate, number_of_sources, number_of_sources));
576 const auto mixer =
577 AudioMixerImpl::Create(std::unique_ptr<OutputRateCalculator>(
578 new CustomRateCalculator(rate)),
579 false);
580
581 std::vector<MockMixerAudioSource> sources(number_of_sources);
582 for (auto& source : sources) {
583 ResetFrame(source.fake_frame());
584 mixer->AddSource(&source);
585 }
586
587 mixer->Mix(number_of_channels, &frame_for_mixing);
588 EXPECT_EQ(rate, frame_for_mixing.sample_rate_hz_);
589 EXPECT_EQ(number_of_channels, frame_for_mixing.num_channels_);
590 }
591 }
592 }
593 }
594
TEST(AudioMixer,MultipleChannelsOneParticipant)595 TEST(AudioMixer, MultipleChannelsOneParticipant) {
596 // Set up a participant with a 6-channel frame, and make sure a 6-channel
597 // frame with the right sample values comes out from the mixer. There are 2
598 // Mix calls because of ramp-up.
599 constexpr size_t kNumberOfChannels = 6;
600 MockMixerAudioSource source;
601 ResetFrame(source.fake_frame());
602 const auto mixer = AudioMixerImpl::Create();
603 mixer->AddSource(&source);
604 mixer->Mix(1, &frame_for_mixing);
605 auto* frame = source.fake_frame();
606 frame->num_channels_ = kNumberOfChannels;
607 std::fill(frame->mutable_data(),
608 frame->mutable_data() + AudioFrame::kMaxDataSizeSamples, 0);
609 for (size_t i = 0; i < kNumberOfChannels; ++i) {
610 frame->mutable_data()[100 * frame->num_channels_ + i] = 1000 * i;
611 }
612
613 mixer->Mix(kNumberOfChannels, &frame_for_mixing);
614
615 EXPECT_EQ(frame_for_mixing.num_channels_, kNumberOfChannels);
616 for (size_t i = 0; i < kNumberOfChannels; ++i) {
617 EXPECT_EQ(frame_for_mixing.data()[100 * frame_for_mixing.num_channels_ + i],
618 static_cast<int16_t>(1000 * i));
619 }
620 }
621
TEST(AudioMixer,MultipleChannelsManyParticipants)622 TEST(AudioMixer, MultipleChannelsManyParticipants) {
623 // Sets up 2 participants. One has a 6-channel frame. Make sure a 6-channel
624 // frame with the right sample values comes out from the mixer. There are 2
625 // Mix calls because of ramp-up.
626 constexpr size_t kNumberOfChannels = 6;
627 MockMixerAudioSource source;
628 const auto mixer = AudioMixerImpl::Create();
629 mixer->AddSource(&source);
630 ResetFrame(source.fake_frame());
631 mixer->Mix(1, &frame_for_mixing);
632 auto* frame = source.fake_frame();
633 frame->num_channels_ = kNumberOfChannels;
634 std::fill(frame->mutable_data(),
635 frame->mutable_data() + AudioFrame::kMaxDataSizeSamples, 0);
636 for (size_t i = 0; i < kNumberOfChannels; ++i) {
637 frame->mutable_data()[100 * frame->num_channels_ + i] = 1000 * i;
638 }
639 MockMixerAudioSource other_source;
640 ResetFrame(other_source.fake_frame());
641 mixer->AddSource(&other_source);
642
643 mixer->Mix(kNumberOfChannels, &frame_for_mixing);
644
645 EXPECT_EQ(frame_for_mixing.num_channels_, kNumberOfChannels);
646 for (size_t i = 0; i < kNumberOfChannels; ++i) {
647 EXPECT_EQ(frame_for_mixing.data()[100 * frame_for_mixing.num_channels_ + i],
648 static_cast<int16_t>(1000 * i));
649 }
650 }
651
TEST(AudioMixer,ShouldIncludeRtpPacketInfoFromAllMixedSources)652 TEST(AudioMixer, ShouldIncludeRtpPacketInfoFromAllMixedSources) {
653 const uint32_t kSsrc0 = 10;
654 const uint32_t kSsrc1 = 11;
655 const uint32_t kSsrc2 = 12;
656 const uint32_t kCsrc0 = 20;
657 const uint32_t kCsrc1 = 21;
658 const uint32_t kCsrc2 = 22;
659 const uint32_t kCsrc3 = 23;
660 const int kAudioLevel0 = 10;
661 const int kAudioLevel1 = 40;
662 const absl::optional<uint32_t> kAudioLevel2 = absl::nullopt;
663 const uint32_t kRtpTimestamp0 = 300;
664 const uint32_t kRtpTimestamp1 = 400;
665 const Timestamp kReceiveTime0 = Timestamp::Millis(10);
666 const Timestamp kReceiveTime1 = Timestamp::Millis(20);
667
668 RtpPacketInfo p0(kSsrc0, {kCsrc0, kCsrc1}, kRtpTimestamp0, kReceiveTime0);
669 p0.set_audio_level(kAudioLevel0);
670 RtpPacketInfo p1(kSsrc1, {kCsrc2}, kRtpTimestamp1, kReceiveTime1);
671 p1.set_audio_level(kAudioLevel1);
672 RtpPacketInfo p2(kSsrc2, {kCsrc3}, kRtpTimestamp1, kReceiveTime1);
673 p2.set_audio_level(kAudioLevel2);
674
675 const auto mixer = AudioMixerImpl::Create();
676
677 MockMixerAudioSource source;
678 source.set_packet_infos(RtpPacketInfos({p0}));
679 mixer->AddSource(&source);
680 ResetFrame(source.fake_frame());
681 mixer->Mix(1, &frame_for_mixing);
682
683 MockMixerAudioSource other_source;
684 other_source.set_packet_infos(RtpPacketInfos({p1, p2}));
685 ResetFrame(other_source.fake_frame());
686 mixer->AddSource(&other_source);
687
688 mixer->Mix(/*number_of_channels=*/1, &frame_for_mixing);
689
690 EXPECT_THAT(frame_for_mixing.packet_infos_, UnorderedElementsAre(p0, p1, p2));
691 }
692
TEST(AudioMixer,MixerShouldIncludeRtpPacketInfoFromMixedSourcesOnly)693 TEST(AudioMixer, MixerShouldIncludeRtpPacketInfoFromMixedSourcesOnly) {
694 const uint32_t kSsrc0 = 10;
695 const uint32_t kSsrc1 = 11;
696 const uint32_t kSsrc2 = 21;
697 const uint32_t kCsrc0 = 30;
698 const uint32_t kCsrc1 = 31;
699 const uint32_t kCsrc2 = 32;
700 const uint32_t kCsrc3 = 33;
701 const int kAudioLevel0 = 10;
702 const absl::optional<uint32_t> kAudioLevelMissing = absl::nullopt;
703 const uint32_t kRtpTimestamp0 = 300;
704 const uint32_t kRtpTimestamp1 = 400;
705 const Timestamp kReceiveTime0 = Timestamp::Millis(10);
706 const Timestamp kReceiveTime1 = Timestamp::Millis(20);
707
708 RtpPacketInfo p0(kSsrc0, {kCsrc0, kCsrc1}, kRtpTimestamp0, kReceiveTime0);
709 p0.set_audio_level(kAudioLevel0);
710 RtpPacketInfo p1(kSsrc1, {kCsrc2}, kRtpTimestamp1, kReceiveTime1);
711 p1.set_audio_level(kAudioLevelMissing);
712 RtpPacketInfo p2(kSsrc2, {kCsrc3}, kRtpTimestamp1, kReceiveTime1);
713 p2.set_audio_level(kAudioLevelMissing);
714
715 const auto mixer = AudioMixerImpl::Create(/*max_sources_to_mix=*/2);
716
717 MockMixerAudioSource source1;
718 source1.set_packet_infos(RtpPacketInfos({p0}));
719 mixer->AddSource(&source1);
720 ResetFrame(source1.fake_frame());
721 mixer->Mix(1, &frame_for_mixing);
722
723 MockMixerAudioSource source2;
724 source2.set_packet_infos(RtpPacketInfos({p1}));
725 ResetFrame(source2.fake_frame());
726 mixer->AddSource(&source2);
727
728 // The mixer prioritizes kVadActive over kVadPassive.
729 // We limit the number of sources to mix to 2 and set the third source's VAD
730 // activity to kVadPassive so that it will not be added to the mix.
731 MockMixerAudioSource source3;
732 source3.set_packet_infos(RtpPacketInfos({p2}));
733 ResetFrame(source3.fake_frame());
734 source3.fake_frame()->vad_activity_ = AudioFrame::kVadPassive;
735 mixer->AddSource(&source3);
736
737 mixer->Mix(/*number_of_channels=*/1, &frame_for_mixing);
738
739 EXPECT_THAT(frame_for_mixing.packet_infos_, UnorderedElementsAre(p0, p1));
740 }
741
742 class HighOutputRateCalculator : public OutputRateCalculator {
743 public:
744 static const int kDefaultFrequency = 76000;
CalculateOutputRateFromRange(rtc::ArrayView<const int> preferred_sample_rates)745 int CalculateOutputRateFromRange(
746 rtc::ArrayView<const int> preferred_sample_rates) override {
747 return kDefaultFrequency;
748 }
~HighOutputRateCalculator()749 ~HighOutputRateCalculator() override {}
750 };
751 const int HighOutputRateCalculator::kDefaultFrequency;
752
TEST(AudioMixerDeathTest,MultipleChannelsAndHighRate)753 TEST(AudioMixerDeathTest, MultipleChannelsAndHighRate) {
754 constexpr size_t kSamplesPerChannel =
755 HighOutputRateCalculator::kDefaultFrequency / 100;
756 // As many channels as an AudioFrame can fit:
757 constexpr size_t kNumberOfChannels =
758 AudioFrame::kMaxDataSizeSamples / kSamplesPerChannel;
759 MockMixerAudioSource source;
760 const auto mixer = AudioMixerImpl::Create(
761 std::make_unique<HighOutputRateCalculator>(), true);
762 mixer->AddSource(&source);
763 ResetFrame(source.fake_frame());
764 mixer->Mix(1, &frame_for_mixing);
765 auto* frame = source.fake_frame();
766 frame->num_channels_ = kNumberOfChannels;
767 frame->sample_rate_hz_ = HighOutputRateCalculator::kDefaultFrequency;
768 frame->samples_per_channel_ = kSamplesPerChannel;
769
770 std::fill(frame->mutable_data(),
771 frame->mutable_data() + AudioFrame::kMaxDataSizeSamples, 0);
772 MockMixerAudioSource other_source;
773 ResetFrame(other_source.fake_frame());
774 auto* other_frame = other_source.fake_frame();
775 other_frame->num_channels_ = kNumberOfChannels;
776 other_frame->sample_rate_hz_ = HighOutputRateCalculator::kDefaultFrequency;
777 other_frame->samples_per_channel_ = kSamplesPerChannel;
778 mixer->AddSource(&other_source);
779
780 #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)
781 EXPECT_DEATH(mixer->Mix(kNumberOfChannels, &frame_for_mixing), "");
782 #elif !RTC_DCHECK_IS_ON
783 mixer->Mix(kNumberOfChannels, &frame_for_mixing);
784 EXPECT_EQ(frame_for_mixing.num_channels_, kNumberOfChannels);
785 EXPECT_EQ(frame_for_mixing.sample_rate_hz_,
786 HighOutputRateCalculator::kDefaultFrequency);
787 #endif
788 }
789
790 } // namespace webrtc
791