1 /*
2  *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 /*
12  * LEFT TO DO:
13  * - WRITE TESTS for the stuff in this  file.
14  * - Check the creation, maybe make it safer by returning an empty optional or
15  *   unique_ptr. --- It looks OK, but RecreateEncoderInstance can perhaps crash
16  *   on a valid config. Can run it in the fuzzer for some time. Should prbl also
17  *   fuzz the config.
18  */
19 
20 #include "modules/audio_coding/codecs/opus/audio_encoder_multi_channel_opus_impl.h"
21 
22 #include <algorithm>
23 #include <memory>
24 #include <string>
25 #include <vector>
26 
27 #include "absl/strings/match.h"
28 #include "modules/audio_coding/codecs/opus/audio_coder_opus_common.h"
29 #include "rtc_base/arraysize.h"
30 #include "rtc_base/checks.h"
31 #include "rtc_base/logging.h"
32 #include "rtc_base/string_to_number.h"
33 
34 namespace webrtc {
35 
36 namespace {
37 
38 // Recommended bitrates for one channel:
39 // 8-12 kb/s for NB speech,
40 // 16-20 kb/s for WB speech,
41 // 28-40 kb/s for FB speech,
42 // 48-64 kb/s for FB mono music, and
43 // 64-128 kb/s for FB stereo music.
44 // The current implementation multiplies these values by the number of channels.
45 constexpr int kOpusBitrateNbBps = 12000;
46 constexpr int kOpusBitrateWbBps = 20000;
47 constexpr int kOpusBitrateFbBps = 32000;
48 
49 constexpr int kDefaultMaxPlaybackRate = 48000;
50 // These two lists must be sorted from low to high
51 #if WEBRTC_OPUS_SUPPORT_120MS_PTIME
52 constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60, 120};
53 #else
54 constexpr int kOpusSupportedFrameLengths[] = {10, 20, 40, 60};
55 #endif
56 
GetBitrateBps(const AudioEncoderMultiChannelOpusConfig & config)57 int GetBitrateBps(const AudioEncoderMultiChannelOpusConfig& config) {
58   RTC_DCHECK(config.IsOk());
59   return config.bitrate_bps;
60 }
GetMaxPlaybackRate(const SdpAudioFormat & format)61 int GetMaxPlaybackRate(const SdpAudioFormat& format) {
62   const auto param = GetFormatParameter<int>(format, "maxplaybackrate");
63   if (param && *param >= 8000) {
64     return std::min(*param, kDefaultMaxPlaybackRate);
65   }
66   return kDefaultMaxPlaybackRate;
67 }
68 
GetFrameSizeMs(const SdpAudioFormat & format)69 int GetFrameSizeMs(const SdpAudioFormat& format) {
70   const auto ptime = GetFormatParameter<int>(format, "ptime");
71   if (ptime.has_value()) {
72     // Pick the next highest supported frame length from
73     // kOpusSupportedFrameLengths.
74     for (const int supported_frame_length : kOpusSupportedFrameLengths) {
75       if (supported_frame_length >= *ptime) {
76         return supported_frame_length;
77       }
78     }
79     // If none was found, return the largest supported frame length.
80     return *(std::end(kOpusSupportedFrameLengths) - 1);
81   }
82 
83   return AudioEncoderOpusConfig::kDefaultFrameSizeMs;
84 }
85 
CalculateDefaultBitrate(int max_playback_rate,size_t num_channels)86 int CalculateDefaultBitrate(int max_playback_rate, size_t num_channels) {
87   const int bitrate = [&] {
88     if (max_playback_rate <= 8000) {
89       return kOpusBitrateNbBps * rtc::dchecked_cast<int>(num_channels);
90     } else if (max_playback_rate <= 16000) {
91       return kOpusBitrateWbBps * rtc::dchecked_cast<int>(num_channels);
92     } else {
93       return kOpusBitrateFbBps * rtc::dchecked_cast<int>(num_channels);
94     }
95   }();
96   RTC_DCHECK_GE(bitrate, AudioEncoderMultiChannelOpusConfig::kMinBitrateBps);
97   return bitrate;
98 }
99 
100 // Get the maxaveragebitrate parameter in string-form, so we can properly figure
101 // out how invalid it is and accurately log invalid values.
CalculateBitrate(int max_playback_rate_hz,size_t num_channels,absl::optional<std::string> bitrate_param)102 int CalculateBitrate(int max_playback_rate_hz,
103                      size_t num_channels,
104                      absl::optional<std::string> bitrate_param) {
105   const int default_bitrate =
106       CalculateDefaultBitrate(max_playback_rate_hz, num_channels);
107 
108   if (bitrate_param) {
109     const auto bitrate = rtc::StringToNumber<int>(*bitrate_param);
110     if (bitrate) {
111       const int chosen_bitrate =
112           std::max(AudioEncoderOpusConfig::kMinBitrateBps,
113                    std::min(*bitrate, AudioEncoderOpusConfig::kMaxBitrateBps));
114       if (bitrate != chosen_bitrate) {
115         RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate " << *bitrate
116                             << " clamped to " << chosen_bitrate;
117       }
118       return chosen_bitrate;
119     }
120     RTC_LOG(LS_WARNING) << "Invalid maxaveragebitrate \"" << *bitrate_param
121                         << "\" replaced by default bitrate " << default_bitrate;
122   }
123 
124   return default_bitrate;
125 }
126 
127 }  // namespace
128 
129 std::unique_ptr<AudioEncoder>
MakeAudioEncoder(const AudioEncoderMultiChannelOpusConfig & config,int payload_type)130 AudioEncoderMultiChannelOpusImpl::MakeAudioEncoder(
131     const AudioEncoderMultiChannelOpusConfig& config,
132     int payload_type) {
133   if (!config.IsOk()) {
134     RTC_DCHECK_NOTREACHED();
135     return nullptr;
136   }
137   return std::make_unique<AudioEncoderMultiChannelOpusImpl>(config,
138                                                             payload_type);
139 }
140 
AudioEncoderMultiChannelOpusImpl(const AudioEncoderMultiChannelOpusConfig & config,int payload_type)141 AudioEncoderMultiChannelOpusImpl::AudioEncoderMultiChannelOpusImpl(
142     const AudioEncoderMultiChannelOpusConfig& config,
143     int payload_type)
144     : payload_type_(payload_type), inst_(nullptr) {
145   RTC_DCHECK(0 <= payload_type && payload_type <= 127);
146 
147   RTC_CHECK(RecreateEncoderInstance(config));
148 }
149 
~AudioEncoderMultiChannelOpusImpl()150 AudioEncoderMultiChannelOpusImpl::~AudioEncoderMultiChannelOpusImpl() {
151   RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_));
152 }
153 
SufficientOutputBufferSize() const154 size_t AudioEncoderMultiChannelOpusImpl::SufficientOutputBufferSize() const {
155   // Calculate the number of bytes we expect the encoder to produce,
156   // then multiply by two to give a wide margin for error.
157   const size_t bytes_per_millisecond =
158       static_cast<size_t>(GetBitrateBps(config_) / (1000 * 8) + 1);
159   const size_t approx_encoded_bytes =
160       Num10msFramesPerPacket() * 10 * bytes_per_millisecond;
161   return 2 * approx_encoded_bytes;
162 }
163 
Reset()164 void AudioEncoderMultiChannelOpusImpl::Reset() {
165   RTC_CHECK(RecreateEncoderInstance(config_));
166 }
167 
168 absl::optional<std::pair<TimeDelta, TimeDelta>>
GetFrameLengthRange() const169 AudioEncoderMultiChannelOpusImpl::GetFrameLengthRange() const {
170   return {{TimeDelta::Millis(config_.frame_size_ms),
171            TimeDelta::Millis(config_.frame_size_ms)}};
172 }
173 
174 // If the given config is OK, recreate the Opus encoder instance with those
175 // settings, save the config, and return true. Otherwise, do nothing and return
176 // false.
RecreateEncoderInstance(const AudioEncoderMultiChannelOpusConfig & config)177 bool AudioEncoderMultiChannelOpusImpl::RecreateEncoderInstance(
178     const AudioEncoderMultiChannelOpusConfig& config) {
179   if (!config.IsOk())
180     return false;
181   config_ = config;
182   if (inst_)
183     RTC_CHECK_EQ(0, WebRtcOpus_EncoderFree(inst_));
184   input_buffer_.clear();
185   input_buffer_.reserve(Num10msFramesPerPacket() * SamplesPer10msFrame());
186   RTC_CHECK_EQ(
187       0, WebRtcOpus_MultistreamEncoderCreate(
188              &inst_, config.num_channels,
189              config.application ==
190                      AudioEncoderMultiChannelOpusConfig::ApplicationMode::kVoip
191                  ? 0
192                  : 1,
193              config.num_streams, config.coupled_streams,
194              config.channel_mapping.data()));
195   const int bitrate = GetBitrateBps(config);
196   RTC_CHECK_EQ(0, WebRtcOpus_SetBitRate(inst_, bitrate));
197   RTC_LOG(LS_VERBOSE) << "Set Opus bitrate to " << bitrate << " bps.";
198   if (config.fec_enabled) {
199     RTC_CHECK_EQ(0, WebRtcOpus_EnableFec(inst_));
200     RTC_LOG(LS_VERBOSE) << "Opus enable FEC";
201   } else {
202     RTC_CHECK_EQ(0, WebRtcOpus_DisableFec(inst_));
203     RTC_LOG(LS_VERBOSE) << "Opus disable FEC";
204   }
205   RTC_CHECK_EQ(
206       0, WebRtcOpus_SetMaxPlaybackRate(inst_, config.max_playback_rate_hz));
207   RTC_LOG(LS_VERBOSE) << "Set Opus playback rate to "
208                       << config.max_playback_rate_hz << " hz.";
209 
210   // Use the DEFAULT complexity.
211   RTC_CHECK_EQ(
212       0, WebRtcOpus_SetComplexity(inst_, AudioEncoderOpusConfig().complexity));
213   RTC_LOG(LS_VERBOSE) << "Set Opus coding complexity to "
214                       << AudioEncoderOpusConfig().complexity;
215 
216   if (config.dtx_enabled) {
217     RTC_CHECK_EQ(0, WebRtcOpus_EnableDtx(inst_));
218     RTC_LOG(LS_VERBOSE) << "Opus enable DTX";
219   } else {
220     RTC_CHECK_EQ(0, WebRtcOpus_DisableDtx(inst_));
221     RTC_LOG(LS_VERBOSE) << "Opus disable DTX";
222   }
223 
224   if (config.cbr_enabled) {
225     RTC_CHECK_EQ(0, WebRtcOpus_EnableCbr(inst_));
226     RTC_LOG(LS_VERBOSE) << "Opus enable CBR";
227   } else {
228     RTC_CHECK_EQ(0, WebRtcOpus_DisableCbr(inst_));
229     RTC_LOG(LS_VERBOSE) << "Opus disable CBR";
230   }
231   num_channels_to_encode_ = NumChannels();
232   next_frame_length_ms_ = config_.frame_size_ms;
233   RTC_LOG(LS_VERBOSE) << "Set Opus frame length to " << config_.frame_size_ms
234                       << " ms";
235   return true;
236 }
237 
238 absl::optional<AudioEncoderMultiChannelOpusConfig>
SdpToConfig(const SdpAudioFormat & format)239 AudioEncoderMultiChannelOpusImpl::SdpToConfig(const SdpAudioFormat& format) {
240   if (!absl::EqualsIgnoreCase(format.name, "multiopus") ||
241       format.clockrate_hz != 48000) {
242     return absl::nullopt;
243   }
244 
245   AudioEncoderMultiChannelOpusConfig config;
246   config.num_channels = format.num_channels;
247   config.frame_size_ms = GetFrameSizeMs(format);
248   config.max_playback_rate_hz = GetMaxPlaybackRate(format);
249   config.fec_enabled = (GetFormatParameter(format, "useinbandfec") == "1");
250   config.dtx_enabled = (GetFormatParameter(format, "usedtx") == "1");
251   config.cbr_enabled = (GetFormatParameter(format, "cbr") == "1");
252   config.bitrate_bps =
253       CalculateBitrate(config.max_playback_rate_hz, config.num_channels,
254                        GetFormatParameter(format, "maxaveragebitrate"));
255   config.application =
256       config.num_channels == 1
257           ? AudioEncoderMultiChannelOpusConfig::ApplicationMode::kVoip
258           : AudioEncoderMultiChannelOpusConfig::ApplicationMode::kAudio;
259 
260   config.supported_frame_lengths_ms.clear();
261   std::copy(std::begin(kOpusSupportedFrameLengths),
262             std::end(kOpusSupportedFrameLengths),
263             std::back_inserter(config.supported_frame_lengths_ms));
264 
265   auto num_streams = GetFormatParameter<int>(format, "num_streams");
266   if (!num_streams.has_value()) {
267     return absl::nullopt;
268   }
269   config.num_streams = *num_streams;
270 
271   auto coupled_streams = GetFormatParameter<int>(format, "coupled_streams");
272   if (!coupled_streams.has_value()) {
273     return absl::nullopt;
274   }
275   config.coupled_streams = *coupled_streams;
276 
277   auto channel_mapping =
278       GetFormatParameter<std::vector<unsigned char>>(format, "channel_mapping");
279   if (!channel_mapping.has_value()) {
280     return absl::nullopt;
281   }
282   config.channel_mapping = *channel_mapping;
283 
284   if (!config.IsOk()) {
285     return absl::nullopt;
286   }
287   return config;
288 }
289 
QueryAudioEncoder(const AudioEncoderMultiChannelOpusConfig & config)290 AudioCodecInfo AudioEncoderMultiChannelOpusImpl::QueryAudioEncoder(
291     const AudioEncoderMultiChannelOpusConfig& config) {
292   RTC_DCHECK(config.IsOk());
293   AudioCodecInfo info(48000, config.num_channels, config.bitrate_bps,
294                       AudioEncoderOpusConfig::kMinBitrateBps,
295                       AudioEncoderOpusConfig::kMaxBitrateBps);
296   info.allow_comfort_noise = false;
297   info.supports_network_adaption = false;
298   return info;
299 }
300 
Num10msFramesPerPacket() const301 size_t AudioEncoderMultiChannelOpusImpl::Num10msFramesPerPacket() const {
302   return static_cast<size_t>(rtc::CheckedDivExact(config_.frame_size_ms, 10));
303 }
SamplesPer10msFrame() const304 size_t AudioEncoderMultiChannelOpusImpl::SamplesPer10msFrame() const {
305   return rtc::CheckedDivExact(48000, 100) * config_.num_channels;
306 }
SampleRateHz() const307 int AudioEncoderMultiChannelOpusImpl::SampleRateHz() const {
308   return 48000;
309 }
NumChannels() const310 size_t AudioEncoderMultiChannelOpusImpl::NumChannels() const {
311   return config_.num_channels;
312 }
Num10MsFramesInNextPacket() const313 size_t AudioEncoderMultiChannelOpusImpl::Num10MsFramesInNextPacket() const {
314   return Num10msFramesPerPacket();
315 }
Max10MsFramesInAPacket() const316 size_t AudioEncoderMultiChannelOpusImpl::Max10MsFramesInAPacket() const {
317   return Num10msFramesPerPacket();
318 }
GetTargetBitrate() const319 int AudioEncoderMultiChannelOpusImpl::GetTargetBitrate() const {
320   return GetBitrateBps(config_);
321 }
322 
EncodeImpl(uint32_t rtp_timestamp,rtc::ArrayView<const int16_t> audio,rtc::Buffer * encoded)323 AudioEncoder::EncodedInfo AudioEncoderMultiChannelOpusImpl::EncodeImpl(
324     uint32_t rtp_timestamp,
325     rtc::ArrayView<const int16_t> audio,
326     rtc::Buffer* encoded) {
327   if (input_buffer_.empty())
328     first_timestamp_in_buffer_ = rtp_timestamp;
329 
330   input_buffer_.insert(input_buffer_.end(), audio.cbegin(), audio.cend());
331   if (input_buffer_.size() <
332       (Num10msFramesPerPacket() * SamplesPer10msFrame())) {
333     return EncodedInfo();
334   }
335   RTC_CHECK_EQ(input_buffer_.size(),
336                Num10msFramesPerPacket() * SamplesPer10msFrame());
337 
338   const size_t max_encoded_bytes = SufficientOutputBufferSize();
339   EncodedInfo info;
340   info.encoded_bytes = encoded->AppendData(
341       max_encoded_bytes, [&](rtc::ArrayView<uint8_t> encoded) {
342         int status = WebRtcOpus_Encode(
343             inst_, &input_buffer_[0],
344             rtc::CheckedDivExact(input_buffer_.size(), config_.num_channels),
345             rtc::saturated_cast<int16_t>(max_encoded_bytes), encoded.data());
346 
347         RTC_CHECK_GE(status, 0);  // Fails only if fed invalid data.
348 
349         return static_cast<size_t>(status);
350       });
351   input_buffer_.clear();
352 
353   // Will use new packet size for next encoding.
354   config_.frame_size_ms = next_frame_length_ms_;
355 
356   info.encoded_timestamp = first_timestamp_in_buffer_;
357   info.payload_type = payload_type_;
358   info.send_even_if_empty = true;  // Allows Opus to send empty packets.
359 
360   info.speech = true;
361   info.encoder_type = CodecType::kOther;
362 
363   return info;
364 }
365 
366 }  // namespace webrtc
367