1 /* 2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef MODULES_AUDIO_PROCESSING_AEC3_SUBTRACTOR_H_ 12 #define MODULES_AUDIO_PROCESSING_AEC3_SUBTRACTOR_H_ 13 14 #include <math.h> 15 #include <stddef.h> 16 17 #include <array> 18 #include <vector> 19 20 #include "api/array_view.h" 21 #include "api/audio/echo_canceller3_config.h" 22 #include "modules/audio_processing/aec3/adaptive_fir_filter.h" 23 #include "modules/audio_processing/aec3/aec3_common.h" 24 #include "modules/audio_processing/aec3/aec3_fft.h" 25 #include "modules/audio_processing/aec3/aec_state.h" 26 #include "modules/audio_processing/aec3/block.h" 27 #include "modules/audio_processing/aec3/coarse_filter_update_gain.h" 28 #include "modules/audio_processing/aec3/echo_path_variability.h" 29 #include "modules/audio_processing/aec3/refined_filter_update_gain.h" 30 #include "modules/audio_processing/aec3/render_buffer.h" 31 #include "modules/audio_processing/aec3/render_signal_analyzer.h" 32 #include "modules/audio_processing/aec3/subtractor_output.h" 33 #include "modules/audio_processing/logging/apm_data_dumper.h" 34 #include "rtc_base/checks.h" 35 36 namespace webrtc { 37 38 // Proves linear echo cancellation functionality 39 class Subtractor { 40 public: 41 Subtractor(const EchoCanceller3Config& config, 42 size_t num_render_channels, 43 size_t num_capture_channels, 44 ApmDataDumper* data_dumper, 45 Aec3Optimization optimization); 46 ~Subtractor(); 47 Subtractor(const Subtractor&) = delete; 48 Subtractor& operator=(const Subtractor&) = delete; 49 50 // Performs the echo subtraction. 51 void Process(const RenderBuffer& render_buffer, 52 const Block& capture, 53 const RenderSignalAnalyzer& render_signal_analyzer, 54 const AecState& aec_state, 55 rtc::ArrayView<SubtractorOutput> outputs); 56 57 void HandleEchoPathChange(const EchoPathVariability& echo_path_variability); 58 59 // Exits the initial state. 60 void ExitInitialState(); 61 62 // Returns the block-wise frequency responses for the refined adaptive 63 // filters. 64 const std::vector<std::vector<std::array<float, kFftLengthBy2Plus1>>>& FilterFrequencyResponses()65 FilterFrequencyResponses() const { 66 return refined_frequency_responses_; 67 } 68 69 // Returns the estimates of the impulse responses for the refined adaptive 70 // filters. FilterImpulseResponses()71 const std::vector<std::vector<float>>& FilterImpulseResponses() const { 72 return refined_impulse_responses_; 73 } 74 DumpFilters()75 void DumpFilters() { 76 data_dumper_->DumpRaw( 77 "aec3_subtractor_h_refined", 78 rtc::ArrayView<const float>( 79 refined_impulse_responses_[0].data(), 80 GetTimeDomainLength( 81 refined_filters_[0]->max_filter_size_partitions()))); 82 if (ApmDataDumper::IsAvailable()) { 83 RTC_DCHECK_GT(coarse_impulse_responses_.size(), 0); 84 data_dumper_->DumpRaw( 85 "aec3_subtractor_h_coarse", 86 rtc::ArrayView<const float>( 87 coarse_impulse_responses_[0].data(), 88 GetTimeDomainLength( 89 coarse_filter_[0]->max_filter_size_partitions()))); 90 } 91 92 refined_filters_[0]->DumpFilter("aec3_subtractor_H_refined"); 93 coarse_filter_[0]->DumpFilter("aec3_subtractor_H_coarse"); 94 } 95 96 private: 97 class FilterMisadjustmentEstimator { 98 public: 99 FilterMisadjustmentEstimator() = default; 100 ~FilterMisadjustmentEstimator() = default; 101 // Update the misadjustment estimator. 102 void Update(const SubtractorOutput& output); 103 // GetMisadjustment() Returns a recommended scale for the filter so the 104 // prediction error energy gets closer to the energy that is seen at the 105 // microphone input. GetMisadjustment()106 float GetMisadjustment() const { 107 RTC_DCHECK_GT(inv_misadjustment_, 0.0f); 108 // It is not aiming to adjust all the estimated mismatch. Instead, 109 // it adjusts half of that estimated mismatch. 110 return 2.f / sqrtf(inv_misadjustment_); 111 } 112 // Returns true if the prediciton error energy is significantly larger 113 // than the microphone signal energy and, therefore, an adjustment is 114 // recommended. IsAdjustmentNeeded()115 bool IsAdjustmentNeeded() const { return inv_misadjustment_ > 10.f; } 116 void Reset(); 117 void Dump(ApmDataDumper* data_dumper) const; 118 119 private: 120 const int n_blocks_ = 4; 121 int n_blocks_acum_ = 0; 122 float e2_acum_ = 0.f; 123 float y2_acum_ = 0.f; 124 float inv_misadjustment_ = 0.f; 125 int overhang_ = 0.f; 126 }; 127 128 const Aec3Fft fft_; 129 ApmDataDumper* data_dumper_; 130 const Aec3Optimization optimization_; 131 const EchoCanceller3Config config_; 132 const size_t num_capture_channels_; 133 const bool use_coarse_filter_reset_hangover_; 134 135 std::vector<std::unique_ptr<AdaptiveFirFilter>> refined_filters_; 136 std::vector<std::unique_ptr<AdaptiveFirFilter>> coarse_filter_; 137 std::vector<std::unique_ptr<RefinedFilterUpdateGain>> refined_gains_; 138 std::vector<std::unique_ptr<CoarseFilterUpdateGain>> coarse_gains_; 139 std::vector<FilterMisadjustmentEstimator> filter_misadjustment_estimators_; 140 std::vector<size_t> poor_coarse_filter_counters_; 141 std::vector<int> coarse_filter_reset_hangover_; 142 std::vector<std::vector<std::array<float, kFftLengthBy2Plus1>>> 143 refined_frequency_responses_; 144 std::vector<std::vector<float>> refined_impulse_responses_; 145 std::vector<std::vector<float>> coarse_impulse_responses_; 146 }; 147 148 } // namespace webrtc 149 150 #endif // MODULES_AUDIO_PROCESSING_AEC3_SUBTRACTOR_H_ 151