1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "sdk/android/src/jni/audio_device/opensles_recorder.h"
12
13 #include <android/log.h>
14
15 #include <memory>
16
17 #include "api/array_view.h"
18 #include "modules/audio_device/fine_audio_buffer.h"
19 #include "rtc_base/arraysize.h"
20 #include "rtc_base/checks.h"
21 #include "rtc_base/platform_thread.h"
22 #include "rtc_base/time_utils.h"
23 #include "sdk/android/src/jni/audio_device/audio_common.h"
24
25 #define TAG "OpenSLESRecorder"
26 #define ALOGV(...) __android_log_print(ANDROID_LOG_VERBOSE, TAG, __VA_ARGS__)
27 #define ALOGD(...) __android_log_print(ANDROID_LOG_DEBUG, TAG, __VA_ARGS__)
28 #define ALOGE(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__)
29 #define ALOGW(...) __android_log_print(ANDROID_LOG_WARN, TAG, __VA_ARGS__)
30 #define ALOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__)
31
32 #define LOG_ON_ERROR(op) \
33 [](SLresult err) { \
34 if (err != SL_RESULT_SUCCESS) { \
35 ALOGE("%s:%d %s failed: %s", __FILE__, __LINE__, #op, \
36 GetSLErrorString(err)); \
37 return true; \
38 } \
39 return false; \
40 }(op)
41
42 namespace webrtc {
43
44 namespace jni {
45
OpenSLESRecorder(const AudioParameters & audio_parameters,rtc::scoped_refptr<OpenSLEngineManager> engine_manager)46 OpenSLESRecorder::OpenSLESRecorder(
47 const AudioParameters& audio_parameters,
48 rtc::scoped_refptr<OpenSLEngineManager> engine_manager)
49 : audio_parameters_(audio_parameters),
50 audio_device_buffer_(nullptr),
51 initialized_(false),
52 recording_(false),
53 engine_manager_(std::move(engine_manager)),
54 engine_(nullptr),
55 recorder_(nullptr),
56 simple_buffer_queue_(nullptr),
57 buffer_index_(0),
58 last_rec_time_(0) {
59 ALOGD("ctor[tid=%d]", rtc::CurrentThreadId());
60 // Detach from this thread since we want to use the checker to verify calls
61 // from the internal audio thread.
62 thread_checker_opensles_.Detach();
63 // Use native audio output parameters provided by the audio manager and
64 // define the PCM format structure.
65 pcm_format_ = CreatePCMConfiguration(audio_parameters_.channels(),
66 audio_parameters_.sample_rate(),
67 audio_parameters_.bits_per_sample());
68 }
69
~OpenSLESRecorder()70 OpenSLESRecorder::~OpenSLESRecorder() {
71 ALOGD("dtor[tid=%d]", rtc::CurrentThreadId());
72 RTC_DCHECK(thread_checker_.IsCurrent());
73 Terminate();
74 DestroyAudioRecorder();
75 engine_ = nullptr;
76 RTC_DCHECK(!engine_);
77 RTC_DCHECK(!recorder_);
78 RTC_DCHECK(!simple_buffer_queue_);
79 }
80
Init()81 int OpenSLESRecorder::Init() {
82 ALOGD("Init[tid=%d]", rtc::CurrentThreadId());
83 RTC_DCHECK(thread_checker_.IsCurrent());
84 if (audio_parameters_.channels() == 2) {
85 ALOGD("Stereo mode is enabled");
86 }
87 return 0;
88 }
89
Terminate()90 int OpenSLESRecorder::Terminate() {
91 ALOGD("Terminate[tid=%d]", rtc::CurrentThreadId());
92 RTC_DCHECK(thread_checker_.IsCurrent());
93 StopRecording();
94 return 0;
95 }
96
InitRecording()97 int OpenSLESRecorder::InitRecording() {
98 ALOGD("InitRecording[tid=%d]", rtc::CurrentThreadId());
99 RTC_DCHECK(thread_checker_.IsCurrent());
100 RTC_DCHECK(!initialized_);
101 RTC_DCHECK(!recording_);
102 if (!ObtainEngineInterface()) {
103 ALOGE("Failed to obtain SL Engine interface");
104 return -1;
105 }
106 CreateAudioRecorder();
107 initialized_ = true;
108 buffer_index_ = 0;
109 return 0;
110 }
111
RecordingIsInitialized() const112 bool OpenSLESRecorder::RecordingIsInitialized() const {
113 return initialized_;
114 }
115
StartRecording()116 int OpenSLESRecorder::StartRecording() {
117 ALOGD("StartRecording[tid=%d]", rtc::CurrentThreadId());
118 RTC_DCHECK(thread_checker_.IsCurrent());
119 RTC_DCHECK(initialized_);
120 RTC_DCHECK(!recording_);
121 if (fine_audio_buffer_) {
122 fine_audio_buffer_->ResetRecord();
123 }
124 // Add buffers to the queue before changing state to SL_RECORDSTATE_RECORDING
125 // to ensure that recording starts as soon as the state is modified. On some
126 // devices, SLAndroidSimpleBufferQueue::Clear() used in Stop() does not flush
127 // the buffers as intended and we therefore check the number of buffers
128 // already queued first. Enqueue() can return SL_RESULT_BUFFER_INSUFFICIENT
129 // otherwise.
130 int num_buffers_in_queue = GetBufferCount();
131 for (int i = 0; i < kNumOfOpenSLESBuffers - num_buffers_in_queue; ++i) {
132 if (!EnqueueAudioBuffer()) {
133 recording_ = false;
134 return -1;
135 }
136 }
137 num_buffers_in_queue = GetBufferCount();
138 RTC_DCHECK_EQ(num_buffers_in_queue, kNumOfOpenSLESBuffers);
139 LogBufferState();
140 // Start audio recording by changing the state to SL_RECORDSTATE_RECORDING.
141 // Given that buffers are already enqueued, recording should start at once.
142 // The macro returns -1 if recording fails to start.
143 last_rec_time_ = rtc::Time();
144 if (LOG_ON_ERROR(
145 (*recorder_)->SetRecordState(recorder_, SL_RECORDSTATE_RECORDING))) {
146 return -1;
147 }
148 recording_ = (GetRecordState() == SL_RECORDSTATE_RECORDING);
149 RTC_DCHECK(recording_);
150 return 0;
151 }
152
StopRecording()153 int OpenSLESRecorder::StopRecording() {
154 ALOGD("StopRecording[tid=%d]", rtc::CurrentThreadId());
155 RTC_DCHECK(thread_checker_.IsCurrent());
156 if (!initialized_ || !recording_) {
157 return 0;
158 }
159 // Stop recording by setting the record state to SL_RECORDSTATE_STOPPED.
160 if (LOG_ON_ERROR(
161 (*recorder_)->SetRecordState(recorder_, SL_RECORDSTATE_STOPPED))) {
162 return -1;
163 }
164 // Clear the buffer queue to get rid of old data when resuming recording.
165 if (LOG_ON_ERROR((*simple_buffer_queue_)->Clear(simple_buffer_queue_))) {
166 return -1;
167 }
168 thread_checker_opensles_.Detach();
169 initialized_ = false;
170 recording_ = false;
171 return 0;
172 }
173
Recording() const174 bool OpenSLESRecorder::Recording() const {
175 return recording_;
176 }
177
AttachAudioBuffer(AudioDeviceBuffer * audio_buffer)178 void OpenSLESRecorder::AttachAudioBuffer(AudioDeviceBuffer* audio_buffer) {
179 ALOGD("AttachAudioBuffer");
180 RTC_DCHECK(thread_checker_.IsCurrent());
181 RTC_CHECK(audio_buffer);
182 audio_device_buffer_ = audio_buffer;
183 // Ensure that the audio device buffer is informed about the native sample
184 // rate used on the recording side.
185 const int sample_rate_hz = audio_parameters_.sample_rate();
186 ALOGD("SetRecordingSampleRate(%d)", sample_rate_hz);
187 audio_device_buffer_->SetRecordingSampleRate(sample_rate_hz);
188 // Ensure that the audio device buffer is informed about the number of
189 // channels preferred by the OS on the recording side.
190 const size_t channels = audio_parameters_.channels();
191 ALOGD("SetRecordingChannels(%zu)", channels);
192 audio_device_buffer_->SetRecordingChannels(channels);
193 // Allocated memory for internal data buffers given existing audio parameters.
194 AllocateDataBuffers();
195 }
196
IsAcousticEchoCancelerSupported() const197 bool OpenSLESRecorder::IsAcousticEchoCancelerSupported() const {
198 return false;
199 }
200
IsNoiseSuppressorSupported() const201 bool OpenSLESRecorder::IsNoiseSuppressorSupported() const {
202 return false;
203 }
204
EnableBuiltInAEC(bool enable)205 int OpenSLESRecorder::EnableBuiltInAEC(bool enable) {
206 ALOGD("EnableBuiltInAEC(%d)", enable);
207 RTC_DCHECK(thread_checker_.IsCurrent());
208 ALOGE("Not implemented");
209 return 0;
210 }
211
EnableBuiltInNS(bool enable)212 int OpenSLESRecorder::EnableBuiltInNS(bool enable) {
213 ALOGD("EnableBuiltInNS(%d)", enable);
214 RTC_DCHECK(thread_checker_.IsCurrent());
215 ALOGE("Not implemented");
216 return 0;
217 }
218
ObtainEngineInterface()219 bool OpenSLESRecorder::ObtainEngineInterface() {
220 ALOGD("ObtainEngineInterface");
221 RTC_DCHECK(thread_checker_.IsCurrent());
222 if (engine_)
223 return true;
224 // Get access to (or create if not already existing) the global OpenSL Engine
225 // object.
226 SLObjectItf engine_object = engine_manager_->GetOpenSLEngine();
227 if (engine_object == nullptr) {
228 ALOGE("Failed to access the global OpenSL engine");
229 return false;
230 }
231 // Get the SL Engine Interface which is implicit.
232 if (LOG_ON_ERROR(
233 (*engine_object)
234 ->GetInterface(engine_object, SL_IID_ENGINE, &engine_))) {
235 return false;
236 }
237 return true;
238 }
239
CreateAudioRecorder()240 bool OpenSLESRecorder::CreateAudioRecorder() {
241 ALOGD("CreateAudioRecorder");
242 RTC_DCHECK(thread_checker_.IsCurrent());
243 if (recorder_object_.Get())
244 return true;
245 RTC_DCHECK(!recorder_);
246 RTC_DCHECK(!simple_buffer_queue_);
247
248 // Audio source configuration.
249 SLDataLocator_IODevice mic_locator = {SL_DATALOCATOR_IODEVICE,
250 SL_IODEVICE_AUDIOINPUT,
251 SL_DEFAULTDEVICEID_AUDIOINPUT, NULL};
252 SLDataSource audio_source = {&mic_locator, NULL};
253
254 // Audio sink configuration.
255 SLDataLocator_AndroidSimpleBufferQueue buffer_queue = {
256 SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE,
257 static_cast<SLuint32>(kNumOfOpenSLESBuffers)};
258 SLDataSink audio_sink = {&buffer_queue, &pcm_format_};
259
260 // Create the audio recorder object (requires the RECORD_AUDIO permission).
261 // Do not realize the recorder yet. Set the configuration first.
262 const SLInterfaceID interface_id[] = {SL_IID_ANDROIDSIMPLEBUFFERQUEUE,
263 SL_IID_ANDROIDCONFIGURATION};
264 const SLboolean interface_required[] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE};
265 if (LOG_ON_ERROR((*engine_)->CreateAudioRecorder(
266 engine_, recorder_object_.Receive(), &audio_source, &audio_sink,
267 arraysize(interface_id), interface_id, interface_required))) {
268 return false;
269 }
270
271 // Configure the audio recorder (before it is realized).
272 SLAndroidConfigurationItf recorder_config;
273 if (LOG_ON_ERROR((recorder_object_->GetInterface(recorder_object_.Get(),
274 SL_IID_ANDROIDCONFIGURATION,
275 &recorder_config)))) {
276 return false;
277 }
278
279 // Uses the default microphone tuned for audio communication.
280 // Note that, SL_ANDROID_RECORDING_PRESET_VOICE_RECOGNITION leads to a fast
281 // track but also excludes usage of required effects like AEC, AGC and NS.
282 // SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION
283 SLint32 stream_type = SL_ANDROID_RECORDING_PRESET_VOICE_COMMUNICATION;
284 if (LOG_ON_ERROR(((*recorder_config)
285 ->SetConfiguration(recorder_config,
286 SL_ANDROID_KEY_RECORDING_PRESET,
287 &stream_type, sizeof(SLint32))))) {
288 return false;
289 }
290
291 // The audio recorder can now be realized (in synchronous mode).
292 if (LOG_ON_ERROR((recorder_object_->Realize(recorder_object_.Get(),
293 SL_BOOLEAN_FALSE)))) {
294 return false;
295 }
296
297 // Get the implicit recorder interface (SL_IID_RECORD).
298 if (LOG_ON_ERROR((recorder_object_->GetInterface(
299 recorder_object_.Get(), SL_IID_RECORD, &recorder_)))) {
300 return false;
301 }
302
303 // Get the simple buffer queue interface (SL_IID_ANDROIDSIMPLEBUFFERQUEUE).
304 // It was explicitly requested.
305 if (LOG_ON_ERROR((recorder_object_->GetInterface(
306 recorder_object_.Get(), SL_IID_ANDROIDSIMPLEBUFFERQUEUE,
307 &simple_buffer_queue_)))) {
308 return false;
309 }
310
311 // Register the input callback for the simple buffer queue.
312 // This callback will be called when receiving new data from the device.
313 if (LOG_ON_ERROR(((*simple_buffer_queue_)
314 ->RegisterCallback(simple_buffer_queue_,
315 SimpleBufferQueueCallback, this)))) {
316 return false;
317 }
318 return true;
319 }
320
DestroyAudioRecorder()321 void OpenSLESRecorder::DestroyAudioRecorder() {
322 ALOGD("DestroyAudioRecorder");
323 RTC_DCHECK(thread_checker_.IsCurrent());
324 if (!recorder_object_.Get())
325 return;
326 (*simple_buffer_queue_)
327 ->RegisterCallback(simple_buffer_queue_, nullptr, nullptr);
328 recorder_object_.Reset();
329 recorder_ = nullptr;
330 simple_buffer_queue_ = nullptr;
331 }
332
SimpleBufferQueueCallback(SLAndroidSimpleBufferQueueItf buffer_queue,void * context)333 void OpenSLESRecorder::SimpleBufferQueueCallback(
334 SLAndroidSimpleBufferQueueItf buffer_queue,
335 void* context) {
336 OpenSLESRecorder* stream = static_cast<OpenSLESRecorder*>(context);
337 stream->ReadBufferQueue();
338 }
339
AllocateDataBuffers()340 void OpenSLESRecorder::AllocateDataBuffers() {
341 ALOGD("AllocateDataBuffers");
342 RTC_DCHECK(thread_checker_.IsCurrent());
343 RTC_DCHECK(!simple_buffer_queue_);
344 RTC_CHECK(audio_device_buffer_);
345 // Create a modified audio buffer class which allows us to deliver any number
346 // of samples (and not only multiple of 10ms) to match the native audio unit
347 // buffer size.
348 ALOGD("frames per native buffer: %zu", audio_parameters_.frames_per_buffer());
349 ALOGD("frames per 10ms buffer: %zu",
350 audio_parameters_.frames_per_10ms_buffer());
351 ALOGD("bytes per native buffer: %zu", audio_parameters_.GetBytesPerBuffer());
352 ALOGD("native sample rate: %d", audio_parameters_.sample_rate());
353 RTC_DCHECK(audio_device_buffer_);
354 fine_audio_buffer_ = std::make_unique<FineAudioBuffer>(audio_device_buffer_);
355 // Allocate queue of audio buffers that stores recorded audio samples.
356 const int buffer_size_samples =
357 audio_parameters_.frames_per_buffer() * audio_parameters_.channels();
358 audio_buffers_.reset(new std::unique_ptr<SLint16[]>[kNumOfOpenSLESBuffers]);
359 for (int i = 0; i < kNumOfOpenSLESBuffers; ++i) {
360 audio_buffers_[i].reset(new SLint16[buffer_size_samples]);
361 }
362 }
363
ReadBufferQueue()364 void OpenSLESRecorder::ReadBufferQueue() {
365 RTC_DCHECK(thread_checker_opensles_.IsCurrent());
366 SLuint32 state = GetRecordState();
367 if (state != SL_RECORDSTATE_RECORDING) {
368 ALOGW("Buffer callback in non-recording state!");
369 return;
370 }
371 // Check delta time between two successive callbacks and provide a warning
372 // if it becomes very large.
373 // TODO(henrika): using 150ms as upper limit but this value is rather random.
374 const uint32_t current_time = rtc::Time();
375 const uint32_t diff = current_time - last_rec_time_;
376 if (diff > 150) {
377 ALOGW("Bad OpenSL ES record timing, dT=%u [ms]", diff);
378 }
379 last_rec_time_ = current_time;
380 // Send recorded audio data to the WebRTC sink.
381 // TODO(henrika): fix delay estimates. It is OK to use fixed values for now
382 // since there is no support to turn off built-in EC in combination with
383 // OpenSL ES anyhow. Hence, as is, the WebRTC based AEC (which would use
384 // these estimates) will never be active.
385 fine_audio_buffer_->DeliverRecordedData(
386 rtc::ArrayView<const int16_t>(
387 audio_buffers_[buffer_index_].get(),
388 audio_parameters_.frames_per_buffer() * audio_parameters_.channels()),
389 25);
390 // Enqueue the utilized audio buffer and use if for recording again.
391 EnqueueAudioBuffer();
392 }
393
EnqueueAudioBuffer()394 bool OpenSLESRecorder::EnqueueAudioBuffer() {
395 SLresult err =
396 (*simple_buffer_queue_)
397 ->Enqueue(
398 simple_buffer_queue_,
399 reinterpret_cast<SLint8*>(audio_buffers_[buffer_index_].get()),
400 audio_parameters_.GetBytesPerBuffer());
401 if (SL_RESULT_SUCCESS != err) {
402 ALOGE("Enqueue failed: %s", GetSLErrorString(err));
403 return false;
404 }
405 buffer_index_ = (buffer_index_ + 1) % kNumOfOpenSLESBuffers;
406 return true;
407 }
408
GetRecordState() const409 SLuint32 OpenSLESRecorder::GetRecordState() const {
410 RTC_DCHECK(recorder_);
411 SLuint32 state;
412 SLresult err = (*recorder_)->GetRecordState(recorder_, &state);
413 if (SL_RESULT_SUCCESS != err) {
414 ALOGE("GetRecordState failed: %s", GetSLErrorString(err));
415 }
416 return state;
417 }
418
GetBufferQueueState() const419 SLAndroidSimpleBufferQueueState OpenSLESRecorder::GetBufferQueueState() const {
420 RTC_DCHECK(simple_buffer_queue_);
421 // state.count: Number of buffers currently in the queue.
422 // state.index: Index of the currently filling buffer. This is a linear index
423 // that keeps a cumulative count of the number of buffers recorded.
424 SLAndroidSimpleBufferQueueState state;
425 SLresult err =
426 (*simple_buffer_queue_)->GetState(simple_buffer_queue_, &state);
427 if (SL_RESULT_SUCCESS != err) {
428 ALOGE("GetState failed: %s", GetSLErrorString(err));
429 }
430 return state;
431 }
432
LogBufferState() const433 void OpenSLESRecorder::LogBufferState() const {
434 SLAndroidSimpleBufferQueueState state = GetBufferQueueState();
435 ALOGD("state.count:%d state.index:%d", state.count, state.index);
436 }
437
GetBufferCount()438 SLuint32 OpenSLESRecorder::GetBufferCount() {
439 SLAndroidSimpleBufferQueueState state = GetBufferQueueState();
440 return state.count;
441 }
442
443 } // namespace jni
444
445 } // namespace webrtc
446