1*d9f75844SAndroid Build Coastguard Worker /* 2*d9f75844SAndroid Build Coastguard Worker * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. 3*d9f75844SAndroid Build Coastguard Worker * 4*d9f75844SAndroid Build Coastguard Worker * Use of this source code is governed by a BSD-style license 5*d9f75844SAndroid Build Coastguard Worker * that can be found in the LICENSE file in the root of the source 6*d9f75844SAndroid Build Coastguard Worker * tree. An additional intellectual property rights grant can be found 7*d9f75844SAndroid Build Coastguard Worker * in the file PATENTS. All contributing project authors may 8*d9f75844SAndroid Build Coastguard Worker * be found in the AUTHORS file in the root of the source tree. 9*d9f75844SAndroid Build Coastguard Worker */ 10*d9f75844SAndroid Build Coastguard Worker 11*d9f75844SAndroid Build Coastguard Worker #ifndef API_AUDIO_AUDIO_FRAME_H_ 12*d9f75844SAndroid Build Coastguard Worker #define API_AUDIO_AUDIO_FRAME_H_ 13*d9f75844SAndroid Build Coastguard Worker 14*d9f75844SAndroid Build Coastguard Worker #include <stddef.h> 15*d9f75844SAndroid Build Coastguard Worker #include <stdint.h> 16*d9f75844SAndroid Build Coastguard Worker 17*d9f75844SAndroid Build Coastguard Worker #include "api/audio/channel_layout.h" 18*d9f75844SAndroid Build Coastguard Worker #include "api/rtp_packet_infos.h" 19*d9f75844SAndroid Build Coastguard Worker 20*d9f75844SAndroid Build Coastguard Worker namespace webrtc { 21*d9f75844SAndroid Build Coastguard Worker 22*d9f75844SAndroid Build Coastguard Worker /* This class holds up to 120 ms of super-wideband (32 kHz) stereo audio. It 23*d9f75844SAndroid Build Coastguard Worker * allows for adding and subtracting frames while keeping track of the resulting 24*d9f75844SAndroid Build Coastguard Worker * states. 25*d9f75844SAndroid Build Coastguard Worker * 26*d9f75844SAndroid Build Coastguard Worker * Notes 27*d9f75844SAndroid Build Coastguard Worker * - This is a de-facto api, not designed for external use. The AudioFrame class 28*d9f75844SAndroid Build Coastguard Worker * is in need of overhaul or even replacement, and anyone depending on it 29*d9f75844SAndroid Build Coastguard Worker * should be prepared for that. 30*d9f75844SAndroid Build Coastguard Worker * - The total number of samples is samples_per_channel_ * num_channels_. 31*d9f75844SAndroid Build Coastguard Worker * - Stereo data is interleaved starting with the left channel. 32*d9f75844SAndroid Build Coastguard Worker */ 33*d9f75844SAndroid Build Coastguard Worker class AudioFrame { 34*d9f75844SAndroid Build Coastguard Worker public: 35*d9f75844SAndroid Build Coastguard Worker // Using constexpr here causes linker errors unless the variable also has an 36*d9f75844SAndroid Build Coastguard Worker // out-of-class definition, which is impractical in this header-only class. 37*d9f75844SAndroid Build Coastguard Worker // (This makes no sense because it compiles as an enum value, which we most 38*d9f75844SAndroid Build Coastguard Worker // certainly cannot take the address of, just fine.) C++17 introduces inline 39*d9f75844SAndroid Build Coastguard Worker // variables which should allow us to switch to constexpr and keep this a 40*d9f75844SAndroid Build Coastguard Worker // header-only class. 41*d9f75844SAndroid Build Coastguard Worker enum : size_t { 42*d9f75844SAndroid Build Coastguard Worker // Stereo, 32 kHz, 120 ms (2 * 32 * 120) 43*d9f75844SAndroid Build Coastguard Worker // Stereo, 192 kHz, 20 ms (2 * 192 * 20) 44*d9f75844SAndroid Build Coastguard Worker kMaxDataSizeSamples = 7680, 45*d9f75844SAndroid Build Coastguard Worker kMaxDataSizeBytes = kMaxDataSizeSamples * sizeof(int16_t), 46*d9f75844SAndroid Build Coastguard Worker }; 47*d9f75844SAndroid Build Coastguard Worker 48*d9f75844SAndroid Build Coastguard Worker enum VADActivity { kVadActive = 0, kVadPassive = 1, kVadUnknown = 2 }; 49*d9f75844SAndroid Build Coastguard Worker enum SpeechType { 50*d9f75844SAndroid Build Coastguard Worker kNormalSpeech = 0, 51*d9f75844SAndroid Build Coastguard Worker kPLC = 1, 52*d9f75844SAndroid Build Coastguard Worker kCNG = 2, 53*d9f75844SAndroid Build Coastguard Worker kPLCCNG = 3, 54*d9f75844SAndroid Build Coastguard Worker kCodecPLC = 5, 55*d9f75844SAndroid Build Coastguard Worker kUndefined = 4 56*d9f75844SAndroid Build Coastguard Worker }; 57*d9f75844SAndroid Build Coastguard Worker 58*d9f75844SAndroid Build Coastguard Worker AudioFrame(); 59*d9f75844SAndroid Build Coastguard Worker 60*d9f75844SAndroid Build Coastguard Worker AudioFrame(const AudioFrame&) = delete; 61*d9f75844SAndroid Build Coastguard Worker AudioFrame& operator=(const AudioFrame&) = delete; 62*d9f75844SAndroid Build Coastguard Worker 63*d9f75844SAndroid Build Coastguard Worker // Resets all members to their default state. 64*d9f75844SAndroid Build Coastguard Worker void Reset(); 65*d9f75844SAndroid Build Coastguard Worker // Same as Reset(), but leaves mute state unchanged. Muting a frame requires 66*d9f75844SAndroid Build Coastguard Worker // the buffer to be zeroed on the next call to mutable_data(). Callers 67*d9f75844SAndroid Build Coastguard Worker // intending to write to the buffer immediately after Reset() can instead use 68*d9f75844SAndroid Build Coastguard Worker // ResetWithoutMuting() to skip this wasteful zeroing. 69*d9f75844SAndroid Build Coastguard Worker void ResetWithoutMuting(); 70*d9f75844SAndroid Build Coastguard Worker 71*d9f75844SAndroid Build Coastguard Worker void UpdateFrame(uint32_t timestamp, 72*d9f75844SAndroid Build Coastguard Worker const int16_t* data, 73*d9f75844SAndroid Build Coastguard Worker size_t samples_per_channel, 74*d9f75844SAndroid Build Coastguard Worker int sample_rate_hz, 75*d9f75844SAndroid Build Coastguard Worker SpeechType speech_type, 76*d9f75844SAndroid Build Coastguard Worker VADActivity vad_activity, 77*d9f75844SAndroid Build Coastguard Worker size_t num_channels = 1); 78*d9f75844SAndroid Build Coastguard Worker 79*d9f75844SAndroid Build Coastguard Worker void CopyFrom(const AudioFrame& src); 80*d9f75844SAndroid Build Coastguard Worker 81*d9f75844SAndroid Build Coastguard Worker // Sets a wall-time clock timestamp in milliseconds to be used for profiling 82*d9f75844SAndroid Build Coastguard Worker // of time between two points in the audio chain. 83*d9f75844SAndroid Build Coastguard Worker // Example: 84*d9f75844SAndroid Build Coastguard Worker // t0: UpdateProfileTimeStamp() 85*d9f75844SAndroid Build Coastguard Worker // t1: ElapsedProfileTimeMs() => t1 - t0 [msec] 86*d9f75844SAndroid Build Coastguard Worker void UpdateProfileTimeStamp(); 87*d9f75844SAndroid Build Coastguard Worker // Returns the time difference between now and when UpdateProfileTimeStamp() 88*d9f75844SAndroid Build Coastguard Worker // was last called. Returns -1 if UpdateProfileTimeStamp() has not yet been 89*d9f75844SAndroid Build Coastguard Worker // called. 90*d9f75844SAndroid Build Coastguard Worker int64_t ElapsedProfileTimeMs() const; 91*d9f75844SAndroid Build Coastguard Worker 92*d9f75844SAndroid Build Coastguard Worker // data() returns a zeroed static buffer if the frame is muted. 93*d9f75844SAndroid Build Coastguard Worker // mutable_frame() always returns a non-static buffer; the first call to 94*d9f75844SAndroid Build Coastguard Worker // mutable_frame() zeros the non-static buffer and marks the frame unmuted. 95*d9f75844SAndroid Build Coastguard Worker const int16_t* data() const; 96*d9f75844SAndroid Build Coastguard Worker int16_t* mutable_data(); 97*d9f75844SAndroid Build Coastguard Worker 98*d9f75844SAndroid Build Coastguard Worker // Prefer to mute frames using AudioFrameOperations::Mute. 99*d9f75844SAndroid Build Coastguard Worker void Mute(); 100*d9f75844SAndroid Build Coastguard Worker // Frame is muted by default. 101*d9f75844SAndroid Build Coastguard Worker bool muted() const; 102*d9f75844SAndroid Build Coastguard Worker max_16bit_samples()103*d9f75844SAndroid Build Coastguard Worker size_t max_16bit_samples() const { return kMaxDataSizeSamples; } samples_per_channel()104*d9f75844SAndroid Build Coastguard Worker size_t samples_per_channel() const { return samples_per_channel_; } num_channels()105*d9f75844SAndroid Build Coastguard Worker size_t num_channels() const { return num_channels_; } channel_layout()106*d9f75844SAndroid Build Coastguard Worker ChannelLayout channel_layout() const { return channel_layout_; } sample_rate_hz()107*d9f75844SAndroid Build Coastguard Worker int sample_rate_hz() const { return sample_rate_hz_; } 108*d9f75844SAndroid Build Coastguard Worker set_absolute_capture_timestamp_ms(int64_t absolute_capture_time_stamp_ms)109*d9f75844SAndroid Build Coastguard Worker void set_absolute_capture_timestamp_ms( 110*d9f75844SAndroid Build Coastguard Worker int64_t absolute_capture_time_stamp_ms) { 111*d9f75844SAndroid Build Coastguard Worker absolute_capture_timestamp_ms_ = absolute_capture_time_stamp_ms; 112*d9f75844SAndroid Build Coastguard Worker } 113*d9f75844SAndroid Build Coastguard Worker absolute_capture_timestamp_ms()114*d9f75844SAndroid Build Coastguard Worker absl::optional<int64_t> absolute_capture_timestamp_ms() const { 115*d9f75844SAndroid Build Coastguard Worker return absolute_capture_timestamp_ms_; 116*d9f75844SAndroid Build Coastguard Worker } 117*d9f75844SAndroid Build Coastguard Worker 118*d9f75844SAndroid Build Coastguard Worker // RTP timestamp of the first sample in the AudioFrame. 119*d9f75844SAndroid Build Coastguard Worker uint32_t timestamp_ = 0; 120*d9f75844SAndroid Build Coastguard Worker // Time since the first frame in milliseconds. 121*d9f75844SAndroid Build Coastguard Worker // -1 represents an uninitialized value. 122*d9f75844SAndroid Build Coastguard Worker int64_t elapsed_time_ms_ = -1; 123*d9f75844SAndroid Build Coastguard Worker // NTP time of the estimated capture time in local timebase in milliseconds. 124*d9f75844SAndroid Build Coastguard Worker // -1 represents an uninitialized value. 125*d9f75844SAndroid Build Coastguard Worker int64_t ntp_time_ms_ = -1; 126*d9f75844SAndroid Build Coastguard Worker size_t samples_per_channel_ = 0; 127*d9f75844SAndroid Build Coastguard Worker int sample_rate_hz_ = 0; 128*d9f75844SAndroid Build Coastguard Worker size_t num_channels_ = 0; 129*d9f75844SAndroid Build Coastguard Worker ChannelLayout channel_layout_ = CHANNEL_LAYOUT_NONE; 130*d9f75844SAndroid Build Coastguard Worker SpeechType speech_type_ = kUndefined; 131*d9f75844SAndroid Build Coastguard Worker VADActivity vad_activity_ = kVadUnknown; 132*d9f75844SAndroid Build Coastguard Worker // Monotonically increasing timestamp intended for profiling of audio frames. 133*d9f75844SAndroid Build Coastguard Worker // Typically used for measuring elapsed time between two different points in 134*d9f75844SAndroid Build Coastguard Worker // the audio path. No lock is used to save resources and we are thread safe 135*d9f75844SAndroid Build Coastguard Worker // by design. 136*d9f75844SAndroid Build Coastguard Worker // TODO([email protected]): consider using absl::optional. 137*d9f75844SAndroid Build Coastguard Worker int64_t profile_timestamp_ms_ = 0; 138*d9f75844SAndroid Build Coastguard Worker 139*d9f75844SAndroid Build Coastguard Worker // Information about packets used to assemble this audio frame. This is needed 140*d9f75844SAndroid Build Coastguard Worker // by `SourceTracker` when the frame is delivered to the RTCRtpReceiver's 141*d9f75844SAndroid Build Coastguard Worker // MediaStreamTrack, in order to implement getContributingSources(). See: 142*d9f75844SAndroid Build Coastguard Worker // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources 143*d9f75844SAndroid Build Coastguard Worker // 144*d9f75844SAndroid Build Coastguard Worker // TODO(bugs.webrtc.org/10757): 145*d9f75844SAndroid Build Coastguard Worker // Note that this information might not be fully accurate since we currently 146*d9f75844SAndroid Build Coastguard Worker // don't have a proper way to track it across the audio sync buffer. The 147*d9f75844SAndroid Build Coastguard Worker // sync buffer is the small sample-holding buffer located after the audio 148*d9f75844SAndroid Build Coastguard Worker // decoder and before where samples are assembled into output frames. 149*d9f75844SAndroid Build Coastguard Worker // 150*d9f75844SAndroid Build Coastguard Worker // `RtpPacketInfos` may also be empty if the audio samples did not come from 151*d9f75844SAndroid Build Coastguard Worker // RTP packets. E.g. if the audio were locally generated by packet loss 152*d9f75844SAndroid Build Coastguard Worker // concealment, comfort noise generation, etc. 153*d9f75844SAndroid Build Coastguard Worker RtpPacketInfos packet_infos_; 154*d9f75844SAndroid Build Coastguard Worker 155*d9f75844SAndroid Build Coastguard Worker private: 156*d9f75844SAndroid Build Coastguard Worker // A permanently zeroed out buffer to represent muted frames. This is a 157*d9f75844SAndroid Build Coastguard Worker // header-only class, so the only way to avoid creating a separate empty 158*d9f75844SAndroid Build Coastguard Worker // buffer per translation unit is to wrap a static in an inline function. 159*d9f75844SAndroid Build Coastguard Worker static const int16_t* empty_data(); 160*d9f75844SAndroid Build Coastguard Worker 161*d9f75844SAndroid Build Coastguard Worker int16_t data_[kMaxDataSizeSamples]; 162*d9f75844SAndroid Build Coastguard Worker bool muted_ = true; 163*d9f75844SAndroid Build Coastguard Worker 164*d9f75844SAndroid Build Coastguard Worker // Absolute capture timestamp when this audio frame was originally captured. 165*d9f75844SAndroid Build Coastguard Worker // This is only valid for audio frames captured on this machine. The absolute 166*d9f75844SAndroid Build Coastguard Worker // capture timestamp of a received frame is found in `packet_infos_`. 167*d9f75844SAndroid Build Coastguard Worker // This timestamp MUST be based on the same clock as rtc::TimeMillis(). 168*d9f75844SAndroid Build Coastguard Worker absl::optional<int64_t> absolute_capture_timestamp_ms_; 169*d9f75844SAndroid Build Coastguard Worker }; 170*d9f75844SAndroid Build Coastguard Worker 171*d9f75844SAndroid Build Coastguard Worker } // namespace webrtc 172*d9f75844SAndroid Build Coastguard Worker 173*d9f75844SAndroid Build Coastguard Worker #endif // API_AUDIO_AUDIO_FRAME_H_ 174