xref: /aosp_15_r20/external/webrtc/api/audio/audio_frame.h (revision d9f758449e529ab9291ac668be2861e7a55c2422)
1*d9f75844SAndroid Build Coastguard Worker /*
2*d9f75844SAndroid Build Coastguard Worker  *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3*d9f75844SAndroid Build Coastguard Worker  *
4*d9f75844SAndroid Build Coastguard Worker  *  Use of this source code is governed by a BSD-style license
5*d9f75844SAndroid Build Coastguard Worker  *  that can be found in the LICENSE file in the root of the source
6*d9f75844SAndroid Build Coastguard Worker  *  tree. An additional intellectual property rights grant can be found
7*d9f75844SAndroid Build Coastguard Worker  *  in the file PATENTS.  All contributing project authors may
8*d9f75844SAndroid Build Coastguard Worker  *  be found in the AUTHORS file in the root of the source tree.
9*d9f75844SAndroid Build Coastguard Worker  */
10*d9f75844SAndroid Build Coastguard Worker 
11*d9f75844SAndroid Build Coastguard Worker #ifndef API_AUDIO_AUDIO_FRAME_H_
12*d9f75844SAndroid Build Coastguard Worker #define API_AUDIO_AUDIO_FRAME_H_
13*d9f75844SAndroid Build Coastguard Worker 
14*d9f75844SAndroid Build Coastguard Worker #include <stddef.h>
15*d9f75844SAndroid Build Coastguard Worker #include <stdint.h>
16*d9f75844SAndroid Build Coastguard Worker 
17*d9f75844SAndroid Build Coastguard Worker #include "api/audio/channel_layout.h"
18*d9f75844SAndroid Build Coastguard Worker #include "api/rtp_packet_infos.h"
19*d9f75844SAndroid Build Coastguard Worker 
20*d9f75844SAndroid Build Coastguard Worker namespace webrtc {
21*d9f75844SAndroid Build Coastguard Worker 
22*d9f75844SAndroid Build Coastguard Worker /* This class holds up to 120 ms of super-wideband (32 kHz) stereo audio. It
23*d9f75844SAndroid Build Coastguard Worker  * allows for adding and subtracting frames while keeping track of the resulting
24*d9f75844SAndroid Build Coastguard Worker  * states.
25*d9f75844SAndroid Build Coastguard Worker  *
26*d9f75844SAndroid Build Coastguard Worker  * Notes
27*d9f75844SAndroid Build Coastguard Worker  * - This is a de-facto api, not designed for external use. The AudioFrame class
28*d9f75844SAndroid Build Coastguard Worker  *   is in need of overhaul or even replacement, and anyone depending on it
29*d9f75844SAndroid Build Coastguard Worker  *   should be prepared for that.
30*d9f75844SAndroid Build Coastguard Worker  * - The total number of samples is samples_per_channel_ * num_channels_.
31*d9f75844SAndroid Build Coastguard Worker  * - Stereo data is interleaved starting with the left channel.
32*d9f75844SAndroid Build Coastguard Worker  */
33*d9f75844SAndroid Build Coastguard Worker class AudioFrame {
34*d9f75844SAndroid Build Coastguard Worker  public:
35*d9f75844SAndroid Build Coastguard Worker   // Using constexpr here causes linker errors unless the variable also has an
36*d9f75844SAndroid Build Coastguard Worker   // out-of-class definition, which is impractical in this header-only class.
37*d9f75844SAndroid Build Coastguard Worker   // (This makes no sense because it compiles as an enum value, which we most
38*d9f75844SAndroid Build Coastguard Worker   // certainly cannot take the address of, just fine.) C++17 introduces inline
39*d9f75844SAndroid Build Coastguard Worker   // variables which should allow us to switch to constexpr and keep this a
40*d9f75844SAndroid Build Coastguard Worker   // header-only class.
41*d9f75844SAndroid Build Coastguard Worker   enum : size_t {
42*d9f75844SAndroid Build Coastguard Worker     // Stereo, 32 kHz, 120 ms (2 * 32 * 120)
43*d9f75844SAndroid Build Coastguard Worker     // Stereo, 192 kHz, 20 ms (2 * 192 * 20)
44*d9f75844SAndroid Build Coastguard Worker     kMaxDataSizeSamples = 7680,
45*d9f75844SAndroid Build Coastguard Worker     kMaxDataSizeBytes = kMaxDataSizeSamples * sizeof(int16_t),
46*d9f75844SAndroid Build Coastguard Worker   };
47*d9f75844SAndroid Build Coastguard Worker 
48*d9f75844SAndroid Build Coastguard Worker   enum VADActivity { kVadActive = 0, kVadPassive = 1, kVadUnknown = 2 };
49*d9f75844SAndroid Build Coastguard Worker   enum SpeechType {
50*d9f75844SAndroid Build Coastguard Worker     kNormalSpeech = 0,
51*d9f75844SAndroid Build Coastguard Worker     kPLC = 1,
52*d9f75844SAndroid Build Coastguard Worker     kCNG = 2,
53*d9f75844SAndroid Build Coastguard Worker     kPLCCNG = 3,
54*d9f75844SAndroid Build Coastguard Worker     kCodecPLC = 5,
55*d9f75844SAndroid Build Coastguard Worker     kUndefined = 4
56*d9f75844SAndroid Build Coastguard Worker   };
57*d9f75844SAndroid Build Coastguard Worker 
58*d9f75844SAndroid Build Coastguard Worker   AudioFrame();
59*d9f75844SAndroid Build Coastguard Worker 
60*d9f75844SAndroid Build Coastguard Worker   AudioFrame(const AudioFrame&) = delete;
61*d9f75844SAndroid Build Coastguard Worker   AudioFrame& operator=(const AudioFrame&) = delete;
62*d9f75844SAndroid Build Coastguard Worker 
63*d9f75844SAndroid Build Coastguard Worker   // Resets all members to their default state.
64*d9f75844SAndroid Build Coastguard Worker   void Reset();
65*d9f75844SAndroid Build Coastguard Worker   // Same as Reset(), but leaves mute state unchanged. Muting a frame requires
66*d9f75844SAndroid Build Coastguard Worker   // the buffer to be zeroed on the next call to mutable_data(). Callers
67*d9f75844SAndroid Build Coastguard Worker   // intending to write to the buffer immediately after Reset() can instead use
68*d9f75844SAndroid Build Coastguard Worker   // ResetWithoutMuting() to skip this wasteful zeroing.
69*d9f75844SAndroid Build Coastguard Worker   void ResetWithoutMuting();
70*d9f75844SAndroid Build Coastguard Worker 
71*d9f75844SAndroid Build Coastguard Worker   void UpdateFrame(uint32_t timestamp,
72*d9f75844SAndroid Build Coastguard Worker                    const int16_t* data,
73*d9f75844SAndroid Build Coastguard Worker                    size_t samples_per_channel,
74*d9f75844SAndroid Build Coastguard Worker                    int sample_rate_hz,
75*d9f75844SAndroid Build Coastguard Worker                    SpeechType speech_type,
76*d9f75844SAndroid Build Coastguard Worker                    VADActivity vad_activity,
77*d9f75844SAndroid Build Coastguard Worker                    size_t num_channels = 1);
78*d9f75844SAndroid Build Coastguard Worker 
79*d9f75844SAndroid Build Coastguard Worker   void CopyFrom(const AudioFrame& src);
80*d9f75844SAndroid Build Coastguard Worker 
81*d9f75844SAndroid Build Coastguard Worker   // Sets a wall-time clock timestamp in milliseconds to be used for profiling
82*d9f75844SAndroid Build Coastguard Worker   // of time between two points in the audio chain.
83*d9f75844SAndroid Build Coastguard Worker   // Example:
84*d9f75844SAndroid Build Coastguard Worker   //   t0: UpdateProfileTimeStamp()
85*d9f75844SAndroid Build Coastguard Worker   //   t1: ElapsedProfileTimeMs() => t1 - t0 [msec]
86*d9f75844SAndroid Build Coastguard Worker   void UpdateProfileTimeStamp();
87*d9f75844SAndroid Build Coastguard Worker   // Returns the time difference between now and when UpdateProfileTimeStamp()
88*d9f75844SAndroid Build Coastguard Worker   // was last called. Returns -1 if UpdateProfileTimeStamp() has not yet been
89*d9f75844SAndroid Build Coastguard Worker   // called.
90*d9f75844SAndroid Build Coastguard Worker   int64_t ElapsedProfileTimeMs() const;
91*d9f75844SAndroid Build Coastguard Worker 
92*d9f75844SAndroid Build Coastguard Worker   // data() returns a zeroed static buffer if the frame is muted.
93*d9f75844SAndroid Build Coastguard Worker   // mutable_frame() always returns a non-static buffer; the first call to
94*d9f75844SAndroid Build Coastguard Worker   // mutable_frame() zeros the non-static buffer and marks the frame unmuted.
95*d9f75844SAndroid Build Coastguard Worker   const int16_t* data() const;
96*d9f75844SAndroid Build Coastguard Worker   int16_t* mutable_data();
97*d9f75844SAndroid Build Coastguard Worker 
98*d9f75844SAndroid Build Coastguard Worker   // Prefer to mute frames using AudioFrameOperations::Mute.
99*d9f75844SAndroid Build Coastguard Worker   void Mute();
100*d9f75844SAndroid Build Coastguard Worker   // Frame is muted by default.
101*d9f75844SAndroid Build Coastguard Worker   bool muted() const;
102*d9f75844SAndroid Build Coastguard Worker 
max_16bit_samples()103*d9f75844SAndroid Build Coastguard Worker   size_t max_16bit_samples() const { return kMaxDataSizeSamples; }
samples_per_channel()104*d9f75844SAndroid Build Coastguard Worker   size_t samples_per_channel() const { return samples_per_channel_; }
num_channels()105*d9f75844SAndroid Build Coastguard Worker   size_t num_channels() const { return num_channels_; }
channel_layout()106*d9f75844SAndroid Build Coastguard Worker   ChannelLayout channel_layout() const { return channel_layout_; }
sample_rate_hz()107*d9f75844SAndroid Build Coastguard Worker   int sample_rate_hz() const { return sample_rate_hz_; }
108*d9f75844SAndroid Build Coastguard Worker 
set_absolute_capture_timestamp_ms(int64_t absolute_capture_time_stamp_ms)109*d9f75844SAndroid Build Coastguard Worker   void set_absolute_capture_timestamp_ms(
110*d9f75844SAndroid Build Coastguard Worker       int64_t absolute_capture_time_stamp_ms) {
111*d9f75844SAndroid Build Coastguard Worker     absolute_capture_timestamp_ms_ = absolute_capture_time_stamp_ms;
112*d9f75844SAndroid Build Coastguard Worker   }
113*d9f75844SAndroid Build Coastguard Worker 
absolute_capture_timestamp_ms()114*d9f75844SAndroid Build Coastguard Worker   absl::optional<int64_t> absolute_capture_timestamp_ms() const {
115*d9f75844SAndroid Build Coastguard Worker     return absolute_capture_timestamp_ms_;
116*d9f75844SAndroid Build Coastguard Worker   }
117*d9f75844SAndroid Build Coastguard Worker 
118*d9f75844SAndroid Build Coastguard Worker   // RTP timestamp of the first sample in the AudioFrame.
119*d9f75844SAndroid Build Coastguard Worker   uint32_t timestamp_ = 0;
120*d9f75844SAndroid Build Coastguard Worker   // Time since the first frame in milliseconds.
121*d9f75844SAndroid Build Coastguard Worker   // -1 represents an uninitialized value.
122*d9f75844SAndroid Build Coastguard Worker   int64_t elapsed_time_ms_ = -1;
123*d9f75844SAndroid Build Coastguard Worker   // NTP time of the estimated capture time in local timebase in milliseconds.
124*d9f75844SAndroid Build Coastguard Worker   // -1 represents an uninitialized value.
125*d9f75844SAndroid Build Coastguard Worker   int64_t ntp_time_ms_ = -1;
126*d9f75844SAndroid Build Coastguard Worker   size_t samples_per_channel_ = 0;
127*d9f75844SAndroid Build Coastguard Worker   int sample_rate_hz_ = 0;
128*d9f75844SAndroid Build Coastguard Worker   size_t num_channels_ = 0;
129*d9f75844SAndroid Build Coastguard Worker   ChannelLayout channel_layout_ = CHANNEL_LAYOUT_NONE;
130*d9f75844SAndroid Build Coastguard Worker   SpeechType speech_type_ = kUndefined;
131*d9f75844SAndroid Build Coastguard Worker   VADActivity vad_activity_ = kVadUnknown;
132*d9f75844SAndroid Build Coastguard Worker   // Monotonically increasing timestamp intended for profiling of audio frames.
133*d9f75844SAndroid Build Coastguard Worker   // Typically used for measuring elapsed time between two different points in
134*d9f75844SAndroid Build Coastguard Worker   // the audio path. No lock is used to save resources and we are thread safe
135*d9f75844SAndroid Build Coastguard Worker   // by design.
136*d9f75844SAndroid Build Coastguard Worker   // TODO([email protected]): consider using absl::optional.
137*d9f75844SAndroid Build Coastguard Worker   int64_t profile_timestamp_ms_ = 0;
138*d9f75844SAndroid Build Coastguard Worker 
139*d9f75844SAndroid Build Coastguard Worker   // Information about packets used to assemble this audio frame. This is needed
140*d9f75844SAndroid Build Coastguard Worker   // by `SourceTracker` when the frame is delivered to the RTCRtpReceiver's
141*d9f75844SAndroid Build Coastguard Worker   // MediaStreamTrack, in order to implement getContributingSources(). See:
142*d9f75844SAndroid Build Coastguard Worker   // https://w3c.github.io/webrtc-pc/#dom-rtcrtpreceiver-getcontributingsources
143*d9f75844SAndroid Build Coastguard Worker   //
144*d9f75844SAndroid Build Coastguard Worker   // TODO(bugs.webrtc.org/10757):
145*d9f75844SAndroid Build Coastguard Worker   //   Note that this information might not be fully accurate since we currently
146*d9f75844SAndroid Build Coastguard Worker   //   don't have a proper way to track it across the audio sync buffer. The
147*d9f75844SAndroid Build Coastguard Worker   //   sync buffer is the small sample-holding buffer located after the audio
148*d9f75844SAndroid Build Coastguard Worker   //   decoder and before where samples are assembled into output frames.
149*d9f75844SAndroid Build Coastguard Worker   //
150*d9f75844SAndroid Build Coastguard Worker   // `RtpPacketInfos` may also be empty if the audio samples did not come from
151*d9f75844SAndroid Build Coastguard Worker   // RTP packets. E.g. if the audio were locally generated by packet loss
152*d9f75844SAndroid Build Coastguard Worker   // concealment, comfort noise generation, etc.
153*d9f75844SAndroid Build Coastguard Worker   RtpPacketInfos packet_infos_;
154*d9f75844SAndroid Build Coastguard Worker 
155*d9f75844SAndroid Build Coastguard Worker  private:
156*d9f75844SAndroid Build Coastguard Worker   // A permanently zeroed out buffer to represent muted frames. This is a
157*d9f75844SAndroid Build Coastguard Worker   // header-only class, so the only way to avoid creating a separate empty
158*d9f75844SAndroid Build Coastguard Worker   // buffer per translation unit is to wrap a static in an inline function.
159*d9f75844SAndroid Build Coastguard Worker   static const int16_t* empty_data();
160*d9f75844SAndroid Build Coastguard Worker 
161*d9f75844SAndroid Build Coastguard Worker   int16_t data_[kMaxDataSizeSamples];
162*d9f75844SAndroid Build Coastguard Worker   bool muted_ = true;
163*d9f75844SAndroid Build Coastguard Worker 
164*d9f75844SAndroid Build Coastguard Worker   // Absolute capture timestamp when this audio frame was originally captured.
165*d9f75844SAndroid Build Coastguard Worker   // This is only valid for audio frames captured on this machine. The absolute
166*d9f75844SAndroid Build Coastguard Worker   // capture timestamp of a received frame is found in `packet_infos_`.
167*d9f75844SAndroid Build Coastguard Worker   // This timestamp MUST be based on the same clock as rtc::TimeMillis().
168*d9f75844SAndroid Build Coastguard Worker   absl::optional<int64_t> absolute_capture_timestamp_ms_;
169*d9f75844SAndroid Build Coastguard Worker };
170*d9f75844SAndroid Build Coastguard Worker 
171*d9f75844SAndroid Build Coastguard Worker }  // namespace webrtc
172*d9f75844SAndroid Build Coastguard Worker 
173*d9f75844SAndroid Build Coastguard Worker #endif  // API_AUDIO_AUDIO_FRAME_H_
174