1 /*
2  *  Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_
12 #define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_
13 
14 #include <map>
15 #include <set>
16 #include <utility>
17 #include <vector>
18 
19 #include "absl/types/optional.h"
20 #include "api/numerics/samples_stats_counter.h"
21 #include "api/units/data_size.h"
22 #include "api/units/timestamp.h"
23 #include "api/video/video_frame.h"
24 #include "api/video/video_frame_type.h"
25 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
26 
27 namespace webrtc {
28 
29 struct ReceiverFrameStats {
30   // Time when last packet of a frame was received.
31   Timestamp received_time = Timestamp::MinusInfinity();
32   Timestamp decode_start_time = Timestamp::MinusInfinity();
33   Timestamp decode_end_time = Timestamp::MinusInfinity();
34   Timestamp rendered_time = Timestamp::MinusInfinity();
35   Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity();
36 
37   // Type and encoded size of received frame.
38   VideoFrameType frame_type = VideoFrameType::kEmptyFrame;
39   DataSize encoded_image_size = DataSize::Bytes(0);
40 
41   absl::optional<int> decoded_frame_width = absl::nullopt;
42   absl::optional<int> decoded_frame_height = absl::nullopt;
43 
44   // Can be not set if frame was dropped in the network.
45   absl::optional<StreamCodecInfo> used_decoder = absl::nullopt;
46 
47   bool dropped = false;
48   bool decoder_failed = false;
49 };
50 
51 // Represents a frame which was sent by sender and is currently on the way to
52 // multiple receivers. Some receivers may receive this frame and some don't.
53 //
54 // Contains all statistic associated with the frame and gathered in multiple
55 // points of the video pipeline.
56 //
57 // Internally may store the copy of the source frame which was sent. In such
58 // case this frame is "alive".
59 class FrameInFlight {
60  public:
61   FrameInFlight(size_t stream,
62                 VideoFrame frame,
63                 Timestamp captured_time,
64                 std::set<size_t> expected_receivers);
65 
stream()66   size_t stream() const { return stream_; }
67   // Returns internal copy of source `VideoFrame` or `absl::nullopt` if it was
68   // removed before.
frame()69   const absl::optional<VideoFrame>& frame() const { return frame_; }
70   // Removes internal copy of the source `VideoFrame` to free up extra memory.
71   // Returns was frame removed or not.
72   bool RemoveFrame();
73   void SetFrameId(uint16_t id);
74 
AddExpectedReceiver(size_t peer)75   void AddExpectedReceiver(size_t peer) { expected_receivers_.insert(peer); }
76 
RemoveExpectedReceiver(size_t peer)77   void RemoveExpectedReceiver(size_t peer) { expected_receivers_.erase(peer); }
78 
79   std::vector<size_t> GetPeersWhichDidntReceive() const;
80 
81   // Returns if all peers which were expected to receive this frame actually
82   // received it or not.
83   bool HaveAllPeersReceived() const;
84 
SetPreEncodeTime(webrtc::Timestamp time)85   void SetPreEncodeTime(webrtc::Timestamp time) { pre_encode_time_ = time; }
86 
87   void OnFrameEncoded(webrtc::Timestamp time,
88                       VideoFrameType frame_type,
89                       DataSize encoded_image_size,
90                       uint32_t target_encode_bitrate,
91                       int qp,
92                       StreamCodecInfo used_encoder);
93 
HasEncodedTime()94   bool HasEncodedTime() const { return encoded_time_.IsFinite(); }
95 
96   void OnFramePreDecode(size_t peer,
97                         webrtc::Timestamp received_time,
98                         webrtc::Timestamp decode_start_time,
99                         VideoFrameType frame_type,
100                         DataSize encoded_image_size);
101 
102   bool HasReceivedTime(size_t peer) const;
103 
104   void OnFrameDecoded(size_t peer,
105                       webrtc::Timestamp time,
106                       int width,
107                       int height,
108                       const StreamCodecInfo& used_decoder);
109   void OnDecoderError(size_t peer, const StreamCodecInfo& used_decoder);
110 
111   bool HasDecodeEndTime(size_t peer) const;
112 
113   void OnFrameRendered(size_t peer, webrtc::Timestamp time);
114 
115   bool HasRenderedTime(size_t peer) const;
116 
117   // Crash if rendered time is not set for specified `peer`.
rendered_time(size_t peer)118   webrtc::Timestamp rendered_time(size_t peer) const {
119     return receiver_stats_.at(peer).rendered_time;
120   }
121 
122   // Marks that frame was dropped and wasn't seen by particular `peer`.
MarkDropped(size_t peer)123   void MarkDropped(size_t peer) { receiver_stats_[peer].dropped = true; }
124   bool IsDropped(size_t peer) const;
125 
SetPrevFrameRenderedTime(size_t peer,webrtc::Timestamp time)126   void SetPrevFrameRenderedTime(size_t peer, webrtc::Timestamp time) {
127     receiver_stats_[peer].prev_frame_rendered_time = time;
128   }
129 
130   FrameStats GetStatsForPeer(size_t peer) const;
131 
132  private:
133   const size_t stream_;
134   // Set of peer's indexes who are expected to receive this frame. This is not
135   // the set of peer's indexes that received the frame. For example, if peer A
136   // was among expected receivers, it received frame and then left the call, A
137   // will be removed from this set, but the Stats for peer A still will be
138   // preserved in the FrameInFlight.
139   //
140   // This set is used to determine if this frame is expected to be received by
141   // any peer or can be safely deleted. It is responsibility of the user of this
142   // object to decide when it should be deleted.
143   std::set<size_t> expected_receivers_;
144   absl::optional<VideoFrame> frame_;
145   // Store frame id separately because `frame_` can be removed when we have too
146   // much memory consuption.
147   uint16_t frame_id_ = VideoFrame::kNotSetId;
148 
149   // Frame events timestamp.
150   Timestamp captured_time_;
151   Timestamp pre_encode_time_ = Timestamp::MinusInfinity();
152   Timestamp encoded_time_ = Timestamp::MinusInfinity();
153   // Type and encoded size of sent frame.
154   VideoFrameType frame_type_ = VideoFrameType::kEmptyFrame;
155   DataSize encoded_image_size_ = DataSize::Bytes(0);
156   uint32_t target_encode_bitrate_ = 0;
157   SamplesStatsCounter qp_values_;
158   // Can be not set if frame was dropped by encoder.
159   absl::optional<StreamCodecInfo> used_encoder_ = absl::nullopt;
160   // Map from the receiver peer's index to frame stats for that peer.
161   std::map<size_t, ReceiverFrameStats> receiver_stats_;
162 };
163 
164 }  // namespace webrtc
165 
166 #endif  // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_
167