1 /* 2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_ 12 #define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_ 13 14 #include <map> 15 #include <set> 16 #include <utility> 17 #include <vector> 18 19 #include "absl/types/optional.h" 20 #include "api/numerics/samples_stats_counter.h" 21 #include "api/units/data_size.h" 22 #include "api/units/timestamp.h" 23 #include "api/video/video_frame.h" 24 #include "api/video/video_frame_type.h" 25 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h" 26 27 namespace webrtc { 28 29 struct ReceiverFrameStats { 30 // Time when last packet of a frame was received. 31 Timestamp received_time = Timestamp::MinusInfinity(); 32 Timestamp decode_start_time = Timestamp::MinusInfinity(); 33 Timestamp decode_end_time = Timestamp::MinusInfinity(); 34 Timestamp rendered_time = Timestamp::MinusInfinity(); 35 Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity(); 36 37 // Type and encoded size of received frame. 38 VideoFrameType frame_type = VideoFrameType::kEmptyFrame; 39 DataSize encoded_image_size = DataSize::Bytes(0); 40 41 absl::optional<int> decoded_frame_width = absl::nullopt; 42 absl::optional<int> decoded_frame_height = absl::nullopt; 43 44 // Can be not set if frame was dropped in the network. 45 absl::optional<StreamCodecInfo> used_decoder = absl::nullopt; 46 47 bool dropped = false; 48 bool decoder_failed = false; 49 }; 50 51 // Represents a frame which was sent by sender and is currently on the way to 52 // multiple receivers. Some receivers may receive this frame and some don't. 53 // 54 // Contains all statistic associated with the frame and gathered in multiple 55 // points of the video pipeline. 56 // 57 // Internally may store the copy of the source frame which was sent. In such 58 // case this frame is "alive". 59 class FrameInFlight { 60 public: 61 FrameInFlight(size_t stream, 62 VideoFrame frame, 63 Timestamp captured_time, 64 std::set<size_t> expected_receivers); 65 stream()66 size_t stream() const { return stream_; } 67 // Returns internal copy of source `VideoFrame` or `absl::nullopt` if it was 68 // removed before. frame()69 const absl::optional<VideoFrame>& frame() const { return frame_; } 70 // Removes internal copy of the source `VideoFrame` to free up extra memory. 71 // Returns was frame removed or not. 72 bool RemoveFrame(); 73 void SetFrameId(uint16_t id); 74 AddExpectedReceiver(size_t peer)75 void AddExpectedReceiver(size_t peer) { expected_receivers_.insert(peer); } 76 RemoveExpectedReceiver(size_t peer)77 void RemoveExpectedReceiver(size_t peer) { expected_receivers_.erase(peer); } 78 79 std::vector<size_t> GetPeersWhichDidntReceive() const; 80 81 // Returns if all peers which were expected to receive this frame actually 82 // received it or not. 83 bool HaveAllPeersReceived() const; 84 SetPreEncodeTime(webrtc::Timestamp time)85 void SetPreEncodeTime(webrtc::Timestamp time) { pre_encode_time_ = time; } 86 87 void OnFrameEncoded(webrtc::Timestamp time, 88 VideoFrameType frame_type, 89 DataSize encoded_image_size, 90 uint32_t target_encode_bitrate, 91 int qp, 92 StreamCodecInfo used_encoder); 93 HasEncodedTime()94 bool HasEncodedTime() const { return encoded_time_.IsFinite(); } 95 96 void OnFramePreDecode(size_t peer, 97 webrtc::Timestamp received_time, 98 webrtc::Timestamp decode_start_time, 99 VideoFrameType frame_type, 100 DataSize encoded_image_size); 101 102 bool HasReceivedTime(size_t peer) const; 103 104 void OnFrameDecoded(size_t peer, 105 webrtc::Timestamp time, 106 int width, 107 int height, 108 const StreamCodecInfo& used_decoder); 109 void OnDecoderError(size_t peer, const StreamCodecInfo& used_decoder); 110 111 bool HasDecodeEndTime(size_t peer) const; 112 113 void OnFrameRendered(size_t peer, webrtc::Timestamp time); 114 115 bool HasRenderedTime(size_t peer) const; 116 117 // Crash if rendered time is not set for specified `peer`. rendered_time(size_t peer)118 webrtc::Timestamp rendered_time(size_t peer) const { 119 return receiver_stats_.at(peer).rendered_time; 120 } 121 122 // Marks that frame was dropped and wasn't seen by particular `peer`. MarkDropped(size_t peer)123 void MarkDropped(size_t peer) { receiver_stats_[peer].dropped = true; } 124 bool IsDropped(size_t peer) const; 125 SetPrevFrameRenderedTime(size_t peer,webrtc::Timestamp time)126 void SetPrevFrameRenderedTime(size_t peer, webrtc::Timestamp time) { 127 receiver_stats_[peer].prev_frame_rendered_time = time; 128 } 129 130 FrameStats GetStatsForPeer(size_t peer) const; 131 132 private: 133 const size_t stream_; 134 // Set of peer's indexes who are expected to receive this frame. This is not 135 // the set of peer's indexes that received the frame. For example, if peer A 136 // was among expected receivers, it received frame and then left the call, A 137 // will be removed from this set, but the Stats for peer A still will be 138 // preserved in the FrameInFlight. 139 // 140 // This set is used to determine if this frame is expected to be received by 141 // any peer or can be safely deleted. It is responsibility of the user of this 142 // object to decide when it should be deleted. 143 std::set<size_t> expected_receivers_; 144 absl::optional<VideoFrame> frame_; 145 // Store frame id separately because `frame_` can be removed when we have too 146 // much memory consuption. 147 uint16_t frame_id_ = VideoFrame::kNotSetId; 148 149 // Frame events timestamp. 150 Timestamp captured_time_; 151 Timestamp pre_encode_time_ = Timestamp::MinusInfinity(); 152 Timestamp encoded_time_ = Timestamp::MinusInfinity(); 153 // Type and encoded size of sent frame. 154 VideoFrameType frame_type_ = VideoFrameType::kEmptyFrame; 155 DataSize encoded_image_size_ = DataSize::Bytes(0); 156 uint32_t target_encode_bitrate_ = 0; 157 SamplesStatsCounter qp_values_; 158 // Can be not set if frame was dropped by encoder. 159 absl::optional<StreamCodecInfo> used_encoder_ = absl::nullopt; 160 // Map from the receiver peer's index to frame stats for that peer. 161 std::map<size_t, ReceiverFrameStats> receiver_stats_; 162 }; 163 164 } // namespace webrtc 165 166 #endif // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAME_IN_FLIGHT_H_ 167