1 /*
2  *  Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_INTERNAL_SHARED_OBJECTS_H_
12 #define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_INTERNAL_SHARED_OBJECTS_H_
13 
14 #include <memory>
15 #include <string>
16 #include <utility>
17 #include <vector>
18 
19 #include "absl/types/optional.h"
20 #include "api/numerics/samples_stats_counter.h"
21 #include "api/units/data_size.h"
22 #include "api/units/timestamp.h"
23 #include "api/video/video_frame.h"
24 #include "api/video/video_frame_type.h"
25 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
26 
27 namespace webrtc {
28 
29 struct InternalStatsKey {
InternalStatsKeyInternalStatsKey30   InternalStatsKey(size_t stream, size_t sender, size_t receiver)
31       : stream(stream), sender(sender), receiver(receiver) {}
32 
33   std::string ToString() const;
34 
35   size_t stream;
36   size_t sender;
37   size_t receiver;
38 };
39 
40 // Required to use InternalStatsKey as std::map key.
41 bool operator<(const InternalStatsKey& a, const InternalStatsKey& b);
42 bool operator==(const InternalStatsKey& a, const InternalStatsKey& b);
43 
44 // Final stats computed for frame after it went through the whole video
45 // pipeline from capturing to rendering or dropping.
46 struct FrameStats {
FrameStatsFrameStats47   FrameStats(uint16_t frame_id, Timestamp captured_time)
48       : frame_id(frame_id), captured_time(captured_time) {}
49 
50   uint16_t frame_id;
51   // Frame events timestamp.
52   Timestamp captured_time;
53   Timestamp pre_encode_time = Timestamp::MinusInfinity();
54   Timestamp encoded_time = Timestamp::MinusInfinity();
55   // Time when last packet of a frame was received.
56   Timestamp received_time = Timestamp::MinusInfinity();
57   Timestamp decode_start_time = Timestamp::MinusInfinity();
58   Timestamp decode_end_time = Timestamp::MinusInfinity();
59   Timestamp rendered_time = Timestamp::MinusInfinity();
60   Timestamp prev_frame_rendered_time = Timestamp::MinusInfinity();
61 
62   VideoFrameType encoded_frame_type = VideoFrameType::kEmptyFrame;
63   DataSize encoded_image_size = DataSize::Bytes(0);
64   VideoFrameType pre_decoded_frame_type = VideoFrameType::kEmptyFrame;
65   DataSize pre_decoded_image_size = DataSize::Bytes(0);
66   uint32_t target_encode_bitrate = 0;
67   // There can be multiple qp values for single video frame when simulcast
68   // or SVC is used. In such case multiple EncodedImage's are created by encoder
69   // and each of it will have its own qp value.
70   SamplesStatsCounter qp_values;
71 
72   absl::optional<int> decoded_frame_width = absl::nullopt;
73   absl::optional<int> decoded_frame_height = absl::nullopt;
74 
75   // Can be not set if frame was dropped by encoder.
76   absl::optional<StreamCodecInfo> used_encoder = absl::nullopt;
77   // Can be not set if frame was dropped in the network.
78   absl::optional<StreamCodecInfo> used_decoder = absl::nullopt;
79 
80   bool decoder_failed = false;
81 };
82 
83 // Describes why comparison was done in overloaded mode (without calculating
84 // PSNR and SSIM).
85 enum class OverloadReason {
86   kNone,
87   // Not enough CPU to process all incoming comparisons.
88   kCpu,
89   // Not enough memory to store captured frames for all comparisons.
90   kMemory
91 };
92 
93 enum class FrameComparisonType {
94   // Comparison for captured and rendered frame.
95   kRegular,
96   // Comparison for captured frame that is known to be dropped somewhere in
97   // video pipeline.
98   kDroppedFrame,
99   // Comparison for captured frame that was still in the video pipeline when
100   // test was stopped. It's unknown is this frame dropped or would it be
101   // delivered if test continue.
102   kFrameInFlight
103 };
104 
105 // Represents comparison between two VideoFrames. Contains video frames itself
106 // and stats. Can be one of two types:
107 //   1. Normal - in this case `captured` is presented and either `rendered` is
108 //      presented and `dropped` is false, either `rendered` is omitted and
109 //      `dropped` is true.
110 //   2. Overloaded - in this case both `captured` and `rendered` are omitted
111 //      because there were too many comparisons in the queue. `dropped` can be
112 //      true or false showing was frame dropped or not.
113 struct FrameComparison {
114   FrameComparison(InternalStatsKey stats_key,
115                   absl::optional<VideoFrame> captured,
116                   absl::optional<VideoFrame> rendered,
117                   FrameComparisonType type,
118                   FrameStats frame_stats,
119                   OverloadReason overload_reason);
120 
121   InternalStatsKey stats_key;
122   // Frames can be omitted if there too many computations waiting in the
123   // queue.
124   absl::optional<VideoFrame> captured;
125   absl::optional<VideoFrame> rendered;
126   FrameComparisonType type;
127   FrameStats frame_stats;
128   OverloadReason overload_reason;
129 };
130 
131 }  // namespace webrtc
132 
133 #endif  // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_INTERNAL_SHARED_OBJECTS_H_
134