1 /*
2  *  Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #ifndef TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_
12 #define TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_
13 
14 #include <deque>
15 #include <map>
16 #include <utility>
17 #include <vector>
18 
19 #include "api/array_view.h"
20 #include "rtc_base/event.h"
21 #include "rtc_base/platform_thread.h"
22 #include "rtc_base/synchronization/mutex.h"
23 #include "system_wrappers/include/clock.h"
24 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_cpu_measurer.h"
25 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
26 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
27 
28 namespace webrtc {
29 
30 struct FramesComparatorStats {
31   // Size of analyzer internal comparisons queue, measured when new element
32   // id added to the queue.
33   SamplesStatsCounter comparisons_queue_size;
34   // Number of performed comparisons of 2 video frames from captured and
35   // rendered streams.
36   int64_t comparisons_done = 0;
37   // Number of cpu overloaded comparisons. Comparison is cpu overloaded if it is
38   // queued when there are too many not processed comparisons in the queue.
39   // Overloaded comparison doesn't include metrics like SSIM and PSNR that
40   // require heavy computations.
41   int64_t cpu_overloaded_comparisons_done = 0;
42   // Number of memory overloaded comparisons. Comparison is memory overloaded if
43   // it is queued when its captured frame was already removed due to high memory
44   // usage for that video stream.
45   int64_t memory_overloaded_comparisons_done = 0;
46 };
47 
48 // Performs comparisons of added frames and tracks frames related statistics.
49 // This class is thread safe.
50 class DefaultVideoQualityAnalyzerFramesComparator {
51  public:
52   // Creates frames comparator.
53   // Frames comparator doesn't use `options.enable_receive_own_stream` for any
54   // purposes, because it's unrelated to its functionality.
55   DefaultVideoQualityAnalyzerFramesComparator(
56       webrtc::Clock* clock,
57       DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer,
58       DefaultVideoQualityAnalyzerOptions options = {})
options_(options)59       : options_(options), clock_(clock), cpu_measurer_(cpu_measurer) {}
~DefaultVideoQualityAnalyzerFramesComparator()60   ~DefaultVideoQualityAnalyzerFramesComparator() { Stop({}); }
61 
62   // Starts frames comparator. This method must be invoked before calling
63   // any other method on this object.
64   void Start(int max_threads_count);
65   // Stops frames comparator. This method will block until all added frame
66   // comparisons will be processed. After `Stop()` is invoked no more new
67   // comparisons can be added to this frames comparator.
68   //
69   // `last_rendered_frame_time` contains timestamps of last rendered frame for
70   //     each (stream, sender, receiver) tuple to properly update time between
71   //     freezes: it has include time from the last freeze until and of call.
72   void Stop(
73       const std::map<InternalStatsKey, Timestamp>& last_rendered_frame_times);
74 
75   // Ensures that stream `stream_index` has stats objects created for all
76   // potential receivers. This method must be called before adding any
77   // frames comparison for that stream.
78   void EnsureStatsForStream(size_t stream_index,
79                             size_t sender_peer_index,
80                             size_t peers_count,
81                             Timestamp captured_time,
82                             Timestamp start_time);
83   // Ensures that newly added participant will have stream stats objects created
84   // for all streams which they can receive. This method must be called before
85   // any frames comparison will be added for the newly added participant.
86   //
87   // `stream_started_time` - start time of each stream for which stats object
88   //     has to be created.
89   // `start_time` - call start time.
90   void RegisterParticipantInCall(
91       rtc::ArrayView<std::pair<InternalStatsKey, Timestamp>>
92           stream_started_time,
93       Timestamp start_time);
94 
95   // `captured` - video frame captured by sender to use for PSNR/SSIM
96   //     computation. If `type` is `FrameComparisonType::kRegular` and
97   //     `captured` is `absl::nullopt` comparison is assumed to be overloaded
98   //     due to memory constraints.
99   // `rendered` - video frame rendered by receiver to use for PSNR/SSIM
100   //     computation. Required only if `type` is
101   //     `FrameComparisonType::kRegular`, but can still be omitted if
102   //     `captured` is `absl::nullopt`.
103   void AddComparison(InternalStatsKey stats_key,
104                      absl::optional<VideoFrame> captured,
105                      absl::optional<VideoFrame> rendered,
106                      FrameComparisonType type,
107                      FrameStats frame_stats);
108   // `skipped_between_rendered` - amount of frames dropped on this stream before
109   //     last received frame and current frame.
110   void AddComparison(InternalStatsKey stats_key,
111                      int skipped_between_rendered,
112                      absl::optional<VideoFrame> captured,
113                      absl::optional<VideoFrame> rendered,
114                      FrameComparisonType type,
115                      FrameStats frame_stats);
116 
stream_stats()117   std::map<InternalStatsKey, StreamStats> stream_stats() const {
118     MutexLock lock(&mutex_);
119     return stream_stats_;
120   }
frames_comparator_stats()121   FramesComparatorStats frames_comparator_stats() const {
122     MutexLock lock(&mutex_);
123     return frames_comparator_stats_;
124   }
125 
126  private:
127   enum State { kNew, kActive, kStopped };
128 
129   void AddComparisonInternal(InternalStatsKey stats_key,
130                              absl::optional<VideoFrame> captured,
131                              absl::optional<VideoFrame> rendered,
132                              FrameComparisonType type,
133                              FrameStats frame_stats)
134       RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
135   void ProcessComparisons();
136   void ProcessComparison(const FrameComparison& comparison);
137   Timestamp Now();
138 
139   const DefaultVideoQualityAnalyzerOptions options_;
140   webrtc::Clock* const clock_;
141   DefaultVideoQualityAnalyzerCpuMeasurer& cpu_measurer_;
142 
143   mutable Mutex mutex_;
144   State state_ RTC_GUARDED_BY(mutex_) = State::kNew;
145   std::map<InternalStatsKey, StreamStats> stream_stats_ RTC_GUARDED_BY(mutex_);
146   std::map<InternalStatsKey, Timestamp> stream_last_freeze_end_time_
147       RTC_GUARDED_BY(mutex_);
148   std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(mutex_);
149   FramesComparatorStats frames_comparator_stats_ RTC_GUARDED_BY(mutex_);
150 
151   std::vector<rtc::PlatformThread> thread_pool_;
152   rtc::Event comparison_available_event_;
153 };
154 
155 }  // namespace webrtc
156 
157 #endif  // TEST_PC_E2E_ANALYZER_VIDEO_DEFAULT_VIDEO_QUALITY_ANALYZER_FRAMES_COMPARATOR_H_
158