1 /*
2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer.h"
12
13 #include <algorithm>
14 #include <map>
15 #include <memory>
16 #include <vector>
17
18 #include "api/rtp_packet_info.h"
19 #include "api/rtp_packet_infos.h"
20 #include "api/test/create_frame_generator.h"
21 #include "api/test/metrics/global_metrics_logger_and_exporter.h"
22 #include "api/video/encoded_image.h"
23 #include "api/video/i420_buffer.h"
24 #include "api/video/video_frame.h"
25 #include "common_video/libyuv/include/webrtc_libyuv.h"
26 #include "rtc_base/strings/string_builder.h"
27 #include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
28 #include "system_wrappers/include/sleep.h"
29 #include "test/gtest.h"
30 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
31
32 namespace webrtc {
33 namespace {
34
35 using ::testing::TestWithParam;
36 using ::testing::ValuesIn;
37
38 using StatsSample = ::webrtc::SamplesStatsCounter::StatsSample;
39
40 constexpr int kAnalyzerMaxThreadsCount = 1;
41 constexpr int kMaxFramesInFlightPerStream = 10;
42 constexpr int kFrameWidth = 320;
43 constexpr int kFrameHeight = 240;
44 constexpr double kMaxSsim = 1;
45 constexpr char kStreamLabel[] = "video-stream";
46 constexpr char kSenderPeerName[] = "alice";
47 constexpr char kReceiverPeerName[] = "bob";
48
AnalyzerOptionsForTest()49 DefaultVideoQualityAnalyzerOptions AnalyzerOptionsForTest() {
50 DefaultVideoQualityAnalyzerOptions options;
51 options.compute_psnr = false;
52 options.compute_ssim = false;
53 options.adjust_cropping_before_comparing_frames = false;
54 options.max_frames_in_flight_per_stream_count = kMaxFramesInFlightPerStream;
55 return options;
56 }
57
NextFrame(test::FrameGeneratorInterface * frame_generator,int64_t timestamp_us)58 VideoFrame NextFrame(test::FrameGeneratorInterface* frame_generator,
59 int64_t timestamp_us) {
60 test::FrameGeneratorInterface::VideoFrameData frame_data =
61 frame_generator->NextFrame();
62 return VideoFrame::Builder()
63 .set_video_frame_buffer(frame_data.buffer)
64 .set_update_rect(frame_data.update_rect)
65 .set_timestamp_us(timestamp_us)
66 .build();
67 }
68
FakeEncode(const VideoFrame & frame)69 EncodedImage FakeEncode(const VideoFrame& frame) {
70 EncodedImage image;
71 std::vector<RtpPacketInfo> packet_infos;
72 packet_infos.push_back(RtpPacketInfo(
73 /*ssrc=*/1,
74 /*csrcs=*/{},
75 /*rtp_timestamp=*/frame.timestamp(),
76 /*receive_time=*/Timestamp::Micros(frame.timestamp_us() + 10000)));
77 image.SetPacketInfos(RtpPacketInfos(packet_infos));
78 return image;
79 }
80
DeepCopy(const VideoFrame & frame)81 VideoFrame DeepCopy(const VideoFrame& frame) {
82 VideoFrame copy = frame;
83 copy.set_video_frame_buffer(
84 I420Buffer::Copy(*frame.video_frame_buffer()->ToI420()));
85 return copy;
86 }
87
GetSortedSamples(const SamplesStatsCounter & counter)88 std::vector<StatsSample> GetSortedSamples(const SamplesStatsCounter& counter) {
89 rtc::ArrayView<const StatsSample> view = counter.GetTimedSamples();
90 std::vector<StatsSample> out(view.begin(), view.end());
91 std::sort(out.begin(), out.end(),
92 [](const StatsSample& a, const StatsSample& b) {
93 return a.time < b.time;
94 });
95 return out;
96 }
97
ToString(const std::vector<StatsSample> & values)98 std::string ToString(const std::vector<StatsSample>& values) {
99 rtc::StringBuilder out;
100 for (const auto& v : values) {
101 out << "{ time_ms=" << v.time.ms() << "; value=" << v.value << "}, ";
102 }
103 return out.str();
104 }
105
FakeCPULoad()106 void FakeCPULoad() {
107 std::vector<int> temp(1000000);
108 for (size_t i = 0; i < temp.size(); ++i) {
109 temp[i] = rand();
110 }
111 std::sort(temp.begin(), temp.end());
112 ASSERT_TRUE(std::is_sorted(temp.begin(), temp.end()));
113 }
114
PassFramesThroughAnalyzer(DefaultVideoQualityAnalyzer & analyzer,absl::string_view sender,absl::string_view stream_label,std::vector<absl::string_view> receivers,int frames_count,test::FrameGeneratorInterface & frame_generator,int interframe_delay_ms=0)115 void PassFramesThroughAnalyzer(DefaultVideoQualityAnalyzer& analyzer,
116 absl::string_view sender,
117 absl::string_view stream_label,
118 std::vector<absl::string_view> receivers,
119 int frames_count,
120 test::FrameGeneratorInterface& frame_generator,
121 int interframe_delay_ms = 0) {
122 for (int i = 0; i < frames_count; ++i) {
123 VideoFrame frame = NextFrame(&frame_generator, /*timestamp_us=*/1);
124 uint16_t frame_id =
125 analyzer.OnFrameCaptured(sender, std::string(stream_label), frame);
126 frame.set_id(frame_id);
127 analyzer.OnFramePreEncode(sender, frame);
128 analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame),
129 VideoQualityAnalyzerInterface::EncoderStats(),
130 false);
131 for (absl::string_view receiver : receivers) {
132 VideoFrame received_frame = DeepCopy(frame);
133 analyzer.OnFramePreDecode(receiver, received_frame.id(),
134 FakeEncode(received_frame));
135 analyzer.OnFrameDecoded(receiver, received_frame,
136 VideoQualityAnalyzerInterface::DecoderStats());
137 analyzer.OnFrameRendered(receiver, received_frame);
138 }
139 if (i < frames_count - 1 && interframe_delay_ms > 0) {
140 SleepMs(interframe_delay_ms);
141 }
142 }
143 }
144
TEST(DefaultVideoQualityAnalyzerTest,MemoryOverloadedAndThenAllFramesReceived)145 TEST(DefaultVideoQualityAnalyzerTest,
146 MemoryOverloadedAndThenAllFramesReceived) {
147 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
148 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
149 /*type=*/absl::nullopt,
150 /*num_squares=*/absl::nullopt);
151
152 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
153 test::GetGlobalMetricsLogger(),
154 AnalyzerOptionsForTest());
155 analyzer.Start("test_case",
156 std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
157 kAnalyzerMaxThreadsCount);
158
159 std::map<uint16_t, VideoFrame> captured_frames;
160 std::vector<uint16_t> frames_order;
161 for (int i = 0; i < kMaxFramesInFlightPerStream * 2; ++i) {
162 VideoFrame frame = NextFrame(frame_generator.get(), i);
163 frame.set_id(
164 analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
165 frames_order.push_back(frame.id());
166 captured_frames.insert({frame.id(), frame});
167 analyzer.OnFramePreEncode(kSenderPeerName, frame);
168 analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
169 VideoQualityAnalyzerInterface::EncoderStats(),
170 false);
171 }
172
173 for (const uint16_t& frame_id : frames_order) {
174 VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
175 analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
176 FakeEncode(received_frame));
177 analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
178 VideoQualityAnalyzerInterface::DecoderStats());
179 analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
180 }
181
182 // Give analyzer some time to process frames on async thread. The computations
183 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
184 // means we have an issue!
185 SleepMs(100);
186 analyzer.Stop();
187
188 AnalyzerStats stats = analyzer.GetAnalyzerStats();
189 EXPECT_EQ(stats.memory_overloaded_comparisons_done,
190 kMaxFramesInFlightPerStream);
191 EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream * 2);
192 FrameCounters frame_counters = analyzer.GetGlobalCounters();
193 EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream * 2);
194 EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream * 2);
195 EXPECT_EQ(frame_counters.dropped, 0);
196 }
197
TEST(DefaultVideoQualityAnalyzerTest,FillMaxMemoryReceiveAllMemoryOverloadedAndThenAllFramesReceived)198 TEST(DefaultVideoQualityAnalyzerTest,
199 FillMaxMemoryReceiveAllMemoryOverloadedAndThenAllFramesReceived) {
200 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
201 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
202 /*type=*/absl::nullopt,
203 /*num_squares=*/absl::nullopt);
204
205 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
206 test::GetGlobalMetricsLogger(),
207 AnalyzerOptionsForTest());
208 analyzer.Start("test_case",
209 std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
210 kAnalyzerMaxThreadsCount);
211
212 std::map<uint16_t, VideoFrame> captured_frames;
213 std::vector<uint16_t> frames_order;
214 // Feel analyzer's memory up to limit
215 for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) {
216 VideoFrame frame = NextFrame(frame_generator.get(), i);
217 frame.set_id(
218 analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
219 frames_order.push_back(frame.id());
220 captured_frames.insert({frame.id(), frame});
221 analyzer.OnFramePreEncode(kSenderPeerName, frame);
222 analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
223 VideoQualityAnalyzerInterface::EncoderStats(),
224 false);
225 }
226
227 // Receive all frames.
228 for (const uint16_t& frame_id : frames_order) {
229 VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
230 analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
231 FakeEncode(received_frame));
232 analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
233 VideoQualityAnalyzerInterface::DecoderStats());
234 analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
235 }
236 frames_order.clear();
237
238 // Give analyzer some time to process frames on async thread. The computations
239 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
240 // means we have an issue!
241 SleepMs(100);
242
243 // Overload analyzer's memory up to limit
244 for (int i = 0; i < 2 * kMaxFramesInFlightPerStream; ++i) {
245 VideoFrame frame = NextFrame(frame_generator.get(), i);
246 frame.set_id(
247 analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
248 frames_order.push_back(frame.id());
249 captured_frames.insert({frame.id(), frame});
250 analyzer.OnFramePreEncode(kSenderPeerName, frame);
251 analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
252 VideoQualityAnalyzerInterface::EncoderStats(),
253 false);
254 }
255
256 // Receive all frames.
257 for (const uint16_t& frame_id : frames_order) {
258 VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
259 analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
260 FakeEncode(received_frame));
261 analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
262 VideoQualityAnalyzerInterface::DecoderStats());
263 analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
264 }
265
266 // Give analyzer some time to process frames on async thread. The computations
267 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
268 // means we have an issue!
269 SleepMs(100);
270 analyzer.Stop();
271
272 AnalyzerStats stats = analyzer.GetAnalyzerStats();
273 EXPECT_EQ(stats.memory_overloaded_comparisons_done,
274 kMaxFramesInFlightPerStream);
275 EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream * 3);
276 FrameCounters frame_counters = analyzer.GetGlobalCounters();
277 EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream * 3);
278 EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream * 3);
279 EXPECT_EQ(frame_counters.dropped, 0);
280 }
281
TEST(DefaultVideoQualityAnalyzerTest,MemoryOverloadedHalfDroppedAndThenHalfFramesReceived)282 TEST(DefaultVideoQualityAnalyzerTest,
283 MemoryOverloadedHalfDroppedAndThenHalfFramesReceived) {
284 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
285 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
286 /*type=*/absl::nullopt,
287 /*num_squares=*/absl::nullopt);
288
289 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
290 test::GetGlobalMetricsLogger(),
291 AnalyzerOptionsForTest());
292 analyzer.Start("test_case",
293 std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
294 kAnalyzerMaxThreadsCount);
295
296 std::map<uint16_t, VideoFrame> captured_frames;
297 std::vector<uint16_t> frames_order;
298 for (int i = 0; i < kMaxFramesInFlightPerStream * 2; ++i) {
299 VideoFrame frame = NextFrame(frame_generator.get(), i);
300 frame.set_id(
301 analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
302 frames_order.push_back(frame.id());
303 captured_frames.insert({frame.id(), frame});
304 analyzer.OnFramePreEncode(kSenderPeerName, frame);
305 analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
306 VideoQualityAnalyzerInterface::EncoderStats(),
307 false);
308 }
309
310 for (size_t i = kMaxFramesInFlightPerStream; i < frames_order.size(); ++i) {
311 uint16_t frame_id = frames_order.at(i);
312 VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
313 analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
314 FakeEncode(received_frame));
315 analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
316 VideoQualityAnalyzerInterface::DecoderStats());
317 analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
318 }
319
320 // Give analyzer some time to process frames on async thread. The computations
321 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
322 // means we have an issue!
323 SleepMs(100);
324 analyzer.Stop();
325
326 AnalyzerStats stats = analyzer.GetAnalyzerStats();
327 EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0);
328 EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream * 2);
329 FrameCounters frame_counters = analyzer.GetGlobalCounters();
330 EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream * 2);
331 EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream);
332 EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream);
333 }
334
TEST(DefaultVideoQualityAnalyzerTest,NormalScenario)335 TEST(DefaultVideoQualityAnalyzerTest, NormalScenario) {
336 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
337 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
338 /*type=*/absl::nullopt,
339 /*num_squares=*/absl::nullopt);
340
341 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
342 test::GetGlobalMetricsLogger(),
343 AnalyzerOptionsForTest());
344 analyzer.Start("test_case",
345 std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
346 kAnalyzerMaxThreadsCount);
347
348 std::map<uint16_t, VideoFrame> captured_frames;
349 std::vector<uint16_t> frames_order;
350 for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) {
351 VideoFrame frame = NextFrame(frame_generator.get(), i);
352 frame.set_id(
353 analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
354 frames_order.push_back(frame.id());
355 captured_frames.insert({frame.id(), frame});
356 analyzer.OnFramePreEncode(kSenderPeerName, frame);
357 analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
358 VideoQualityAnalyzerInterface::EncoderStats(),
359 false);
360 }
361
362 for (size_t i = 1; i < frames_order.size(); i += 2) {
363 uint16_t frame_id = frames_order.at(i);
364 VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
365 analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
366 FakeEncode(received_frame));
367 analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
368 VideoQualityAnalyzerInterface::DecoderStats());
369 analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
370 }
371
372 // Give analyzer some time to process frames on async thread. The computations
373 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
374 // means we have an issue!
375 SleepMs(100);
376 analyzer.Stop();
377
378 AnalyzerStats stats = analyzer.GetAnalyzerStats();
379 EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0);
380 EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream);
381
382 std::vector<StatsSample> frames_in_flight_sizes =
383 GetSortedSamples(stats.frames_in_flight_left_count);
384 EXPECT_EQ(frames_in_flight_sizes.back().value, 0)
385 << ToString(frames_in_flight_sizes);
386
387 FrameCounters frame_counters = analyzer.GetGlobalCounters();
388 EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream);
389 EXPECT_EQ(frame_counters.received, kMaxFramesInFlightPerStream / 2);
390 EXPECT_EQ(frame_counters.decoded, kMaxFramesInFlightPerStream / 2);
391 EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream / 2);
392 EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream / 2);
393 }
394
TEST(DefaultVideoQualityAnalyzerTest,OneFrameReceivedTwice)395 TEST(DefaultVideoQualityAnalyzerTest, OneFrameReceivedTwice) {
396 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
397 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
398 /*type=*/absl::nullopt,
399 /*num_squares=*/absl::nullopt);
400
401 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
402 test::GetGlobalMetricsLogger(),
403 AnalyzerOptionsForTest());
404 analyzer.Start("test_case",
405 std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
406 kAnalyzerMaxThreadsCount);
407
408 VideoFrame captured_frame = NextFrame(frame_generator.get(), 0);
409 captured_frame.set_id(
410 analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, captured_frame));
411 analyzer.OnFramePreEncode(kSenderPeerName, captured_frame);
412 analyzer.OnFrameEncoded(kSenderPeerName, captured_frame.id(),
413 FakeEncode(captured_frame),
414 VideoQualityAnalyzerInterface::EncoderStats(), false);
415
416 VideoFrame received_frame = DeepCopy(captured_frame);
417 analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
418 FakeEncode(received_frame));
419 analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
420 VideoQualityAnalyzerInterface::DecoderStats());
421 analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
422
423 received_frame = DeepCopy(captured_frame);
424 analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
425 FakeEncode(received_frame));
426 analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
427 VideoQualityAnalyzerInterface::DecoderStats());
428 analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
429
430 // Give analyzer some time to process frames on async thread. The computations
431 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
432 // means we have an issue!
433 SleepMs(100);
434 analyzer.Stop();
435
436 AnalyzerStats stats = analyzer.GetAnalyzerStats();
437 EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0);
438 EXPECT_EQ(stats.comparisons_done, 1);
439
440 FrameCounters frame_counters = analyzer.GetGlobalCounters();
441 EXPECT_EQ(frame_counters.captured, 1);
442 EXPECT_EQ(frame_counters.received, 1);
443 EXPECT_EQ(frame_counters.decoded, 1);
444 EXPECT_EQ(frame_counters.rendered, 1);
445 EXPECT_EQ(frame_counters.dropped, 0);
446 }
447
TEST(DefaultVideoQualityAnalyzerTest,NormalScenario2Receivers)448 TEST(DefaultVideoQualityAnalyzerTest, NormalScenario2Receivers) {
449 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
450 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
451 /*type=*/absl::nullopt,
452 /*num_squares=*/absl::nullopt);
453
454 constexpr char kAlice[] = "alice";
455 constexpr char kBob[] = "bob";
456 constexpr char kCharlie[] = "charlie";
457
458 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
459 test::GetGlobalMetricsLogger(),
460 AnalyzerOptionsForTest());
461 analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie},
462 kAnalyzerMaxThreadsCount);
463
464 std::map<uint16_t, VideoFrame> captured_frames;
465 std::vector<uint16_t> frames_order;
466 for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) {
467 VideoFrame frame = NextFrame(frame_generator.get(), i);
468 frame.set_id(analyzer.OnFrameCaptured(kAlice, kStreamLabel, frame));
469 frames_order.push_back(frame.id());
470 captured_frames.insert({frame.id(), frame});
471 analyzer.OnFramePreEncode(kAlice, frame);
472 SleepMs(20);
473 analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame),
474 VideoQualityAnalyzerInterface::EncoderStats(),
475 false);
476 }
477
478 SleepMs(50);
479
480 for (size_t i = 1; i < frames_order.size(); i += 2) {
481 uint16_t frame_id = frames_order.at(i);
482 VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
483 analyzer.OnFramePreDecode(kBob, received_frame.id(),
484 FakeEncode(received_frame));
485 SleepMs(30);
486 analyzer.OnFrameDecoded(kBob, received_frame,
487 VideoQualityAnalyzerInterface::DecoderStats());
488 SleepMs(10);
489 analyzer.OnFrameRendered(kBob, received_frame);
490 }
491
492 for (size_t i = 1; i < frames_order.size(); i += 2) {
493 uint16_t frame_id = frames_order.at(i);
494 VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
495 analyzer.OnFramePreDecode(kCharlie, received_frame.id(),
496 FakeEncode(received_frame));
497 SleepMs(40);
498 analyzer.OnFrameDecoded(kCharlie, received_frame,
499 VideoQualityAnalyzerInterface::DecoderStats());
500 SleepMs(5);
501 analyzer.OnFrameRendered(kCharlie, received_frame);
502 }
503
504 // Give analyzer some time to process frames on async thread. The computations
505 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
506 // means we have an issue!
507 SleepMs(100);
508 analyzer.Stop();
509
510 AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats();
511 EXPECT_EQ(analyzer_stats.memory_overloaded_comparisons_done, 0);
512 EXPECT_EQ(analyzer_stats.comparisons_done, kMaxFramesInFlightPerStream * 2);
513
514 FrameCounters frame_counters = analyzer.GetGlobalCounters();
515 EXPECT_EQ(frame_counters.captured, kMaxFramesInFlightPerStream);
516 EXPECT_EQ(frame_counters.received, kMaxFramesInFlightPerStream);
517 EXPECT_EQ(frame_counters.decoded, kMaxFramesInFlightPerStream);
518 EXPECT_EQ(frame_counters.rendered, kMaxFramesInFlightPerStream);
519 EXPECT_EQ(frame_counters.dropped, kMaxFramesInFlightPerStream);
520
521 VideoStreamsInfo streams_info = analyzer.GetKnownStreams();
522 EXPECT_EQ(streams_info.GetStreams(), std::set<std::string>{kStreamLabel});
523 EXPECT_EQ(streams_info.GetStreams(kAlice),
524 std::set<std::string>{kStreamLabel});
525 EXPECT_EQ(streams_info.GetSender(kStreamLabel), kAlice);
526 EXPECT_EQ(streams_info.GetReceivers(kStreamLabel),
527 (std::set<std::string>{kBob, kCharlie}));
528
529 EXPECT_EQ(streams_info.GetStatsKeys().size(), 2lu);
530 for (auto stream_key : streams_info.GetStatsKeys()) {
531 FrameCounters stream_conters =
532 analyzer.GetPerStreamCounters().at(stream_key);
533 // On some devices the pipeline can be too slow, so we actually can't
534 // force real constraints here. Lets just check, that at least 1
535 // frame passed whole pipeline.
536 EXPECT_GE(stream_conters.captured, 10);
537 EXPECT_GE(stream_conters.pre_encoded, 10);
538 EXPECT_GE(stream_conters.encoded, 10);
539 EXPECT_GE(stream_conters.received, 5);
540 EXPECT_GE(stream_conters.decoded, 5);
541 EXPECT_GE(stream_conters.rendered, 5);
542 EXPECT_GE(stream_conters.dropped, 5);
543 }
544
545 std::map<StatsKey, StreamStats> stats = analyzer.GetStats();
546 const StatsKey kAliceBobStats(kStreamLabel, kBob);
547 const StatsKey kAliceCharlieStats(kStreamLabel, kCharlie);
548 EXPECT_EQ(stats.size(), 2lu);
549 {
550 auto it = stats.find(kAliceBobStats);
551 EXPECT_FALSE(it == stats.end());
552 ASSERT_FALSE(it->second.encode_time_ms.IsEmpty());
553 EXPECT_GE(it->second.encode_time_ms.GetMin(), 20);
554 ASSERT_FALSE(it->second.decode_time_ms.IsEmpty());
555 EXPECT_GE(it->second.decode_time_ms.GetMin(), 30);
556 ASSERT_FALSE(it->second.resolution_of_decoded_frame.IsEmpty());
557 EXPECT_GE(it->second.resolution_of_decoded_frame.GetMin(),
558 kFrameWidth * kFrameHeight - 1);
559 EXPECT_LE(it->second.resolution_of_decoded_frame.GetMax(),
560 kFrameWidth * kFrameHeight + 1);
561 }
562 {
563 auto it = stats.find(kAliceCharlieStats);
564 EXPECT_FALSE(it == stats.end());
565 ASSERT_FALSE(it->second.encode_time_ms.IsEmpty());
566 EXPECT_GE(it->second.encode_time_ms.GetMin(), 20);
567 ASSERT_FALSE(it->second.decode_time_ms.IsEmpty());
568 EXPECT_GE(it->second.decode_time_ms.GetMin(), 30);
569 ASSERT_FALSE(it->second.resolution_of_decoded_frame.IsEmpty());
570 EXPECT_GE(it->second.resolution_of_decoded_frame.GetMin(),
571 kFrameWidth * kFrameHeight - 1);
572 EXPECT_LE(it->second.resolution_of_decoded_frame.GetMax(),
573 kFrameWidth * kFrameHeight + 1);
574 }
575 }
576
577 // Test the case which can happen when SFU is switching from one layer to
578 // another, so the same frame can be received twice by the same peer.
TEST(DefaultVideoQualityAnalyzerTest,OneFrameReceivedTwiceBySamePeerWith2Receivers)579 TEST(DefaultVideoQualityAnalyzerTest,
580 OneFrameReceivedTwiceBySamePeerWith2Receivers) {
581 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
582 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
583 /*type=*/absl::nullopt,
584 /*num_squares=*/absl::nullopt);
585
586 constexpr char kAlice[] = "alice";
587 constexpr char kBob[] = "bob";
588 constexpr char kCharlie[] = "charlie";
589
590 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
591 test::GetGlobalMetricsLogger(),
592 AnalyzerOptionsForTest());
593 analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie},
594 kAnalyzerMaxThreadsCount);
595
596 VideoFrame captured_frame = NextFrame(frame_generator.get(), 0);
597 captured_frame.set_id(
598 analyzer.OnFrameCaptured(kAlice, kStreamLabel, captured_frame));
599 analyzer.OnFramePreEncode(kAlice, captured_frame);
600 analyzer.OnFrameEncoded(kAlice, captured_frame.id(),
601 FakeEncode(captured_frame),
602 VideoQualityAnalyzerInterface::EncoderStats(), false);
603
604 VideoFrame received_frame = DeepCopy(captured_frame);
605 analyzer.OnFramePreDecode(kBob, received_frame.id(),
606 FakeEncode(received_frame));
607 analyzer.OnFrameDecoded(kBob, received_frame,
608 VideoQualityAnalyzerInterface::DecoderStats());
609 analyzer.OnFrameRendered(kBob, received_frame);
610
611 received_frame = DeepCopy(captured_frame);
612 analyzer.OnFramePreDecode(kBob, received_frame.id(),
613 FakeEncode(received_frame));
614 analyzer.OnFrameDecoded(kBob, received_frame,
615 VideoQualityAnalyzerInterface::DecoderStats());
616 analyzer.OnFrameRendered(kBob, received_frame);
617
618 // Give analyzer some time to process frames on async thread. The computations
619 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
620 // means we have an issue!
621 SleepMs(100);
622 analyzer.Stop();
623
624 AnalyzerStats stats = analyzer.GetAnalyzerStats();
625 EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0);
626 // We have 2 comparisons here because 1 for the frame received by Bob and
627 // 1 for the frame in flight from Alice to Charlie.
628 EXPECT_EQ(stats.comparisons_done, 2);
629
630 FrameCounters frame_counters = analyzer.GetGlobalCounters();
631 EXPECT_EQ(frame_counters.captured, 1);
632 EXPECT_EQ(frame_counters.received, 1);
633 EXPECT_EQ(frame_counters.decoded, 1);
634 EXPECT_EQ(frame_counters.rendered, 1);
635 EXPECT_EQ(frame_counters.dropped, 0);
636 }
637
TEST(DefaultVideoQualityAnalyzerTest,HeavyQualityMetricsFromEqualFrames)638 TEST(DefaultVideoQualityAnalyzerTest, HeavyQualityMetricsFromEqualFrames) {
639 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
640 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
641 /*type=*/absl::nullopt,
642 /*num_squares=*/absl::nullopt);
643
644 DefaultVideoQualityAnalyzerOptions analyzer_options;
645 analyzer_options.compute_psnr = true;
646 analyzer_options.compute_ssim = true;
647 analyzer_options.adjust_cropping_before_comparing_frames = false;
648 analyzer_options.max_frames_in_flight_per_stream_count =
649 kMaxFramesInFlightPerStream;
650 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
651 test::GetGlobalMetricsLogger(),
652 analyzer_options);
653 analyzer.Start("test_case",
654 std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
655 kAnalyzerMaxThreadsCount);
656
657 for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) {
658 VideoFrame frame = NextFrame(frame_generator.get(), i);
659 frame.set_id(
660 analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
661 analyzer.OnFramePreEncode(kSenderPeerName, frame);
662 analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
663 VideoQualityAnalyzerInterface::EncoderStats(),
664 false);
665
666 VideoFrame received_frame = DeepCopy(frame);
667 analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
668 FakeEncode(received_frame));
669 analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
670 VideoQualityAnalyzerInterface::DecoderStats());
671 analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
672 }
673
674 // Give analyzer some time to process frames on async thread. Heavy metrics
675 // computation is turned on, so giving some extra time to be sure that
676 // computatio have ended.
677 SleepMs(500);
678 analyzer.Stop();
679
680 AnalyzerStats stats = analyzer.GetAnalyzerStats();
681 EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0);
682 EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream);
683
684 std::vector<StatsSample> frames_in_flight_sizes =
685 GetSortedSamples(stats.frames_in_flight_left_count);
686 EXPECT_EQ(frames_in_flight_sizes.back().value, 0)
687 << ToString(frames_in_flight_sizes);
688
689 std::map<StatsKey, StreamStats> stream_stats = analyzer.GetStats();
690 const StatsKey kAliceBobStats(kStreamLabel, kReceiverPeerName);
691 EXPECT_EQ(stream_stats.size(), 1lu);
692
693 auto it = stream_stats.find(kAliceBobStats);
694 EXPECT_GE(it->second.psnr.GetMin(), kPerfectPSNR);
695 EXPECT_GE(it->second.ssim.GetMin(), kMaxSsim);
696 }
697
TEST(DefaultVideoQualityAnalyzerTest,HeavyQualityMetricsFromShiftedFramesWithAdjustment)698 TEST(DefaultVideoQualityAnalyzerTest,
699 HeavyQualityMetricsFromShiftedFramesWithAdjustment) {
700 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
701 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
702 /*type=*/absl::nullopt,
703 /*num_squares=*/absl::nullopt);
704
705 DefaultVideoQualityAnalyzerOptions analyzer_options;
706 analyzer_options.compute_psnr = true;
707 analyzer_options.compute_ssim = true;
708 analyzer_options.adjust_cropping_before_comparing_frames = true;
709 analyzer_options.max_frames_in_flight_per_stream_count =
710 kMaxFramesInFlightPerStream;
711 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
712 test::GetGlobalMetricsLogger(),
713 analyzer_options);
714 analyzer.Start("test_case",
715 std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
716 kAnalyzerMaxThreadsCount);
717
718 for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) {
719 VideoFrame frame = NextFrame(frame_generator.get(), i);
720 frame.set_id(
721 analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
722 analyzer.OnFramePreEncode(kSenderPeerName, frame);
723 analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
724 VideoQualityAnalyzerInterface::EncoderStats(),
725 false);
726
727 VideoFrame received_frame = frame;
728 // Shift frame by a few pixels.
729 test::CropRegion crop_region{0, 1, 3, 0};
730 rtc::scoped_refptr<VideoFrameBuffer> cropped_buffer =
731 CropAndZoom(crop_region, received_frame.video_frame_buffer()->ToI420());
732 received_frame.set_video_frame_buffer(cropped_buffer);
733
734 analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
735 FakeEncode(received_frame));
736 analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
737 VideoQualityAnalyzerInterface::DecoderStats());
738 analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
739 }
740
741 // Give analyzer some time to process frames on async thread. Heavy metrics
742 // computation is turned on, so giving some extra time to be sure that
743 // computatio have ended.
744 SleepMs(500);
745 analyzer.Stop();
746
747 AnalyzerStats stats = analyzer.GetAnalyzerStats();
748 EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0);
749 EXPECT_EQ(stats.comparisons_done, kMaxFramesInFlightPerStream);
750
751 std::vector<StatsSample> frames_in_flight_sizes =
752 GetSortedSamples(stats.frames_in_flight_left_count);
753 EXPECT_EQ(frames_in_flight_sizes.back().value, 0)
754 << ToString(frames_in_flight_sizes);
755
756 std::map<StatsKey, StreamStats> stream_stats = analyzer.GetStats();
757 const StatsKey kAliceBobStats(kStreamLabel, kReceiverPeerName);
758 EXPECT_EQ(stream_stats.size(), 1lu);
759
760 auto it = stream_stats.find(kAliceBobStats);
761 EXPECT_GE(it->second.psnr.GetMin(), kPerfectPSNR);
762 EXPECT_GE(it->second.ssim.GetMin(), kMaxSsim);
763 }
764
TEST(DefaultVideoQualityAnalyzerTest,CpuUsage)765 TEST(DefaultVideoQualityAnalyzerTest, CpuUsage) {
766 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
767 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
768 /*type=*/absl::nullopt,
769 /*num_squares=*/absl::nullopt);
770
771 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
772 test::GetGlobalMetricsLogger(),
773 AnalyzerOptionsForTest());
774 analyzer.Start("test_case",
775 std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
776 kAnalyzerMaxThreadsCount);
777
778 std::map<uint16_t, VideoFrame> captured_frames;
779 std::vector<uint16_t> frames_order;
780 for (int i = 0; i < kMaxFramesInFlightPerStream; ++i) {
781 VideoFrame frame = NextFrame(frame_generator.get(), i);
782 frame.set_id(
783 analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
784 frames_order.push_back(frame.id());
785 captured_frames.insert({frame.id(), frame});
786 analyzer.OnFramePreEncode(kSenderPeerName, frame);
787 analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
788 VideoQualityAnalyzerInterface::EncoderStats(),
789 false);
790 }
791
792 // Windows CPU clock has low accuracy. We need to fake some additional load to
793 // be sure that the clock ticks (https://crbug.com/webrtc/12249).
794 FakeCPULoad();
795
796 for (size_t i = 1; i < frames_order.size(); i += 2) {
797 uint16_t frame_id = frames_order.at(i);
798 VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
799 analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
800 FakeEncode(received_frame));
801 analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
802 VideoQualityAnalyzerInterface::DecoderStats());
803 analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
804 }
805
806 // Give analyzer some time to process frames on async thread. The computations
807 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
808 // means we have an issue!
809 SleepMs(100);
810 analyzer.Stop();
811
812 double cpu_usage = analyzer.GetCpuUsagePercent();
813 ASSERT_GT(cpu_usage, 0);
814
815 SleepMs(100);
816 analyzer.Stop();
817
818 EXPECT_EQ(analyzer.GetCpuUsagePercent(), cpu_usage);
819 }
820
TEST(DefaultVideoQualityAnalyzerTest,RuntimeParticipantsAdding)821 TEST(DefaultVideoQualityAnalyzerTest, RuntimeParticipantsAdding) {
822 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
823 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
824 /*type=*/absl::nullopt,
825 /*num_squares=*/absl::nullopt);
826
827 constexpr char kAlice[] = "alice";
828 constexpr char kBob[] = "bob";
829 constexpr char kCharlie[] = "charlie";
830 constexpr char kKatie[] = "katie";
831
832 constexpr int kFramesCount = 9;
833 constexpr int kOneThirdFrames = kFramesCount / 3;
834 constexpr int kTwoThirdFrames = 2 * kOneThirdFrames;
835
836 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
837 test::GetGlobalMetricsLogger(),
838 AnalyzerOptionsForTest());
839 analyzer.Start("test_case", {}, kAnalyzerMaxThreadsCount);
840
841 std::map<uint16_t, VideoFrame> captured_frames;
842 std::vector<uint16_t> frames_order;
843 analyzer.RegisterParticipantInCall(kAlice);
844 analyzer.RegisterParticipantInCall(kBob);
845
846 // Alice is sending frames.
847 for (int i = 0; i < kFramesCount; ++i) {
848 VideoFrame frame = NextFrame(frame_generator.get(), i);
849 frame.set_id(analyzer.OnFrameCaptured(kAlice, kStreamLabel, frame));
850 frames_order.push_back(frame.id());
851 captured_frames.insert({frame.id(), frame});
852 analyzer.OnFramePreEncode(kAlice, frame);
853 analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame),
854 VideoQualityAnalyzerInterface::EncoderStats(),
855 false);
856 }
857
858 // Bob receives one third of the sent frames.
859 for (int i = 0; i < kOneThirdFrames; ++i) {
860 uint16_t frame_id = frames_order.at(i);
861 VideoFrame received_frame = DeepCopy(captured_frames.at(frame_id));
862 analyzer.OnFramePreDecode(kBob, received_frame.id(),
863 FakeEncode(received_frame));
864 analyzer.OnFrameDecoded(kBob, received_frame,
865 VideoQualityAnalyzerInterface::DecoderStats());
866 analyzer.OnFrameRendered(kBob, received_frame);
867 }
868
869 analyzer.RegisterParticipantInCall(kCharlie);
870 analyzer.RegisterParticipantInCall(kKatie);
871
872 // New participants were dynamically added. Bob and Charlie receive second
873 // third of the sent frames. Katie drops the frames.
874 for (int i = kOneThirdFrames; i < kTwoThirdFrames; ++i) {
875 uint16_t frame_id = frames_order.at(i);
876 VideoFrame bob_received_frame = DeepCopy(captured_frames.at(frame_id));
877 analyzer.OnFramePreDecode(kBob, bob_received_frame.id(),
878 FakeEncode(bob_received_frame));
879 analyzer.OnFrameDecoded(kBob, bob_received_frame,
880 VideoQualityAnalyzerInterface::DecoderStats());
881 analyzer.OnFrameRendered(kBob, bob_received_frame);
882
883 VideoFrame charlie_received_frame = DeepCopy(captured_frames.at(frame_id));
884 analyzer.OnFramePreDecode(kCharlie, charlie_received_frame.id(),
885 FakeEncode(charlie_received_frame));
886 analyzer.OnFrameDecoded(kCharlie, charlie_received_frame,
887 VideoQualityAnalyzerInterface::DecoderStats());
888 analyzer.OnFrameRendered(kCharlie, charlie_received_frame);
889 }
890
891 // Bob, Charlie and Katie receive the rest of the sent frames.
892 for (int i = kTwoThirdFrames; i < kFramesCount; ++i) {
893 uint16_t frame_id = frames_order.at(i);
894 VideoFrame bob_received_frame = DeepCopy(captured_frames.at(frame_id));
895 analyzer.OnFramePreDecode(kBob, bob_received_frame.id(),
896 FakeEncode(bob_received_frame));
897 analyzer.OnFrameDecoded(kBob, bob_received_frame,
898 VideoQualityAnalyzerInterface::DecoderStats());
899 analyzer.OnFrameRendered(kBob, bob_received_frame);
900
901 VideoFrame charlie_received_frame = DeepCopy(captured_frames.at(frame_id));
902 analyzer.OnFramePreDecode(kCharlie, charlie_received_frame.id(),
903 FakeEncode(charlie_received_frame));
904 analyzer.OnFrameDecoded(kCharlie, charlie_received_frame,
905 VideoQualityAnalyzerInterface::DecoderStats());
906 analyzer.OnFrameRendered(kCharlie, charlie_received_frame);
907
908 VideoFrame katie_received_frame = DeepCopy(captured_frames.at(frame_id));
909 analyzer.OnFramePreDecode(kKatie, katie_received_frame.id(),
910 FakeEncode(katie_received_frame));
911 analyzer.OnFrameDecoded(kKatie, katie_received_frame,
912 VideoQualityAnalyzerInterface::DecoderStats());
913 analyzer.OnFrameRendered(kKatie, katie_received_frame);
914 }
915
916 // Give analyzer some time to process frames on async thread. The computations
917 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
918 // means we have an issue!
919 SleepMs(100);
920 analyzer.Stop();
921
922 AnalyzerStats stats = analyzer.GetAnalyzerStats();
923 EXPECT_EQ(stats.memory_overloaded_comparisons_done, 0);
924 EXPECT_EQ(stats.comparisons_done, kFramesCount + 2 * kTwoThirdFrames);
925
926 std::vector<StatsSample> frames_in_flight_sizes =
927 GetSortedSamples(stats.frames_in_flight_left_count);
928 EXPECT_EQ(frames_in_flight_sizes.back().value, 0)
929 << ToString(frames_in_flight_sizes);
930
931 FrameCounters frame_counters = analyzer.GetGlobalCounters();
932 EXPECT_EQ(frame_counters.captured, kFramesCount);
933 EXPECT_EQ(frame_counters.received, 2 * kFramesCount);
934 EXPECT_EQ(frame_counters.decoded, 2 * kFramesCount);
935 EXPECT_EQ(frame_counters.rendered, 2 * kFramesCount);
936 EXPECT_EQ(frame_counters.dropped, kOneThirdFrames);
937
938 const StatsKey kAliceBobStats(kStreamLabel, kBob);
939 const StatsKey kAliceCharlieStats(kStreamLabel, kCharlie);
940 const StatsKey kAliceKatieStats(kStreamLabel, kKatie);
941 EXPECT_EQ(analyzer.GetKnownStreams().GetStatsKeys(),
942 (std::set<StatsKey>{kAliceBobStats, kAliceCharlieStats,
943 kAliceKatieStats}));
944 {
945 FrameCounters stream_conters =
946 analyzer.GetPerStreamCounters().at(kAliceBobStats);
947 EXPECT_EQ(stream_conters.captured, kFramesCount);
948 EXPECT_EQ(stream_conters.pre_encoded, kFramesCount);
949 EXPECT_EQ(stream_conters.encoded, kFramesCount);
950 EXPECT_EQ(stream_conters.received, kFramesCount);
951 EXPECT_EQ(stream_conters.decoded, kFramesCount);
952 EXPECT_EQ(stream_conters.rendered, kFramesCount);
953 }
954 {
955 FrameCounters stream_conters =
956 analyzer.GetPerStreamCounters().at(kAliceCharlieStats);
957 EXPECT_EQ(stream_conters.captured, kFramesCount);
958 EXPECT_EQ(stream_conters.pre_encoded, kFramesCount);
959 EXPECT_EQ(stream_conters.encoded, kFramesCount);
960 EXPECT_EQ(stream_conters.received, kTwoThirdFrames);
961 EXPECT_EQ(stream_conters.decoded, kTwoThirdFrames);
962 EXPECT_EQ(stream_conters.rendered, kTwoThirdFrames);
963 }
964 {
965 FrameCounters stream_conters =
966 analyzer.GetPerStreamCounters().at(kAliceKatieStats);
967 EXPECT_EQ(stream_conters.captured, kFramesCount);
968 EXPECT_EQ(stream_conters.pre_encoded, kFramesCount);
969 EXPECT_EQ(stream_conters.encoded, kFramesCount);
970 EXPECT_EQ(stream_conters.received, kOneThirdFrames);
971 EXPECT_EQ(stream_conters.decoded, kOneThirdFrames);
972 EXPECT_EQ(stream_conters.rendered, kOneThirdFrames);
973 }
974 }
975
TEST(DefaultVideoQualityAnalyzerTest,SimulcastFrameWasFullyReceivedByAllPeersBeforeEncodeFinish)976 TEST(DefaultVideoQualityAnalyzerTest,
977 SimulcastFrameWasFullyReceivedByAllPeersBeforeEncodeFinish) {
978 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
979 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
980 /*type=*/absl::nullopt,
981 /*num_squares=*/absl::nullopt);
982
983 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
984 test::GetGlobalMetricsLogger(),
985 AnalyzerOptionsForTest());
986 constexpr char kAlice[] = "alice";
987 constexpr char kBob[] = "bob";
988 constexpr char kCharlie[] = "charlie";
989 analyzer.Start("test_case", std::vector<std::string>{kAlice, kBob, kCharlie},
990 kAnalyzerMaxThreadsCount);
991
992 VideoFrame frame = NextFrame(frame_generator.get(), 1);
993
994 frame.set_id(analyzer.OnFrameCaptured(kAlice, kStreamLabel, frame));
995 analyzer.OnFramePreEncode(kAlice, frame);
996 // Encode 1st simulcast layer
997 analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame),
998 VideoQualityAnalyzerInterface::EncoderStats(), false);
999
1000 // Receive by Bob
1001 VideoFrame received_frame = DeepCopy(frame);
1002 analyzer.OnFramePreDecode(kBob, received_frame.id(),
1003 FakeEncode(received_frame));
1004 analyzer.OnFrameDecoded(kBob, received_frame,
1005 VideoQualityAnalyzerInterface::DecoderStats());
1006 analyzer.OnFrameRendered(kBob, received_frame);
1007 // Receive by Charlie
1008 received_frame = DeepCopy(frame);
1009 analyzer.OnFramePreDecode(kCharlie, received_frame.id(),
1010 FakeEncode(received_frame));
1011 analyzer.OnFrameDecoded(kCharlie, received_frame,
1012 VideoQualityAnalyzerInterface::DecoderStats());
1013 analyzer.OnFrameRendered(kCharlie, received_frame);
1014
1015 // Encode 2nd simulcast layer
1016 analyzer.OnFrameEncoded(kAlice, frame.id(), FakeEncode(frame),
1017 VideoQualityAnalyzerInterface::EncoderStats(), false);
1018
1019 // Give analyzer some time to process frames on async thread. The computations
1020 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1021 // means we have an issue!
1022 SleepMs(100);
1023 analyzer.Stop();
1024
1025 AnalyzerStats stats = analyzer.GetAnalyzerStats();
1026 EXPECT_EQ(stats.comparisons_done, 2);
1027
1028 std::vector<StatsSample> frames_in_flight_sizes =
1029 GetSortedSamples(stats.frames_in_flight_left_count);
1030 EXPECT_EQ(frames_in_flight_sizes.back().value, 0)
1031 << ToString(frames_in_flight_sizes);
1032
1033 FrameCounters frame_counters = analyzer.GetGlobalCounters();
1034 EXPECT_EQ(frame_counters.captured, 1);
1035 EXPECT_EQ(frame_counters.rendered, 2);
1036 }
1037
TEST(DefaultVideoQualityAnalyzerTest,FrameCanBeReceivedBySenderAfterItWasReceivedByReceiver)1038 TEST(DefaultVideoQualityAnalyzerTest,
1039 FrameCanBeReceivedBySenderAfterItWasReceivedByReceiver) {
1040 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1041 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1042 /*type=*/absl::nullopt,
1043 /*num_squares=*/absl::nullopt);
1044
1045 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
1046 options.enable_receive_own_stream = true;
1047 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1048 test::GetGlobalMetricsLogger(), options);
1049 analyzer.Start("test_case",
1050 std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
1051 kAnalyzerMaxThreadsCount);
1052
1053 std::vector<VideoFrame> frames;
1054 for (int i = 0; i < 3; ++i) {
1055 VideoFrame frame = NextFrame(frame_generator.get(), 1);
1056 frame.set_id(
1057 analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
1058 frames.push_back(frame);
1059 analyzer.OnFramePreEncode(kSenderPeerName, frame);
1060 analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
1061 VideoQualityAnalyzerInterface::EncoderStats(),
1062 false);
1063 }
1064
1065 // Receive by 2nd peer.
1066 for (VideoFrame& frame : frames) {
1067 VideoFrame received_frame = DeepCopy(frame);
1068 analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
1069 FakeEncode(received_frame));
1070 analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
1071 VideoQualityAnalyzerInterface::DecoderStats());
1072 analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
1073 }
1074
1075 // Check that we still have that frame in flight.
1076 AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats();
1077 std::vector<StatsSample> frames_in_flight_sizes =
1078 GetSortedSamples(analyzer_stats.frames_in_flight_left_count);
1079 EXPECT_EQ(frames_in_flight_sizes.back().value, 3)
1080 << "Expected that frame is still in flight, "
1081 << "because it wasn't received by sender"
1082 << ToString(frames_in_flight_sizes);
1083
1084 // Receive by sender
1085 for (VideoFrame& frame : frames) {
1086 VideoFrame received_frame = DeepCopy(frame);
1087 analyzer.OnFramePreDecode(kSenderPeerName, received_frame.id(),
1088 FakeEncode(received_frame));
1089 analyzer.OnFrameDecoded(kSenderPeerName, received_frame,
1090 VideoQualityAnalyzerInterface::DecoderStats());
1091 analyzer.OnFrameRendered(kSenderPeerName, received_frame);
1092 }
1093
1094 // Give analyzer some time to process frames on async thread. The computations
1095 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1096 // means we have an issue!
1097 SleepMs(100);
1098 analyzer.Stop();
1099
1100 analyzer_stats = analyzer.GetAnalyzerStats();
1101 EXPECT_EQ(analyzer_stats.comparisons_done, 6);
1102
1103 frames_in_flight_sizes =
1104 GetSortedSamples(analyzer_stats.frames_in_flight_left_count);
1105 EXPECT_EQ(frames_in_flight_sizes.back().value, 0)
1106 << ToString(frames_in_flight_sizes);
1107
1108 FrameCounters frame_counters = analyzer.GetGlobalCounters();
1109 EXPECT_EQ(frame_counters.captured, 3);
1110 EXPECT_EQ(frame_counters.rendered, 6);
1111
1112 EXPECT_EQ(analyzer.GetStats().size(), 2lu);
1113 {
1114 FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
1115 StatsKey(kStreamLabel, kReceiverPeerName));
1116 EXPECT_EQ(stream_conters.captured, 3);
1117 EXPECT_EQ(stream_conters.pre_encoded, 3);
1118 EXPECT_EQ(stream_conters.encoded, 3);
1119 EXPECT_EQ(stream_conters.received, 3);
1120 EXPECT_EQ(stream_conters.decoded, 3);
1121 EXPECT_EQ(stream_conters.rendered, 3);
1122 }
1123 {
1124 FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
1125 StatsKey(kStreamLabel, kSenderPeerName));
1126 EXPECT_EQ(stream_conters.captured, 3);
1127 EXPECT_EQ(stream_conters.pre_encoded, 3);
1128 EXPECT_EQ(stream_conters.encoded, 3);
1129 EXPECT_EQ(stream_conters.received, 3);
1130 EXPECT_EQ(stream_conters.decoded, 3);
1131 EXPECT_EQ(stream_conters.rendered, 3);
1132 }
1133 }
1134
TEST(DefaultVideoQualityAnalyzerTest,FrameCanBeReceivedByReceiverAfterItWasReceivedBySender)1135 TEST(DefaultVideoQualityAnalyzerTest,
1136 FrameCanBeReceivedByReceiverAfterItWasReceivedBySender) {
1137 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1138 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1139 /*type=*/absl::nullopt,
1140 /*num_squares=*/absl::nullopt);
1141
1142 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
1143 options.enable_receive_own_stream = true;
1144 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1145 test::GetGlobalMetricsLogger(), options);
1146 analyzer.Start("test_case",
1147 std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
1148 kAnalyzerMaxThreadsCount);
1149
1150 std::vector<VideoFrame> frames;
1151 for (int i = 0; i < 3; ++i) {
1152 VideoFrame frame = NextFrame(frame_generator.get(), 1);
1153 frame.set_id(
1154 analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
1155 frames.push_back(frame);
1156 analyzer.OnFramePreEncode(kSenderPeerName, frame);
1157 analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
1158 VideoQualityAnalyzerInterface::EncoderStats(),
1159 false);
1160 }
1161
1162 // Receive by sender
1163 for (VideoFrame& frame : frames) {
1164 VideoFrame received_frame = DeepCopy(frame);
1165 analyzer.OnFramePreDecode(kSenderPeerName, received_frame.id(),
1166 FakeEncode(received_frame));
1167 analyzer.OnFrameDecoded(kSenderPeerName, received_frame,
1168 VideoQualityAnalyzerInterface::DecoderStats());
1169 analyzer.OnFrameRendered(kSenderPeerName, received_frame);
1170 }
1171
1172 // Check that we still have that frame in flight.
1173 AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats();
1174 std::vector<StatsSample> frames_in_flight_sizes =
1175 GetSortedSamples(analyzer_stats.frames_in_flight_left_count);
1176 EXPECT_EQ(frames_in_flight_sizes.back().value, 3)
1177 << "Expected that frame is still in flight, "
1178 << "because it wasn't received by sender"
1179 << ToString(frames_in_flight_sizes);
1180
1181 // Receive by 2nd peer.
1182 for (VideoFrame& frame : frames) {
1183 VideoFrame received_frame = DeepCopy(frame);
1184 analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
1185 FakeEncode(received_frame));
1186 analyzer.OnFrameDecoded(kReceiverPeerName, received_frame,
1187 VideoQualityAnalyzerInterface::DecoderStats());
1188 analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
1189 }
1190
1191 // Give analyzer some time to process frames on async thread. The computations
1192 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1193 // means we have an issue!
1194 SleepMs(100);
1195 analyzer.Stop();
1196
1197 analyzer_stats = analyzer.GetAnalyzerStats();
1198 EXPECT_EQ(analyzer_stats.comparisons_done, 6);
1199
1200 frames_in_flight_sizes =
1201 GetSortedSamples(analyzer_stats.frames_in_flight_left_count);
1202 EXPECT_EQ(frames_in_flight_sizes.back().value, 0)
1203 << ToString(frames_in_flight_sizes);
1204
1205 FrameCounters frame_counters = analyzer.GetGlobalCounters();
1206 EXPECT_EQ(frame_counters.captured, 3);
1207 EXPECT_EQ(frame_counters.rendered, 6);
1208
1209 EXPECT_EQ(analyzer.GetStats().size(), 2lu);
1210 {
1211 FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
1212 StatsKey(kStreamLabel, kReceiverPeerName));
1213 EXPECT_EQ(stream_conters.captured, 3);
1214 EXPECT_EQ(stream_conters.pre_encoded, 3);
1215 EXPECT_EQ(stream_conters.encoded, 3);
1216 EXPECT_EQ(stream_conters.received, 3);
1217 EXPECT_EQ(stream_conters.decoded, 3);
1218 EXPECT_EQ(stream_conters.rendered, 3);
1219 }
1220 {
1221 FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
1222 StatsKey(kStreamLabel, kSenderPeerName));
1223 EXPECT_EQ(stream_conters.captured, 3);
1224 EXPECT_EQ(stream_conters.pre_encoded, 3);
1225 EXPECT_EQ(stream_conters.encoded, 3);
1226 EXPECT_EQ(stream_conters.received, 3);
1227 EXPECT_EQ(stream_conters.decoded, 3);
1228 EXPECT_EQ(stream_conters.rendered, 3);
1229 }
1230 }
1231
TEST(DefaultVideoQualityAnalyzerTest,CodecTrackedCorrectly)1232 TEST(DefaultVideoQualityAnalyzerTest, CodecTrackedCorrectly) {
1233 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1234 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1235 /*type=*/absl::nullopt,
1236 /*num_squares=*/absl::nullopt);
1237
1238 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1239 test::GetGlobalMetricsLogger(),
1240 AnalyzerOptionsForTest());
1241 analyzer.Start("test_case",
1242 std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
1243 kAnalyzerMaxThreadsCount);
1244
1245 VideoQualityAnalyzerInterface::EncoderStats encoder_stats;
1246 std::vector<std::string> codec_names = {"codec_1", "codec_2"};
1247 std::vector<VideoFrame> frames;
1248 // Send 3 frame for each codec.
1249 for (size_t i = 0; i < codec_names.size(); ++i) {
1250 for (size_t j = 0; j < 3; ++j) {
1251 VideoFrame frame = NextFrame(frame_generator.get(), 3 * i + j);
1252 frame.set_id(
1253 analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
1254 analyzer.OnFramePreEncode(kSenderPeerName, frame);
1255 encoder_stats.encoder_name = codec_names[i];
1256 analyzer.OnFrameEncoded(kSenderPeerName, frame.id(), FakeEncode(frame),
1257 encoder_stats, false);
1258 frames.push_back(std::move(frame));
1259 }
1260 }
1261
1262 // Receive 3 frame for each codec.
1263 VideoQualityAnalyzerInterface::DecoderStats decoder_stats;
1264 for (size_t i = 0; i < codec_names.size(); ++i) {
1265 for (size_t j = 0; j < 3; ++j) {
1266 VideoFrame received_frame = DeepCopy(frames[3 * i + j]);
1267 analyzer.OnFramePreDecode(kReceiverPeerName, received_frame.id(),
1268 FakeEncode(received_frame));
1269 decoder_stats.decoder_name = codec_names[i];
1270 analyzer.OnFrameDecoded(kReceiverPeerName, received_frame, decoder_stats);
1271 analyzer.OnFrameRendered(kReceiverPeerName, received_frame);
1272 }
1273 }
1274
1275 // Give analyzer some time to process frames on async thread. The computations
1276 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1277 // means we have an issue!
1278 SleepMs(100);
1279 analyzer.Stop();
1280
1281 std::map<StatsKey, StreamStats> stats = analyzer.GetStats();
1282 ASSERT_EQ(stats.size(), 1lu);
1283 const StreamStats& stream_stats =
1284 stats.at(StatsKey(kStreamLabel, kReceiverPeerName));
1285 ASSERT_EQ(stream_stats.encoders.size(), 2lu);
1286 EXPECT_EQ(stream_stats.encoders[0].codec_name, codec_names[0]);
1287 EXPECT_EQ(stream_stats.encoders[0].first_frame_id, frames[0].id());
1288 EXPECT_EQ(stream_stats.encoders[0].last_frame_id, frames[2].id());
1289 EXPECT_EQ(stream_stats.encoders[1].codec_name, codec_names[1]);
1290 EXPECT_EQ(stream_stats.encoders[1].first_frame_id, frames[3].id());
1291 EXPECT_EQ(stream_stats.encoders[1].last_frame_id, frames[5].id());
1292
1293 ASSERT_EQ(stream_stats.decoders.size(), 2lu);
1294 EXPECT_EQ(stream_stats.decoders[0].codec_name, codec_names[0]);
1295 EXPECT_EQ(stream_stats.decoders[0].first_frame_id, frames[0].id());
1296 EXPECT_EQ(stream_stats.decoders[0].last_frame_id, frames[2].id());
1297 EXPECT_EQ(stream_stats.decoders[1].codec_name, codec_names[1]);
1298 EXPECT_EQ(stream_stats.decoders[1].first_frame_id, frames[3].id());
1299 EXPECT_EQ(stream_stats.decoders[1].last_frame_id, frames[5].id());
1300 }
1301
TEST(DefaultVideoQualityAnalyzerTest,FramesInFlightAreCorrectlySentToTheComparatorAfterStop)1302 TEST(DefaultVideoQualityAnalyzerTest,
1303 FramesInFlightAreCorrectlySentToTheComparatorAfterStop) {
1304 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1305 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1306 /*type=*/absl::nullopt,
1307 /*num_squares=*/absl::nullopt);
1308
1309 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
1310 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1311 test::GetGlobalMetricsLogger(), options);
1312 analyzer.Start("test_case",
1313 std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
1314 kAnalyzerMaxThreadsCount);
1315
1316 // There are 7 different timings inside frame stats: captured, pre_encode,
1317 // encoded, received, decode_start, decode_end, rendered. captured is always
1318 // set and received is set together with decode_start. So we create 6
1319 // different frames, where for each frame next timings will be set
1320 // * 1st - all of them set
1321 // * 2nd - captured, pre_encode, encoded, received, decode_start, decode_end
1322 // * 3rd - captured, pre_encode, encoded, received, decode_start
1323 // * 4th - captured, pre_encode, encoded
1324 // * 5th - captured, pre_encode
1325 // * 6th - captured
1326 std::vector<VideoFrame> frames;
1327 // Sender side actions
1328 for (int i = 0; i < 6; ++i) {
1329 VideoFrame frame = NextFrame(frame_generator.get(), 1);
1330 frame.set_id(
1331 analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
1332 frames.push_back(frame);
1333 }
1334 for (int i = 0; i < 5; ++i) {
1335 analyzer.OnFramePreEncode(kSenderPeerName, frames[i]);
1336 }
1337 for (int i = 0; i < 4; ++i) {
1338 analyzer.OnFrameEncoded(
1339 kSenderPeerName, frames[i].id(), FakeEncode(frames[i]),
1340 VideoQualityAnalyzerInterface::EncoderStats(), false);
1341 }
1342
1343 // Receiver side actions
1344 for (int i = 0; i < 3; ++i) {
1345 analyzer.OnFramePreDecode(kReceiverPeerName, frames[i].id(),
1346 FakeEncode(frames[i]));
1347 }
1348 for (int i = 0; i < 2; ++i) {
1349 analyzer.OnFrameDecoded(kReceiverPeerName, DeepCopy(frames[i]),
1350 VideoQualityAnalyzerInterface::DecoderStats());
1351 }
1352 for (int i = 0; i < 1; ++i) {
1353 analyzer.OnFrameRendered(kReceiverPeerName, DeepCopy(frames[i]));
1354 }
1355
1356 // Give analyzer some time to process frames on async thread. The computations
1357 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1358 // means we have an issue!
1359 SleepMs(100);
1360 analyzer.Stop();
1361
1362 AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats();
1363 EXPECT_EQ(analyzer_stats.comparisons_done, 6);
1364
1365 // The last frames in flight size has to reflect the amount of frame in flight
1366 // before all of them were sent to the comparison when Stop() was invoked.
1367 std::vector<StatsSample> frames_in_flight_sizes =
1368 GetSortedSamples(analyzer_stats.frames_in_flight_left_count);
1369 EXPECT_EQ(frames_in_flight_sizes.back().value, 5)
1370 << ToString(frames_in_flight_sizes);
1371
1372 FrameCounters frame_counters = analyzer.GetGlobalCounters();
1373 EXPECT_EQ(frame_counters.captured, 6);
1374 EXPECT_EQ(frame_counters.pre_encoded, 5);
1375 EXPECT_EQ(frame_counters.encoded, 4);
1376 EXPECT_EQ(frame_counters.received, 3);
1377 EXPECT_EQ(frame_counters.decoded, 2);
1378 EXPECT_EQ(frame_counters.rendered, 1);
1379
1380 EXPECT_EQ(analyzer.GetStats().size(), 1lu);
1381 {
1382 FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
1383 StatsKey(kStreamLabel, kReceiverPeerName));
1384 EXPECT_EQ(stream_conters.captured, 6);
1385 EXPECT_EQ(stream_conters.pre_encoded, 5);
1386 EXPECT_EQ(stream_conters.encoded, 4);
1387 EXPECT_EQ(stream_conters.received, 3);
1388 EXPECT_EQ(stream_conters.decoded, 2);
1389 EXPECT_EQ(stream_conters.rendered, 1);
1390 }
1391 }
1392
TEST(DefaultVideoQualityAnalyzerTest,FramesInFlightAreCorrectlySentToTheComparatorAfterStopForSenderAndReceiver)1393 TEST(
1394 DefaultVideoQualityAnalyzerTest,
1395 FramesInFlightAreCorrectlySentToTheComparatorAfterStopForSenderAndReceiver) {
1396 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1397 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1398 /*type=*/absl::nullopt,
1399 /*num_squares=*/absl::nullopt);
1400
1401 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
1402 options.enable_receive_own_stream = true;
1403 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1404 test::GetGlobalMetricsLogger(), options);
1405 analyzer.Start("test_case",
1406 std::vector<std::string>{kSenderPeerName, kReceiverPeerName},
1407 kAnalyzerMaxThreadsCount);
1408
1409 // There are 7 different timings inside frame stats: captured, pre_encode,
1410 // encoded, received, decode_start, decode_end, rendered. captured is always
1411 // set and received is set together with decode_start. So we create 6
1412 // different frames, where for each frame next timings will be set
1413 // * 1st - all of them set
1414 // * 2nd - captured, pre_encode, encoded, received, decode_start, decode_end
1415 // * 3rd - captured, pre_encode, encoded, received, decode_start
1416 // * 4th - captured, pre_encode, encoded
1417 // * 5th - captured, pre_encode
1418 // * 6th - captured
1419 std::vector<VideoFrame> frames;
1420 // Sender side actions
1421 for (int i = 0; i < 6; ++i) {
1422 VideoFrame frame = NextFrame(frame_generator.get(), 1);
1423 frame.set_id(
1424 analyzer.OnFrameCaptured(kSenderPeerName, kStreamLabel, frame));
1425 frames.push_back(frame);
1426 }
1427 for (int i = 0; i < 5; ++i) {
1428 analyzer.OnFramePreEncode(kSenderPeerName, frames[i]);
1429 }
1430 for (int i = 0; i < 4; ++i) {
1431 analyzer.OnFrameEncoded(
1432 kSenderPeerName, frames[i].id(), FakeEncode(frames[i]),
1433 VideoQualityAnalyzerInterface::EncoderStats(), false);
1434 }
1435
1436 // Receiver side actions
1437 for (int i = 0; i < 3; ++i) {
1438 analyzer.OnFramePreDecode(kSenderPeerName, frames[i].id(),
1439 FakeEncode(frames[i]));
1440 analyzer.OnFramePreDecode(kReceiverPeerName, frames[i].id(),
1441 FakeEncode(frames[i]));
1442 }
1443 for (int i = 0; i < 2; ++i) {
1444 analyzer.OnFrameDecoded(kSenderPeerName, DeepCopy(frames[i]),
1445 VideoQualityAnalyzerInterface::DecoderStats());
1446 analyzer.OnFrameDecoded(kReceiverPeerName, DeepCopy(frames[i]),
1447 VideoQualityAnalyzerInterface::DecoderStats());
1448 }
1449 for (int i = 0; i < 1; ++i) {
1450 analyzer.OnFrameRendered(kSenderPeerName, DeepCopy(frames[i]));
1451 analyzer.OnFrameRendered(kReceiverPeerName, DeepCopy(frames[i]));
1452 }
1453
1454 // Give analyzer some time to process frames on async thread. The computations
1455 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1456 // means we have an issue!
1457 SleepMs(100);
1458 analyzer.Stop();
1459
1460 AnalyzerStats analyzer_stats = analyzer.GetAnalyzerStats();
1461 EXPECT_EQ(analyzer_stats.comparisons_done, 12);
1462
1463 // The last frames in flight size has to reflect the amount of frame in flight
1464 // before all of them were sent to the comparison when Stop() was invoked.
1465 std::vector<StatsSample> frames_in_flight_sizes =
1466 GetSortedSamples(analyzer_stats.frames_in_flight_left_count);
1467 EXPECT_EQ(frames_in_flight_sizes.back().value, 5)
1468 << ToString(frames_in_flight_sizes);
1469
1470 FrameCounters frame_counters = analyzer.GetGlobalCounters();
1471 EXPECT_EQ(frame_counters.captured, 6);
1472 EXPECT_EQ(frame_counters.pre_encoded, 5);
1473 EXPECT_EQ(frame_counters.encoded, 4);
1474 EXPECT_EQ(frame_counters.received, 6);
1475 EXPECT_EQ(frame_counters.decoded, 4);
1476 EXPECT_EQ(frame_counters.rendered, 2);
1477
1478 EXPECT_EQ(analyzer.GetStats().size(), 2lu);
1479 {
1480 FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
1481 StatsKey(kStreamLabel, kReceiverPeerName));
1482 EXPECT_EQ(stream_conters.captured, 6);
1483 EXPECT_EQ(stream_conters.pre_encoded, 5);
1484 EXPECT_EQ(stream_conters.encoded, 4);
1485 EXPECT_EQ(stream_conters.received, 3);
1486 EXPECT_EQ(stream_conters.decoded, 2);
1487 EXPECT_EQ(stream_conters.rendered, 1);
1488 }
1489 {
1490 FrameCounters stream_conters = analyzer.GetPerStreamCounters().at(
1491 StatsKey(kStreamLabel, kSenderPeerName));
1492 EXPECT_EQ(stream_conters.captured, 6);
1493 EXPECT_EQ(stream_conters.pre_encoded, 5);
1494 EXPECT_EQ(stream_conters.encoded, 4);
1495 EXPECT_EQ(stream_conters.received, 3);
1496 EXPECT_EQ(stream_conters.decoded, 2);
1497 EXPECT_EQ(stream_conters.rendered, 1);
1498 }
1499 }
1500
TEST(DefaultVideoQualityAnalyzerTest,GetStreamFrames)1501 TEST(DefaultVideoQualityAnalyzerTest, GetStreamFrames) {
1502 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1503 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1504 /*type=*/absl::nullopt,
1505 /*num_squares=*/absl::nullopt);
1506
1507 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
1508 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1509 test::GetGlobalMetricsLogger(), options);
1510 analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
1511 kAnalyzerMaxThreadsCount);
1512
1513 // The order in which peers captured frames and passed them to analyzer.
1514 std::vector<std::string> frame_capturers_sequence{
1515 "alice", "alice", "bob", "bob", "bob",
1516 "bob", "bob", "alice", "alice", "alice",
1517 };
1518
1519 std::map<std::string, std::vector<uint16_t>> stream_to_frame_ids;
1520 stream_to_frame_ids.emplace("alice_video", std::vector<uint16_t>{});
1521 stream_to_frame_ids.emplace("bob_video", std::vector<uint16_t>{});
1522
1523 std::vector<VideoFrame> frames;
1524 for (const std::string& sender : frame_capturers_sequence) {
1525 VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
1526 uint16_t frame_id =
1527 analyzer.OnFrameCaptured(sender, sender + "_video", frame);
1528 frame.set_id(frame_id);
1529 stream_to_frame_ids.find(sender + "_video")->second.push_back(frame_id);
1530 frames.push_back(frame);
1531 analyzer.OnFramePreEncode(sender, frame);
1532 analyzer.OnFrameEncoded(sender, frame.id(), FakeEncode(frame),
1533 VideoQualityAnalyzerInterface::EncoderStats(),
1534 false);
1535 }
1536 // We don't need to receive frames for stats to be gathered correctly.
1537
1538 // Give analyzer some time to process frames on async thread. The computations
1539 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1540 // means we have an issue!
1541 SleepMs(100);
1542 analyzer.Stop();
1543
1544 EXPECT_EQ(analyzer.GetStreamFrames(), stream_to_frame_ids);
1545 }
1546
TEST(DefaultVideoQualityAnalyzerTest,ReceiverReceivedFramesWhenSenderRemoved)1547 TEST(DefaultVideoQualityAnalyzerTest, ReceiverReceivedFramesWhenSenderRemoved) {
1548 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1549 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1550 /*type=*/absl::nullopt,
1551 /*num_squares=*/absl::nullopt);
1552
1553 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
1554 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1555 test::GetGlobalMetricsLogger(), options);
1556 analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
1557 kAnalyzerMaxThreadsCount);
1558
1559 VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
1560 uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
1561 frame.set_id(frame_id);
1562 analyzer.OnFramePreEncode("alice", frame);
1563 analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
1564 VideoQualityAnalyzerInterface::EncoderStats(), false);
1565
1566 analyzer.UnregisterParticipantInCall("alice");
1567
1568 analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame));
1569 analyzer.OnFrameDecoded("bob", DeepCopy(frame),
1570 VideoQualityAnalyzerInterface::DecoderStats());
1571 analyzer.OnFrameRendered("bob", DeepCopy(frame));
1572
1573 // Give analyzer some time to process frames on async thread. The computations
1574 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1575 // means we have an issue!
1576 SleepMs(100);
1577 analyzer.Stop();
1578
1579 FrameCounters stream_conters =
1580 analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
1581 EXPECT_EQ(stream_conters.captured, 1);
1582 EXPECT_EQ(stream_conters.pre_encoded, 1);
1583 EXPECT_EQ(stream_conters.encoded, 1);
1584 EXPECT_EQ(stream_conters.received, 1);
1585 EXPECT_EQ(stream_conters.decoded, 1);
1586 EXPECT_EQ(stream_conters.rendered, 1);
1587 }
1588
TEST(DefaultVideoQualityAnalyzerTest,ReceiverReceivedFramesWhenSenderRemovedWithSelfview)1589 TEST(DefaultVideoQualityAnalyzerTest,
1590 ReceiverReceivedFramesWhenSenderRemovedWithSelfview) {
1591 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1592 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1593 /*type=*/absl::nullopt,
1594 /*num_squares=*/absl::nullopt);
1595
1596 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
1597 options.enable_receive_own_stream = true;
1598 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1599 test::GetGlobalMetricsLogger(), options);
1600 analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
1601 kAnalyzerMaxThreadsCount);
1602
1603 VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
1604 uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
1605 frame.set_id(frame_id);
1606 analyzer.OnFramePreEncode("alice", frame);
1607 analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
1608 VideoQualityAnalyzerInterface::EncoderStats(), false);
1609
1610 analyzer.UnregisterParticipantInCall("alice");
1611
1612 analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame));
1613 analyzer.OnFrameDecoded("bob", DeepCopy(frame),
1614 VideoQualityAnalyzerInterface::DecoderStats());
1615 analyzer.OnFrameRendered("bob", DeepCopy(frame));
1616
1617 // Give analyzer some time to process frames on async thread. The computations
1618 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1619 // means we have an issue!
1620 SleepMs(100);
1621 analyzer.Stop();
1622
1623 FrameCounters stream_conters =
1624 analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
1625 EXPECT_EQ(stream_conters.captured, 1);
1626 EXPECT_EQ(stream_conters.pre_encoded, 1);
1627 EXPECT_EQ(stream_conters.encoded, 1);
1628 EXPECT_EQ(stream_conters.received, 1);
1629 EXPECT_EQ(stream_conters.decoded, 1);
1630 EXPECT_EQ(stream_conters.rendered, 1);
1631 }
1632
TEST(DefaultVideoQualityAnalyzerTest,SenderReceivedFramesWhenReceiverRemovedWithSelfview)1633 TEST(DefaultVideoQualityAnalyzerTest,
1634 SenderReceivedFramesWhenReceiverRemovedWithSelfview) {
1635 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1636 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1637 /*type=*/absl::nullopt,
1638 /*num_squares=*/absl::nullopt);
1639
1640 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
1641 options.enable_receive_own_stream = true;
1642 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1643 test::GetGlobalMetricsLogger(), options);
1644 analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
1645 kAnalyzerMaxThreadsCount);
1646
1647 VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
1648 uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
1649 frame.set_id(frame_id);
1650 analyzer.OnFramePreEncode("alice", frame);
1651 analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
1652 VideoQualityAnalyzerInterface::EncoderStats(), false);
1653
1654 analyzer.UnregisterParticipantInCall("bob");
1655
1656 analyzer.OnFramePreDecode("alice", frame.id(), FakeEncode(frame));
1657 analyzer.OnFrameDecoded("alice", DeepCopy(frame),
1658 VideoQualityAnalyzerInterface::DecoderStats());
1659 analyzer.OnFrameRendered("alice", DeepCopy(frame));
1660
1661 // Give analyzer some time to process frames on async thread. The computations
1662 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1663 // means we have an issue!
1664 SleepMs(100);
1665 analyzer.Stop();
1666
1667 FrameCounters stream_conters =
1668 analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "alice"));
1669 EXPECT_EQ(stream_conters.captured, 1);
1670 EXPECT_EQ(stream_conters.pre_encoded, 1);
1671 EXPECT_EQ(stream_conters.encoded, 1);
1672 EXPECT_EQ(stream_conters.received, 1);
1673 EXPECT_EQ(stream_conters.decoded, 1);
1674 EXPECT_EQ(stream_conters.rendered, 1);
1675 }
1676
TEST(DefaultVideoQualityAnalyzerTest,SenderAndReceiverReceivedFramesWhenReceiverRemovedWithSelfview)1677 TEST(DefaultVideoQualityAnalyzerTest,
1678 SenderAndReceiverReceivedFramesWhenReceiverRemovedWithSelfview) {
1679 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1680 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1681 /*type=*/absl::nullopt,
1682 /*num_squares=*/absl::nullopt);
1683
1684 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
1685 options.enable_receive_own_stream = true;
1686 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1687 test::GetGlobalMetricsLogger(), options);
1688 analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
1689 kAnalyzerMaxThreadsCount);
1690
1691 VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
1692 uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
1693 frame.set_id(frame_id);
1694 analyzer.OnFramePreEncode("alice", frame);
1695 analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
1696 VideoQualityAnalyzerInterface::EncoderStats(), false);
1697
1698 analyzer.OnFramePreDecode("bob", frame.id(), FakeEncode(frame));
1699 analyzer.OnFrameDecoded("bob", DeepCopy(frame),
1700 VideoQualityAnalyzerInterface::DecoderStats());
1701 analyzer.OnFrameRendered("bob", DeepCopy(frame));
1702
1703 analyzer.UnregisterParticipantInCall("bob");
1704
1705 analyzer.OnFramePreDecode("alice", frame.id(), FakeEncode(frame));
1706 analyzer.OnFrameDecoded("alice", DeepCopy(frame),
1707 VideoQualityAnalyzerInterface::DecoderStats());
1708 analyzer.OnFrameRendered("alice", DeepCopy(frame));
1709
1710 // Give analyzer some time to process frames on async thread. The computations
1711 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1712 // means we have an issue!
1713 SleepMs(100);
1714 analyzer.Stop();
1715
1716 FrameCounters alice_alice_stream_conters =
1717 analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "alice"));
1718 EXPECT_EQ(alice_alice_stream_conters.captured, 1);
1719 EXPECT_EQ(alice_alice_stream_conters.pre_encoded, 1);
1720 EXPECT_EQ(alice_alice_stream_conters.encoded, 1);
1721 EXPECT_EQ(alice_alice_stream_conters.received, 1);
1722 EXPECT_EQ(alice_alice_stream_conters.decoded, 1);
1723 EXPECT_EQ(alice_alice_stream_conters.rendered, 1);
1724
1725 FrameCounters alice_bob_stream_conters =
1726 analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
1727 EXPECT_EQ(alice_bob_stream_conters.captured, 1);
1728 EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 1);
1729 EXPECT_EQ(alice_bob_stream_conters.encoded, 1);
1730 EXPECT_EQ(alice_bob_stream_conters.received, 1);
1731 EXPECT_EQ(alice_bob_stream_conters.decoded, 1);
1732 EXPECT_EQ(alice_bob_stream_conters.rendered, 1);
1733 }
1734
TEST(DefaultVideoQualityAnalyzerTest,ReceiverRemovedBeforeCapturing2ndFrame)1735 TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeCapturing2ndFrame) {
1736 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1737 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1738 /*type=*/absl::nullopt,
1739 /*num_squares=*/absl::nullopt);
1740
1741 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
1742 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1743 test::GetGlobalMetricsLogger(), options);
1744 analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
1745 kAnalyzerMaxThreadsCount);
1746
1747 PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"},
1748 /*frames_count=*/1, *frame_generator);
1749 analyzer.UnregisterParticipantInCall("bob");
1750 PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {},
1751 /*frames_count=*/1, *frame_generator);
1752
1753 // Give analyzer some time to process frames on async thread. The computations
1754 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1755 // means we have an issue!
1756 SleepMs(100);
1757 analyzer.Stop();
1758
1759 FrameCounters global_stream_conters = analyzer.GetGlobalCounters();
1760 EXPECT_EQ(global_stream_conters.captured, 2);
1761 EXPECT_EQ(global_stream_conters.pre_encoded, 2);
1762 EXPECT_EQ(global_stream_conters.encoded, 2);
1763 EXPECT_EQ(global_stream_conters.received, 1);
1764 EXPECT_EQ(global_stream_conters.decoded, 1);
1765 EXPECT_EQ(global_stream_conters.rendered, 1);
1766 FrameCounters stream_conters =
1767 analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
1768 EXPECT_EQ(stream_conters.captured, 2);
1769 EXPECT_EQ(stream_conters.pre_encoded, 2);
1770 EXPECT_EQ(stream_conters.encoded, 2);
1771 EXPECT_EQ(stream_conters.received, 1);
1772 EXPECT_EQ(stream_conters.decoded, 1);
1773 EXPECT_EQ(stream_conters.rendered, 1);
1774 }
1775
TEST(DefaultVideoQualityAnalyzerTest,ReceiverRemovedBeforePreEncoded)1776 TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforePreEncoded) {
1777 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1778 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1779 /*type=*/absl::nullopt,
1780 /*num_squares=*/absl::nullopt);
1781
1782 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
1783 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1784 test::GetGlobalMetricsLogger(), options);
1785 analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
1786 kAnalyzerMaxThreadsCount);
1787
1788 VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
1789 uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
1790 frame.set_id(frame_id);
1791 analyzer.UnregisterParticipantInCall("bob");
1792 analyzer.OnFramePreEncode("alice", frame);
1793 analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
1794 VideoQualityAnalyzerInterface::EncoderStats(), false);
1795
1796 // Give analyzer some time to process frames on async thread. The computations
1797 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1798 // means we have an issue!
1799 SleepMs(100);
1800 analyzer.Stop();
1801
1802 FrameCounters global_stream_conters = analyzer.GetGlobalCounters();
1803 EXPECT_EQ(global_stream_conters.captured, 1);
1804 EXPECT_EQ(global_stream_conters.pre_encoded, 1);
1805 EXPECT_EQ(global_stream_conters.encoded, 1);
1806 EXPECT_EQ(global_stream_conters.received, 0);
1807 EXPECT_EQ(global_stream_conters.decoded, 0);
1808 EXPECT_EQ(global_stream_conters.rendered, 0);
1809 FrameCounters stream_conters =
1810 analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
1811 EXPECT_EQ(stream_conters.captured, 1);
1812 EXPECT_EQ(stream_conters.pre_encoded, 1);
1813 EXPECT_EQ(stream_conters.encoded, 1);
1814 EXPECT_EQ(stream_conters.received, 0);
1815 EXPECT_EQ(stream_conters.decoded, 0);
1816 EXPECT_EQ(stream_conters.rendered, 0);
1817 }
1818
TEST(DefaultVideoQualityAnalyzerTest,ReceiverRemovedBeforeEncoded)1819 TEST(DefaultVideoQualityAnalyzerTest, ReceiverRemovedBeforeEncoded) {
1820 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1821 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1822 /*type=*/absl::nullopt,
1823 /*num_squares=*/absl::nullopt);
1824
1825 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
1826 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1827 test::GetGlobalMetricsLogger(), options);
1828 analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
1829 kAnalyzerMaxThreadsCount);
1830
1831 VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
1832 uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
1833 frame.set_id(frame_id);
1834 analyzer.OnFramePreEncode("alice", frame);
1835 analyzer.UnregisterParticipantInCall("bob");
1836 analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
1837 VideoQualityAnalyzerInterface::EncoderStats(), false);
1838
1839 // Give analyzer some time to process frames on async thread. The computations
1840 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1841 // means we have an issue!
1842 SleepMs(100);
1843 analyzer.Stop();
1844
1845 FrameCounters global_stream_conters = analyzer.GetGlobalCounters();
1846 EXPECT_EQ(global_stream_conters.captured, 1);
1847 EXPECT_EQ(global_stream_conters.pre_encoded, 1);
1848 EXPECT_EQ(global_stream_conters.encoded, 1);
1849 EXPECT_EQ(global_stream_conters.received, 0);
1850 EXPECT_EQ(global_stream_conters.decoded, 0);
1851 EXPECT_EQ(global_stream_conters.rendered, 0);
1852 FrameCounters stream_conters =
1853 analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
1854 EXPECT_EQ(stream_conters.captured, 1);
1855 EXPECT_EQ(stream_conters.pre_encoded, 1);
1856 EXPECT_EQ(stream_conters.encoded, 1);
1857 EXPECT_EQ(stream_conters.received, 0);
1858 EXPECT_EQ(stream_conters.decoded, 0);
1859 EXPECT_EQ(stream_conters.rendered, 0);
1860 }
1861
TEST(DefaultVideoQualityAnalyzerTest,ReceiverRemovedBetweenSimulcastLayersEncoded)1862 TEST(DefaultVideoQualityAnalyzerTest,
1863 ReceiverRemovedBetweenSimulcastLayersEncoded) {
1864 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1865 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1866 /*type=*/absl::nullopt,
1867 /*num_squares=*/absl::nullopt);
1868
1869 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
1870 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1871 test::GetGlobalMetricsLogger(), options);
1872 analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
1873 kAnalyzerMaxThreadsCount);
1874
1875 VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
1876 uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
1877 frame.set_id(frame_id);
1878 analyzer.OnFramePreEncode("alice", frame);
1879 // 1st simulcast layer encoded
1880 analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
1881 VideoQualityAnalyzerInterface::EncoderStats(), false);
1882 analyzer.UnregisterParticipantInCall("bob");
1883 // 2nd simulcast layer encoded
1884 analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
1885 VideoQualityAnalyzerInterface::EncoderStats(), false);
1886
1887 // Give analyzer some time to process frames on async thread. The computations
1888 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1889 // means we have an issue!
1890 SleepMs(100);
1891 analyzer.Stop();
1892
1893 FrameCounters global_stream_conters = analyzer.GetGlobalCounters();
1894 EXPECT_EQ(global_stream_conters.captured, 1);
1895 EXPECT_EQ(global_stream_conters.pre_encoded, 1);
1896 EXPECT_EQ(global_stream_conters.encoded, 1);
1897 EXPECT_EQ(global_stream_conters.received, 0);
1898 EXPECT_EQ(global_stream_conters.decoded, 0);
1899 EXPECT_EQ(global_stream_conters.rendered, 0);
1900 FrameCounters stream_conters =
1901 analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
1902 EXPECT_EQ(stream_conters.captured, 1);
1903 EXPECT_EQ(stream_conters.pre_encoded, 1);
1904 EXPECT_EQ(stream_conters.encoded, 1);
1905 EXPECT_EQ(stream_conters.received, 0);
1906 EXPECT_EQ(stream_conters.decoded, 0);
1907 EXPECT_EQ(stream_conters.rendered, 0);
1908 }
1909
TEST(DefaultVideoQualityAnalyzerTest,UnregisterOneAndRegisterAnother)1910 TEST(DefaultVideoQualityAnalyzerTest, UnregisterOneAndRegisterAnother) {
1911 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1912 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1913 /*type=*/absl::nullopt,
1914 /*num_squares=*/absl::nullopt);
1915
1916 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
1917 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1918 test::GetGlobalMetricsLogger(), options);
1919 analyzer.Start("test_case",
1920 std::vector<std::string>{"alice", "bob", "charlie"},
1921 kAnalyzerMaxThreadsCount);
1922
1923 PassFramesThroughAnalyzer(analyzer, "alice", "alice_video",
1924 {"bob", "charlie"},
1925 /*frames_count=*/2, *frame_generator);
1926 analyzer.UnregisterParticipantInCall("bob");
1927 analyzer.RegisterParticipantInCall("david");
1928 PassFramesThroughAnalyzer(analyzer, "alice", "alice_video",
1929 {"charlie", "david"},
1930 /*frames_count=*/4, *frame_generator);
1931
1932 // Give analyzer some time to process frames on async thread. The computations
1933 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1934 // means we have an issue!
1935 SleepMs(100);
1936 analyzer.Stop();
1937
1938 FrameCounters global_stream_conters = analyzer.GetGlobalCounters();
1939 EXPECT_EQ(global_stream_conters.captured, 6);
1940 EXPECT_EQ(global_stream_conters.pre_encoded, 6);
1941 EXPECT_EQ(global_stream_conters.encoded, 6);
1942 EXPECT_EQ(global_stream_conters.received, 12);
1943 EXPECT_EQ(global_stream_conters.decoded, 12);
1944 EXPECT_EQ(global_stream_conters.rendered, 12);
1945 FrameCounters alice_bob_stream_conters =
1946 analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
1947 EXPECT_EQ(alice_bob_stream_conters.captured, 6);
1948 EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 6);
1949 EXPECT_EQ(alice_bob_stream_conters.encoded, 6);
1950 EXPECT_EQ(alice_bob_stream_conters.received, 2);
1951 EXPECT_EQ(alice_bob_stream_conters.decoded, 2);
1952 EXPECT_EQ(alice_bob_stream_conters.rendered, 2);
1953 FrameCounters alice_charlie_stream_conters =
1954 analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "charlie"));
1955 EXPECT_EQ(alice_charlie_stream_conters.captured, 6);
1956 EXPECT_EQ(alice_charlie_stream_conters.pre_encoded, 6);
1957 EXPECT_EQ(alice_charlie_stream_conters.encoded, 6);
1958 EXPECT_EQ(alice_charlie_stream_conters.received, 6);
1959 EXPECT_EQ(alice_charlie_stream_conters.decoded, 6);
1960 EXPECT_EQ(alice_charlie_stream_conters.rendered, 6);
1961 FrameCounters alice_david_stream_conters =
1962 analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "david"));
1963 EXPECT_EQ(alice_david_stream_conters.captured, 6);
1964 EXPECT_EQ(alice_david_stream_conters.pre_encoded, 6);
1965 EXPECT_EQ(alice_david_stream_conters.encoded, 6);
1966 EXPECT_EQ(alice_david_stream_conters.received, 4);
1967 EXPECT_EQ(alice_david_stream_conters.decoded, 4);
1968 EXPECT_EQ(alice_david_stream_conters.rendered, 4);
1969 }
1970
TEST(DefaultVideoQualityAnalyzerTest,UnregisterOneAndRegisterAnotherRegisterBack)1971 TEST(DefaultVideoQualityAnalyzerTest,
1972 UnregisterOneAndRegisterAnotherRegisterBack) {
1973 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
1974 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
1975 /*type=*/absl::nullopt,
1976 /*num_squares=*/absl::nullopt);
1977
1978 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
1979 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
1980 test::GetGlobalMetricsLogger(), options);
1981 analyzer.Start("test_case",
1982 std::vector<std::string>{"alice", "bob", "charlie"},
1983 kAnalyzerMaxThreadsCount);
1984
1985 PassFramesThroughAnalyzer(analyzer, "alice", "alice_video",
1986 {"bob", "charlie"},
1987 /*frames_count=*/2, *frame_generator);
1988 analyzer.UnregisterParticipantInCall("bob");
1989 PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"charlie"},
1990 /*frames_count=*/4, *frame_generator);
1991 analyzer.RegisterParticipantInCall("bob");
1992 PassFramesThroughAnalyzer(analyzer, "alice", "alice_video",
1993 {"bob", "charlie"},
1994 /*frames_count=*/6, *frame_generator);
1995
1996 // Give analyzer some time to process frames on async thread. The computations
1997 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
1998 // means we have an issue!
1999 SleepMs(100);
2000 analyzer.Stop();
2001
2002 FrameCounters global_stream_conters = analyzer.GetGlobalCounters();
2003 EXPECT_EQ(global_stream_conters.captured, 12);
2004 EXPECT_EQ(global_stream_conters.pre_encoded, 12);
2005 EXPECT_EQ(global_stream_conters.encoded, 12);
2006 EXPECT_EQ(global_stream_conters.received, 20);
2007 EXPECT_EQ(global_stream_conters.decoded, 20);
2008 EXPECT_EQ(global_stream_conters.rendered, 20);
2009 FrameCounters alice_bob_stream_conters =
2010 analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "bob"));
2011 EXPECT_EQ(alice_bob_stream_conters.captured, 12);
2012 EXPECT_EQ(alice_bob_stream_conters.pre_encoded, 12);
2013 EXPECT_EQ(alice_bob_stream_conters.encoded, 12);
2014 EXPECT_EQ(alice_bob_stream_conters.received, 8);
2015 EXPECT_EQ(alice_bob_stream_conters.decoded, 8);
2016 EXPECT_EQ(alice_bob_stream_conters.rendered, 8);
2017 FrameCounters alice_charlie_stream_conters =
2018 analyzer.GetPerStreamCounters().at(StatsKey("alice_video", "charlie"));
2019 EXPECT_EQ(alice_charlie_stream_conters.captured, 12);
2020 EXPECT_EQ(alice_charlie_stream_conters.pre_encoded, 12);
2021 EXPECT_EQ(alice_charlie_stream_conters.encoded, 12);
2022 EXPECT_EQ(alice_charlie_stream_conters.received, 12);
2023 EXPECT_EQ(alice_charlie_stream_conters.decoded, 12);
2024 EXPECT_EQ(alice_charlie_stream_conters.rendered, 12);
2025 }
2026
TEST(DefaultVideoQualityAnalyzerTest,FramesInFlightAreAccountedForUnregisterPeers)2027 TEST(DefaultVideoQualityAnalyzerTest,
2028 FramesInFlightAreAccountedForUnregisterPeers) {
2029 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
2030 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
2031 /*type=*/absl::nullopt,
2032 /*num_squares=*/absl::nullopt);
2033
2034 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
2035 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
2036 test::GetGlobalMetricsLogger(), options);
2037 analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
2038 kAnalyzerMaxThreadsCount);
2039
2040 // Add one frame in flight which has encode time >= 10ms.
2041 VideoFrame frame = NextFrame(frame_generator.get(), /*timestamp_us=*/1);
2042 uint16_t frame_id = analyzer.OnFrameCaptured("alice", "alice_video", frame);
2043 frame.set_id(frame_id);
2044 analyzer.OnFramePreEncode("alice", frame);
2045 SleepMs(10);
2046 analyzer.OnFrameEncoded("alice", frame.id(), FakeEncode(frame),
2047 VideoQualityAnalyzerInterface::EncoderStats(), false);
2048
2049 analyzer.UnregisterParticipantInCall("bob");
2050
2051 // Give analyzer some time to process frames on async thread. The computations
2052 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
2053 // means we have an issue!
2054 SleepMs(100);
2055 analyzer.Stop();
2056
2057 StreamStats stats = analyzer.GetStats().at(StatsKey("alice_video", "bob"));
2058 ASSERT_EQ(stats.encode_time_ms.NumSamples(), 1);
2059 EXPECT_GE(stats.encode_time_ms.GetAverage(), 10);
2060 }
2061
TEST(DefaultVideoQualityAnalyzerTest,InfraMetricsAreReportedWhenRequested)2062 TEST(DefaultVideoQualityAnalyzerTest, InfraMetricsAreReportedWhenRequested) {
2063 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
2064 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
2065 /*type=*/absl::nullopt,
2066 /*num_squares=*/absl::nullopt);
2067
2068 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
2069 options.report_infra_metrics = true;
2070 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
2071 test::GetGlobalMetricsLogger(), options);
2072 analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
2073 kAnalyzerMaxThreadsCount);
2074
2075 PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"},
2076 /*frames_count=*/1, *frame_generator);
2077
2078 // Give analyzer some time to process frames on async thread. The computations
2079 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
2080 // means we have an issue!
2081 SleepMs(100);
2082 analyzer.Stop();
2083
2084 AnalyzerStats stats = analyzer.GetAnalyzerStats();
2085 EXPECT_EQ(stats.on_frame_captured_processing_time_ms.NumSamples(), 1);
2086 EXPECT_EQ(stats.on_frame_pre_encode_processing_time_ms.NumSamples(), 1);
2087 EXPECT_EQ(stats.on_frame_encoded_processing_time_ms.NumSamples(), 1);
2088 EXPECT_EQ(stats.on_frame_pre_decode_processing_time_ms.NumSamples(), 1);
2089 EXPECT_EQ(stats.on_frame_decoded_processing_time_ms.NumSamples(), 1);
2090 EXPECT_EQ(stats.on_frame_rendered_processing_time_ms.NumSamples(), 1);
2091 EXPECT_EQ(stats.on_decoder_error_processing_time_ms.NumSamples(), 0);
2092 }
2093
TEST(DefaultVideoQualityAnalyzerTest,InfraMetricsNotCollectedByDefault)2094 TEST(DefaultVideoQualityAnalyzerTest, InfraMetricsNotCollectedByDefault) {
2095 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
2096 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
2097 /*type=*/absl::nullopt,
2098 /*num_squares=*/absl::nullopt);
2099
2100 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
2101 options.report_infra_metrics = false;
2102 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
2103 test::GetGlobalMetricsLogger(), options);
2104 analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
2105 kAnalyzerMaxThreadsCount);
2106
2107 PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"},
2108 /*frames_count=*/1, *frame_generator);
2109
2110 // Give analyzer some time to process frames on async thread. The computations
2111 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
2112 // means we have an issue!
2113 SleepMs(100);
2114 analyzer.Stop();
2115
2116 AnalyzerStats stats = analyzer.GetAnalyzerStats();
2117 EXPECT_EQ(stats.on_frame_captured_processing_time_ms.NumSamples(), 0);
2118 EXPECT_EQ(stats.on_frame_pre_encode_processing_time_ms.NumSamples(), 0);
2119 EXPECT_EQ(stats.on_frame_encoded_processing_time_ms.NumSamples(), 0);
2120 EXPECT_EQ(stats.on_frame_pre_decode_processing_time_ms.NumSamples(), 0);
2121 EXPECT_EQ(stats.on_frame_decoded_processing_time_ms.NumSamples(), 0);
2122 EXPECT_EQ(stats.on_frame_rendered_processing_time_ms.NumSamples(), 0);
2123 EXPECT_EQ(stats.on_decoder_error_processing_time_ms.NumSamples(), 0);
2124 }
2125
TEST(DefaultVideoQualityAnalyzerTest,FrameDroppedByDecoderIsAccountedCorrectly)2126 TEST(DefaultVideoQualityAnalyzerTest,
2127 FrameDroppedByDecoderIsAccountedCorrectly) {
2128 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
2129 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
2130 /*type=*/absl::nullopt,
2131 /*num_squares=*/absl::nullopt);
2132
2133 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
2134 options.report_infra_metrics = false;
2135 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
2136 test::GetGlobalMetricsLogger(), options);
2137 analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
2138 kAnalyzerMaxThreadsCount);
2139
2140 VideoFrame to_be_dropped_frame =
2141 NextFrame(frame_generator.get(), /*timestamp_us=*/1);
2142 uint16_t frame_id =
2143 analyzer.OnFrameCaptured("alice", "alice_video", to_be_dropped_frame);
2144 to_be_dropped_frame.set_id(frame_id);
2145 analyzer.OnFramePreEncode("alice", to_be_dropped_frame);
2146 analyzer.OnFrameEncoded("alice", to_be_dropped_frame.id(),
2147 FakeEncode(to_be_dropped_frame),
2148 VideoQualityAnalyzerInterface::EncoderStats(), false);
2149 VideoFrame received_to_be_dropped_frame = DeepCopy(to_be_dropped_frame);
2150 analyzer.OnFramePreDecode("bob", received_to_be_dropped_frame.id(),
2151 FakeEncode(received_to_be_dropped_frame));
2152 PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"},
2153 /*frames_count=*/1, *frame_generator);
2154
2155 // Give analyzer some time to process frames on async thread. The computations
2156 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
2157 // means we have an issue!
2158 SleepMs(100);
2159 analyzer.Stop();
2160
2161 StreamStats stats = analyzer.GetStats().at(StatsKey("alice_video", "bob"));
2162 ASSERT_EQ(stats.dropped_by_phase[FrameDropPhase::kByDecoder], 1);
2163 }
2164
2165 class DefaultVideoQualityAnalyzerTimeBetweenFreezesTest
2166 : public TestWithParam<bool> {};
2167
TEST_P(DefaultVideoQualityAnalyzerTimeBetweenFreezesTest,TimeBetweenFreezesIsEqualToStreamDurationWhenThereAreNoFeeezes)2168 TEST_P(DefaultVideoQualityAnalyzerTimeBetweenFreezesTest,
2169 TimeBetweenFreezesIsEqualToStreamDurationWhenThereAreNoFeeezes) {
2170 std::unique_ptr<test::FrameGeneratorInterface> frame_generator =
2171 test::CreateSquareFrameGenerator(kFrameWidth, kFrameHeight,
2172 /*type=*/absl::nullopt,
2173 /*num_squares=*/absl::nullopt);
2174
2175 DefaultVideoQualityAnalyzerOptions options = AnalyzerOptionsForTest();
2176 DefaultVideoQualityAnalyzer analyzer(Clock::GetRealTimeClock(),
2177 test::GetGlobalMetricsLogger(), options);
2178 analyzer.Start("test_case", std::vector<std::string>{"alice", "bob"},
2179 kAnalyzerMaxThreadsCount);
2180
2181 PassFramesThroughAnalyzer(analyzer, "alice", "alice_video", {"bob"},
2182 /*frames_count=*/5, *frame_generator,
2183 /*interframe_delay_ms=*/50);
2184 if (GetParam()) {
2185 analyzer.UnregisterParticipantInCall("bob");
2186 }
2187
2188 // Give analyzer some time to process frames on async thread. The computations
2189 // have to be fast (heavy metrics are disabled!), so if doesn't fit 100ms it
2190 // means we have an issue!
2191 SleepMs(50);
2192 analyzer.Stop();
2193
2194 StreamStats stats = analyzer.GetStats().at(StatsKey("alice_video", "bob"));
2195 ASSERT_EQ(stats.time_between_freezes_ms.NumSamples(), 1);
2196 EXPECT_GE(stats.time_between_freezes_ms.GetAverage(), 200);
2197 }
2198
2199 INSTANTIATE_TEST_SUITE_P(WithRegisteredAndUnregisteredPeerAtTheEndOfTheCall,
2200 DefaultVideoQualityAnalyzerTimeBetweenFreezesTest,
2201 ValuesIn({true, false}));
2202
2203 } // namespace
2204 } // namespace webrtc
2205