1 /*
2 * Copyright 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "test/scenario/stats_collection.h"
11
12 #include "test/gtest.h"
13 #include "test/scenario/scenario.h"
14
15 namespace webrtc {
16 namespace test {
17 namespace {
CreateAnalyzedStream(Scenario * s,NetworkSimulationConfig network_config,VideoQualityAnalyzer * analyzer,CallStatsCollectors * collectors)18 void CreateAnalyzedStream(Scenario* s,
19 NetworkSimulationConfig network_config,
20 VideoQualityAnalyzer* analyzer,
21 CallStatsCollectors* collectors) {
22 VideoStreamConfig config;
23 config.encoder.codec = VideoStreamConfig::Encoder::Codec::kVideoCodecVP8;
24 config.encoder.implementation =
25 VideoStreamConfig::Encoder::Implementation::kSoftware;
26 config.hooks.frame_pair_handlers = {analyzer->Handler()};
27 auto* caller = s->CreateClient("caller", CallClientConfig());
28 auto* callee = s->CreateClient("callee", CallClientConfig());
29 auto route =
30 s->CreateRoutes(caller, {s->CreateSimulationNode(network_config)}, callee,
31 {s->CreateSimulationNode(NetworkSimulationConfig())});
32 VideoStreamPair* video = s->CreateVideoStream(route->forward(), config);
33 auto* audio = s->CreateAudioStream(route->forward(), AudioStreamConfig());
34 s->Every(TimeDelta::Seconds(1), [=] {
35 collectors->call.AddStats(caller->GetStats());
36
37 VideoSendStream::Stats send_stats;
38 caller->SendTask([&]() { send_stats = video->send()->GetStats(); });
39 collectors->video_send.AddStats(send_stats, s->Now());
40
41 AudioReceiveStreamInterface::Stats receive_stats;
42 caller->SendTask([&]() { receive_stats = audio->receive()->GetStats(); });
43 collectors->audio_receive.AddStats(receive_stats);
44
45 // Querying the video stats from within the expected runtime environment
46 // (i.e. the TQ that belongs to the CallClient, not the Scenario TQ that
47 // we're currently on).
48 VideoReceiveStreamInterface::Stats video_receive_stats;
49 auto* video_stream = video->receive();
50 callee->SendTask([&video_stream, &video_receive_stats]() {
51 video_receive_stats = video_stream->GetStats();
52 });
53 collectors->video_receive.AddStats(video_receive_stats);
54 });
55 }
56 } // namespace
57
TEST(ScenarioAnalyzerTest,PsnrIsHighWhenNetworkIsGood)58 TEST(ScenarioAnalyzerTest, PsnrIsHighWhenNetworkIsGood) {
59 VideoQualityAnalyzer analyzer;
60 CallStatsCollectors stats;
61 {
62 Scenario s;
63 NetworkSimulationConfig good_network;
64 good_network.bandwidth = DataRate::KilobitsPerSec(1000);
65 CreateAnalyzedStream(&s, good_network, &analyzer, &stats);
66 s.RunFor(TimeDelta::Seconds(3));
67 }
68 // This is a change detecting test, the targets are based on previous runs and
69 // might change due to changes in configuration and encoder etc. The main
70 // purpose is to show how the stats can be used. To avoid being overly
71 // sensistive to change, the ranges are chosen to be quite large.
72 EXPECT_NEAR(analyzer.stats().psnr_with_freeze.Mean(), 43, 10);
73 EXPECT_NEAR(stats.call.stats().target_rate.Mean().kbps(), 700, 300);
74 EXPECT_NEAR(stats.video_send.stats().media_bitrate.Mean().kbps(), 500, 200);
75 EXPECT_NEAR(stats.video_receive.stats().resolution.Mean(), 180, 10);
76 EXPECT_NEAR(stats.audio_receive.stats().jitter_buffer.Mean().ms(), 40, 20);
77 }
78
TEST(ScenarioAnalyzerTest,PsnrIsLowWhenNetworkIsBad)79 TEST(ScenarioAnalyzerTest, PsnrIsLowWhenNetworkIsBad) {
80 VideoQualityAnalyzer analyzer;
81 CallStatsCollectors stats;
82 {
83 Scenario s;
84 NetworkSimulationConfig bad_network;
85 bad_network.bandwidth = DataRate::KilobitsPerSec(100);
86 bad_network.loss_rate = 0.02;
87 CreateAnalyzedStream(&s, bad_network, &analyzer, &stats);
88 s.RunFor(TimeDelta::Seconds(3));
89 }
90 // This is a change detecting test, the targets are based on previous runs and
91 // might change due to changes in configuration and encoder etc.
92 EXPECT_NEAR(analyzer.stats().psnr_with_freeze.Mean(), 20, 10);
93 EXPECT_NEAR(stats.call.stats().target_rate.Mean().kbps(), 75, 50);
94 EXPECT_NEAR(stats.video_send.stats().media_bitrate.Mean().kbps(), 100, 50);
95 EXPECT_NEAR(stats.video_receive.stats().resolution.Mean(), 180, 10);
96 EXPECT_NEAR(stats.audio_receive.stats().jitter_buffer.Mean().ms(), 250, 200);
97 }
98
TEST(ScenarioAnalyzerTest,CountsCapturedButNotRendered)99 TEST(ScenarioAnalyzerTest, CountsCapturedButNotRendered) {
100 VideoQualityAnalyzer analyzer;
101 CallStatsCollectors stats;
102 {
103 Scenario s;
104 NetworkSimulationConfig long_delays;
105 long_delays.delay = TimeDelta::Seconds(5);
106 CreateAnalyzedStream(&s, long_delays, &analyzer, &stats);
107 // Enough time to send frames but not enough to deliver.
108 s.RunFor(TimeDelta::Millis(100));
109 }
110 EXPECT_GE(analyzer.stats().capture.count, 1);
111 EXPECT_EQ(analyzer.stats().render.count, 0);
112 }
113 } // namespace test
114 } // namespace webrtc
115