1 /*
2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "test/pc/e2e/analyzer/video/analyzing_video_sink.h"
11
12 #include <memory>
13 #include <set>
14 #include <utility>
15
16 #include "absl/memory/memory.h"
17 #include "absl/strings/string_view.h"
18 #include "absl/types/optional.h"
19 #include "api/test/pclf/media_configuration.h"
20 #include "api/test/video/video_frame_writer.h"
21 #include "api/units/timestamp.h"
22 #include "api/video/i420_buffer.h"
23 #include "api/video/video_frame.h"
24 #include "rtc_base/checks.h"
25 #include "rtc_base/logging.h"
26 #include "rtc_base/synchronization/mutex.h"
27 #include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h"
28 #include "test/pc/e2e/analyzer/video/video_dumping.h"
29 #include "test/testsupport/fixed_fps_video_frame_writer_adapter.h"
30 #include "test/video_renderer.h"
31
32 namespace webrtc {
33 namespace webrtc_pc_e2e {
34
AnalyzingVideoSink(absl::string_view peer_name,Clock * clock,VideoQualityAnalyzerInterface & analyzer,AnalyzingVideoSinksHelper & sinks_helper,const VideoSubscription & subscription,bool report_infra_stats)35 AnalyzingVideoSink::AnalyzingVideoSink(absl::string_view peer_name,
36 Clock* clock,
37 VideoQualityAnalyzerInterface& analyzer,
38 AnalyzingVideoSinksHelper& sinks_helper,
39 const VideoSubscription& subscription,
40 bool report_infra_stats)
41 : peer_name_(peer_name),
42 report_infra_stats_(report_infra_stats),
43 clock_(clock),
44 analyzer_(&analyzer),
45 sinks_helper_(&sinks_helper),
46 subscription_(subscription) {}
47
UpdateSubscription(const VideoSubscription & subscription)48 void AnalyzingVideoSink::UpdateSubscription(
49 const VideoSubscription& subscription) {
50 // For peers with changed resolutions we need to close current writers and
51 // open new ones. This is done by removing existing sinks, which will force
52 // creation of the new sinks when next frame will be received.
53 std::set<test::VideoFrameWriter*> writers_to_close;
54 {
55 MutexLock lock(&mutex_);
56 subscription_ = subscription;
57 for (auto it = stream_sinks_.cbegin(); it != stream_sinks_.cend();) {
58 absl::optional<VideoResolution> new_requested_resolution =
59 subscription_.GetResolutionForPeer(it->second.sender_peer_name);
60 if (!new_requested_resolution.has_value() ||
61 (*new_requested_resolution != it->second.resolution)) {
62 RTC_LOG(LS_INFO) << peer_name_ << ": Subscribed resolution for stream "
63 << it->first << " from " << it->second.sender_peer_name
64 << " was updated from "
65 << it->second.resolution.ToString() << " to "
66 << new_requested_resolution->ToString()
67 << ". Repopulating all video sinks and recreating "
68 << "requested video writers";
69 writers_to_close.insert(it->second.video_frame_writer);
70 it = stream_sinks_.erase(it);
71 } else {
72 ++it;
73 }
74 }
75 }
76 sinks_helper_->CloseAndRemoveVideoWriters(writers_to_close);
77 }
78
OnFrame(const VideoFrame & frame)79 void AnalyzingVideoSink::OnFrame(const VideoFrame& frame) {
80 if (IsDummyFrame(frame)) {
81 // This is dummy frame, so we don't need to process it further.
82 return;
83 }
84
85 if (frame.id() == VideoFrame::kNotSetId) {
86 // If frame ID is unknown we can't get required render resolution, so pass
87 // to the analyzer in the actual resolution of the frame.
88 AnalyzeFrame(frame);
89 } else {
90 std::string stream_label = analyzer_->GetStreamLabel(frame.id());
91 MutexLock lock(&mutex_);
92 Timestamp processing_started = clock_->CurrentTime();
93 SinksDescriptor* sinks_descriptor = PopulateSinks(stream_label);
94 RTC_CHECK(sinks_descriptor != nullptr);
95
96 VideoFrame scaled_frame =
97 ScaleVideoFrame(frame, sinks_descriptor->resolution);
98 AnalyzeFrame(scaled_frame);
99 for (auto& sink : sinks_descriptor->sinks) {
100 sink->OnFrame(scaled_frame);
101 }
102 Timestamp processing_finished = clock_->CurrentTime();
103
104 if (report_infra_stats_) {
105 stats_.analyzing_sink_processing_time_ms.AddSample(
106 (processing_finished - processing_started).ms<double>());
107 }
108 }
109 }
110
stats() const111 AnalyzingVideoSink::Stats AnalyzingVideoSink::stats() const {
112 MutexLock lock(&mutex_);
113 return stats_;
114 }
115
ScaleVideoFrame(const VideoFrame & frame,const VideoResolution & required_resolution)116 VideoFrame AnalyzingVideoSink::ScaleVideoFrame(
117 const VideoFrame& frame,
118 const VideoResolution& required_resolution) {
119 Timestamp processing_started = clock_->CurrentTime();
120 if (required_resolution.width() == static_cast<size_t>(frame.width()) &&
121 required_resolution.height() == static_cast<size_t>(frame.height())) {
122 if (report_infra_stats_) {
123 stats_.scaling_tims_ms.AddSample(
124 (clock_->CurrentTime() - processing_started).ms<double>());
125 }
126 return frame;
127 }
128
129 // We allow some difference in the aspect ration because when decoder
130 // downscales video stream it may round up some dimensions to make them even,
131 // ex: 960x540 -> 480x270 -> 240x136 instead of 240x135.
132 RTC_CHECK_LE(std::abs(static_cast<double>(required_resolution.width()) /
133 required_resolution.height() -
134 static_cast<double>(frame.width()) / frame.height()),
135 0.1)
136 << peer_name_
137 << ": Received frame has too different aspect ratio compared to "
138 << "requested video resolution: required resolution="
139 << required_resolution.ToString()
140 << "; actual resolution=" << frame.width() << "x" << frame.height();
141
142 rtc::scoped_refptr<I420Buffer> scaled_buffer(I420Buffer::Create(
143 required_resolution.width(), required_resolution.height()));
144 scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420());
145
146 VideoFrame scaled_frame = frame;
147 scaled_frame.set_video_frame_buffer(scaled_buffer);
148 if (report_infra_stats_) {
149 stats_.scaling_tims_ms.AddSample(
150 (clock_->CurrentTime() - processing_started).ms<double>());
151 }
152 return scaled_frame;
153 }
154
AnalyzeFrame(const VideoFrame & frame)155 void AnalyzingVideoSink::AnalyzeFrame(const VideoFrame& frame) {
156 VideoFrame frame_copy = frame;
157 frame_copy.set_video_frame_buffer(
158 I420Buffer::Copy(*frame.video_frame_buffer()->ToI420()));
159 analyzer_->OnFrameRendered(peer_name_, frame_copy);
160 }
161
PopulateSinks(absl::string_view stream_label)162 AnalyzingVideoSink::SinksDescriptor* AnalyzingVideoSink::PopulateSinks(
163 absl::string_view stream_label) {
164 // Fast pass: sinks already exists.
165 auto sinks_it = stream_sinks_.find(std::string(stream_label));
166 if (sinks_it != stream_sinks_.end()) {
167 return &sinks_it->second;
168 }
169
170 // Slow pass: we need to create and save sinks
171 absl::optional<std::pair<std::string, VideoConfig>> peer_and_config =
172 sinks_helper_->GetPeerAndConfig(stream_label);
173 RTC_CHECK(peer_and_config.has_value())
174 << "No video config for stream " << stream_label;
175 const std::string& sender_peer_name = peer_and_config->first;
176 const VideoConfig& config = peer_and_config->second;
177
178 absl::optional<VideoResolution> resolution =
179 subscription_.GetResolutionForPeer(sender_peer_name);
180 if (!resolution.has_value()) {
181 RTC_LOG(LS_ERROR) << peer_name_ << " received stream " << stream_label
182 << " from " << sender_peer_name
183 << " for which they were not subscribed";
184 resolution = config.GetResolution();
185 }
186 if (!resolution->IsRegular()) {
187 RTC_LOG(LS_ERROR) << peer_name_ << " received stream " << stream_label
188 << " from " << sender_peer_name
189 << " for which resolution wasn't resolved";
190 resolution = config.GetResolution();
191 }
192
193 RTC_CHECK(resolution.has_value());
194
195 SinksDescriptor sinks_descriptor(sender_peer_name, *resolution);
196 if (config.output_dump_options.has_value()) {
197 std::unique_ptr<test::VideoFrameWriter> writer =
198 config.output_dump_options->CreateOutputDumpVideoFrameWriter(
199 stream_label, peer_name_, *resolution);
200 if (config.output_dump_use_fixed_framerate) {
201 writer = std::make_unique<test::FixedFpsVideoFrameWriterAdapter>(
202 resolution->fps(), clock_, std::move(writer));
203 }
204 sinks_descriptor.sinks.push_back(std::make_unique<VideoWriter>(
205 writer.get(), config.output_dump_options->sampling_modulo()));
206 sinks_descriptor.video_frame_writer =
207 sinks_helper_->AddVideoWriter(std::move(writer));
208 }
209 if (config.show_on_screen) {
210 sinks_descriptor.sinks.push_back(
211 absl::WrapUnique(test::VideoRenderer::Create(
212 (*config.stream_label + "-render").c_str(), resolution->width(),
213 resolution->height())));
214 }
215 return &stream_sinks_.emplace(stream_label, std::move(sinks_descriptor))
216 .first->second;
217 }
218
219 } // namespace webrtc_pc_e2e
220 } // namespace webrtc
221