1 /*
2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "test/pc/e2e/analyzer/video/quality_analyzing_video_decoder.h"
12
13 #include <cstdint>
14 #include <cstring>
15 #include <memory>
16 #include <utility>
17
18 #include "absl/strings/string_view.h"
19 #include "absl/types/optional.h"
20 #include "api/video/i420_buffer.h"
21 #include "api/video/video_frame.h"
22 #include "modules/video_coding/include/video_error_codes.h"
23 #include "rtc_base/logging.h"
24 #include "test/pc/e2e/analyzer/video/simulcast_dummy_buffer_helper.h"
25
26 namespace webrtc {
27 namespace webrtc_pc_e2e {
28
QualityAnalyzingVideoDecoder(absl::string_view peer_name,std::unique_ptr<VideoDecoder> delegate,EncodedImageDataExtractor * extractor,VideoQualityAnalyzerInterface * analyzer)29 QualityAnalyzingVideoDecoder::QualityAnalyzingVideoDecoder(
30 absl::string_view peer_name,
31 std::unique_ptr<VideoDecoder> delegate,
32 EncodedImageDataExtractor* extractor,
33 VideoQualityAnalyzerInterface* analyzer)
34 : peer_name_(peer_name),
35 implementation_name_("AnalyzingDecoder-" +
36 std::string(delegate->ImplementationName())),
37 delegate_(std::move(delegate)),
38 extractor_(extractor),
39 analyzer_(analyzer) {
40 analyzing_callback_ = std::make_unique<DecoderCallback>(this);
41 }
42 QualityAnalyzingVideoDecoder::~QualityAnalyzingVideoDecoder() = default;
43
Configure(const Settings & settings)44 bool QualityAnalyzingVideoDecoder::Configure(const Settings& settings) {
45 {
46 MutexLock lock(&mutex_);
47 codec_name_ = std::string(CodecTypeToPayloadString(settings.codec_type())) +
48 "_" + delegate_->GetDecoderInfo().implementation_name;
49 }
50 return delegate_->Configure(settings);
51 }
52
Decode(const EncodedImage & input_image,bool missing_frames,int64_t render_time_ms)53 int32_t QualityAnalyzingVideoDecoder::Decode(const EncodedImage& input_image,
54 bool missing_frames,
55 int64_t render_time_ms) {
56 // Image extractor extracts id from provided EncodedImage and also returns
57 // the image with the original buffer. Buffer can be modified in place, so
58 // owner of original buffer will be responsible for deleting it, or extractor
59 // can create a new buffer. In such case extractor will be responsible for
60 // deleting it.
61 EncodedImageExtractionResult out = extractor_->ExtractData(input_image);
62
63 if (out.discard) {
64 // To partly emulate behavior of Selective Forwarding Unit (SFU) in the
65 // test, on receiver side we will "discard" frames from irrelevant streams.
66 // When all encoded images were marked to discarded, black frame have to be
67 // returned. Because simulcast streams will be received by receiver as 3
68 // different independent streams we don't want that irrelevant streams
69 // affect video quality metrics and also we don't want to use CPU time to
70 // decode them to prevent regressions on relevant streams. Also we can't
71 // just drop frame, because in such case, receiving part will be confused
72 // with all frames missing and will request a key frame, which will result
73 // into extra load on network and sender side. Because of it, discarded
74 // image will be always decoded as black frame and will be passed to
75 // callback directly without reaching decoder and video quality analyzer.
76 //
77 // For more details see QualityAnalyzingVideoEncoder.
78 return analyzing_callback_->IrrelevantSimulcastStreamDecoded(
79 out.id.value_or(VideoFrame::kNotSetId), input_image.Timestamp());
80 }
81
82 EncodedImage* origin_image;
83 {
84 MutexLock lock(&mutex_);
85 // Store id to be able to retrieve it in analyzing callback.
86 timestamp_to_frame_id_.insert({input_image.Timestamp(), out.id});
87 // Store encoded image to prevent its destruction while it is used in
88 // decoder.
89 origin_image = &(
90 decoding_images_.insert({input_image.Timestamp(), std::move(out.image)})
91 .first->second);
92 }
93 // We can safely dereference `origin_image`, because it can be removed from
94 // the map only after `delegate_` Decode method will be invoked. Image will
95 // be removed inside DecodedImageCallback, which can be done on separate
96 // thread.
97 analyzer_->OnFramePreDecode(
98 peer_name_, out.id.value_or(VideoFrame::kNotSetId), *origin_image);
99 int32_t result =
100 delegate_->Decode(*origin_image, missing_frames, render_time_ms);
101 if (result != WEBRTC_VIDEO_CODEC_OK) {
102 // If delegate decoder failed, then cleanup data for this image.
103 VideoQualityAnalyzerInterface::DecoderStats stats;
104 {
105 MutexLock lock(&mutex_);
106 timestamp_to_frame_id_.erase(input_image.Timestamp());
107 decoding_images_.erase(input_image.Timestamp());
108 stats.decoder_name = codec_name_;
109 }
110 analyzer_->OnDecoderError(
111 peer_name_, out.id.value_or(VideoFrame::kNotSetId), result, stats);
112 }
113 return result;
114 }
115
RegisterDecodeCompleteCallback(DecodedImageCallback * callback)116 int32_t QualityAnalyzingVideoDecoder::RegisterDecodeCompleteCallback(
117 DecodedImageCallback* callback) {
118 analyzing_callback_->SetDelegateCallback(callback);
119 return delegate_->RegisterDecodeCompleteCallback(analyzing_callback_.get());
120 }
121
Release()122 int32_t QualityAnalyzingVideoDecoder::Release() {
123 // Release decoder first. During release process it can still decode some
124 // frames, so we don't take a lock to prevent deadlock.
125 int32_t result = delegate_->Release();
126
127 MutexLock lock(&mutex_);
128 analyzing_callback_->SetDelegateCallback(nullptr);
129 timestamp_to_frame_id_.clear();
130 decoding_images_.clear();
131 return result;
132 }
133
GetDecoderInfo() const134 VideoDecoder::DecoderInfo QualityAnalyzingVideoDecoder::GetDecoderInfo() const {
135 DecoderInfo info = delegate_->GetDecoderInfo();
136 info.implementation_name = implementation_name_;
137 return info;
138 }
139
ImplementationName() const140 const char* QualityAnalyzingVideoDecoder::ImplementationName() const {
141 return implementation_name_.c_str();
142 }
143
DecoderCallback(QualityAnalyzingVideoDecoder * decoder)144 QualityAnalyzingVideoDecoder::DecoderCallback::DecoderCallback(
145 QualityAnalyzingVideoDecoder* decoder)
146 : decoder_(decoder), delegate_callback_(nullptr) {}
147 QualityAnalyzingVideoDecoder::DecoderCallback::~DecoderCallback() = default;
148
SetDelegateCallback(DecodedImageCallback * delegate)149 void QualityAnalyzingVideoDecoder::DecoderCallback::SetDelegateCallback(
150 DecodedImageCallback* delegate) {
151 MutexLock lock(&callback_mutex_);
152 delegate_callback_ = delegate;
153 }
154
155 // We have to implement all next 3 methods because we don't know which one
156 // exactly is implemented in `delegate_callback_`, so we need to call the same
157 // method on `delegate_callback_`, as was called on `this` callback.
Decoded(VideoFrame & decodedImage)158 int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded(
159 VideoFrame& decodedImage) {
160 decoder_->OnFrameDecoded(&decodedImage, /*decode_time_ms=*/absl::nullopt,
161 /*qp=*/absl::nullopt);
162
163 MutexLock lock(&callback_mutex_);
164 RTC_DCHECK(delegate_callback_);
165 return delegate_callback_->Decoded(decodedImage);
166 }
167
Decoded(VideoFrame & decodedImage,int64_t decode_time_ms)168 int32_t QualityAnalyzingVideoDecoder::DecoderCallback::Decoded(
169 VideoFrame& decodedImage,
170 int64_t decode_time_ms) {
171 decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, /*qp=*/absl::nullopt);
172
173 MutexLock lock(&callback_mutex_);
174 RTC_DCHECK(delegate_callback_);
175 return delegate_callback_->Decoded(decodedImage, decode_time_ms);
176 }
177
Decoded(VideoFrame & decodedImage,absl::optional<int32_t> decode_time_ms,absl::optional<uint8_t> qp)178 void QualityAnalyzingVideoDecoder::DecoderCallback::Decoded(
179 VideoFrame& decodedImage,
180 absl::optional<int32_t> decode_time_ms,
181 absl::optional<uint8_t> qp) {
182 decoder_->OnFrameDecoded(&decodedImage, decode_time_ms, qp);
183
184 MutexLock lock(&callback_mutex_);
185 RTC_DCHECK(delegate_callback_);
186 delegate_callback_->Decoded(decodedImage, decode_time_ms, qp);
187 }
188
189 int32_t
IrrelevantSimulcastStreamDecoded(uint16_t frame_id,uint32_t timestamp_ms)190 QualityAnalyzingVideoDecoder::DecoderCallback::IrrelevantSimulcastStreamDecoded(
191 uint16_t frame_id,
192 uint32_t timestamp_ms) {
193 webrtc::VideoFrame dummy_frame =
194 webrtc::VideoFrame::Builder()
195 .set_video_frame_buffer(GetDummyFrameBuffer())
196 .set_timestamp_rtp(timestamp_ms)
197 .set_id(frame_id)
198 .build();
199 MutexLock lock(&callback_mutex_);
200 RTC_DCHECK(delegate_callback_);
201 delegate_callback_->Decoded(dummy_frame, absl::nullopt, absl::nullopt);
202 return WEBRTC_VIDEO_CODEC_OK;
203 }
204
205 rtc::scoped_refptr<webrtc::VideoFrameBuffer>
GetDummyFrameBuffer()206 QualityAnalyzingVideoDecoder::DecoderCallback::GetDummyFrameBuffer() {
207 if (!dummy_frame_buffer_) {
208 dummy_frame_buffer_ = CreateDummyFrameBuffer();
209 }
210
211 return dummy_frame_buffer_;
212 }
213
OnFrameDecoded(VideoFrame * frame,absl::optional<int32_t> decode_time_ms,absl::optional<uint8_t> qp)214 void QualityAnalyzingVideoDecoder::OnFrameDecoded(
215 VideoFrame* frame,
216 absl::optional<int32_t> decode_time_ms,
217 absl::optional<uint8_t> qp) {
218 absl::optional<uint16_t> frame_id;
219 std::string codec_name;
220 {
221 MutexLock lock(&mutex_);
222 auto it = timestamp_to_frame_id_.find(frame->timestamp());
223 if (it == timestamp_to_frame_id_.end()) {
224 // Ensure, that we have info about this frame. It can happen that for some
225 // reasons decoder response, that it failed to decode, when we were
226 // posting frame to it, but then call the callback for this frame.
227 RTC_LOG(LS_ERROR) << "QualityAnalyzingVideoDecoder::OnFrameDecoded: No "
228 "frame id for frame for frame->timestamp()="
229 << frame->timestamp();
230 return;
231 }
232 frame_id = it->second;
233 timestamp_to_frame_id_.erase(it);
234 decoding_images_.erase(frame->timestamp());
235 codec_name = codec_name_;
236 }
237 // Set frame id to the value, that was extracted from corresponding encoded
238 // image.
239 frame->set_id(frame_id.value_or(VideoFrame::kNotSetId));
240 VideoQualityAnalyzerInterface::DecoderStats stats;
241 stats.decoder_name = codec_name;
242 stats.decode_time_ms = decode_time_ms;
243 analyzer_->OnFrameDecoded(peer_name_, *frame, stats);
244 }
245
QualityAnalyzingVideoDecoderFactory(absl::string_view peer_name,std::unique_ptr<VideoDecoderFactory> delegate,EncodedImageDataExtractor * extractor,VideoQualityAnalyzerInterface * analyzer)246 QualityAnalyzingVideoDecoderFactory::QualityAnalyzingVideoDecoderFactory(
247 absl::string_view peer_name,
248 std::unique_ptr<VideoDecoderFactory> delegate,
249 EncodedImageDataExtractor* extractor,
250 VideoQualityAnalyzerInterface* analyzer)
251 : peer_name_(peer_name),
252 delegate_(std::move(delegate)),
253 extractor_(extractor),
254 analyzer_(analyzer) {}
255 QualityAnalyzingVideoDecoderFactory::~QualityAnalyzingVideoDecoderFactory() =
256 default;
257
258 std::vector<SdpVideoFormat>
GetSupportedFormats() const259 QualityAnalyzingVideoDecoderFactory::GetSupportedFormats() const {
260 return delegate_->GetSupportedFormats();
261 }
262
263 std::unique_ptr<VideoDecoder>
CreateVideoDecoder(const SdpVideoFormat & format)264 QualityAnalyzingVideoDecoderFactory::CreateVideoDecoder(
265 const SdpVideoFormat& format) {
266 std::unique_ptr<VideoDecoder> decoder = delegate_->CreateVideoDecoder(format);
267 return std::make_unique<QualityAnalyzingVideoDecoder>(
268 peer_name_, std::move(decoder), extractor_, analyzer_);
269 }
270
271 } // namespace webrtc_pc_e2e
272 } // namespace webrtc
273