1 /* 2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ 12 #define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ 13 14 #include <map> 15 #include <memory> 16 #include <string> 17 #include <vector> 18 19 #include "absl/strings/string_view.h" 20 #include "absl/types/optional.h" 21 #include "api/test/video_quality_analyzer_interface.h" 22 #include "api/video/encoded_image.h" 23 #include "api/video/video_frame.h" 24 #include "api/video_codecs/sdp_video_format.h" 25 #include "api/video_codecs/video_decoder.h" 26 #include "api/video_codecs/video_decoder_factory.h" 27 #include "rtc_base/synchronization/mutex.h" 28 #include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" 29 30 namespace webrtc { 31 namespace webrtc_pc_e2e { 32 33 // QualityAnalyzingVideoDecoder is used to wrap origin video decoder and inject 34 // VideoQualityAnalyzerInterface before and after decoder. 35 // 36 // QualityAnalyzingVideoDecoder propagates all calls to the origin decoder. 37 // It registers its own DecodedImageCallback in the origin decoder and will 38 // store user specified callback inside itself. 39 // 40 // When Decode(...) will be invoked, quality decoder first will extract frame id 41 // from passed EncodedImage with EncodedImageIdExtracor that was specified in 42 // constructor, then will call video quality analyzer, with correct 43 // EncodedImage and only then will pass image to origin decoder. 44 // 45 // When origin decoder decodes the image it will call quality decoder's special 46 // callback, where video analyzer will be called again and then decoded frame 47 // will be passed to origin callback, provided by user. 48 // 49 // Quality decoder registers its own callback in origin decoder, at the same 50 // time the user registers their callback in quality decoder. 51 class QualityAnalyzingVideoDecoder : public VideoDecoder { 52 public: 53 QualityAnalyzingVideoDecoder(absl::string_view peer_name, 54 std::unique_ptr<VideoDecoder> delegate, 55 EncodedImageDataExtractor* extractor, 56 VideoQualityAnalyzerInterface* analyzer); 57 ~QualityAnalyzingVideoDecoder() override; 58 59 // Methods of VideoDecoder interface. 60 bool Configure(const Settings& settings) override; 61 int32_t Decode(const EncodedImage& input_image, 62 bool missing_frames, 63 int64_t render_time_ms) override; 64 int32_t RegisterDecodeCompleteCallback( 65 DecodedImageCallback* callback) override; 66 int32_t Release() override; 67 DecoderInfo GetDecoderInfo() const override; 68 const char* ImplementationName() const override; 69 70 private: 71 class DecoderCallback : public DecodedImageCallback { 72 public: 73 explicit DecoderCallback(QualityAnalyzingVideoDecoder* decoder); 74 ~DecoderCallback() override; 75 76 void SetDelegateCallback(DecodedImageCallback* delegate); 77 78 // Methods of DecodedImageCallback interface. 79 int32_t Decoded(VideoFrame& decodedImage) override; 80 int32_t Decoded(VideoFrame& decodedImage, int64_t decode_time_ms) override; 81 void Decoded(VideoFrame& decodedImage, 82 absl::optional<int32_t> decode_time_ms, 83 absl::optional<uint8_t> qp) override; 84 85 int32_t IrrelevantSimulcastStreamDecoded(uint16_t frame_id, 86 uint32_t timestamp_ms); 87 88 private: 89 rtc::scoped_refptr<webrtc::VideoFrameBuffer> GetDummyFrameBuffer(); 90 91 QualityAnalyzingVideoDecoder* const decoder_; 92 93 rtc::scoped_refptr<webrtc::VideoFrameBuffer> dummy_frame_buffer_; 94 95 Mutex callback_mutex_; 96 DecodedImageCallback* delegate_callback_ RTC_GUARDED_BY(callback_mutex_); 97 }; 98 99 void OnFrameDecoded(VideoFrame* frame, 100 absl::optional<int32_t> decode_time_ms, 101 absl::optional<uint8_t> qp); 102 103 const std::string peer_name_; 104 const std::string implementation_name_; 105 std::unique_ptr<VideoDecoder> delegate_; 106 EncodedImageDataExtractor* const extractor_; 107 VideoQualityAnalyzerInterface* const analyzer_; 108 std::unique_ptr<DecoderCallback> analyzing_callback_; 109 110 // VideoDecoder interface assumes async delivery of decoded video frames. 111 // This lock is used to protect shared state, that have to be propagated 112 // from received EncodedImage to resulted VideoFrame. 113 Mutex mutex_; 114 115 // Name of the video codec type used. Ex: VP8, VP9, H264 etc. 116 std::string codec_name_ RTC_GUARDED_BY(mutex_); 117 std::map<uint32_t, absl::optional<uint16_t>> timestamp_to_frame_id_ 118 RTC_GUARDED_BY(mutex_); 119 // Stores currently being decoded images by timestamp. Because 120 // EncodedImageDataExtractor can create new copy on EncodedImage we need to 121 // ensure, that this image won't be deleted during async decoding. To do it 122 // all images are putted into this map and removed from here inside callback. 123 std::map<uint32_t, EncodedImage> decoding_images_ RTC_GUARDED_BY(mutex_); 124 }; 125 126 // Produces QualityAnalyzingVideoDecoder, which hold decoders, produced by 127 // specified factory as delegates. Forwards all other calls to specified 128 // factory. 129 class QualityAnalyzingVideoDecoderFactory : public VideoDecoderFactory { 130 public: 131 QualityAnalyzingVideoDecoderFactory( 132 absl::string_view peer_name, 133 std::unique_ptr<VideoDecoderFactory> delegate, 134 EncodedImageDataExtractor* extractor, 135 VideoQualityAnalyzerInterface* analyzer); 136 ~QualityAnalyzingVideoDecoderFactory() override; 137 138 // Methods of VideoDecoderFactory interface. 139 std::vector<SdpVideoFormat> GetSupportedFormats() const override; 140 std::unique_ptr<VideoDecoder> CreateVideoDecoder( 141 const SdpVideoFormat& format) override; 142 143 private: 144 const std::string peer_name_; 145 std::unique_ptr<VideoDecoderFactory> delegate_; 146 EncodedImageDataExtractor* const extractor_; 147 VideoQualityAnalyzerInterface* const analyzer_; 148 }; 149 150 } // namespace webrtc_pc_e2e 151 } // namespace webrtc 152 153 #endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_DECODER_H_ 154