1 /*
2  *  Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_frames_comparator.h"
12 
13 #include <algorithm>
14 #include <map>
15 #include <string>
16 #include <utility>
17 #include <vector>
18 
19 #include "absl/types/optional.h"
20 #include "api/array_view.h"
21 #include "api/scoped_refptr.h"
22 #include "api/video/i420_buffer.h"
23 #include "api/video/video_frame_type.h"
24 #include "common_video/libyuv/include/webrtc_libyuv.h"
25 #include "rtc_base/checks.h"
26 #include "rtc_base/platform_thread.h"
27 #include "rtc_base/synchronization/mutex.h"
28 #include "rtc_tools/frame_analyzer/video_geometry_aligner.h"
29 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_internal_shared_objects.h"
30 #include "test/pc/e2e/analyzer/video/default_video_quality_analyzer_shared_objects.h"
31 #include "test/pc/e2e/metric_metadata_keys.h"
32 
33 namespace webrtc {
34 namespace {
35 
36 using ::webrtc::webrtc_pc_e2e::SampleMetadataKey;
37 
38 constexpr TimeDelta kFreezeThreshold = TimeDelta::Millis(150);
39 constexpr int kMaxActiveComparisons = 10;
40 
StatsSample(double value,Timestamp sampling_time,std::map<std::string,std::string> metadata)41 SamplesStatsCounter::StatsSample StatsSample(
42     double value,
43     Timestamp sampling_time,
44     std::map<std::string, std::string> metadata) {
45   return SamplesStatsCounter::StatsSample{value, sampling_time,
46                                           std::move(metadata)};
47 }
48 
StatsSample(TimeDelta duration,Timestamp sampling_time,std::map<std::string,std::string> metadata)49 SamplesStatsCounter::StatsSample StatsSample(
50     TimeDelta duration,
51     Timestamp sampling_time,
52     std::map<std::string, std::string> metadata) {
53   return SamplesStatsCounter::StatsSample{duration.ms<double>(), sampling_time,
54                                           std::move(metadata)};
55 }
56 
ValidateFrameComparison(FrameComparison comparison)57 FrameComparison ValidateFrameComparison(FrameComparison comparison) {
58   RTC_DCHECK(comparison.frame_stats.captured_time.IsFinite())
59       << "Any comparison has to have finite captured_time";
60   switch (comparison.type) {
61     case FrameComparisonType::kRegular:
62       // Regular comparison has to have all FrameStats filled in.
63       RTC_DCHECK(comparison.captured.has_value() ||
64                  comparison.overload_reason != OverloadReason::kNone)
65           << "Regular comparison has to have captured frame if it's not "
66           << "overloaded comparison";
67       RTC_DCHECK(comparison.rendered.has_value() ||
68                  comparison.overload_reason != OverloadReason::kNone)
69           << "rendered frame has to be presented if it's not overloaded "
70           << "comparison";
71       RTC_DCHECK(comparison.frame_stats.pre_encode_time.IsFinite())
72           << "Regular comparison has to have finite pre_encode_time";
73       RTC_DCHECK(comparison.frame_stats.encoded_time.IsFinite())
74           << "Regular comparison has to have finite encoded_time";
75       RTC_DCHECK(comparison.frame_stats.received_time.IsFinite())
76           << "Regular comparison has to have finite received_time";
77       RTC_DCHECK(comparison.frame_stats.decode_start_time.IsFinite())
78           << "Regular comparison has to have finite decode_start_time";
79       RTC_DCHECK(comparison.frame_stats.decode_end_time.IsFinite())
80           << "Regular comparison has to have finite decode_end_time";
81       RTC_DCHECK(comparison.frame_stats.rendered_time.IsFinite())
82           << "Regular comparison has to have finite rendered_time";
83       RTC_DCHECK(comparison.frame_stats.decoded_frame_width.has_value())
84           << "Regular comparison has to have decoded_frame_width";
85       RTC_DCHECK(comparison.frame_stats.decoded_frame_height.has_value())
86           << "Regular comparison has to have decoded_frame_height";
87       RTC_DCHECK(comparison.frame_stats.used_encoder.has_value())
88           << "Regular comparison has to have used_encoder";
89       RTC_DCHECK(comparison.frame_stats.used_decoder.has_value())
90           << "Regular comparison has to have used_decoder";
91       RTC_DCHECK(!comparison.frame_stats.decoder_failed)
92           << "Regular comparison can't have decoder failure";
93       break;
94     case FrameComparisonType::kDroppedFrame:
95       // Frame can be dropped before encoder, by encoder, inside network or
96       // after decoder.
97       RTC_DCHECK(!comparison.captured.has_value())
98           << "Dropped frame comparison can't have captured frame";
99       RTC_DCHECK(!comparison.rendered.has_value())
100           << "Dropped frame comparison can't have rendered frame";
101 
102       if (comparison.frame_stats.encoded_time.IsFinite()) {
103         RTC_DCHECK(comparison.frame_stats.used_encoder.has_value())
104             << "Dropped frame comparison has to have used_encoder when "
105             << "encoded_time is set";
106         RTC_DCHECK(comparison.frame_stats.pre_encode_time.IsFinite())
107             << "Dropped frame comparison has to have finite pre_encode_time "
108             << "when encoded_time is finite.";
109       }
110 
111       if (comparison.frame_stats.decode_end_time.IsFinite() ||
112           comparison.frame_stats.decoder_failed) {
113         RTC_DCHECK(comparison.frame_stats.received_time.IsFinite())
114             << "Dropped frame comparison has to have received_time when "
115             << "decode_end_time is set or decoder_failed is true";
116         RTC_DCHECK(comparison.frame_stats.decode_start_time.IsFinite())
117             << "Dropped frame comparison has to have decode_start_time when "
118             << "decode_end_time is set or decoder_failed is true";
119         RTC_DCHECK(comparison.frame_stats.used_decoder.has_value())
120             << "Dropped frame comparison has to have used_decoder when "
121             << "decode_end_time is set or decoder_failed is true";
122       } else if (comparison.frame_stats.decode_end_time.IsFinite()) {
123         RTC_DCHECK(comparison.frame_stats.decoded_frame_width.has_value())
124             << "Dropped frame comparison has to have decoded_frame_width when "
125             << "decode_end_time is set";
126         RTC_DCHECK(comparison.frame_stats.decoded_frame_height.has_value())
127             << "Dropped frame comparison has to have decoded_frame_height when "
128             << "decode_end_time is set";
129       }
130       RTC_DCHECK(!comparison.frame_stats.rendered_time.IsFinite())
131           << "Dropped frame comparison can't have rendered_time";
132       break;
133     case FrameComparisonType::kFrameInFlight:
134       // Frame in flight comparison may miss almost any FrameStats, but if
135       // stats for stage X are set, then stats for stage X - 1 also has to be
136       // set. Also these frames were never rendered.
137       RTC_DCHECK(!comparison.captured.has_value())
138           << "Frame in flight comparison can't have captured frame";
139       RTC_DCHECK(!comparison.rendered.has_value())
140           << "Frame in flight comparison can't have rendered frame";
141       RTC_DCHECK(!comparison.frame_stats.rendered_time.IsFinite())
142           << "Frame in flight comparison can't have rendered_time";
143 
144       if (comparison.frame_stats.decode_end_time.IsFinite() ||
145           comparison.frame_stats.decoder_failed) {
146         RTC_DCHECK(comparison.frame_stats.used_decoder.has_value())
147             << "Frame in flight comparison has to have used_decoder when "
148             << "decode_end_time is set or decoder_failed is true.";
149         RTC_DCHECK(comparison.frame_stats.decode_start_time.IsFinite())
150             << "Frame in flight comparison has to have finite "
151             << "decode_start_time when decode_end_time is finite or "
152             << "decoder_failed is true.";
153       }
154       if (comparison.frame_stats.decode_end_time.IsFinite()) {
155         RTC_DCHECK(comparison.frame_stats.decoded_frame_width.has_value())
156             << "Frame in flight comparison has to have decoded_frame_width "
157             << "when decode_end_time is set.";
158         RTC_DCHECK(comparison.frame_stats.decoded_frame_height.has_value())
159             << "Frame in flight comparison has to have decoded_frame_height "
160             << "when decode_end_time is set.";
161       }
162       if (comparison.frame_stats.decode_start_time.IsFinite()) {
163         RTC_DCHECK(comparison.frame_stats.received_time.IsFinite())
164             << "Frame in flight comparison has to have finite received_time "
165             << "when decode_start_time is finite.";
166       }
167       if (comparison.frame_stats.received_time.IsFinite()) {
168         RTC_DCHECK(comparison.frame_stats.encoded_time.IsFinite())
169             << "Frame in flight comparison has to have finite encoded_time "
170             << "when received_time is finite.";
171       }
172       if (comparison.frame_stats.encoded_time.IsFinite()) {
173         RTC_DCHECK(comparison.frame_stats.used_encoder.has_value())
174             << "Frame in flight comparison has to have used_encoder when "
175             << "encoded_time is set";
176         RTC_DCHECK(comparison.frame_stats.pre_encode_time.IsFinite())
177             << "Frame in flight comparison has to have finite pre_encode_time "
178             << "when encoded_time is finite.";
179       }
180       break;
181   }
182   return comparison;
183 }
184 
185 }  // namespace
186 
Start(int max_threads_count)187 void DefaultVideoQualityAnalyzerFramesComparator::Start(int max_threads_count) {
188   for (int i = 0; i < max_threads_count; i++) {
189     thread_pool_.push_back(rtc::PlatformThread::SpawnJoinable(
190         [this] { ProcessComparisons(); },
191         "DefaultVideoQualityAnalyzerFramesComparator-" + std::to_string(i)));
192   }
193   {
194     MutexLock lock(&mutex_);
195     RTC_CHECK_EQ(state_, State::kNew) << "Frames comparator is already started";
196     state_ = State::kActive;
197   }
198   cpu_measurer_.StartMeasuringCpuProcessTime();
199 }
200 
Stop(const std::map<InternalStatsKey,Timestamp> & last_rendered_frame_times)201 void DefaultVideoQualityAnalyzerFramesComparator::Stop(
202     const std::map<InternalStatsKey, Timestamp>& last_rendered_frame_times) {
203   {
204     MutexLock lock(&mutex_);
205     if (state_ == State::kStopped) {
206       return;
207     }
208     RTC_CHECK_EQ(state_, State::kActive)
209         << "Frames comparator has to be started before it will be used";
210     state_ = State::kStopped;
211   }
212   cpu_measurer_.StopMeasuringCpuProcessTime();
213   comparison_available_event_.Set();
214   thread_pool_.clear();
215 
216   {
217     MutexLock lock(&mutex_);
218     // Perform final Metrics update. On this place analyzer is stopped and no
219     // one holds any locks.
220 
221     // Time between freezes.
222     // Count time since the last freeze to the end of the call as time
223     // between freezes.
224     for (auto& entry : last_rendered_frame_times) {
225       const InternalStatsKey& stats_key = entry.first;
226       const Timestamp& last_rendered_frame_time = entry.second;
227 
228       // If there are no freezes in the call we have to report
229       // time_between_freezes_ms as call duration and in such case
230       // `last_rendered_frame_time` for this stream will be stream start time.
231       // If there is freeze, then we need add time from last rendered frame
232       // to last freeze end as time between freezes.
233       stream_stats_.at(stats_key).time_between_freezes_ms.AddSample(StatsSample(
234           last_rendered_frame_time - stream_last_freeze_end_time_.at(stats_key),
235           Now(), /*metadata=*/{}));
236     }
237   }
238 }
239 
EnsureStatsForStream(size_t stream_index,size_t sender_peer_index,size_t peers_count,Timestamp captured_time,Timestamp start_time)240 void DefaultVideoQualityAnalyzerFramesComparator::EnsureStatsForStream(
241     size_t stream_index,
242     size_t sender_peer_index,
243     size_t peers_count,
244     Timestamp captured_time,
245     Timestamp start_time) {
246   MutexLock lock(&mutex_);
247   RTC_CHECK_EQ(state_, State::kActive)
248       << "Frames comparator has to be started before it will be used";
249 
250   for (size_t i = 0; i < peers_count; ++i) {
251     if (i == sender_peer_index && !options_.enable_receive_own_stream) {
252       continue;
253     }
254     InternalStatsKey stats_key(stream_index, sender_peer_index, i);
255     if (stream_stats_.find(stats_key) == stream_stats_.end()) {
256       stream_stats_.insert({stats_key, StreamStats(captured_time)});
257       // Assume that the first freeze was before first stream frame captured.
258       // This way time before the first freeze would be counted as time
259       // between freezes.
260       stream_last_freeze_end_time_.insert({stats_key, start_time});
261     } else {
262       // When we see some `stream_label` for the first time we need to create
263       // stream stats object for it and set up some states, but we need to do
264       // it only once and for all receivers, so on the next frame on the same
265       // `stream_label` we can be sure, that it's already done and we needn't
266       // to scan though all peers again.
267       break;
268     }
269   }
270 }
271 
RegisterParticipantInCall(rtc::ArrayView<std::pair<InternalStatsKey,Timestamp>> stream_started_time,Timestamp start_time)272 void DefaultVideoQualityAnalyzerFramesComparator::RegisterParticipantInCall(
273     rtc::ArrayView<std::pair<InternalStatsKey, Timestamp>> stream_started_time,
274     Timestamp start_time) {
275   MutexLock lock(&mutex_);
276   RTC_CHECK_EQ(state_, State::kActive)
277       << "Frames comparator has to be started before it will be used";
278 
279   for (const std::pair<InternalStatsKey, Timestamp>& pair :
280        stream_started_time) {
281     stream_stats_.insert({pair.first, StreamStats(pair.second)});
282     stream_last_freeze_end_time_.insert({pair.first, start_time});
283   }
284 }
285 
AddComparison(InternalStatsKey stats_key,absl::optional<VideoFrame> captured,absl::optional<VideoFrame> rendered,FrameComparisonType type,FrameStats frame_stats)286 void DefaultVideoQualityAnalyzerFramesComparator::AddComparison(
287     InternalStatsKey stats_key,
288     absl::optional<VideoFrame> captured,
289     absl::optional<VideoFrame> rendered,
290     FrameComparisonType type,
291     FrameStats frame_stats) {
292   MutexLock lock(&mutex_);
293   RTC_CHECK_EQ(state_, State::kActive)
294       << "Frames comparator has to be started before it will be used";
295   AddComparisonInternal(std::move(stats_key), std::move(captured),
296                         std::move(rendered), type, std::move(frame_stats));
297 }
298 
AddComparison(InternalStatsKey stats_key,int skipped_between_rendered,absl::optional<VideoFrame> captured,absl::optional<VideoFrame> rendered,FrameComparisonType type,FrameStats frame_stats)299 void DefaultVideoQualityAnalyzerFramesComparator::AddComparison(
300     InternalStatsKey stats_key,
301     int skipped_between_rendered,
302     absl::optional<VideoFrame> captured,
303     absl::optional<VideoFrame> rendered,
304     FrameComparisonType type,
305     FrameStats frame_stats) {
306   MutexLock lock(&mutex_);
307   RTC_CHECK_EQ(state_, State::kActive)
308       << "Frames comparator has to be started before it will be used";
309   stream_stats_.at(stats_key).skipped_between_rendered.AddSample(
310       StatsSample(skipped_between_rendered, Now(),
311                   /*metadata=*/
312                   {{SampleMetadataKey::kFrameIdMetadataKey,
313                     std::to_string(frame_stats.frame_id)}}));
314   AddComparisonInternal(std::move(stats_key), std::move(captured),
315                         std::move(rendered), type, std::move(frame_stats));
316 }
317 
AddComparisonInternal(InternalStatsKey stats_key,absl::optional<VideoFrame> captured,absl::optional<VideoFrame> rendered,FrameComparisonType type,FrameStats frame_stats)318 void DefaultVideoQualityAnalyzerFramesComparator::AddComparisonInternal(
319     InternalStatsKey stats_key,
320     absl::optional<VideoFrame> captured,
321     absl::optional<VideoFrame> rendered,
322     FrameComparisonType type,
323     FrameStats frame_stats) {
324   cpu_measurer_.StartExcludingCpuThreadTime();
325   frames_comparator_stats_.comparisons_queue_size.AddSample(
326       StatsSample(comparisons_.size(), Now(), /*metadata=*/{}));
327   // If there too many computations waiting in the queue, we won't provide
328   // frames itself to make future computations lighter.
329   if (comparisons_.size() >= kMaxActiveComparisons) {
330     comparisons_.emplace_back(ValidateFrameComparison(
331         FrameComparison(std::move(stats_key), /*captured=*/absl::nullopt,
332                         /*rendered=*/absl::nullopt, type,
333                         std::move(frame_stats), OverloadReason::kCpu)));
334   } else {
335     OverloadReason overload_reason = OverloadReason::kNone;
336     if (!captured && type == FrameComparisonType::kRegular) {
337       overload_reason = OverloadReason::kMemory;
338     }
339     comparisons_.emplace_back(ValidateFrameComparison(FrameComparison(
340         std::move(stats_key), std::move(captured), std::move(rendered), type,
341         std::move(frame_stats), overload_reason)));
342   }
343   comparison_available_event_.Set();
344   cpu_measurer_.StopExcludingCpuThreadTime();
345 }
346 
ProcessComparisons()347 void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparisons() {
348   while (true) {
349     // Try to pick next comparison to perform from the queue.
350     absl::optional<FrameComparison> comparison = absl::nullopt;
351     bool more_new_comparisons_expected;
352     {
353       MutexLock lock(&mutex_);
354       if (!comparisons_.empty()) {
355         comparison = comparisons_.front();
356         comparisons_.pop_front();
357         if (!comparisons_.empty()) {
358           comparison_available_event_.Set();
359         }
360       }
361       // If state is stopped => no new frame comparisons are expected.
362       more_new_comparisons_expected = state_ != State::kStopped;
363     }
364     if (!comparison) {
365       if (!more_new_comparisons_expected) {
366         comparison_available_event_.Set();
367         return;
368       }
369       comparison_available_event_.Wait(TimeDelta::Seconds(1));
370       continue;
371     }
372 
373     cpu_measurer_.StartExcludingCpuThreadTime();
374     ProcessComparison(comparison.value());
375     cpu_measurer_.StopExcludingCpuThreadTime();
376   }
377 }
378 
ProcessComparison(const FrameComparison & comparison)379 void DefaultVideoQualityAnalyzerFramesComparator::ProcessComparison(
380     const FrameComparison& comparison) {
381   // Comparison is checked to be valid before adding, so we can use this
382   // assumptions during computations.
383 
384   // Perform expensive psnr and ssim calculations while not holding lock.
385   double psnr = -1.0;
386   double ssim = -1.0;
387   if ((options_.compute_psnr || options_.compute_ssim) &&
388       comparison.captured.has_value() && comparison.rendered.has_value()) {
389     rtc::scoped_refptr<I420BufferInterface> reference_buffer =
390         comparison.captured->video_frame_buffer()->ToI420();
391     rtc::scoped_refptr<I420BufferInterface> test_buffer =
392         comparison.rendered->video_frame_buffer()->ToI420();
393     if (options_.adjust_cropping_before_comparing_frames) {
394       test_buffer = ScaleVideoFrameBuffer(
395           *test_buffer, reference_buffer->width(), reference_buffer->height());
396       reference_buffer = test::AdjustCropping(reference_buffer, test_buffer);
397     }
398     if (options_.compute_psnr) {
399       psnr = options_.use_weighted_psnr
400                  ? I420WeightedPSNR(*reference_buffer, *test_buffer)
401                  : I420PSNR(*reference_buffer, *test_buffer);
402     }
403     if (options_.compute_ssim) {
404       ssim = I420SSIM(*reference_buffer, *test_buffer);
405     }
406   }
407 
408   const FrameStats& frame_stats = comparison.frame_stats;
409 
410   MutexLock lock(&mutex_);
411   auto stats_it = stream_stats_.find(comparison.stats_key);
412   RTC_CHECK(stats_it != stream_stats_.end()) << comparison.stats_key.ToString();
413   StreamStats* stats = &stats_it->second;
414 
415   frames_comparator_stats_.comparisons_done++;
416   if (comparison.overload_reason == OverloadReason::kCpu) {
417     frames_comparator_stats_.cpu_overloaded_comparisons_done++;
418   } else if (comparison.overload_reason == OverloadReason::kMemory) {
419     frames_comparator_stats_.memory_overloaded_comparisons_done++;
420   }
421 
422   std::map<std::string, std::string> metadata;
423   metadata.emplace(SampleMetadataKey::kFrameIdMetadataKey,
424                    std::to_string(frame_stats.frame_id));
425 
426   if (psnr > 0) {
427     stats->psnr.AddSample(
428         StatsSample(psnr, frame_stats.rendered_time, metadata));
429   }
430   if (ssim > 0) {
431     stats->ssim.AddSample(
432         StatsSample(ssim, frame_stats.received_time, metadata));
433   }
434   stats->capture_frame_rate.AddEvent(frame_stats.captured_time);
435 
436   // Compute dropped phase for dropped frame
437   if (comparison.type == FrameComparisonType::kDroppedFrame) {
438     FrameDropPhase dropped_phase;
439     if (frame_stats.decode_end_time.IsFinite()) {
440       dropped_phase = FrameDropPhase::kAfterDecoder;
441     } else if (frame_stats.decode_start_time.IsFinite()) {
442       dropped_phase = FrameDropPhase::kByDecoder;
443     } else if (frame_stats.encoded_time.IsFinite()) {
444       dropped_phase = FrameDropPhase::kTransport;
445     } else if (frame_stats.pre_encode_time.IsFinite()) {
446       dropped_phase = FrameDropPhase::kByEncoder;
447     } else {
448       dropped_phase = FrameDropPhase::kBeforeEncoder;
449     }
450     stats->dropped_by_phase[dropped_phase]++;
451   }
452 
453   if (frame_stats.encoded_time.IsFinite()) {
454     stats->encode_time_ms.AddSample(
455         StatsSample(frame_stats.encoded_time - frame_stats.pre_encode_time,
456                     frame_stats.encoded_time, metadata));
457     stats->encode_frame_rate.AddEvent(frame_stats.encoded_time);
458     stats->total_encoded_images_payload +=
459         frame_stats.encoded_image_size.bytes();
460     stats->target_encode_bitrate.AddSample(StatsSample(
461         frame_stats.target_encode_bitrate, frame_stats.encoded_time, metadata));
462     for (SamplesStatsCounter::StatsSample qp :
463          frame_stats.qp_values.GetTimedSamples()) {
464       qp.metadata = metadata;
465       stats->qp.AddSample(std::move(qp));
466     }
467 
468     // Stats sliced on encoded frame type.
469     if (frame_stats.encoded_frame_type == VideoFrameType::kVideoFrameKey) {
470       ++stats->num_send_key_frames;
471     }
472   }
473   // Next stats can be calculated only if frame was received on remote side.
474   if (comparison.type != FrameComparisonType::kDroppedFrame ||
475       comparison.frame_stats.decoder_failed) {
476     if (frame_stats.rendered_time.IsFinite()) {
477       stats->total_delay_incl_transport_ms.AddSample(
478           StatsSample(frame_stats.rendered_time - frame_stats.captured_time,
479                       frame_stats.received_time, metadata));
480       stats->receive_to_render_time_ms.AddSample(
481           StatsSample(frame_stats.rendered_time - frame_stats.received_time,
482                       frame_stats.rendered_time, metadata));
483     }
484     if (frame_stats.decode_start_time.IsFinite()) {
485       stats->transport_time_ms.AddSample(
486           StatsSample(frame_stats.decode_start_time - frame_stats.encoded_time,
487                       frame_stats.decode_start_time, metadata));
488 
489       // Stats sliced on decoded frame type.
490       if (frame_stats.pre_decoded_frame_type ==
491           VideoFrameType::kVideoFrameKey) {
492         ++stats->num_recv_key_frames;
493         stats->recv_key_frame_size_bytes.AddSample(
494             StatsSample(frame_stats.pre_decoded_image_size.bytes(),
495                         frame_stats.decode_start_time, metadata));
496       } else if (frame_stats.pre_decoded_frame_type ==
497                  VideoFrameType::kVideoFrameDelta) {
498         stats->recv_delta_frame_size_bytes.AddSample(
499             StatsSample(frame_stats.pre_decoded_image_size.bytes(),
500                         frame_stats.decode_start_time, metadata));
501       }
502     }
503     if (frame_stats.decode_end_time.IsFinite()) {
504       stats->decode_time_ms.AddSample(StatsSample(
505           frame_stats.decode_end_time - frame_stats.decode_start_time,
506           frame_stats.decode_end_time, metadata));
507       stats->resolution_of_decoded_frame.AddSample(
508           StatsSample(*comparison.frame_stats.decoded_frame_width *
509                           *comparison.frame_stats.decoded_frame_height,
510                       frame_stats.decode_end_time, metadata));
511     }
512 
513     if (frame_stats.prev_frame_rendered_time.IsFinite() &&
514         frame_stats.rendered_time.IsFinite()) {
515       TimeDelta time_between_rendered_frames =
516           frame_stats.rendered_time - frame_stats.prev_frame_rendered_time;
517       stats->time_between_rendered_frames_ms.AddSample(StatsSample(
518           time_between_rendered_frames, frame_stats.rendered_time, metadata));
519       TimeDelta average_time_between_rendered_frames = TimeDelta::Millis(
520           stats->time_between_rendered_frames_ms.GetAverage());
521       if (time_between_rendered_frames >
522           std::max(kFreezeThreshold + average_time_between_rendered_frames,
523                    3 * average_time_between_rendered_frames)) {
524         stats->freeze_time_ms.AddSample(StatsSample(
525             time_between_rendered_frames, frame_stats.rendered_time, metadata));
526         auto freeze_end_it =
527             stream_last_freeze_end_time_.find(comparison.stats_key);
528         RTC_DCHECK(freeze_end_it != stream_last_freeze_end_time_.end());
529         stats->time_between_freezes_ms.AddSample(StatsSample(
530             frame_stats.prev_frame_rendered_time - freeze_end_it->second,
531             frame_stats.rendered_time, metadata));
532         freeze_end_it->second = frame_stats.rendered_time;
533       }
534     }
535   }
536   // Compute stream codec info.
537   if (frame_stats.used_encoder.has_value()) {
538     if (stats->encoders.empty() || stats->encoders.back().codec_name !=
539                                        frame_stats.used_encoder->codec_name) {
540       stats->encoders.push_back(*frame_stats.used_encoder);
541     }
542     stats->encoders.back().last_frame_id =
543         frame_stats.used_encoder->last_frame_id;
544     stats->encoders.back().switched_from_at =
545         frame_stats.used_encoder->switched_from_at;
546   }
547 
548   if (frame_stats.used_decoder.has_value()) {
549     if (stats->decoders.empty() || stats->decoders.back().codec_name !=
550                                        frame_stats.used_decoder->codec_name) {
551       stats->decoders.push_back(*frame_stats.used_decoder);
552     }
553     stats->decoders.back().last_frame_id =
554         frame_stats.used_decoder->last_frame_id;
555     stats->decoders.back().switched_from_at =
556         frame_stats.used_decoder->switched_from_at;
557   }
558 }
559 
Now()560 Timestamp DefaultVideoQualityAnalyzerFramesComparator::Now() {
561   return clock_->CurrentTime();
562 }
563 
564 }  // namespace webrtc
565