1 /*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "media/base/video_broadcaster.h"
12
13 #include <algorithm>
14 #include <vector>
15
16 #include "absl/types/optional.h"
17 #include "api/video/i420_buffer.h"
18 #include "api/video/video_rotation.h"
19 #include "media/base/video_common.h"
20 #include "rtc_base/checks.h"
21 #include "rtc_base/logging.h"
22
23 namespace rtc {
24
25 VideoBroadcaster::VideoBroadcaster() = default;
26 VideoBroadcaster::~VideoBroadcaster() = default;
27
AddOrUpdateSink(VideoSinkInterface<webrtc::VideoFrame> * sink,const VideoSinkWants & wants)28 void VideoBroadcaster::AddOrUpdateSink(
29 VideoSinkInterface<webrtc::VideoFrame>* sink,
30 const VideoSinkWants& wants) {
31 RTC_DCHECK(sink != nullptr);
32 webrtc::MutexLock lock(&sinks_and_wants_lock_);
33 if (!FindSinkPair(sink)) {
34 // `Sink` is a new sink, which didn't receive previous frame.
35 previous_frame_sent_to_all_sinks_ = false;
36
37 if (last_constraints_.has_value()) {
38 RTC_LOG(LS_INFO) << __func__ << " forwarding stored constraints min_fps "
39 << last_constraints_->min_fps.value_or(-1) << " max_fps "
40 << last_constraints_->max_fps.value_or(-1);
41 sink->OnConstraintsChanged(*last_constraints_);
42 }
43 }
44 VideoSourceBase::AddOrUpdateSink(sink, wants);
45 UpdateWants();
46 }
47
RemoveSink(VideoSinkInterface<webrtc::VideoFrame> * sink)48 void VideoBroadcaster::RemoveSink(
49 VideoSinkInterface<webrtc::VideoFrame>* sink) {
50 RTC_DCHECK(sink != nullptr);
51 webrtc::MutexLock lock(&sinks_and_wants_lock_);
52 VideoSourceBase::RemoveSink(sink);
53 UpdateWants();
54 }
55
frame_wanted() const56 bool VideoBroadcaster::frame_wanted() const {
57 webrtc::MutexLock lock(&sinks_and_wants_lock_);
58 return !sink_pairs().empty();
59 }
60
wants() const61 VideoSinkWants VideoBroadcaster::wants() const {
62 webrtc::MutexLock lock(&sinks_and_wants_lock_);
63 return current_wants_;
64 }
65
OnFrame(const webrtc::VideoFrame & frame)66 void VideoBroadcaster::OnFrame(const webrtc::VideoFrame& frame) {
67 webrtc::MutexLock lock(&sinks_and_wants_lock_);
68 bool current_frame_was_discarded = false;
69 for (auto& sink_pair : sink_pairs()) {
70 if (sink_pair.wants.rotation_applied &&
71 frame.rotation() != webrtc::kVideoRotation_0) {
72 // Calls to OnFrame are not synchronized with changes to the sink wants.
73 // When rotation_applied is set to true, one or a few frames may get here
74 // with rotation still pending. Protect sinks that don't expect any
75 // pending rotation.
76 RTC_LOG(LS_VERBOSE) << "Discarding frame with unexpected rotation.";
77 sink_pair.sink->OnDiscardedFrame();
78 current_frame_was_discarded = true;
79 continue;
80 }
81 if (sink_pair.wants.black_frames) {
82 webrtc::VideoFrame black_frame =
83 webrtc::VideoFrame::Builder()
84 .set_video_frame_buffer(
85 GetBlackFrameBuffer(frame.width(), frame.height()))
86 .set_rotation(frame.rotation())
87 .set_timestamp_us(frame.timestamp_us())
88 .set_id(frame.id())
89 .build();
90 sink_pair.sink->OnFrame(black_frame);
91 } else if (!previous_frame_sent_to_all_sinks_ && frame.has_update_rect()) {
92 // Since last frame was not sent to some sinks, no reliable update
93 // information is available, so we need to clear the update rect.
94 webrtc::VideoFrame copy = frame;
95 copy.clear_update_rect();
96 sink_pair.sink->OnFrame(copy);
97 } else {
98 sink_pair.sink->OnFrame(frame);
99 }
100 }
101 previous_frame_sent_to_all_sinks_ = !current_frame_was_discarded;
102 }
103
OnDiscardedFrame()104 void VideoBroadcaster::OnDiscardedFrame() {
105 webrtc::MutexLock lock(&sinks_and_wants_lock_);
106 for (auto& sink_pair : sink_pairs()) {
107 sink_pair.sink->OnDiscardedFrame();
108 }
109 }
110
ProcessConstraints(const webrtc::VideoTrackSourceConstraints & constraints)111 void VideoBroadcaster::ProcessConstraints(
112 const webrtc::VideoTrackSourceConstraints& constraints) {
113 webrtc::MutexLock lock(&sinks_and_wants_lock_);
114 RTC_LOG(LS_INFO) << __func__ << " min_fps "
115 << constraints.min_fps.value_or(-1) << " max_fps "
116 << constraints.max_fps.value_or(-1) << " broadcasting to "
117 << sink_pairs().size() << " sinks.";
118 last_constraints_ = constraints;
119 for (auto& sink_pair : sink_pairs())
120 sink_pair.sink->OnConstraintsChanged(constraints);
121 }
122
UpdateWants()123 void VideoBroadcaster::UpdateWants() {
124 VideoSinkWants wants;
125 wants.rotation_applied = false;
126 wants.resolution_alignment = 1;
127 wants.aggregates.emplace(VideoSinkWants::Aggregates());
128 wants.is_active = false;
129
130 // TODO(webrtc:14451) : I think it makes sense to always
131 // "ignore" encoders that are not active. But that would
132 // probably require a controlled roll out with a field trials?
133 // To play it safe, only ignore inactive encoders is there is an
134 // active encoder using the new api (requested_resolution),
135 // this means that there is only a behavioural change when using new
136 // api.
137 bool ignore_inactive_encoders_old_api = false;
138 for (auto& sink : sink_pairs()) {
139 if (sink.wants.is_active && sink.wants.requested_resolution.has_value()) {
140 ignore_inactive_encoders_old_api = true;
141 break;
142 }
143 }
144
145 for (auto& sink : sink_pairs()) {
146 if (!sink.wants.is_active &&
147 (sink.wants.requested_resolution || ignore_inactive_encoders_old_api)) {
148 continue;
149 }
150 // wants.rotation_applied == ANY(sink.wants.rotation_applied)
151 if (sink.wants.rotation_applied) {
152 wants.rotation_applied = true;
153 }
154 // wants.max_pixel_count == MIN(sink.wants.max_pixel_count)
155 if (sink.wants.max_pixel_count < wants.max_pixel_count) {
156 wants.max_pixel_count = sink.wants.max_pixel_count;
157 }
158 // Select the minimum requested target_pixel_count, if any, of all sinks so
159 // that we don't over utilize the resources for any one.
160 // TODO(sprang): Consider using the median instead, since the limit can be
161 // expressed by max_pixel_count.
162 if (sink.wants.target_pixel_count &&
163 (!wants.target_pixel_count ||
164 (*sink.wants.target_pixel_count < *wants.target_pixel_count))) {
165 wants.target_pixel_count = sink.wants.target_pixel_count;
166 }
167 // Select the minimum for the requested max framerates.
168 if (sink.wants.max_framerate_fps < wants.max_framerate_fps) {
169 wants.max_framerate_fps = sink.wants.max_framerate_fps;
170 }
171 wants.resolution_alignment = cricket::LeastCommonMultiple(
172 wants.resolution_alignment, sink.wants.resolution_alignment);
173
174 // Pick MAX(requested_resolution) since the actual can be downscaled
175 // in encoder instead.
176 if (sink.wants.requested_resolution) {
177 if (!wants.requested_resolution) {
178 wants.requested_resolution = sink.wants.requested_resolution;
179 } else {
180 wants.requested_resolution->width =
181 std::max(wants.requested_resolution->width,
182 sink.wants.requested_resolution->width);
183 wants.requested_resolution->height =
184 std::max(wants.requested_resolution->height,
185 sink.wants.requested_resolution->height);
186 }
187 } else if (sink.wants.is_active) {
188 wants.aggregates->any_active_without_requested_resolution = true;
189 }
190
191 wants.is_active |= sink.wants.is_active;
192 }
193
194 if (wants.target_pixel_count &&
195 *wants.target_pixel_count >= wants.max_pixel_count) {
196 wants.target_pixel_count.emplace(wants.max_pixel_count);
197 }
198 current_wants_ = wants;
199 }
200
201 const rtc::scoped_refptr<webrtc::VideoFrameBuffer>&
GetBlackFrameBuffer(int width,int height)202 VideoBroadcaster::GetBlackFrameBuffer(int width, int height) {
203 if (!black_frame_buffer_ || black_frame_buffer_->width() != width ||
204 black_frame_buffer_->height() != height) {
205 rtc::scoped_refptr<webrtc::I420Buffer> buffer =
206 webrtc::I420Buffer::Create(width, height);
207 webrtc::I420Buffer::SetBlack(buffer.get());
208 black_frame_buffer_ = buffer;
209 }
210
211 return black_frame_buffer_;
212 }
213
214 } // namespace rtc
215