1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/desktop_capture/desktop_and_cursor_composer.h"
12
13 #include <stdint.h>
14 #include <string.h>
15
16 #include <memory>
17 #include <utility>
18 #include <vector>
19
20 #include "modules/desktop_capture/desktop_capturer.h"
21 #include "modules/desktop_capture/desktop_frame.h"
22 #include "modules/desktop_capture/mouse_cursor.h"
23 #include "modules/desktop_capture/shared_desktop_frame.h"
24 #include "rtc_base/arraysize.h"
25 #include "test/gmock.h"
26 #include "test/gtest.h"
27
28 namespace webrtc {
29
30 namespace {
31
32 using testing::ElementsAre;
33
34 const int kFrameXCoord = 100;
35 const int kFrameYCoord = 200;
36 const int kScreenWidth = 100;
37 const int kScreenHeight = 100;
38 const int kCursorWidth = 10;
39 const int kCursorHeight = 10;
40
41 const int kTestCursorSize = 3;
42 const uint32_t kTestCursorData[kTestCursorSize][kTestCursorSize] = {
43 {
44 0xffffffff,
45 0x99990000,
46 0xaa222222,
47 },
48 {
49 0x88008800,
50 0xaa0000aa,
51 0xaa333333,
52 },
53 {
54 0x00000000,
55 0xaa0000aa,
56 0xaa333333,
57 },
58 };
59
GetFakeFramePixelValue(const DesktopVector & p)60 uint32_t GetFakeFramePixelValue(const DesktopVector& p) {
61 uint32_t r = 100 + p.x();
62 uint32_t g = 100 + p.y();
63 uint32_t b = 100 + p.x() + p.y();
64 return b + (g << 8) + (r << 16) + 0xff000000;
65 }
66
GetFramePixel(const DesktopFrame & frame,const DesktopVector & pos)67 uint32_t GetFramePixel(const DesktopFrame& frame, const DesktopVector& pos) {
68 return *reinterpret_cast<uint32_t*>(frame.GetFrameDataAtPos(pos));
69 }
70
71 // Blends two pixel values taking into account alpha.
BlendPixels(uint32_t dest,uint32_t src)72 uint32_t BlendPixels(uint32_t dest, uint32_t src) {
73 uint8_t alpha = 255 - ((src & 0xff000000) >> 24);
74 uint32_t r =
75 ((dest & 0x00ff0000) >> 16) * alpha / 255 + ((src & 0x00ff0000) >> 16);
76 uint32_t g =
77 ((dest & 0x0000ff00) >> 8) * alpha / 255 + ((src & 0x0000ff00) >> 8);
78 uint32_t b = (dest & 0x000000ff) * alpha / 255 + (src & 0x000000ff);
79 return b + (g << 8) + (r << 16) + 0xff000000;
80 }
81
CreateTestFrame(int width=kScreenWidth,int height=kScreenHeight)82 DesktopFrame* CreateTestFrame(int width = kScreenWidth,
83 int height = kScreenHeight) {
84 DesktopFrame* frame = new BasicDesktopFrame(DesktopSize(width, height));
85 uint32_t* data = reinterpret_cast<uint32_t*>(frame->data());
86 for (int y = 0; y < height; ++y) {
87 for (int x = 0; x < width; ++x) {
88 *(data++) = GetFakeFramePixelValue(DesktopVector(x, y));
89 }
90 }
91 return frame;
92 }
93
CreateTestCursor(DesktopVector hotspot)94 MouseCursor* CreateTestCursor(DesktopVector hotspot) {
95 std::unique_ptr<DesktopFrame> image(
96 new BasicDesktopFrame(DesktopSize(kCursorWidth, kCursorHeight)));
97 uint32_t* data = reinterpret_cast<uint32_t*>(image->data());
98 // Set four pixels near the hotspot and leave all other blank.
99 for (int y = 0; y < kTestCursorSize; ++y) {
100 for (int x = 0; x < kTestCursorSize; ++x) {
101 data[(hotspot.y() + y) * kCursorWidth + (hotspot.x() + x)] =
102 kTestCursorData[y][x];
103 }
104 }
105 return new MouseCursor(image.release(), hotspot);
106 }
107
108 class FakeScreenCapturer : public DesktopCapturer {
109 public:
FakeScreenCapturer()110 FakeScreenCapturer() {}
111
Start(Callback * callback)112 void Start(Callback* callback) override { callback_ = callback; }
113
CaptureFrame()114 void CaptureFrame() override {
115 callback_->OnCaptureResult(
116 next_frame_ ? Result::SUCCESS : Result::ERROR_TEMPORARY,
117 std::move(next_frame_));
118 }
119
SetNextFrame(std::unique_ptr<DesktopFrame> next_frame)120 void SetNextFrame(std::unique_ptr<DesktopFrame> next_frame) {
121 next_frame_ = std::move(next_frame);
122 }
123
IsOccluded(const DesktopVector & pos)124 bool IsOccluded(const DesktopVector& pos) override { return is_occluded_; }
125
set_is_occluded(bool value)126 void set_is_occluded(bool value) { is_occluded_ = value; }
127
128 private:
129 Callback* callback_ = nullptr;
130
131 std::unique_ptr<DesktopFrame> next_frame_;
132 bool is_occluded_ = false;
133 };
134
135 class FakeMouseMonitor : public MouseCursorMonitor {
136 public:
FakeMouseMonitor()137 FakeMouseMonitor() : changed_(true) {}
138
SetState(CursorState state,const DesktopVector & pos)139 void SetState(CursorState state, const DesktopVector& pos) {
140 state_ = state;
141 position_ = pos;
142 }
143
SetHotspot(const DesktopVector & hotspot)144 void SetHotspot(const DesktopVector& hotspot) {
145 if (!hotspot_.equals(hotspot))
146 changed_ = true;
147 hotspot_ = hotspot;
148 }
149
Init(Callback * callback,Mode mode)150 void Init(Callback* callback, Mode mode) override { callback_ = callback; }
151
Capture()152 void Capture() override {
153 if (changed_) {
154 callback_->OnMouseCursor(CreateTestCursor(hotspot_));
155 }
156 callback_->OnMouseCursorPosition(position_);
157 }
158
159 private:
160 Callback* callback_;
161 CursorState state_;
162 DesktopVector position_;
163 DesktopVector hotspot_;
164 bool changed_;
165 };
166
VerifyFrame(const DesktopFrame & frame,MouseCursorMonitor::CursorState state,const DesktopVector & pos)167 void VerifyFrame(const DesktopFrame& frame,
168 MouseCursorMonitor::CursorState state,
169 const DesktopVector& pos) {
170 // Verify that all other pixels are set to their original values.
171 DesktopRect image_rect =
172 DesktopRect::MakeWH(kTestCursorSize, kTestCursorSize);
173 image_rect.Translate(pos);
174
175 for (int y = 0; y < kScreenHeight; ++y) {
176 for (int x = 0; x < kScreenWidth; ++x) {
177 DesktopVector p(x, y);
178 if (state == MouseCursorMonitor::INSIDE && image_rect.Contains(p)) {
179 EXPECT_EQ(BlendPixels(GetFakeFramePixelValue(p),
180 kTestCursorData[y - pos.y()][x - pos.x()]),
181 GetFramePixel(frame, p));
182 } else {
183 EXPECT_EQ(GetFakeFramePixelValue(p), GetFramePixel(frame, p));
184 }
185 }
186 }
187 }
188
189 } // namespace
190
operator ==(const DesktopRect & left,const DesktopRect & right)191 bool operator==(const DesktopRect& left, const DesktopRect& right) {
192 return left.equals(right);
193 }
194
operator <<(std::ostream & out,const DesktopRect & rect)195 std::ostream& operator<<(std::ostream& out, const DesktopRect& rect) {
196 out << "{" << rect.left() << "+" << rect.top() << "-" << rect.width() << "x"
197 << rect.height() << "}";
198 return out;
199 }
200
201 class DesktopAndCursorComposerTest : public ::testing::Test,
202 public DesktopCapturer::Callback {
203 public:
DesktopAndCursorComposerTest(bool include_cursor=true)204 explicit DesktopAndCursorComposerTest(bool include_cursor = true)
205 : fake_screen_(new FakeScreenCapturer()),
206 fake_cursor_(include_cursor ? new FakeMouseMonitor() : nullptr),
207 blender_(fake_screen_, fake_cursor_) {
208 blender_.Start(this);
209 }
210
211 // DesktopCapturer::Callback interface
OnCaptureResult(DesktopCapturer::Result result,std::unique_ptr<DesktopFrame> frame)212 void OnCaptureResult(DesktopCapturer::Result result,
213 std::unique_ptr<DesktopFrame> frame) override {
214 frame_ = std::move(frame);
215 }
216
217 protected:
218 // Owned by `blender_`.
219 FakeScreenCapturer* fake_screen_;
220 FakeMouseMonitor* fake_cursor_;
221
222 DesktopAndCursorComposer blender_;
223 std::unique_ptr<DesktopFrame> frame_;
224 };
225
226 class DesktopAndCursorComposerNoCursorMonitorTest
227 : public DesktopAndCursorComposerTest {
228 public:
DesktopAndCursorComposerNoCursorMonitorTest()229 DesktopAndCursorComposerNoCursorMonitorTest()
230 : DesktopAndCursorComposerTest(false) {}
231 };
232
TEST_F(DesktopAndCursorComposerTest,CursorShouldBeIgnoredIfNoFrameCaptured)233 TEST_F(DesktopAndCursorComposerTest, CursorShouldBeIgnoredIfNoFrameCaptured) {
234 struct {
235 int x, y;
236 int hotspot_x, hotspot_y;
237 bool inside;
238 } tests[] = {
239 {0, 0, 0, 0, true}, {50, 50, 0, 0, true}, {100, 50, 0, 0, true},
240 {50, 100, 0, 0, true}, {100, 100, 0, 0, true}, {0, 0, 2, 5, true},
241 {1, 1, 2, 5, true}, {50, 50, 2, 5, true}, {100, 100, 2, 5, true},
242 {0, 0, 5, 2, true}, {50, 50, 5, 2, true}, {100, 100, 5, 2, true},
243 {0, 0, 0, 0, false},
244 };
245
246 for (size_t i = 0; i < arraysize(tests); i++) {
247 SCOPED_TRACE(i);
248
249 DesktopVector hotspot(tests[i].hotspot_x, tests[i].hotspot_y);
250 fake_cursor_->SetHotspot(hotspot);
251
252 MouseCursorMonitor::CursorState state = tests[i].inside
253 ? MouseCursorMonitor::INSIDE
254 : MouseCursorMonitor::OUTSIDE;
255 DesktopVector pos(tests[i].x, tests[i].y);
256 fake_cursor_->SetState(state, pos);
257
258 std::unique_ptr<SharedDesktopFrame> frame(
259 SharedDesktopFrame::Wrap(CreateTestFrame()));
260
261 blender_.CaptureFrame();
262 // If capturer captured nothing, then cursor should be ignored, not matter
263 // its state or position.
264 EXPECT_EQ(frame_, nullptr);
265 }
266 }
267
TEST_F(DesktopAndCursorComposerTest,CursorShouldBeIgnoredIfFrameMayContainIt)268 TEST_F(DesktopAndCursorComposerTest, CursorShouldBeIgnoredIfFrameMayContainIt) {
269 // We can't use a shared frame because we need to detect modifications
270 // compared to a control.
271 std::unique_ptr<DesktopFrame> control_frame(CreateTestFrame());
272 control_frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord));
273
274 struct {
275 int x;
276 int y;
277 bool may_contain_cursor;
278 } tests[] = {
279 {100, 200, true},
280 {100, 200, false},
281 {150, 250, true},
282 {150, 250, false},
283 };
284
285 for (size_t i = 0; i < arraysize(tests); i++) {
286 SCOPED_TRACE(i);
287
288 std::unique_ptr<DesktopFrame> frame(CreateTestFrame());
289 frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord));
290 frame->set_may_contain_cursor(tests[i].may_contain_cursor);
291 fake_screen_->SetNextFrame(std::move(frame));
292
293 const DesktopVector abs_pos(tests[i].x, tests[i].y);
294 fake_cursor_->SetState(MouseCursorMonitor::INSIDE, abs_pos);
295 blender_.CaptureFrame();
296
297 // If the frame may already have contained the cursor, then `CaptureFrame()`
298 // should not have modified it, so it should be the same as the control.
299 EXPECT_TRUE(frame_);
300 const DesktopVector rel_pos(abs_pos.subtract(control_frame->top_left()));
301 if (tests[i].may_contain_cursor) {
302 EXPECT_EQ(
303 *reinterpret_cast<uint32_t*>(frame_->GetFrameDataAtPos(rel_pos)),
304 *reinterpret_cast<uint32_t*>(
305 control_frame->GetFrameDataAtPos(rel_pos)));
306
307 } else {
308 // `CaptureFrame()` should have modified the frame to have the cursor.
309 EXPECT_NE(
310 *reinterpret_cast<uint32_t*>(frame_->GetFrameDataAtPos(rel_pos)),
311 *reinterpret_cast<uint32_t*>(
312 control_frame->GetFrameDataAtPos(rel_pos)));
313 EXPECT_TRUE(frame_->may_contain_cursor());
314 }
315 }
316 }
317
TEST_F(DesktopAndCursorComposerTest,CursorShouldBeIgnoredIfItIsOutOfDesktopFrame)318 TEST_F(DesktopAndCursorComposerTest,
319 CursorShouldBeIgnoredIfItIsOutOfDesktopFrame) {
320 std::unique_ptr<SharedDesktopFrame> frame(
321 SharedDesktopFrame::Wrap(CreateTestFrame()));
322 frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord));
323 // The frame covers (100, 200) - (200, 300).
324
325 struct {
326 int x;
327 int y;
328 } tests[] = {
329 {0, 0}, {50, 50}, {50, 150}, {100, 150}, {50, 200},
330 {99, 200}, {100, 199}, {200, 300}, {200, 299}, {199, 300},
331 {-1, -1}, {-10000, -10000}, {10000, 10000},
332 };
333 for (size_t i = 0; i < arraysize(tests); i++) {
334 SCOPED_TRACE(i);
335
336 fake_screen_->SetNextFrame(frame->Share());
337 // The CursorState is ignored when using absolute cursor position.
338 fake_cursor_->SetState(MouseCursorMonitor::OUTSIDE,
339 DesktopVector(tests[i].x, tests[i].y));
340 blender_.CaptureFrame();
341 VerifyFrame(*frame_, MouseCursorMonitor::OUTSIDE, DesktopVector(0, 0));
342 }
343 }
344
TEST_F(DesktopAndCursorComposerTest,IsOccludedShouldBeConsidered)345 TEST_F(DesktopAndCursorComposerTest, IsOccludedShouldBeConsidered) {
346 std::unique_ptr<SharedDesktopFrame> frame(
347 SharedDesktopFrame::Wrap(CreateTestFrame()));
348 frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord));
349 // The frame covers (100, 200) - (200, 300).
350
351 struct {
352 int x;
353 int y;
354 } tests[] = {
355 {100, 200}, {101, 200}, {100, 201}, {101, 201}, {150, 250}, {199, 299},
356 };
357 fake_screen_->set_is_occluded(true);
358 for (size_t i = 0; i < arraysize(tests); i++) {
359 SCOPED_TRACE(i);
360
361 fake_screen_->SetNextFrame(frame->Share());
362 // The CursorState is ignored when using absolute cursor position.
363 fake_cursor_->SetState(MouseCursorMonitor::OUTSIDE,
364 DesktopVector(tests[i].x, tests[i].y));
365 blender_.CaptureFrame();
366 VerifyFrame(*frame_, MouseCursorMonitor::OUTSIDE, DesktopVector());
367 }
368 }
369
TEST_F(DesktopAndCursorComposerTest,CursorIncluded)370 TEST_F(DesktopAndCursorComposerTest, CursorIncluded) {
371 std::unique_ptr<SharedDesktopFrame> frame(
372 SharedDesktopFrame::Wrap(CreateTestFrame()));
373 frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord));
374 // The frame covers (100, 200) - (200, 300).
375
376 struct {
377 int x;
378 int y;
379 } tests[] = {
380 {100, 200}, {101, 200}, {100, 201}, {101, 201}, {150, 250}, {199, 299},
381 };
382 for (size_t i = 0; i < arraysize(tests); i++) {
383 SCOPED_TRACE(i);
384
385 const DesktopVector abs_pos(tests[i].x, tests[i].y);
386 const DesktopVector rel_pos(abs_pos.subtract(frame->top_left()));
387
388 fake_screen_->SetNextFrame(frame->Share());
389 // The CursorState is ignored when using absolute cursor position.
390 fake_cursor_->SetState(MouseCursorMonitor::OUTSIDE, abs_pos);
391 blender_.CaptureFrame();
392 VerifyFrame(*frame_, MouseCursorMonitor::INSIDE, rel_pos);
393
394 // Verify that the cursor is erased before the frame buffer is returned to
395 // the screen capturer.
396 frame_.reset();
397 VerifyFrame(*frame, MouseCursorMonitor::OUTSIDE, DesktopVector());
398 }
399 }
400
TEST_F(DesktopAndCursorComposerNoCursorMonitorTest,UpdatedRegionIncludesOldAndNewCursorRectsIfMoved)401 TEST_F(DesktopAndCursorComposerNoCursorMonitorTest,
402 UpdatedRegionIncludesOldAndNewCursorRectsIfMoved) {
403 std::unique_ptr<SharedDesktopFrame> frame(
404 SharedDesktopFrame::Wrap(CreateTestFrame()));
405 DesktopRect first_cursor_rect;
406 {
407 // Block to scope test_cursor, which is invalidated by OnMouseCursor.
408 MouseCursor* test_cursor = CreateTestCursor(DesktopVector(0, 0));
409 first_cursor_rect = DesktopRect::MakeSize(test_cursor->image()->size());
410 blender_.OnMouseCursor(test_cursor);
411 }
412 blender_.OnMouseCursorPosition(DesktopVector(0, 0));
413 fake_screen_->SetNextFrame(frame->Share());
414 blender_.CaptureFrame();
415
416 DesktopVector cursor_move_offset(1, 1);
417 DesktopRect second_cursor_rect = first_cursor_rect;
418 second_cursor_rect.Translate(cursor_move_offset);
419 blender_.OnMouseCursorPosition(cursor_move_offset);
420 fake_screen_->SetNextFrame(frame->Share());
421 blender_.CaptureFrame();
422
423 EXPECT_TRUE(frame->updated_region().is_empty());
424 DesktopRegion expected_region;
425 expected_region.AddRect(first_cursor_rect);
426 expected_region.AddRect(second_cursor_rect);
427 EXPECT_TRUE(frame_->updated_region().Equals(expected_region));
428 }
429
TEST_F(DesktopAndCursorComposerNoCursorMonitorTest,UpdatedRegionIncludesOldAndNewCursorRectsIfShapeChanged)430 TEST_F(DesktopAndCursorComposerNoCursorMonitorTest,
431 UpdatedRegionIncludesOldAndNewCursorRectsIfShapeChanged) {
432 std::unique_ptr<SharedDesktopFrame> frame(
433 SharedDesktopFrame::Wrap(CreateTestFrame()));
434 DesktopRect first_cursor_rect;
435 {
436 // Block to scope test_cursor, which is invalidated by OnMouseCursor.
437 MouseCursor* test_cursor = CreateTestCursor(DesktopVector(0, 0));
438 first_cursor_rect = DesktopRect::MakeSize(test_cursor->image()->size());
439 blender_.OnMouseCursor(test_cursor);
440 }
441 blender_.OnMouseCursorPosition(DesktopVector(0, 0));
442 fake_screen_->SetNextFrame(frame->Share());
443 blender_.CaptureFrame();
444
445 // Create a second cursor, the same shape as the first. Since the code doesn't
446 // compare the cursor pixels, this is sufficient, and avoids needing two test
447 // cursor bitmaps.
448 DesktopRect second_cursor_rect;
449 {
450 MouseCursor* test_cursor = CreateTestCursor(DesktopVector(0, 0));
451 second_cursor_rect = DesktopRect::MakeSize(test_cursor->image()->size());
452 blender_.OnMouseCursor(test_cursor);
453 }
454 fake_screen_->SetNextFrame(frame->Share());
455 blender_.CaptureFrame();
456
457 EXPECT_TRUE(frame->updated_region().is_empty());
458 DesktopRegion expected_region;
459 expected_region.AddRect(first_cursor_rect);
460 expected_region.AddRect(second_cursor_rect);
461 EXPECT_TRUE(frame_->updated_region().Equals(expected_region));
462 }
463
TEST_F(DesktopAndCursorComposerNoCursorMonitorTest,UpdatedRegionUnchangedIfCursorUnchanged)464 TEST_F(DesktopAndCursorComposerNoCursorMonitorTest,
465 UpdatedRegionUnchangedIfCursorUnchanged) {
466 std::unique_ptr<SharedDesktopFrame> frame(
467 SharedDesktopFrame::Wrap(CreateTestFrame()));
468 blender_.OnMouseCursor(CreateTestCursor(DesktopVector(0, 0)));
469 blender_.OnMouseCursorPosition(DesktopVector(0, 0));
470 fake_screen_->SetNextFrame(frame->Share());
471 blender_.CaptureFrame();
472 fake_screen_->SetNextFrame(frame->Share());
473 blender_.CaptureFrame();
474
475 EXPECT_TRUE(frame->updated_region().is_empty());
476 EXPECT_TRUE(frame_->updated_region().is_empty());
477 }
478
479 } // namespace webrtc
480