1 /*
2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "modules/desktop_capture/win/wgc_capture_session.h"
12
13 #include <DispatcherQueue.h>
14 #include <windows.graphics.capture.interop.h>
15 #include <windows.graphics.directX.direct3d11.interop.h>
16 #include <windows.graphics.h>
17 #include <wrl/client.h>
18 #include <wrl/event.h>
19
20 #include <memory>
21 #include <utility>
22 #include <vector>
23
24 #include "modules/desktop_capture/win/wgc_desktop_frame.h"
25 #include "rtc_base/checks.h"
26 #include "rtc_base/logging.h"
27 #include "rtc_base/time_utils.h"
28 #include "rtc_base/win/create_direct3d_device.h"
29 #include "rtc_base/win/get_activation_factory.h"
30 #include "system_wrappers/include/metrics.h"
31
32 using Microsoft::WRL::ComPtr;
33 namespace WGC = ABI::Windows::Graphics::Capture;
34
35 namespace webrtc {
36 namespace {
37
38 // We must use a BGRA pixel format that has 4 bytes per pixel, as required by
39 // the DesktopFrame interface.
40 constexpr auto kPixelFormat = ABI::Windows::Graphics::DirectX::
41 DirectXPixelFormat::DirectXPixelFormat_B8G8R8A8UIntNormalized;
42
43 // The maximum time `GetFrame` will wait for a frame to arrive, if we don't have
44 // any in the pool.
45 constexpr TimeDelta kMaxWaitForFrame = TimeDelta::Millis(50);
46 constexpr TimeDelta kMaxWaitForFirstFrame = TimeDelta::Millis(500);
47
48 // These values are persisted to logs. Entries should not be renumbered and
49 // numeric values should never be reused.
50 enum class StartCaptureResult {
51 kSuccess = 0,
52 kSourceClosed = 1,
53 kAddClosedFailed = 2,
54 kDxgiDeviceCastFailed = 3,
55 kD3dDelayLoadFailed = 4,
56 kD3dDeviceCreationFailed = 5,
57 kFramePoolActivationFailed = 6,
58 // kFramePoolCastFailed = 7, (deprecated)
59 // kGetItemSizeFailed = 8, (deprecated)
60 kCreateFramePoolFailed = 9,
61 kCreateCaptureSessionFailed = 10,
62 kStartCaptureFailed = 11,
63 kMaxValue = kStartCaptureFailed
64 };
65
66 // These values are persisted to logs. Entries should not be renumbered and
67 // numeric values should never be reused.
68 enum class GetFrameResult {
69 kSuccess = 0,
70 kItemClosed = 1,
71 kTryGetNextFrameFailed = 2,
72 kFrameDropped = 3,
73 kGetSurfaceFailed = 4,
74 kDxgiInterfaceAccessFailed = 5,
75 kTexture2dCastFailed = 6,
76 kCreateMappedTextureFailed = 7,
77 kMapFrameFailed = 8,
78 kGetContentSizeFailed = 9,
79 kResizeMappedTextureFailed = 10,
80 kRecreateFramePoolFailed = 11,
81 kMaxValue = kRecreateFramePoolFailed
82 };
83
RecordStartCaptureResult(StartCaptureResult error)84 void RecordStartCaptureResult(StartCaptureResult error) {
85 RTC_HISTOGRAM_ENUMERATION(
86 "WebRTC.DesktopCapture.Win.WgcCaptureSessionStartResult",
87 static_cast<int>(error), static_cast<int>(StartCaptureResult::kMaxValue));
88 }
89
RecordGetFrameResult(GetFrameResult error)90 void RecordGetFrameResult(GetFrameResult error) {
91 RTC_HISTOGRAM_ENUMERATION(
92 "WebRTC.DesktopCapture.Win.WgcCaptureSessionGetFrameResult",
93 static_cast<int>(error), static_cast<int>(GetFrameResult::kMaxValue));
94 }
95
96 } // namespace
97
WgcCaptureSession(ComPtr<ID3D11Device> d3d11_device,ComPtr<WGC::IGraphicsCaptureItem> item,ABI::Windows::Graphics::SizeInt32 size)98 WgcCaptureSession::WgcCaptureSession(ComPtr<ID3D11Device> d3d11_device,
99 ComPtr<WGC::IGraphicsCaptureItem> item,
100 ABI::Windows::Graphics::SizeInt32 size)
101 : d3d11_device_(std::move(d3d11_device)),
102 item_(std::move(item)),
103 size_(size) {}
~WgcCaptureSession()104 WgcCaptureSession::~WgcCaptureSession() {
105 RemoveEventHandlers();
106 }
107
StartCapture(const DesktopCaptureOptions & options)108 HRESULT WgcCaptureSession::StartCapture(const DesktopCaptureOptions& options) {
109 RTC_DCHECK_RUN_ON(&sequence_checker_);
110 RTC_DCHECK(!is_capture_started_);
111
112 if (item_closed_) {
113 RTC_LOG(LS_ERROR) << "The target source has been closed.";
114 RecordStartCaptureResult(StartCaptureResult::kSourceClosed);
115 return E_ABORT;
116 }
117
118 RTC_DCHECK(d3d11_device_);
119 RTC_DCHECK(item_);
120
121 // Listen for the Closed event, to detect if the source we are capturing is
122 // closed (e.g. application window is closed or monitor is disconnected). If
123 // it is, we should abort the capture.
124 item_closed_token_ = std::make_unique<EventRegistrationToken>();
125 auto closed_handler =
126 Microsoft::WRL::Callback<ABI::Windows::Foundation::ITypedEventHandler<
127 WGC::GraphicsCaptureItem*, IInspectable*>>(
128 this, &WgcCaptureSession::OnItemClosed);
129 HRESULT hr =
130 item_->add_Closed(closed_handler.Get(), item_closed_token_.get());
131 if (FAILED(hr)) {
132 RecordStartCaptureResult(StartCaptureResult::kAddClosedFailed);
133 return hr;
134 }
135
136 ComPtr<IDXGIDevice> dxgi_device;
137 hr = d3d11_device_->QueryInterface(IID_PPV_ARGS(&dxgi_device));
138 if (FAILED(hr)) {
139 RecordStartCaptureResult(StartCaptureResult::kDxgiDeviceCastFailed);
140 return hr;
141 }
142
143 if (!ResolveCoreWinRTDirect3DDelayload()) {
144 RecordStartCaptureResult(StartCaptureResult::kD3dDelayLoadFailed);
145 return E_FAIL;
146 }
147
148 hr = CreateDirect3DDeviceFromDXGIDevice(dxgi_device.Get(), &direct3d_device_);
149 if (FAILED(hr)) {
150 RecordStartCaptureResult(StartCaptureResult::kD3dDeviceCreationFailed);
151 return hr;
152 }
153
154 ComPtr<WGC::IDirect3D11CaptureFramePoolStatics> frame_pool_statics;
155 hr = GetActivationFactory<
156 WGC::IDirect3D11CaptureFramePoolStatics,
157 RuntimeClass_Windows_Graphics_Capture_Direct3D11CaptureFramePool>(
158 &frame_pool_statics);
159 if (FAILED(hr)) {
160 RecordStartCaptureResult(StartCaptureResult::kFramePoolActivationFailed);
161 return hr;
162 }
163
164 hr = frame_pool_statics->Create(direct3d_device_.Get(), kPixelFormat,
165 kNumBuffers, size_, &frame_pool_);
166 if (FAILED(hr)) {
167 RecordStartCaptureResult(StartCaptureResult::kCreateFramePoolFailed);
168 return hr;
169 }
170
171 frames_in_pool_ = 0;
172
173 // Because `WgcCapturerWin` created a `DispatcherQueue`, and we created
174 // `frame_pool_` via `Create`, the `FrameArrived` event will be delivered on
175 // the current thread.
176 frame_arrived_token_ = std::make_unique<EventRegistrationToken>();
177 auto frame_arrived_handler =
178 Microsoft::WRL::Callback<ABI::Windows::Foundation::ITypedEventHandler<
179 WGC::Direct3D11CaptureFramePool*, IInspectable*>>(
180 this, &WgcCaptureSession::OnFrameArrived);
181 hr = frame_pool_->add_FrameArrived(frame_arrived_handler.Get(),
182 frame_arrived_token_.get());
183
184 hr = frame_pool_->CreateCaptureSession(item_.Get(), &session_);
185 if (FAILED(hr)) {
186 RecordStartCaptureResult(StartCaptureResult::kCreateCaptureSessionFailed);
187 return hr;
188 }
189
190 if (!options.prefer_cursor_embedded()) {
191 ComPtr<ABI::Windows::Graphics::Capture::IGraphicsCaptureSession2> session2;
192 if (SUCCEEDED(session_->QueryInterface(
193 ABI::Windows::Graphics::Capture::IID_IGraphicsCaptureSession2,
194 &session2))) {
195 session2->put_IsCursorCaptureEnabled(false);
196 }
197 }
198
199 hr = session_->StartCapture();
200 if (FAILED(hr)) {
201 RTC_LOG(LS_ERROR) << "Failed to start CaptureSession: " << hr;
202 RecordStartCaptureResult(StartCaptureResult::kStartCaptureFailed);
203 return hr;
204 }
205
206 RecordStartCaptureResult(StartCaptureResult::kSuccess);
207
208 is_capture_started_ = true;
209 return hr;
210 }
211
GetFrame(std::unique_ptr<DesktopFrame> * output_frame)212 HRESULT WgcCaptureSession::GetFrame(
213 std::unique_ptr<DesktopFrame>* output_frame) {
214 RTC_DCHECK_RUN_ON(&sequence_checker_);
215
216 if (item_closed_) {
217 RTC_LOG(LS_ERROR) << "The target source has been closed.";
218 RecordGetFrameResult(GetFrameResult::kItemClosed);
219 return E_ABORT;
220 }
221
222 RTC_DCHECK(is_capture_started_);
223
224 if (frames_in_pool_ < 1)
225 wait_for_frame_event_.Wait(first_frame_ ? kMaxWaitForFirstFrame
226 : kMaxWaitForFrame);
227
228 ComPtr<WGC::IDirect3D11CaptureFrame> capture_frame;
229 HRESULT hr = frame_pool_->TryGetNextFrame(&capture_frame);
230 if (FAILED(hr)) {
231 RTC_LOG(LS_ERROR) << "TryGetNextFrame failed: " << hr;
232 RecordGetFrameResult(GetFrameResult::kTryGetNextFrameFailed);
233 return hr;
234 }
235
236 if (!capture_frame) {
237 RecordGetFrameResult(GetFrameResult::kFrameDropped);
238 return hr;
239 }
240
241 first_frame_ = false;
242 --frames_in_pool_;
243
244 // We need to get `capture_frame` as an `ID3D11Texture2D` so that we can get
245 // the raw image data in the format required by the `DesktopFrame` interface.
246 ComPtr<ABI::Windows::Graphics::DirectX::Direct3D11::IDirect3DSurface>
247 d3d_surface;
248 hr = capture_frame->get_Surface(&d3d_surface);
249 if (FAILED(hr)) {
250 RecordGetFrameResult(GetFrameResult::kGetSurfaceFailed);
251 return hr;
252 }
253
254 ComPtr<Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess>
255 direct3DDxgiInterfaceAccess;
256 hr = d3d_surface->QueryInterface(IID_PPV_ARGS(&direct3DDxgiInterfaceAccess));
257 if (FAILED(hr)) {
258 RecordGetFrameResult(GetFrameResult::kDxgiInterfaceAccessFailed);
259 return hr;
260 }
261
262 ComPtr<ID3D11Texture2D> texture_2D;
263 hr = direct3DDxgiInterfaceAccess->GetInterface(IID_PPV_ARGS(&texture_2D));
264 if (FAILED(hr)) {
265 RecordGetFrameResult(GetFrameResult::kTexture2dCastFailed);
266 return hr;
267 }
268
269 if (!mapped_texture_) {
270 hr = CreateMappedTexture(texture_2D);
271 if (FAILED(hr)) {
272 RecordGetFrameResult(GetFrameResult::kCreateMappedTextureFailed);
273 return hr;
274 }
275 }
276
277 // We need to copy `texture_2D` into `mapped_texture_` as the latter has the
278 // D3D11_CPU_ACCESS_READ flag set, which lets us access the image data.
279 // Otherwise it would only be readable by the GPU.
280 ComPtr<ID3D11DeviceContext> d3d_context;
281 d3d11_device_->GetImmediateContext(&d3d_context);
282
283 ABI::Windows::Graphics::SizeInt32 new_size;
284 hr = capture_frame->get_ContentSize(&new_size);
285 if (FAILED(hr)) {
286 RecordGetFrameResult(GetFrameResult::kGetContentSizeFailed);
287 return hr;
288 }
289
290 // If the size changed, we must resize `mapped_texture_` and `frame_pool_` to
291 // fit the new size. This must be done before `CopySubresourceRegion` so that
292 // the textures are the same size.
293 if (size_.Height != new_size.Height || size_.Width != new_size.Width) {
294 hr = CreateMappedTexture(texture_2D, new_size.Width, new_size.Height);
295 if (FAILED(hr)) {
296 RecordGetFrameResult(GetFrameResult::kResizeMappedTextureFailed);
297 return hr;
298 }
299
300 hr = frame_pool_->Recreate(direct3d_device_.Get(), kPixelFormat,
301 kNumBuffers, new_size);
302 if (FAILED(hr)) {
303 RecordGetFrameResult(GetFrameResult::kRecreateFramePoolFailed);
304 return hr;
305 }
306 }
307
308 // If the size has changed since the last capture, we must be sure to use
309 // the smaller dimensions. Otherwise we might overrun our buffer, or
310 // read stale data from the last frame.
311 int image_height = std::min(size_.Height, new_size.Height);
312 int image_width = std::min(size_.Width, new_size.Width);
313
314 D3D11_BOX copy_region;
315 copy_region.left = 0;
316 copy_region.top = 0;
317 copy_region.right = image_width;
318 copy_region.bottom = image_height;
319 // Our textures are 2D so we just want one "slice" of the box.
320 copy_region.front = 0;
321 copy_region.back = 1;
322 d3d_context->CopySubresourceRegion(mapped_texture_.Get(),
323 /*dst_subresource_index=*/0, /*dst_x=*/0,
324 /*dst_y=*/0, /*dst_z=*/0, texture_2D.Get(),
325 /*src_subresource_index=*/0, ©_region);
326
327 D3D11_MAPPED_SUBRESOURCE map_info;
328 hr = d3d_context->Map(mapped_texture_.Get(), /*subresource_index=*/0,
329 D3D11_MAP_READ, /*D3D11_MAP_FLAG_DO_NOT_WAIT=*/0,
330 &map_info);
331 if (FAILED(hr)) {
332 RecordGetFrameResult(GetFrameResult::kMapFrameFailed);
333 return hr;
334 }
335
336 int row_data_length = image_width * DesktopFrame::kBytesPerPixel;
337
338 // Make a copy of the data pointed to by `map_info.pData` so we are free to
339 // unmap our texture.
340 uint8_t* src_data = static_cast<uint8_t*>(map_info.pData);
341 std::vector<uint8_t> image_data;
342 image_data.resize(image_height * row_data_length);
343 uint8_t* image_data_ptr = image_data.data();
344 for (int i = 0; i < image_height; i++) {
345 memcpy(image_data_ptr, src_data, row_data_length);
346 image_data_ptr += row_data_length;
347 src_data += map_info.RowPitch;
348 }
349
350 d3d_context->Unmap(mapped_texture_.Get(), 0);
351
352 // Transfer ownership of `image_data` to the output_frame.
353 DesktopSize size(image_width, image_height);
354 *output_frame = std::make_unique<WgcDesktopFrame>(size, row_data_length,
355 std::move(image_data));
356
357 size_ = new_size;
358 RecordGetFrameResult(GetFrameResult::kSuccess);
359 return hr;
360 }
361
CreateMappedTexture(ComPtr<ID3D11Texture2D> src_texture,UINT width,UINT height)362 HRESULT WgcCaptureSession::CreateMappedTexture(
363 ComPtr<ID3D11Texture2D> src_texture,
364 UINT width,
365 UINT height) {
366 RTC_DCHECK_RUN_ON(&sequence_checker_);
367
368 D3D11_TEXTURE2D_DESC src_desc;
369 src_texture->GetDesc(&src_desc);
370 D3D11_TEXTURE2D_DESC map_desc;
371 map_desc.Width = width == 0 ? src_desc.Width : width;
372 map_desc.Height = height == 0 ? src_desc.Height : height;
373 map_desc.MipLevels = src_desc.MipLevels;
374 map_desc.ArraySize = src_desc.ArraySize;
375 map_desc.Format = src_desc.Format;
376 map_desc.SampleDesc = src_desc.SampleDesc;
377 map_desc.Usage = D3D11_USAGE_STAGING;
378 map_desc.BindFlags = 0;
379 map_desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
380 map_desc.MiscFlags = 0;
381 return d3d11_device_->CreateTexture2D(&map_desc, nullptr, &mapped_texture_);
382 }
383
OnFrameArrived(WGC::IDirect3D11CaptureFramePool * sender,IInspectable * event_args)384 HRESULT WgcCaptureSession::OnFrameArrived(
385 WGC::IDirect3D11CaptureFramePool* sender,
386 IInspectable* event_args) {
387 RTC_DCHECK_RUN_ON(&sequence_checker_);
388 RTC_DCHECK_LT(frames_in_pool_, kNumBuffers);
389 ++frames_in_pool_;
390 wait_for_frame_event_.Set();
391 return S_OK;
392 }
393
OnItemClosed(WGC::IGraphicsCaptureItem * sender,IInspectable * event_args)394 HRESULT WgcCaptureSession::OnItemClosed(WGC::IGraphicsCaptureItem* sender,
395 IInspectable* event_args) {
396 RTC_DCHECK_RUN_ON(&sequence_checker_);
397
398 RTC_LOG(LS_INFO) << "Capture target has been closed.";
399 item_closed_ = true;
400 is_capture_started_ = false;
401
402 RemoveEventHandlers();
403
404 mapped_texture_ = nullptr;
405 session_ = nullptr;
406 frame_pool_ = nullptr;
407 direct3d_device_ = nullptr;
408 item_ = nullptr;
409 d3d11_device_ = nullptr;
410
411 return S_OK;
412 }
413
RemoveEventHandlers()414 void WgcCaptureSession::RemoveEventHandlers() {
415 HRESULT hr;
416 if (frame_pool_ && frame_arrived_token_) {
417 hr = frame_pool_->remove_FrameArrived(*frame_arrived_token_);
418 frame_arrived_token_.reset();
419 if (FAILED(hr)) {
420 RTC_LOG(LS_WARNING) << "Failed to remove FrameArrived event handler: "
421 << hr;
422 }
423 }
424 if (item_ && item_closed_token_) {
425 hr = item_->remove_Closed(*item_closed_token_);
426 item_closed_token_.reset();
427 if (FAILED(hr))
428 RTC_LOG(LS_WARNING) << "Failed to remove Closed event handler: " << hr;
429 }
430 }
431
432 } // namespace webrtc
433