DirectX based screen capturer logic

webrtc/modules/desktop_capture/win/desktop_frame_win_dxgi.cc

+/*
+ *  Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/desktop_capture/win/desktop_frame_win_dxgi.h"
+
+#include <string.h>
+
+#include <comdef.h>
+#include <D3D11.h>
+
+#include "webrtc/base/checks.h"
+#include "webrtc/base/scoped_ptr.h"
+#include "webrtc/system_wrappers/include/logging.h"
+
+namespace webrtc {
+
+using std::unique_ptr;
+using Microsoft::WRL::ComPtr;
+
+namespace {
+
+ID3D11Device* device = nullptr;
+ID3D11DeviceContext* context = nullptr;
+
+}  // namespace
+
+void DesktopFrameWinDxgi::Initialize(ID3D11Device* dev,
+                                     ID3D11DeviceContext* ctx) {
+  RTC_DCHECK(dev != nullptr);
+  RTC_DCHECK(ctx != nullptr);
+  RTC_DCHECK(device == nullptr);
+  RTC_DCHECK(context == nullptr);
+
+  device = dev;
+  context = ctx;
+}
+
+DesktopFrameWinDxgi::DesktopFrameWinDxgi(DesktopSize size,
+                                         int stride,
+                                         SharedMemory* shared_memory)
+    : DesktopFrame(size,
+                   stride,
+                   nullptr,  // Data will always be returned from rect_.pBits.
+                   shared_memory) {
+  RTC_DCHECK(device != nullptr);
+  RTC_DCHECK(context != nullptr);
+}
+
+DesktopFrameWinDxgi::~DesktopFrameWinDxgi() {}
+
+unique_ptr<DesktopFrameWinDxgi> DesktopFrameWinDxgi::Create(
+    DesktopSize size,
+    SharedMemoryFactory* shared_memory_factory) {
+  int stride = size.width() * DesktopFrame::kBytesPerPixel;
+  int buffer_size = stride * size.height();
+  rtc::scoped_ptr<SharedMemory> shared_memory;
+  if (shared_memory_factory != nullptr) {
+    shared_memory = shared_memory_factory->CreateSharedMemory(buffer_size);
+    if (!shared_memory) {
+      LOG(LS_ERROR) << "Failed to create shared memory with size "
+                    << buffer_size;
+      // Well, we are still able to work without shared_memory.
+    }
+  } else {
+    LOG(LS_WARNING) << "No SharedMemoryFactory provided";
+  }
+
+  return unique_ptr<DesktopFrameWinDxgi>(
+      new DesktopFrameWinDxgi(size, stride, shared_memory.release()));
+}
+
+bool DesktopFrameWinDxgi::Capture(const DXGI_OUTDUPL_FRAME_INFO& frame_info,
+                                  IDXGIResource* resource) {
+  if (resource == nullptr || frame_info.AccumulatedFrames == 0) {
+    // Nothing updated, but current data is still valid.
+    return false;
+  }
+
+  ComPtr<ID3D11Texture2D> texture;
+  _com_error err(resource->QueryInterface(
+      __uuidof(ID3D11Texture2D),
+      reinterpret_cast<void**>(texture.GetAddressOf())));
+  if (err.Error() != S_OK || !texture) {
+    LOG(LS_ERROR) << "Failed to convert IDXGIResource to ID3D11Texture2D, "
+                     "error "
+                  << err.ErrorMessage() << ", code " << err.Error();
+    return false;
+  }
+
+  // AcquireNextFrame returns a CPU inaccessible IDXGIResource, so we need to
+  // make a copy.
+  if (!CreateTexture(texture.Get())) {
+    return false;
+  }
+
+  context->CopyResource(static_cast<ID3D11Resource*>(stage_.Get()),
+                        static_cast<ID3D11Resource*>(texture.Get()));
+
+  rect_ = {0};
+  err = _com_error(surface_->Map(&rect_, DXGI_MAP_READ));
+  if (err.Error() != S_OK) {
+    LOG(LS_ERROR) << "Failed to map the IDXGISurface to a bitmap, error "
+                  << err.ErrorMessage() << ", code " << err.Error();
+    return false;
+  }
+
+  if (rect_.Pitch != stride()) {
+    // Unfortunately we cannot check whether the buffer size is also consistent.
+    LOG(LS_ERROR) << "System returns a different size of bitmap, with pitch "
+                  << rect_.Pitch;
+    return false;
+  }
+
+  if (shared_memory() != nullptr) {

[Sergey Ulanov, 2016/04/08 21:22:26]

There are two cases:
 1. SharedMemoryFactory is set. In this case generated frames need to be copied
to the shared memory.
 2. SharedMemoryFactory is not set. Data doesn't need to be copied out of the
buffers allocated by DirectX.

You are trying to support both cases here and this makes this CL rather
convoluted. Previously it was guaranteed that the buffer used in DesktopFrame
never changes. You had to change DesktopFrame, and so data() is now mutable.
This makes it impossible to provide some guarantees about SharedDesktopFrame.
E.g. previously it was possible to use SharedDesktopFrame on two separate
threads and now it isn't safe anymore as the buffer may be moved.

Also note that in case (2) there is no reason to use ScreenCaptureFrameQueue -
you can just can create new instance of DesktopFrameWinDxgi for every captured
frame. It's only useful for (1). 

Here is what I suggest:
 1. Keep the current DesktopFrame semantics, which guarantees that data() always
returns the same value and cannot change, i.e. buffer for a frame cannot be
moved.
 2. In the new capturer always create a new instance or DesktopFrameWinDxgi for
every captured frame.
 3. If SharedMemoryFactory is not set then just return that DesktopFrameWinDxgi.
 4. If SharedMemoryFactory is set then use ScreenCaptureFrameQueue with
SharedMemoryDesktopFrame and copy changed regions from DesktopFrameWinDxgi.

[Hzj_jie, 2016/04/11 22:19:16]

There are two reasons I used current solution.
1. in ScreenCaptureFrameQueue::MoveToNextFrame(), it's guaranteed current frame
is not shared anymore. I believe it means it's safe for us to update it.
2. If we create a new instance of DesktopFrameWinDxgi, Windows needs to allocate
and zero ~6M memory (use an 1400 * 1050 screen resolution as an example). It's
not a trivial cost for high fps driving DirectX.

[Sergey Ulanov, 2016/04/12 03:00:46]

ScreenCaptureFrameQueue is not the only place where SharedDesktopFrame is used.
E.g. it's also used in some tests. I'm not sure if the change in semantics is
safe there. And even if SharedDesktopFrame was used only ScreenCaptureFrameQueue
I would like to keep the current DesktopFrame behavior which guarantees that
data(), size() and stride() always return the same value for the same object.
Currently this code takes a texture from DirectX and copies it to a different
texture called stage_. Then it reads back from stage_. Why is it not possible to
read directly from the texture that we get from DirectX?
In either case I do agree with that we should avoid creation of a new texture
for each frame. We could share the same texture between multiple
DesktopFrameWinDxgi instances, but another possibility is to create a new set of
DesktopFrameWinDxgi every time screen size changes - this will still guarantee
that an instance DesktopFrameWinDxgi always references the same texture.

DesktopFrame objects are supposed to be simple and should not contain any
complex logic like you have here. There are two things that this class is
currently responsible for that I think should be handled elsewhere:
 - Allocation of new texture when size changes
 - Allocation of shared memory (can be replaced with SharedMemoryDesktopFrame). 

[Hzj_jie, 2016/04/12 17:40:40]

For the reason we make a copy to stage_, it's a Windows implementation
limitation, the original texture returns from IDXGIOutputDuplication is pointing
to GPU memory, and won't be able to access from CPU. So the CopyResource
function runs on GPU to copy data from GPU memory to CPU memory.

Yes, I have also considered to create a set of stage textures, but the problem
is, we need to know the queue length of ScreenCaptureFrameQueue. Another
approach is to extend ScreenCaptureFrameQueue to
ScreenCaptureFrameQueueWithContext<T>, and each frame can have exactly one T
(ID3D11Texture in this case) object.

Let's talk about it later.

+    // If there is a shared_memory, we need to copy the data to it, so other
+    // processes can use it.
+    memcpy(shared_memory()->data(), rect_.pBits, shared_memory()->size());

[Sergey Ulanov, 2016/04/08 21:22:26]

This copies the whole frame every time. It should be possible to avoid it by
copying only changing parts. E.g. take a look at how it's implemented in
ScreenCapturerX11. ScreenCaptureFrameQueue accumulates list of changes between
pair of frames, which should be useful here. It's another reason why this code
doesn't belong in this class.

[Hzj_jie, 2016/04/11 22:19:16]

What's my initial plan is to move all variables from DesktopFrame to a new
class, say SimpleDesktopFrame. And provide a CopyUpdatedRegionTo function to
avoid the duplicate SynchronizeFrame logic to be implemented in different
ScreenCapturer classes.
And going forward, I believe both Differ, SharedMemory and
ScreenCapturerFrameQueue can be removed from ScreenCapturer. i.e. A
ScreenCapturer does not need to take care about how to maintain the queue of
frames, differentiate frames, or copy changed region to shared memory. These
logic do not have relationship with ScreenCapturer, instead, they can be shared
in a platform and implementation independent class.

[Sergey Ulanov, 2016/04/12 03:00:47]

IMO ScreenCapturerFrameQueue would a better place for this functionality.
These are great ideas. Particularly separating Differ to a separate layer would
be useful. But keep in mind that it's not always possible to separate these
functions into separate layers without making the capturers less efficient. In
particular handling shared memory in the GDI capturer allows to avoid copying
the data from the OS buffer. X11 and OSX capturer get damage notifications from
the OS and capture only changed regions, so you do need Differ and
ScreenCaptureFrameQueue there. 

+  }
+  // surface_->Unmap will be called next time we capture an image to avoid
+  // memory copy without shared_memory.
+  return true;
+}
+
+bool DesktopFrameWinDxgi::CreateTexture(ID3D11Texture2D* texture) {
+  RTC_DCHECK(texture != nullptr);
+  D3D11_TEXTURE2D_DESC desc;

[Sergey Ulanov, 2016/04/08 21:22:26]

memset the struct to 0?

[Hzj_jie, 2016/04/11 22:19:16]

Done.

+  texture->GetDesc(&desc);
+  // TODO(zijiehe): Debug
+  // desc.ArraySize = 1;  // Capture only one frame.
+  desc.BindFlags = 0;
+  desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
+  // desc.MipLevels = 1;
+  desc.MiscFlags = 0;
+  // desc.SampleDesc.Count = 1;
+  // desc.SampleDesc.Quality = 0;

[Sergey Ulanov, 2016/04/08 21:22:26]

remove commented code?

[Hzj_jie, 2016/04/11 22:19:16]

Yes, should be, sorry.

+  desc.Usage = D3D11_USAGE_STAGING;
+  if (stage_) {
+    // Make sure stage_ and surface_ are always pointing to a same object.
+    // We need an ID3D11Texture2D instance for
+    // ID3D11DeviceContext::CopySubresourceRegion, but an IDXGISurface for
+    // IDXGISurface::Map.
+    {
+      ComPtr<IUnknown> left;
+      ComPtr<IUnknown> right;
+      bool left_result = SUCCEEDED(stage_.As(&left));
+      bool right_result = SUCCEEDED(surface_.As(&right));
+      RTC_DCHECK(left_result);
+      RTC_DCHECK(right_result);
+      RTC_DCHECK(left.Get() == right.Get());
+    }
+
+    // This buffer should be used already.
+    _com_error err = _com_error(surface_->Unmap());
+    if (err.Error() == S_OK) {
+      D3D11_TEXTURE2D_DESC orgi_desc;
+      stage_->GetDesc(&orgi_desc);
+      if (memcmp(&desc, &orgi_desc, sizeof(D3D11_TEXTURE2D_DESC)) == 0) {
+        return true;
+      }
+    } else {
+      // Let's recreate stage_ and surface_ later.
+      LOG(LS_ERROR) << "Failed to unmap surface, error " << err.ErrorMessage()
+                    << ", code " << err.Error();
+    }
+
+    stage_.Reset();
+    surface_.Reset();
+  } else {
+    RTC_DCHECK(!surface_);
+  }
+
+  _com_error err = _com_error(
+      device->CreateTexture2D(&desc, nullptr, stage_.GetAddressOf()));
+  if (err.Error() != S_OK || !stage_) {
+    LOG(LS_ERROR) << "Failed to create a new ID3D11Texture2D as stage, "
+                     "error "
+                  << err.ErrorMessage() << ", code " << err.Error();
+    return false;
+  }
+
+  err = _com_error(stage_.As(&surface_));
+  if (err.Error() != S_OK || !surface_) {
+    LOG(LS_ERROR) << "Failed to convert ID3D11Texture2D to IDXGISurface, "
+                     "error "
+                  << err.ErrorMessage() << ", code " << err.Error();
+    return false;
+  }
+
+  return true;
+}
+
+}  // namespace webrtc