Tab Capture: Backing store readbacks to YV12 VideoFrames.

ui/surface/accelerated_surface_win.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "ui/surface/accelerated_surface_win.h"
 
 #include <windows.h>
 #include <algorithm>
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/callback.h"
 #include "base/command_line.h"
 #include "base/debug/trace_event.h"
 #include "base/file_path.h"
 #include "base/lazy_instance.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/message_loop_proxy.h"
 #include "base/scoped_native_library.h"
 #include "base/stringprintf.h"
 #include "base/synchronization/waitable_event.h"
 #include "base/threading/thread.h"
 #include "base/threading/thread_restrictions.h"
-#include "base/time.h"
 #include "base/win/wrapped_window_proc.h"
+#include "media/base/video_frame.h"
+#include "media/base/video_util.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 #include "ui/base/win/dpi.h"
 #include "ui/base/win/hwnd_util.h"
 #include "ui/base/win/shell.h"
 #include "ui/gfx/rect.h"
 #include "ui/gl/gl_switches.h"
 #include "ui/surface/accelerated_surface_transformer_win.h"
 #include "ui/surface/d3d9_utils_win.h"
 #include "ui/surface/surface_switches.h"
 
@@ skipping 11 matching lines @@
 bool DoFirstShowPresentWithGDI() {
   return CommandLine::ForCurrentProcess()->HasSwitch(
       switches::kDoFirstShowPresentWithGDI);
 }
 
 bool DoAllShowPresentWithGDI() {
   return CommandLine::ForCurrentProcess()->HasSwitch(
       switches::kDoAllShowPresentWithGDI);
 }
 
+// Lock a D3D surface, and invoke a copier function on the result.
+bool LockAndCopyPlane(IDirect3DSurface9* src_surface,
+                      media::VideoFrame* dst_frame,
+                      void (*copier)(const uint8* source,

[scherkus (not reviewing), 2013/02/05 22:40:44]

nit: typedef + docs?
typedef void(*CopyPlaneFunc)(const blah blah blah);

[ncarter (slow), 2013/02/06 23:54:44]

I got rid of the function pointer altogether by simply exposing the
CopyPlane(size_t plane_index ... ) helper function in video_util.h. Make sure
you have a look at that.

+                                     int stride,
+                                     int rows,
+                                     media::VideoFrame* frame)) {
+  gfx::Size src_size = d3d_utils::GetSize(src_surface);
+
+  D3DLOCKED_RECT locked_rect;
+  {
+    TRACE_EVENT0("gpu", "LockRect");
+    HRESULT hr = src_surface->LockRect(&locked_rect, NULL,
+                                       D3DLOCK_READONLY | D3DLOCK_NOSYSLOCK);
+    if (FAILED(hr))
+      return false;
+  }
+
+  {
+    TRACE_EVENT0("gpu", "memcpy");
+    uint8* src = reinterpret_cast<uint8*>(locked_rect.pBits);
+    int src_stride = locked_rect.Pitch;
+    copier(src, src_stride, src_size.height(), dst_frame);
+  }
+  src_surface->UnlockRect();
+  return true;
+}
+
+// Return the largest centered rectangle with the same aspect ratio of |content|
+// that fits entirely inside of |bounds|.
+gfx::Rect ComputeLetterboxRegion(
+    const gfx::Rect& bounds,
+    const gfx::Size& content) {
+  int64 x = static_cast<int64>(content.width()) * bounds.height();
+  int64 y = static_cast<int64>(bounds.width()) * content.height();
+
+  gfx::Size letterbox(bounds.width(), bounds.height());
+  if (y < x)
+    letterbox.set_height(static_cast<int>(y / content.width()));
+  else
+    letterbox.set_width(static_cast<int>(x / content.height()));
+  gfx::Rect result = bounds;
+  result.ClampToCenteredSize(letterbox);
+  return result;
+}
+
 }  // namespace
 
 // A PresentThread is a thread that is dedicated to presenting surfaces to a
 // window. It owns a Direct3D device and a Direct3D query for this purpose.
 class PresentThread : public base::Thread,
                       public base::RefCountedThreadSafe<PresentThread> {
  public:
   explicit PresentThread(const char* name);
 
   IDirect3DDevice9Ex* device() { return device_.get(); }
@@ skipping 244 matching lines @@
   present_thread_->message_loop()->PostTask(
       FROM_HERE,
       base::Bind(&AcceleratedPresenter::DoCopyToAndAcknowledge,
                  this,
                  requested_src_subrect,
                  dst_size,
                  base::MessageLoopProxy::current(),
                  callback));
 }
 
+void AcceleratedPresenter::AsyncCopyToVideoFrame(
+    const gfx::Rect& requested_src_subrect,
+    const gfx::Size& dst_size,
+    const base::Callback<void(media::VideoFrame*)>& callback) {
+  present_thread_->message_loop()->PostTask(
+      FROM_HERE,
+      base::Bind(&AcceleratedPresenter::DoCopyToVideoFrameAndAcknowledge,
+                 this,
+                 requested_src_subrect,
+                 dst_size,
+                 base::MessageLoopProxy::current(),
+                 callback));
+}
+
 void AcceleratedPresenter::DoCopyToAndAcknowledge(
     const gfx::Rect& src_subrect,
     const gfx::Size& dst_size,
     scoped_refptr<base::SingleThreadTaskRunner> callback_runner,
     const base::Callback<void(bool, const SkBitmap&)>& callback) {
 
   SkBitmap target;
-  bool result = DoCopyTo(src_subrect, dst_size, &target);
+  bool result = DoCopyToARGB(src_subrect, dst_size, &target);
   if (!result)
     target.reset();
   callback_runner->PostTask(
       FROM_HERE,
       base::Bind(callback, result, target));
 }
 
-bool AcceleratedPresenter::DoCopyTo(const gfx::Rect& requested_src_subrect,
-                                    const gfx::Size& dst_size,
-                                    SkBitmap* bitmap) {
+void AcceleratedPresenter::DoCopyToVideoFrameAndAcknowledge(
+    const gfx::Rect& src_subrect,
+    const gfx::Size& dst_size,
+    scoped_refptr<base::SingleThreadTaskRunner> callback_runner,
+    const base::Callback<void(media::VideoFrame*)>& callback) {
+

[scherkus (not reviewing), 2013/02/05 22:40:44]

nit: remove blank line

[ncarter (slow), 2013/02/06 23:54:44]

Done.

+  scoped_refptr<media::VideoFrame> target(
+      media::VideoFrame::CreateFrame(
+          media::VideoFrame::YV12,
+          dst_size,
+          gfx::Rect(dst_size),
+          dst_size,
+          base::TimeDelta()));
+  bool result = DoCopyToYUV(src_subrect, dst_size, target);
+  if (!result)
+    target = media::VideoFrame::CreateEmptyFrame();

[scherkus (not reviewing), 2013/02/05 22:40:44]

I've never been a huge fan of the whole EMPTY frame business (you can end up
passing an empty frame all the way through the stack and into code that blindly
assumes its, say, YV12)

I haven't looked at all the callers, but do you feel having a
always-non-null-but-potentially-EMPTY VideoFrame is better than
potentially-null-but-always-YV12?

[ncarter (slow), 2013/02/06 23:54:44]

I'm moving to a model where VideoFrame is allocated and passed in by the
original caller, the callback just receives a |bool status|, and we never
synthesize empty frames. I think this will address your concern.

+  callback_runner->PostTask(FROM_HERE, base::Bind(callback, target));
+}
+
+bool AcceleratedPresenter::DoCopyToARGB(const gfx::Rect& requested_src_subrect,
+                                        const gfx::Size& dst_size,
+                                        SkBitmap* bitmap) {
   TRACE_EVENT2(
       "gpu", "CopyTo",
       "width", dst_size.width(),
       "height", dst_size.height());
 
   base::AutoLock locked(lock_);
 
-  TRACE_EVENT0("gpu", "CopyTo_locked");
-
   if (!swap_chain_)
     return false;
 
   AcceleratedSurfaceTransformer* gpu_ops =
       present_thread_->surface_transformer();
 
   base::win::ScopedComPtr<IDirect3DSurface9> back_buffer;
   HRESULT hr = swap_chain_->GetBackBuffer(0,
                                           D3DBACKBUFFER_TYPE_MONO,
                                           back_buffer.Receive());
@@ skipping 24 matching lines @@
                                                    dst_size,
                                                    final_surface.Receive())) {
       LOG(ERROR) << "Failed to create temporary lockable surface";
       return false;
     }
   }
 
   {
     // Let the surface transformer start the resize into |final_surface|.
     TRACE_EVENT0("gpu", "ResizeBilinear");
-    if (!gpu_ops->ResizeBilinear(back_buffer, src_subrect, final_surface)) {
+    if (!gpu_ops->ResizeBilinear(back_buffer, src_subrect,
+                                 final_surface, gfx::Rect(dst_size))) {
       LOG(ERROR) << "Failed to resize bilinear";
       return false;
     }
   }
 
   D3DLOCKED_RECT locked_rect;
 
   // Empirical evidence seems to suggest that LockRect and memcpy are faster
   // than would be GetRenderTargetData to an offscreen surface wrapping *buf.
   {
@@ skipping 20 matching lines @@
 
     memcpy(reinterpret_cast<int8*>(bitmap->getPixels()),
            reinterpret_cast<int8*>(locked_rect.pBits),
            locked_rect.Pitch * dst_size.height());
   }
   final_surface->UnlockRect();
 
   return true;
 }
 
+bool AcceleratedPresenter::DoCopyToYUV(const gfx::Rect& requested_src_subrect,
+                                       const gfx::Size& dst_size,
+                                       media::VideoFrame* frame) {
+  TRACE_EVENT2(
+      "gpu", "CopyToYUV",
+      "width", dst_size.width(),
+      "height", dst_size.height());
+
+  base::AutoLock locked(lock_);
+
+  if (!swap_chain_)
+    return false;
+
+  AcceleratedSurfaceTransformer* gpu_ops =
+      present_thread_->surface_transformer();
+
+  base::win::ScopedComPtr<IDirect3DSurface9> back_buffer;
+  HRESULT hr = swap_chain_->GetBackBuffer(0,
+                                          D3DBACKBUFFER_TYPE_MONO,
+                                          back_buffer.Receive());
+  if (FAILED(hr))
+    return false;
+
+  D3DSURFACE_DESC desc;
+  hr = back_buffer->GetDesc(&desc);
+  if (FAILED(hr))
+    return false;
+
+  const gfx::Size back_buffer_size(desc.Width, desc.Height);
+  if (back_buffer_size.IsEmpty())
+    return false;
+
+  // With window resizing, it's possible that the back buffer is smaller than
+  // the requested src subset. Clip to the actual back buffer.
+  gfx::Rect src_subrect = requested_src_subrect;
+  src_subrect.Intersect(gfx::Rect(back_buffer_size));
+
+  base::win::ScopedComPtr<IDirect3DTexture9> resized_as_texture;
+  base::win::ScopedComPtr<IDirect3DSurface9> resized;
+  {
+    TRACE_EVENT0("gpu", "CreateTemporaryRenderTargetTexture");
+    if (!d3d_utils::CreateTemporaryRenderTargetTexture(
+            present_thread_->device(),
+            dst_size,
+            resized_as_texture.Receive(),
+            resized.Receive())) {
+      return false;
+    }
+  }
+
+  // Shrink the source to fit entirely in the destination while preserving
+  // aspect ratio. Fill in any margin with black.
+  // TODO(nick): It would be more efficient all around to implement
+  // letterboxing as a memset() on the dst.
+  gfx::Rect letterbox = ComputeLetterboxRegion(gfx::Rect(dst_size),
+                                               src_subrect.size());
+  if (letterbox != gfx::Rect(dst_size)) {
+    TRACE_EVENT0("gpu", "Letterbox");
+    present_thread_->device()->ColorFill(resized, NULL, 0xFF000000);
+  }
+
+  {
+    TRACE_EVENT0("gpu", "ResizeBilinear");
+    if (!gpu_ops->ResizeBilinear(back_buffer, src_subrect, resized, letterbox))
+      return false;
+  }
+
+  base::win::ScopedComPtr<IDirect3DSurface9> y, u, v;
+  {
+    TRACE_EVENT0("gpu", "TransformRGBToYV12");
+    if (!gpu_ops->TransformRGBToYV12(resized_as_texture,
+                                     dst_size,
+                                     y.Receive(), u.Receive(), v.Receive())) {
+      return false;
+    }
+  }
+
+  if (!LockAndCopyPlane(y, frame, &media::CopyYPlane))
+    return false;
+  if (!LockAndCopyPlane(u, frame, &media::CopyUPlane))
+    return false;
+  if (!LockAndCopyPlane(v, frame, &media::CopyVPlane))
+    return false;
+  return true;
+}
+
 void AcceleratedPresenter::Suspend() {
   present_thread_->message_loop()->PostTask(
       FROM_HERE,
       base::Bind(&AcceleratedPresenter::DoSuspend,
                  this));
 }
 
 void AcceleratedPresenter::WasHidden() {
   base::AutoLock locked(lock_);
   hidden_ = true;
@@ skipping 409 matching lines @@
   presenter_->Present(dc);
 }
 
 void AcceleratedSurface::AsyncCopyTo(
     const gfx::Rect& src_subrect,
     const gfx::Size& dst_size,
     const base::Callback<void(bool, const SkBitmap&)>& callback) {
   presenter_->AsyncCopyTo(src_subrect, dst_size, callback);
 }
 
+void AcceleratedSurface::AsyncCopyToVideoFrame(
+    const gfx::Rect& src_subrect,
+    const gfx::Size& dst_size,
+    const base::Callback<void(media::VideoFrame*)>& callback) {
+  presenter_->AsyncCopyToVideoFrame(src_subrect, dst_size, callback);
+}
+
 void AcceleratedSurface::Suspend() {
   presenter_->Suspend();
 }
 
 void AcceleratedSurface::WasHidden() {
   presenter_->WasHidden();
 }
 
 void AcceleratedSurface::SetIsSessionLocked(bool locked) {
   presenter_->SetIsSessionLocked(locked);
 }