cast: support cursor rendering for tab capture

content/browser/media/capture/web_contents_video_capture_device.cc

Index: content/browser/media/capture/web_contents_video_capture_device.cc
diff --git a/content/browser/media/capture/web_contents_video_capture_device.cc b/content/browser/media/capture/web_contents_video_capture_device.cc
index 64312d9a7c92fd296700e79464d0ecb1d8d004f6..2e2dca435900771796b45897ccf50f4ac3090b1e 100644
--- a/content/browser/media/capture/web_contents_video_capture_device.cc
+++ b/content/browser/media/capture/web_contents_video_capture_device.cc
@@ -50,6 +50,8 @@
 
 #include "content/browser/media/capture/web_contents_video_capture_device.h"
 
+#include <algorithm>
+
 #include "base/basictypes.h"
 #include "base/bind.h"
 #include "base/callback_helpers.h"
@@ -63,6 +65,7 @@
 #include "base/threading/thread.h"
 #include "base/threading/thread_checker.h"
 #include "base/time/time.h"
+#include "content/browser/media/capture/cursor_renderer.h"
 #include "content/browser/media/capture/web_contents_capture_util.h"
 #include "content/browser/media/capture/web_contents_tracker.h"
 #include "content/browser/renderer_host/render_widget_host_impl.h"
@@ -84,6 +87,10 @@
 #include "ui/gfx/geometry/dip_util.h"
 #include "ui/gfx/geometry/size_conversions.h"
 
+#if defined(USE_AURA)
+#include "content/browser/media/capture/cursor_renderer_aura.h"
+#endif
+
 namespace content {
 
 namespace {
@@ -119,10 +126,13 @@ class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber {
  public:
   FrameSubscriber(media::VideoCaptureOracle::Event event_type,
                   const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle,
-                  VideoFrameDeliveryLog* delivery_log)
+                  VideoFrameDeliveryLog* delivery_log,
+                  base::WeakPtr<content::CursorRenderer> cursor_renderer)
       : event_type_(event_type),
         oracle_proxy_(oracle),
-        delivery_log_(delivery_log) {}
+        delivery_log_(delivery_log),
+        cursor_renderer_(cursor_renderer),
+        weak_ptr_factory_(this) {}
 
   bool ShouldCaptureFrame(
       const gfx::Rect& damage_rect,
@@ -131,10 +141,23 @@ class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber {
       RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback*
           deliver_frame_cb) override;
 
+  static void DidCaptureFrame(
+      base::WeakPtr<FrameSubscriber> frame_subscriber_,
+      const media::ThreadSafeCaptureOracle::CaptureFrameCallback&
+          capture_frame_cb,
+      const scoped_refptr<media::VideoFrame>& frame,
+      base::TimeTicks timestamp,
+      const gfx::Rect& region_in_frame,
+      bool success);
+
  private:
   const media::VideoCaptureOracle::Event event_type_;
   scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_;
   VideoFrameDeliveryLog* const delivery_log_;
+  // We need a weak pointer since FrameSubscriber is owned externally and
+  // may outlive the cursor renderer.
+  base::WeakPtr<CursorRenderer> cursor_renderer_;
+  base::WeakPtrFactory<FrameSubscriber> weak_ptr_factory_;
 };
 
 // ContentCaptureSubscription is the relationship between a RenderWidgetHost
@@ -177,10 +200,15 @@ class ContentCaptureSubscription {
   const int render_widget_id_;
 
   VideoFrameDeliveryLog delivery_log_;
-  FrameSubscriber timer_subscriber_;
+  scoped_ptr<FrameSubscriber> timer_subscriber_;
   CaptureCallback capture_callback_;
   base::Timer timer_;
 
+  // Responsible for tracking the cursor state and input events to make
+  // decisions and then render the mouse cursor on the video frame after
+  // capture is completed.
+  scoped_ptr<content::CursorRenderer> cursor_renderer_;
+
   DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription);
 };
 
@@ -193,9 +221,10 @@ class ContentCaptureSubscription {
 // these activities is not possible. This operation may be expensive (tens to
 // hundreds of milliseconds), so the caller should ensure that it runs on a
 // thread where such a pause would cause UI jank.
-void RenderVideoFrame(const SkBitmap& input,
-                      const scoped_refptr<media::VideoFrame>& output,
-                      const base::Callback<void(bool)>& done_cb);
+void RenderVideoFrame(
+    const SkBitmap& input,
+    const scoped_refptr<media::VideoFrame>& output,
+    const base::Callback<void(const gfx::Rect&, bool)>& done_cb);
 
 // Renews capture subscriptions based on feedback from WebContentsTracker, and
 // also executes copying of the backing store on the UI BrowserThread.
@@ -249,6 +278,7 @@ class WebContentsCaptureMachine : public media::VideoCaptureMachine {
       const base::TimeTicks& start_time,
       const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
           deliver_frame_cb,
+      const gfx::Rect& region_in_frame,
       bool success);
 
   // Remove the old subscription, and attempt to start a new one if |had_target|
@@ -304,16 +334,44 @@ bool FrameSubscriber::ShouldCaptureFrame(
                "instance", this);
 
   media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb;
+
   bool oracle_decision = oracle_proxy_->ObserveEventAndDecideCapture(
       event_type_, damage_rect, present_time, storage, &capture_frame_cb);
 
   if (!capture_frame_cb.is_null())
-    *deliver_frame_cb = base::Bind(capture_frame_cb, *storage);
+    *deliver_frame_cb =
+        base::Bind(&FrameSubscriber::DidCaptureFrame,
+                   weak_ptr_factory_.GetWeakPtr(), capture_frame_cb, *storage);
   if (oracle_decision)
     delivery_log_->ChronicleFrameDelivery(present_time);
   return oracle_decision;
 }
 
+void FrameSubscriber::DidCaptureFrame(
+    base::WeakPtr<FrameSubscriber> frame_subscriber_,
+    const media::ThreadSafeCaptureOracle::CaptureFrameCallback&
+        capture_frame_cb,
+    const scoped_refptr<media::VideoFrame>& frame,
+    base::TimeTicks timestamp,
+    const gfx::Rect& region_in_frame,
+    bool success) {
+  // We can get a callback in the shutdown sequence for the browser main loop
+  // and this can result in a DCHECK failure. Avoid this by doing DCHECK only
+  // on success.
+  if (success) {
+    DCHECK_CURRENTLY_ON(BrowserThread::UI);
+    // TODO(isheriff): Unclear if taking a snapshot of cursor here affects user
+    // experience in any particular scenarios. Doing it prior to capture may
+    // require evaluating region_in_frame in this file.
+    if (frame_subscriber_ && frame_subscriber_->cursor_renderer_ && success) {

[miu, 2015/11/06 00:22:54]

Don't need the "&& success" at the end anymore.

[Irfan, 2015/11/06 17:49:26]

Done.

+      if (frame_subscriber_->cursor_renderer_->SnapshotCursorState(
+              region_in_frame))
+        frame_subscriber_->cursor_renderer_->RenderOnVideoFrame(frame);
+    }
+  }
+  capture_frame_cb.Run(frame, timestamp, success);
+}
+
 ContentCaptureSubscription::ContentCaptureSubscription(
     const RenderWidgetHost& source,
     const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy,
@@ -321,20 +379,30 @@ ContentCaptureSubscription::ContentCaptureSubscription(
     : render_process_id_(source.GetProcess()->GetID()),
       render_widget_id_(source.GetRoutingID()),
       delivery_log_(),
-      timer_subscriber_(media::VideoCaptureOracle::kTimerPoll, oracle_proxy,
-                        &delivery_log_),
       capture_callback_(capture_callback),
       timer_(true, true) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 
   RenderWidgetHostView* const view = source.GetView();
+#if defined(USE_AURA)
+  if (view)
+    cursor_renderer_.reset(
+        new content::CursorRendererAura(view->GetNativeView()));
+#endif
+  timer_subscriber_.reset(new FrameSubscriber(
+      media::VideoCaptureOracle::kTimerPoll, oracle_proxy, &delivery_log_,
+      cursor_renderer_ ? cursor_renderer_->GetWeakPtr()
+                       : base::WeakPtr<CursorRenderer>()));
 
   // Subscribe to compositor updates. These will be serviced directly by the
   // oracle.
   if (view) {
     scoped_ptr<RenderWidgetHostViewFrameSubscriber> subscriber(
         new FrameSubscriber(media::VideoCaptureOracle::kCompositorUpdate,
-            oracle_proxy, &delivery_log_));
+                            oracle_proxy, &delivery_log_,
+                            cursor_renderer_
+                                ? cursor_renderer_->GetWeakPtr()
+                                : base::WeakPtr<CursorRenderer>()));
     view->BeginFrameSubscription(subscriber.Pass());
   }
 
@@ -370,18 +438,45 @@ void ContentCaptureSubscription::OnTimer() {
   RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb;
 
   const base::TimeTicks start_time = base::TimeTicks::Now();
-  if (timer_subscriber_.ShouldCaptureFrame(gfx::Rect(),
-                                           start_time,
-                                           &frame,
-                                           &deliver_frame_cb)) {
+  if (timer_subscriber_->ShouldCaptureFrame(gfx::Rect(), start_time, &frame,
+                                            &deliver_frame_cb)) {
     capture_callback_.Run(start_time, frame, deliver_frame_cb);
   }
 }
 
-void RenderVideoFrame(const SkBitmap& input,
-                      const scoped_refptr<media::VideoFrame>& output,
-                      const base::Callback<void(bool)>& done_cb) {
-  base::ScopedClosureRunner failure_handler(base::Bind(done_cb, false));
+// ScopedClosureRunOnBrowserThread ensures that the closure always executes
+// (on the browser thread) no matter how the current scope exits.
+class ScopedClosureRunOnBrowserThread {

[miu, 2015/11/06 00:22:54]

There's a common solution to this problem: media::BindToCurrentLoop().

What I suggest is, in WebContentsCaptureMachine::DidCopyFromBackingStore(), make
the render_thread_->PostTask() statement look like this:

#include "media/base/bind_to_current_loop.h"

render_thread_->task_runner()->PostTask(
    FROM_HERE,
    media::BindToCurrentLoop(
        base::Bind(&RenderVideoFrame, bitmap, target,
                              base::Bind(deliver_frame_cb, start_time))));

This puts an implicit "trampoline" back to the main thread when the callback is
run.  Then, you can just use the base::ScopedClosureRunner in RenderVideoFrame()
like it was before.

[Irfan, 2015/11/06 17:49:26]

Aha, excellent

+ public:
+  ScopedClosureRunOnBrowserThread(const base::Closure& closure);
+  ~ScopedClosureRunOnBrowserThread();
+  void Release();
+
+ private:
+  base::Closure closure_;
+
+  DISALLOW_COPY_AND_ASSIGN(ScopedClosureRunOnBrowserThread);
+};
+
+ScopedClosureRunOnBrowserThread::ScopedClosureRunOnBrowserThread(
+    const base::Closure& closure)
+    : closure_(closure) {}
+
+ScopedClosureRunOnBrowserThread::~ScopedClosureRunOnBrowserThread() {
+  if (!closure_.is_null())
+    BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, closure_);
+}
+
+void ScopedClosureRunOnBrowserThread::Release() {
+  closure_.Reset();
+}
+
+void RenderVideoFrame(
+    const SkBitmap& input,
+    const scoped_refptr<media::VideoFrame>& output,
+    const base::Callback<void(const gfx::Rect&, bool)>& done_cb) {
+  ScopedClosureRunOnBrowserThread failure_handler(
+      base::Bind(done_cb, gfx::Rect(), false));
 
   SkAutoLockPixels locker(input);
 
@@ -412,7 +507,6 @@ void RenderVideoFrame(const SkBitmap& input,
   SkBitmap scaled_bitmap;
   if (input.width() != region_in_frame.width() ||
       input.height() != region_in_frame.height()) {
-
     skia::ImageOperations::ResizeMethod method;
     if (input.width() < region_in_frame.width() ||
         input.height() < region_in_frame.height()) {
@@ -452,8 +546,9 @@ void RenderVideoFrame(const SkBitmap& input,
   }
 
   // The result is now ready.
-  ignore_result(failure_handler.Release());
-  done_cb.Run(true);
+  failure_handler.Release();
+  BrowserThread::PostTask(BrowserThread::UI, FROM_HERE,
+                          base::Bind(done_cb, region_in_frame, true));
 }
 
 VideoFrameDeliveryLog::VideoFrameDeliveryLog()
@@ -597,7 +692,7 @@ void WebContentsCaptureMachine::Capture(
   RenderWidgetHostViewBase* view =
       rwh ? static_cast<RenderWidgetHostViewBase*>(rwh->GetView()) : NULL;
   if (!view) {
-    deliver_frame_cb.Run(base::TimeTicks(), false);
+    deliver_frame_cb.Run(base::TimeTicks(), gfx::Rect(), false);
     return;
   }
 
@@ -722,7 +817,7 @@ void WebContentsCaptureMachine::DidCopyFromBackingStore(
   } else {
     // Capture can fail due to transient issues, so just skip this frame.
     DVLOG(1) << "CopyFromBackingStore failed; skipping frame.";
-    deliver_frame_cb.Run(start_time, false);
+    deliver_frame_cb.Run(start_time, gfx::Rect(), false);
   }
 }
 
@@ -730,6 +825,7 @@ void WebContentsCaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame(
     const base::TimeTicks& start_time,
     const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
         deliver_frame_cb,
+    const gfx::Rect& region_in_frame,
     bool success) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   base::TimeTicks now = base::TimeTicks::Now();
@@ -740,7 +836,7 @@ void WebContentsCaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame(
     // Capture can fail due to transient issues, so just skip this frame.
     DVLOG(1) << "CopyFromCompositingSurface failed; skipping frame.";
   }
-  deliver_frame_cb.Run(start_time, success);
+  deliver_frame_cb.Run(start_time, region_in_frame, success);
 }
 
 void WebContentsCaptureMachine::RenewFrameSubscription(bool had_target) {