cast: support cursor rendering for tab capture

content/browser/media/capture/web_contents_video_capture_device.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // Implementation notes: This needs to work on a variety of hardware
 // configurations where the speed of the CPU and GPU greatly affect overall
 // performance. Spanning several threads, the process of capturing has been
 // split up into four conceptual stages:
 //
 //   1. Reserve Buffer: Before a frame can be captured, a slot in the client's
@@ skipping 32 matching lines @@
 //
 // In the above example, both capturing and rendering *each* take almost the
 // full 33 ms available between frames, yet we see that the required throughput
 // is obtained.
 //
 // Turning on verbose logging will cause the effective frame rate to be logged
 // at 5-second intervals.
 
 #include "content/browser/media/capture/web_contents_video_capture_device.h"
 
+#include <algorithm>
+
 #include "base/basictypes.h"
 #include "base/bind.h"
 #include "base/callback_helpers.h"
 #include "base/location.h"
 #include "base/logging.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/memory/weak_ptr.h"
 #include "base/metrics/histogram.h"
 #include "base/sequenced_task_runner.h"
 #include "base/single_thread_task_runner.h"
 #include "base/threading/thread.h"
 #include "base/threading/thread_checker.h"
 #include "base/time/time.h"
+#include "content/browser/media/capture/cursor_renderer.h"
 #include "content/browser/media/capture/web_contents_capture_util.h"
 #include "content/browser/media/capture/web_contents_tracker.h"
 #include "content/browser/renderer_host/render_widget_host_impl.h"
 #include "content/browser/renderer_host/render_widget_host_view_base.h"
 #include "content/public/browser/browser_thread.h"
 #include "content/public/browser/render_process_host.h"
 #include "content/public/browser/render_widget_host_view.h"
 #include "content/public/browser/render_widget_host_view_frame_subscriber.h"
 #include "content/public/browser/web_contents.h"
 #include "media/base/video_capture_types.h"
 #include "media/base/video_util.h"
 #include "media/capture/content/screen_capture_device_core.h"
 #include "media/capture/content/thread_safe_capture_oracle.h"
 #include "media/capture/content/video_capture_oracle.h"
 #include "skia/ext/image_operations.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 #include "third_party/skia/include/core/SkColor.h"
 #include "ui/base/layout.h"
 #include "ui/gfx/geometry/dip_util.h"
 #include "ui/gfx/geometry/size_conversions.h"
 
+#if defined(USE_AURA)
+#include "content/browser/media/capture/cursor_renderer_aura.h"
+#endif
+
 namespace content {
 
 namespace {
 
 void DeleteOnWorkerThread(scoped_ptr<base::Thread> render_thread,
                           const base::Closure& callback) {
   render_thread.reset();
 
   // After thread join call the callback on UI thread.
   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, callback);
@@ skipping 15 matching lines @@
 
   DISALLOW_COPY_AND_ASSIGN(VideoFrameDeliveryLog);
 };
 
 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible
 // with RenderWidgetHostViewFrameSubscriber. We create one per event type.
 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber {
  public:
   FrameSubscriber(media::VideoCaptureOracle::Event event_type,
                   const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle,
-                  VideoFrameDeliveryLog* delivery_log)
+                  VideoFrameDeliveryLog* delivery_log,
+                  base::WeakPtr<content::CursorRenderer> cursor_renderer)
       : event_type_(event_type),
         oracle_proxy_(oracle),
-        delivery_log_(delivery_log) {}
+        delivery_log_(delivery_log),
+        cursor_renderer_(cursor_renderer),
+        weak_ptr_factory_(this) {}
 
   bool ShouldCaptureFrame(
       const gfx::Rect& damage_rect,
       base::TimeTicks present_time,
       scoped_refptr<media::VideoFrame>* storage,
       RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback*
           deliver_frame_cb) override;
 
+  static void DidCaptureFrame(
+      base::WeakPtr<FrameSubscriber> frame_subscriber_,
+      const media::ThreadSafeCaptureOracle::CaptureFrameCallback&
+          capture_frame_cb,
+      const scoped_refptr<media::VideoFrame>& frame,
+      base::TimeTicks timestamp,
+      const gfx::Rect& region_in_frame,
+      bool success);
+
  private:
   const media::VideoCaptureOracle::Event event_type_;
   scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_;
   VideoFrameDeliveryLog* const delivery_log_;
+  // We need a weak pointer since FrameSubscriber is owned externally and
+  // may outlive the cursor renderer.
+  base::WeakPtr<CursorRenderer> cursor_renderer_;
+  base::WeakPtrFactory<FrameSubscriber> weak_ptr_factory_;
 };
 
 // ContentCaptureSubscription is the relationship between a RenderWidgetHost
 // whose content is updating, a subscriber that is deciding which of these
 // updates to capture (and where to deliver them to), and a callback that
 // knows how to do the capture and prepare the result for delivery.
 //
 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in
 // the RenderWidgetHostView, to process compositor updates, and (b) running a
 // timer to possibly initiate forced, non-event-driven captures needed by
@@ skipping 22 matching lines @@
  private:
   void OnTimer();
 
   // Maintain a weak reference to the RenderWidgetHost (via its routing ID),
   // since the instance could be destroyed externally during the lifetime of
   // |this|.
   const int render_process_id_;
   const int render_widget_id_;
 
   VideoFrameDeliveryLog delivery_log_;
-  FrameSubscriber timer_subscriber_;
+  scoped_ptr<FrameSubscriber> timer_subscriber_;
   CaptureCallback capture_callback_;
   base::Timer timer_;
 
+  // Responsible for tracking the cursor state and input events to make
+  // decisions and then render the mouse cursor on the video frame after
+  // capture is completed.
+  scoped_ptr<content::CursorRenderer> cursor_renderer_;
+
   DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription);
 };
 
 // Render the SkBitmap |input| into the given VideoFrame buffer |output|, then
 // invoke |done_cb| to indicate success or failure. |input| is expected to be
 // ARGB. |output| must be YV12 or I420. Colorspace conversion is always done.
 // Scaling and letterboxing will be done as needed.
 //
 // This software implementation should be used only when GPU acceleration of
 // these activities is not possible. This operation may be expensive (tens to
 // hundreds of milliseconds), so the caller should ensure that it runs on a
 // thread where such a pause would cause UI jank.
-void RenderVideoFrame(const SkBitmap& input,
-                      const scoped_refptr<media::VideoFrame>& output,
-                      const base::Callback<void(bool)>& done_cb);
+void RenderVideoFrame(
+    const SkBitmap& input,
+    const scoped_refptr<media::VideoFrame>& output,
+    const base::Callback<void(const gfx::Rect&, bool)>& done_cb);
 
 // Renews capture subscriptions based on feedback from WebContentsTracker, and
 // also executes copying of the backing store on the UI BrowserThread.
 class WebContentsCaptureMachine : public media::VideoCaptureMachine {
  public:
   WebContentsCaptureMachine(int render_process_id,
                             int main_render_frame_id,
                             bool enable_auto_throttling);
   ~WebContentsCaptureMachine() override;
 
@@ skipping 33 matching lines @@
       const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
           deliver_frame_cb,
       const SkBitmap& bitmap,
       ReadbackResponse response);
 
   // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame().
   void DidCopyFromCompositingSurfaceToVideoFrame(
       const base::TimeTicks& start_time,
       const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
           deliver_frame_cb,
+      const gfx::Rect& region_in_frame,
       bool success);
 
   // Remove the old subscription, and attempt to start a new one if |had_target|
   // is true.
   void RenewFrameSubscription(bool had_target);
 
   // Called whenever the render widget is resized.
   void UpdateCaptureSize();
 
   // Parameters saved in constructor.
@@ skipping 35 matching lines @@
 
 bool FrameSubscriber::ShouldCaptureFrame(
     const gfx::Rect& damage_rect,
     base::TimeTicks present_time,
     scoped_refptr<media::VideoFrame>* storage,
     DeliverFrameCallback* deliver_frame_cb) {
   TRACE_EVENT1("gpu.capture", "FrameSubscriber::ShouldCaptureFrame",
                "instance", this);
 
   media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb;
+
   bool oracle_decision = oracle_proxy_->ObserveEventAndDecideCapture(
       event_type_, damage_rect, present_time, storage, &capture_frame_cb);
 
   if (!capture_frame_cb.is_null())
-    *deliver_frame_cb = base::Bind(capture_frame_cb, *storage);
+    *deliver_frame_cb =
+        base::Bind(&FrameSubscriber::DidCaptureFrame,
+                   weak_ptr_factory_.GetWeakPtr(), capture_frame_cb, *storage);
   if (oracle_decision)
     delivery_log_->ChronicleFrameDelivery(present_time);
   return oracle_decision;
 }
 
+void FrameSubscriber::DidCaptureFrame(
+    base::WeakPtr<FrameSubscriber> frame_subscriber_,
+    const media::ThreadSafeCaptureOracle::CaptureFrameCallback&
+        capture_frame_cb,
+    const scoped_refptr<media::VideoFrame>& frame,
+    base::TimeTicks timestamp,
+    const gfx::Rect& region_in_frame,
+    bool success) {
+  // We can get a callback in the shutdown sequence for the browser main loop
+  // and this can result in a DCHECK failure. Avoid this by doing DCHECK only
+  // on success.
+  if (success) {
+    DCHECK_CURRENTLY_ON(BrowserThread::UI);
+    // TODO(isheriff): Unclear if taking a snapshot of cursor here affects user
+    // experience in any particular scenarios. Doing it prior to capture may
+    // require evaluating region_in_frame in this file.
+    if (frame_subscriber_ && frame_subscriber_->cursor_renderer_ && success) {

[miu, 2015/11/06 00:22:54]

Don't need the "&& success" at the end anymore.

[Irfan, 2015/11/06 17:49:26]

Done.

+      if (frame_subscriber_->cursor_renderer_->SnapshotCursorState(
+              region_in_frame))
+        frame_subscriber_->cursor_renderer_->RenderOnVideoFrame(frame);
+    }
+  }
+  capture_frame_cb.Run(frame, timestamp, success);
+}
+
 ContentCaptureSubscription::ContentCaptureSubscription(
     const RenderWidgetHost& source,
     const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy,
     const CaptureCallback& capture_callback)
     : render_process_id_(source.GetProcess()->GetID()),
       render_widget_id_(source.GetRoutingID()),
       delivery_log_(),
-      timer_subscriber_(media::VideoCaptureOracle::kTimerPoll, oracle_proxy,
-                        &delivery_log_),
       capture_callback_(capture_callback),
       timer_(true, true) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 
   RenderWidgetHostView* const view = source.GetView();
+#if defined(USE_AURA)
+  if (view)
+    cursor_renderer_.reset(
+        new content::CursorRendererAura(view->GetNativeView()));
+#endif
+  timer_subscriber_.reset(new FrameSubscriber(
+      media::VideoCaptureOracle::kTimerPoll, oracle_proxy, &delivery_log_,
+      cursor_renderer_ ? cursor_renderer_->GetWeakPtr()
+                       : base::WeakPtr<CursorRenderer>()));
 
   // Subscribe to compositor updates. These will be serviced directly by the
   // oracle.
   if (view) {
     scoped_ptr<RenderWidgetHostViewFrameSubscriber> subscriber(
         new FrameSubscriber(media::VideoCaptureOracle::kCompositorUpdate,
-            oracle_proxy, &delivery_log_));
+                            oracle_proxy, &delivery_log_,
+                            cursor_renderer_
+                                ? cursor_renderer_->GetWeakPtr()
+                                : base::WeakPtr<CursorRenderer>()));
     view->BeginFrameSubscription(subscriber.Pass());
   }
 
   // Subscribe to timer events. This instance will service these as well.
   timer_.Start(FROM_HERE,
                std::max(oracle_proxy->min_capture_period(),
                         base::TimeDelta::FromMilliseconds(media
                             ::VideoCaptureOracle::kMinTimerPollPeriodMillis)),
                base::Bind(&ContentCaptureSubscription::OnTimer,
                           base::Unretained(this)));
@@ skipping 15 matching lines @@
 }
 
 void ContentCaptureSubscription::OnTimer() {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   TRACE_EVENT0("gpu.capture", "ContentCaptureSubscription::OnTimer");
 
   scoped_refptr<media::VideoFrame> frame;
   RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb;
 
   const base::TimeTicks start_time = base::TimeTicks::Now();
-  if (timer_subscriber_.ShouldCaptureFrame(gfx::Rect(),
-                                           start_time,
-                                           &frame,
-                                           &deliver_frame_cb)) {
+  if (timer_subscriber_->ShouldCaptureFrame(gfx::Rect(), start_time, &frame,
+                                            &deliver_frame_cb)) {
     capture_callback_.Run(start_time, frame, deliver_frame_cb);
   }
 }
 
-void RenderVideoFrame(const SkBitmap& input,
-                      const scoped_refptr<media::VideoFrame>& output,
-                      const base::Callback<void(bool)>& done_cb) {
-  base::ScopedClosureRunner failure_handler(base::Bind(done_cb, false));
+// ScopedClosureRunOnBrowserThread ensures that the closure always executes
+// (on the browser thread) no matter how the current scope exits.
+class ScopedClosureRunOnBrowserThread {

[miu, 2015/11/06 00:22:54]

There's a common solution to this problem: media::BindToCurrentLoop().

What I suggest is, in WebContentsCaptureMachine::DidCopyFromBackingStore(), make
the render_thread_->PostTask() statement look like this:

#include "media/base/bind_to_current_loop.h"

render_thread_->task_runner()->PostTask(
    FROM_HERE,
    media::BindToCurrentLoop(
        base::Bind(&RenderVideoFrame, bitmap, target,
                              base::Bind(deliver_frame_cb, start_time))));

This puts an implicit "trampoline" back to the main thread when the callback is
run.  Then, you can just use the base::ScopedClosureRunner in RenderVideoFrame()
like it was before.

[Irfan, 2015/11/06 17:49:26]

Aha, excellent

+ public:
+  ScopedClosureRunOnBrowserThread(const base::Closure& closure);
+  ~ScopedClosureRunOnBrowserThread();
+  void Release();
+
+ private:
+  base::Closure closure_;
+
+  DISALLOW_COPY_AND_ASSIGN(ScopedClosureRunOnBrowserThread);
+};
+
+ScopedClosureRunOnBrowserThread::ScopedClosureRunOnBrowserThread(
+    const base::Closure& closure)
+    : closure_(closure) {}
+
+ScopedClosureRunOnBrowserThread::~ScopedClosureRunOnBrowserThread() {
+  if (!closure_.is_null())
+    BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, closure_);
+}
+
+void ScopedClosureRunOnBrowserThread::Release() {
+  closure_.Reset();
+}
+
+void RenderVideoFrame(
+    const SkBitmap& input,
+    const scoped_refptr<media::VideoFrame>& output,
+    const base::Callback<void(const gfx::Rect&, bool)>& done_cb) {
+  ScopedClosureRunOnBrowserThread failure_handler(
+      base::Bind(done_cb, gfx::Rect(), false));
 
   SkAutoLockPixels locker(input);
 
   // Sanity-check the captured bitmap.
   if (input.empty() ||
       !input.readyToDraw() ||
       input.colorType() != kN32_SkColorType ||
       input.width() < 2 || input.height() < 2) {
     DVLOG(1) << "input unacceptable (size="
              << input.getSize()
@@ skipping 10 matching lines @@
 
   // Calculate the width and height of the content region in the |output|, based
   // on the aspect ratio of |input|.
   const gfx::Rect region_in_frame = media::ComputeLetterboxRegion(
       output->visible_rect(), gfx::Size(input.width(), input.height()));
 
   // Scale the bitmap to the required size, if necessary.
   SkBitmap scaled_bitmap;
   if (input.width() != region_in_frame.width() ||
       input.height() != region_in_frame.height()) {
-
     skia::ImageOperations::ResizeMethod method;
     if (input.width() < region_in_frame.width() ||
         input.height() < region_in_frame.height()) {
       // Avoid box filtering when magnifying, because it's actually
       // nearest-neighbor.
       method = skia::ImageOperations::RESIZE_HAMMING1;
     } else {
       method = skia::ImageOperations::RESIZE_BOX;
     }
 
@@ skipping 19 matching lines @@
 
     SkAutoLockPixels scaled_bitmap_locker(scaled_bitmap);
     media::CopyRGBToVideoFrame(
         reinterpret_cast<uint8*>(scaled_bitmap.getPixels()),
         scaled_bitmap.rowBytes(),
         region_in_yv12_frame,
         output.get());
   }
 
   // The result is now ready.
-  ignore_result(failure_handler.Release());
-  done_cb.Run(true);
+  failure_handler.Release();
+  BrowserThread::PostTask(BrowserThread::UI, FROM_HERE,
+                          base::Bind(done_cb, region_in_frame, true));
 }
 
 VideoFrameDeliveryLog::VideoFrameDeliveryLog()
     : last_frame_rate_log_time_(),
       count_frames_rendered_(0) {
 }
 
 void VideoFrameDeliveryLog::ChronicleFrameDelivery(base::TimeTicks frame_time) {
   // Log frame rate, if verbose logging is turned on.
   static const base::TimeDelta kFrameRateLogInterval =
@@ skipping 123 matching lines @@
     const base::TimeTicks& start_time,
     const scoped_refptr<media::VideoFrame>& target,
     const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
         deliver_frame_cb) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 
   RenderWidgetHost* rwh = tracker_->GetTargetRenderWidgetHost();
   RenderWidgetHostViewBase* view =
       rwh ? static_cast<RenderWidgetHostViewBase*>(rwh->GetView()) : NULL;
   if (!view) {
-    deliver_frame_cb.Run(base::TimeTicks(), false);
+    deliver_frame_cb.Run(base::TimeTicks(), gfx::Rect(), false);
     return;
   }
 
   gfx::Size view_size = view->GetViewBounds().size();
   if (view_size != last_view_size_) {
     last_view_size_ = view_size;
 
     // Measure the number of kilopixels.
     UMA_HISTOGRAM_COUNTS_10000(
         "TabCapture.ViewChangeKiloPixels",
@@ skipping 104 matching lines @@
   if (response == READBACK_SUCCESS) {
     UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeBitmap", now - start_time);
     TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", target.get(),
                                  "Render");
     render_thread_->task_runner()->PostTask(
         FROM_HERE, base::Bind(&RenderVideoFrame, bitmap, target,
                               base::Bind(deliver_frame_cb, start_time)));
   } else {
     // Capture can fail due to transient issues, so just skip this frame.
     DVLOG(1) << "CopyFromBackingStore failed; skipping frame.";
-    deliver_frame_cb.Run(start_time, false);
+    deliver_frame_cb.Run(start_time, gfx::Rect(), false);
   }
 }
 
 void WebContentsCaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame(
     const base::TimeTicks& start_time,
     const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
         deliver_frame_cb,
+    const gfx::Rect& region_in_frame,
     bool success) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   base::TimeTicks now = base::TimeTicks::Now();
 
   if (success) {
     UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeVideoFrame", now - start_time);
   } else {
     // Capture can fail due to transient issues, so just skip this frame.
     DVLOG(1) << "CopyFromCompositingSurface failed; skipping frame.";
   }
-  deliver_frame_cb.Run(start_time, success);
+  deliver_frame_cb.Run(start_time, region_in_frame, success);
 }
 
 void WebContentsCaptureMachine::RenewFrameSubscription(bool had_target) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 
   RenderWidgetHost* const rwh =
       had_target ? tracker_->GetTargetRenderWidgetHost() : nullptr;
 
   // Always destroy the old subscription before creating a new one.
   const bool had_subscription = !!subscription_;
@@ skipping 82 matching lines @@
     scoped_ptr<Client> client) {
   DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString();
   core_->AllocateAndStart(params, client.Pass());
 }
 
 void WebContentsVideoCaptureDevice::StopAndDeAllocate() {
   core_->StopAndDeAllocate();
 }
 
 }  // namespace content