cast: support cursor rendering for tab capture

content/browser/media/capture/web_contents_video_capture_device.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // Implementation notes: This needs to work on a variety of hardware
 // configurations where the speed of the CPU and GPU greatly affect overall
 // performance. Spanning several threads, the process of capturing has been
 // split up into four conceptual stages:
 //
 //   1. Reserve Buffer: Before a frame can be captured, a slot in the client's
@@ skipping 32 matching lines @@
 //
 // In the above example, both capturing and rendering *each* take almost the
 // full 33 ms available between frames, yet we see that the required throughput
 // is obtained.
 //
 // Turning on verbose logging will cause the effective frame rate to be logged
 // at 5-second intervals.
 
 #include "content/browser/media/capture/web_contents_video_capture_device.h"
 
+#include <algorithm>

[xjz, 2015/10/22 17:08:42]

Still think no need to include this.

[Irfan, 2015/10/26 23:10:32]

This was a lint error unrelated to my code change

+
 #include "base/basictypes.h"
 #include "base/bind.h"
 #include "base/callback_helpers.h"
 #include "base/location.h"
 #include "base/logging.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/memory/weak_ptr.h"
 #include "base/metrics/histogram.h"
 #include "base/sequenced_task_runner.h"
 #include "base/single_thread_task_runner.h"
 #include "base/threading/thread.h"
 #include "base/threading/thread_checker.h"
 #include "base/time/time.h"
+#include "content/browser/media/capture/cursor_renderer.h"
 #include "content/browser/media/capture/web_contents_capture_util.h"
 #include "content/browser/media/capture/web_contents_tracker.h"
 #include "content/browser/renderer_host/render_widget_host_impl.h"
 #include "content/browser/renderer_host/render_widget_host_view_base.h"
 #include "content/public/browser/browser_thread.h"
 #include "content/public/browser/render_process_host.h"
 #include "content/public/browser/render_widget_host_view.h"
 #include "content/public/browser/render_widget_host_view_frame_subscriber.h"
 #include "content/public/browser/web_contents.h"
 #include "media/base/video_capture_types.h"
 #include "media/base/video_util.h"
 #include "media/capture/content/screen_capture_device_core.h"
 #include "media/capture/content/thread_safe_capture_oracle.h"
 #include "media/capture/content/video_capture_oracle.h"
 #include "skia/ext/image_operations.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 #include "third_party/skia/include/core/SkColor.h"
 #include "ui/base/layout.h"
 #include "ui/gfx/geometry/dip_util.h"
 #include "ui/gfx/geometry/size_conversions.h"
 
+#if defined(USE_AURA)
+#include "content/browser/media/capture/cursor_renderer_aura.h"
+#else
+#include "content/browser/media/capture/cursor_renderer_dummy.h"

[miu, 2015/10/23 01:55:13]

Suggest you don't provide the #else clause.  See comments below.

[Irfan, 2015/10/26 23:10:31]

Done.

+#endif
+
 namespace content {
 
 namespace {
 
 void DeleteOnWorkerThread(scoped_ptr<base::Thread> render_thread,
                           const base::Closure& callback) {
   render_thread.reset();
 
   // After thread join call the callback on UI thread.
   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, callback);
@@ skipping 15 matching lines @@
 
   DISALLOW_COPY_AND_ASSIGN(VideoFrameDeliveryLog);
 };
 
 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible
 // with RenderWidgetHostViewFrameSubscriber. We create one per event type.
 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber {
  public:
   FrameSubscriber(media::VideoCaptureOracle::Event event_type,
                   const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle,
-                  VideoFrameDeliveryLog* delivery_log)
+                  VideoFrameDeliveryLog* delivery_log,
+                  base::WeakPtr<content::CursorRenderer> cursor_renderer)
       : event_type_(event_type),
         oracle_proxy_(oracle),
-        delivery_log_(delivery_log) {}
+        delivery_log_(delivery_log),
+        cursor_renderer_(cursor_renderer),
+        weak_ptr_factory_(this) {}
 
   bool ShouldCaptureFrame(
       const gfx::Rect& damage_rect,
       base::TimeTicks present_time,
       scoped_refptr<media::VideoFrame>* storage,
       RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback*
           deliver_frame_cb) override;
 
+  static void DidCaptureFrame(
+      base::WeakPtr<FrameSubscriber> frame_subscriber_,
+      media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb,

[miu, 2015/10/23 01:55:13]

Please pass the callback by const reference.  The use of base::Bind() will
create a copy.

[Irfan, 2015/10/26 23:10:32]

Done.

+      const scoped_refptr<media::VideoFrame>& frame,
+      base::TimeTicks timestamp,
+      gfx::Rect region_in_frame,

[miu, 2015/10/23 01:55:13]

The gfx::Rect should be passed by const reference as well.

[Irfan, 2015/10/26 23:10:32]

Done.

+      bool success);
+
  private:
   const media::VideoCaptureOracle::Event event_type_;
   scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_;
   VideoFrameDeliveryLog* const delivery_log_;
+  base::WeakPtr<content::CursorRenderer> cursor_renderer_;

[miu, 2015/10/23 01:55:13]

Please document why we need a weak pointer to the CursorRenderer (because
FrameSubscriber is owned externally, and may outlive it; and if FrameSubscriber
owned it, there'd be two duplicate instances).

[miu, 2015/10/23 01:55:13]

No need for content:: qualifier.

[Irfan, 2015/10/26 23:10:31]

Done.

[Irfan, 2015/10/26 23:10:32]

Done.

+  base::WeakPtrFactory<FrameSubscriber> weak_ptr_factory_;
 };
 
 // ContentCaptureSubscription is the relationship between a RenderWidgetHost
 // whose content is updating, a subscriber that is deciding which of these
 // updates to capture (and where to deliver them to), and a callback that
 // knows how to do the capture and prepare the result for delivery.
 //
 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in
 // the RenderWidgetHostView, to process compositor updates, and (b) running a
 // timer to possibly initiate forced, non-event-driven captures needed by
@@ skipping 22 matching lines @@
  private:
   void OnTimer();
 
   // Maintain a weak reference to the RenderWidgetHost (via its routing ID),
   // since the instance could be destroyed externally during the lifetime of
   // |this|.
   const int render_process_id_;
   const int render_widget_id_;
 
   VideoFrameDeliveryLog delivery_log_;
-  FrameSubscriber timer_subscriber_;
+  scoped_ptr<FrameSubscriber> timer_subscriber_;
   CaptureCallback capture_callback_;
   base::Timer timer_;
 
+  // Responsible for tracking the cursor state and input events to make
+  // decisions and then render the mouse cursor on the video frame after
+  // capture is completed.
+  scoped_ptr<content::CursorRenderer> cursor_renderer_;
+
   DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription);
 };
 
 // Render the SkBitmap |input| into the given VideoFrame buffer |output|, then
 // invoke |done_cb| to indicate success or failure. |input| is expected to be
 // ARGB. |output| must be YV12 or I420. Colorspace conversion is always done.
 // Scaling and letterboxing will be done as needed.
 //
 // This software implementation should be used only when GPU acceleration of
 // these activities is not possible. This operation may be expensive (tens to
 // hundreds of milliseconds), so the caller should ensure that it runs on a
 // thread where such a pause would cause UI jank.
 void RenderVideoFrame(const SkBitmap& input,
                       const scoped_refptr<media::VideoFrame>& output,
-                      const base::Callback<void(bool)>& done_cb);
+                      const base::Callback<void(gfx::Rect, bool)>& done_cb);
 
 // Renews capture subscriptions based on feedback from WebContentsTracker, and
 // also executes copying of the backing store on the UI BrowserThread.
 class WebContentsCaptureMachine : public media::VideoCaptureMachine {
  public:
   WebContentsCaptureMachine(int render_process_id,
                             int main_render_frame_id,
                             bool enable_auto_throttling);
   ~WebContentsCaptureMachine() override;
 
@@ skipping 33 matching lines @@
       const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
           deliver_frame_cb,
       const SkBitmap& bitmap,
       ReadbackResponse response);
 
   // Response callback for RWHVP::CopyFromCompositingSurfaceToVideoFrame().
   void DidCopyFromCompositingSurfaceToVideoFrame(
       const base::TimeTicks& start_time,
       const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
           deliver_frame_cb,
+      gfx::Rect region_in_frame,
       bool success);
 
   // Remove the old subscription, and attempt to start a new one if |had_target|
   // is true.
   void RenewFrameSubscription(bool had_target);
 
   // Called whenever the render widget is resized.
   void UpdateCaptureSize();
 
   // Parameters saved in constructor.
@@ skipping 35 matching lines @@
 
 bool FrameSubscriber::ShouldCaptureFrame(
     const gfx::Rect& damage_rect,
     base::TimeTicks present_time,
     scoped_refptr<media::VideoFrame>* storage,
     DeliverFrameCallback* deliver_frame_cb) {
   TRACE_EVENT1("gpu.capture", "FrameSubscriber::ShouldCaptureFrame",
                "instance", this);
 
   media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb;
+
   bool oracle_decision = oracle_proxy_->ObserveEventAndDecideCapture(
       event_type_, damage_rect, present_time, storage, &capture_frame_cb);
 
   if (!capture_frame_cb.is_null())
-    *deliver_frame_cb = base::Bind(capture_frame_cb, *storage);
+    *deliver_frame_cb =
+        base::Bind(&FrameSubscriber::DidCaptureFrame,
+                   weak_ptr_factory_.GetWeakPtr(), capture_frame_cb, *storage);
   if (oracle_decision)
     delivery_log_->ChronicleFrameDelivery(present_time);
   return oracle_decision;
 }
 
+void FrameSubscriber::DidCaptureFrame(
+    base::WeakPtr<FrameSubscriber> frame_subscriber_,
+    media::ThreadSafeCaptureOracle::CaptureFrameCallback capture_frame_cb,
+    const scoped_refptr<media::VideoFrame>& frame,
+    base::TimeTicks timestamp,
+    gfx::Rect region_in_frame,
+    bool success) {
+  DCHECK_CURRENTLY_ON(BrowserThread::UI);
+  if (frame_subscriber_ && frame_subscriber_->cursor_renderer_ && success) {
+    if (frame_subscriber_->cursor_renderer_->Update(region_in_frame))

[miu, 2015/10/23 01:55:13]

Some considerations on the timing of the call to Update():

In AuraWindowCaptureMachine, Update() is being called to snapshot the cursor and
its position before the YUV readback.  Here, in FrameSubscriber, it's being
called after readback.  Since the operations on the GPU could take up to 100 ms
(or more), you might be snapshotting significantly old data and drawing the
cursor in an old spot.

Going even deeper, maybe AuraWindowCaptureMachine is doing this wrong too. 
Perhaps the call to Update() for both AWCM and FrameSubscriber should be at the
moment the CopyOutputRequest is being issued?

[Irfan, 2015/10/26 23:10:32]

I went with a comment to see if it affects any scenarios. Doing it here prior to
capture requires figuring region_in_frame calculations and am not sure about
restructuring that since we wanted cursor rendering out of delegated_frame_host
in first place.

+      frame_subscriber_->cursor_renderer_->RenderOnVideoFrame(frame);
+  }
+  capture_frame_cb.Run(frame, timestamp, success);
+}
+
 ContentCaptureSubscription::ContentCaptureSubscription(
     const RenderWidgetHost& source,
     const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy,
     const CaptureCallback& capture_callback)
     : render_process_id_(source.GetProcess()->GetID()),
       render_widget_id_(source.GetRoutingID()),
       delivery_log_(),
-      timer_subscriber_(media::VideoCaptureOracle::kTimerPoll, oracle_proxy,
-                        &delivery_log_),
       capture_callback_(capture_callback),
       timer_(true, true) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 
   RenderWidgetHostView* const view = source.GetView();
+  if (view)
+#if defined(USE_AURA)
+    cursor_renderer_.reset(
+        new content::CursorRendererAura(view->GetNativeView()));
+#else
+    cursor_renderer_.reset(

[miu, 2015/10/23 01:55:13]

Please remove the #else clause here.  In Chromium code, we try hard not to waste
memory (even for a simple no-op stub).  It's fine if |cursor_renderer_| is null.

For passing the WeakPtr to the frame subscriber constructors, you can do
something like:

  base::WeakPtr<CursorRenderer> weak_cursor_renderer;
#if defined(USE_AURA)
  cursor_renderer_.reset(new CursorRendererAura(...));
  weak_cursor_renderer_ = cursor_renderer_->GetWeakPtr();
#endif

[Irfan, 2015/10/26 23:10:32]

Done.

+        new content::CursorRendererDummy(view->GetNativeView()));
+#endif
+  timer_subscriber_.reset(
+      new FrameSubscriber(media::VideoCaptureOracle::kTimerPoll, oracle_proxy,
+                          &delivery_log_, cursor_renderer_->GetWeakPtr()));
 
   // Subscribe to compositor updates. These will be serviced directly by the
   // oracle.
   if (view) {
     scoped_ptr<RenderWidgetHostViewFrameSubscriber> subscriber(
         new FrameSubscriber(media::VideoCaptureOracle::kCompositorUpdate,
-            oracle_proxy, &delivery_log_));
+                            oracle_proxy, &delivery_log_,
+                            cursor_renderer_->GetWeakPtr()));
     view->BeginFrameSubscription(subscriber.Pass());
   }
 
   // Subscribe to timer events. This instance will service these as well.
   timer_.Start(FROM_HERE,
                std::max(oracle_proxy->min_capture_period(),
                         base::TimeDelta::FromMilliseconds(media
                             ::VideoCaptureOracle::kMinTimerPollPeriodMillis)),
                base::Bind(&ContentCaptureSubscription::OnTimer,
                           base::Unretained(this)));
@@ skipping 15 matching lines @@
 }
 
 void ContentCaptureSubscription::OnTimer() {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   TRACE_EVENT0("gpu.capture", "ContentCaptureSubscription::OnTimer");
 
   scoped_refptr<media::VideoFrame> frame;
   RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback deliver_frame_cb;
 
   const base::TimeTicks start_time = base::TimeTicks::Now();
-  if (timer_subscriber_.ShouldCaptureFrame(gfx::Rect(),
-                                           start_time,
-                                           &frame,
-                                           &deliver_frame_cb)) {
+  if (timer_subscriber_->ShouldCaptureFrame(gfx::Rect(), start_time, &frame,
+                                            &deliver_frame_cb)) {
     capture_callback_.Run(start_time, frame, deliver_frame_cb);
   }
 }
 
 void RenderVideoFrame(const SkBitmap& input,
                       const scoped_refptr<media::VideoFrame>& output,
-                      const base::Callback<void(bool)>& done_cb) {
-  base::ScopedClosureRunner failure_handler(base::Bind(done_cb, false));
+                      const base::Callback<void(gfx::Rect, bool)>& done_cb) {
+  base::ScopedClosureRunner failure_handler(
+      base::Bind(done_cb, gfx::Rect(), false));
 
   SkAutoLockPixels locker(input);
 
   // Sanity-check the captured bitmap.
   if (input.empty() ||
       !input.readyToDraw() ||
       input.colorType() != kN32_SkColorType ||
       input.width() < 2 || input.height() < 2) {
     DVLOG(1) << "input unacceptable (size="
              << input.getSize()
@@ skipping 10 matching lines @@
 
   // Calculate the width and height of the content region in the |output|, based
   // on the aspect ratio of |input|.
   const gfx::Rect region_in_frame = media::ComputeLetterboxRegion(
       output->visible_rect(), gfx::Size(input.width(), input.height()));
 
   // Scale the bitmap to the required size, if necessary.
   SkBitmap scaled_bitmap;
   if (input.width() != region_in_frame.width() ||
       input.height() != region_in_frame.height()) {
-
     skia::ImageOperations::ResizeMethod method;
     if (input.width() < region_in_frame.width() ||
         input.height() < region_in_frame.height()) {
       // Avoid box filtering when magnifying, because it's actually
       // nearest-neighbor.
       method = skia::ImageOperations::RESIZE_HAMMING1;
     } else {
       method = skia::ImageOperations::RESIZE_BOX;
     }
 
@@ skipping 20 matching lines @@
     SkAutoLockPixels scaled_bitmap_locker(scaled_bitmap);
     media::CopyRGBToVideoFrame(
         reinterpret_cast<uint8*>(scaled_bitmap.getPixels()),
         scaled_bitmap.rowBytes(),
         region_in_yv12_frame,
         output.get());
   }
 
   // The result is now ready.
   ignore_result(failure_handler.Release());
-  done_cb.Run(true);
+  done_cb.Run(region_in_frame, true);
 }
 
 VideoFrameDeliveryLog::VideoFrameDeliveryLog()
     : last_frame_rate_log_time_(),
       count_frames_rendered_(0) {
 }
 
 void VideoFrameDeliveryLog::ChronicleFrameDelivery(base::TimeTicks frame_time) {
   // Log frame rate, if verbose logging is turned on.
   static const base::TimeDelta kFrameRateLogInterval =
@@ skipping 103 matching lines @@
   // The following cancels any outstanding callbacks and causes IsStarted() to
   // return false from here onward.
   weak_ptr_factory_.InvalidateWeakPtrs();
 
   // Note: RenewFrameSubscription() must be called before stopping |tracker_| so
   // the web_contents() can be notified that the capturing is ending.
   RenewFrameSubscription(false);
   tracker_->Stop();
 
   // The render thread cannot be stopped on the UI thread, so post a message
-  // to the thread pool used for blocking operations.
+  // to the thread pool used; for blocking operations.

[xjz, 2015/10/22 17:08:42]

nit: remove ';'.

   if (render_thread_.get()) {
     BrowserThread::PostBlockingPoolTask(
         FROM_HERE,
         base::Bind(&DeleteOnWorkerThread, base::Passed(&render_thread_),
                    callback));
   }
 }
 
 void WebContentsCaptureMachine::Capture(
     const base::TimeTicks& start_time,
     const scoped_refptr<media::VideoFrame>& target,
     const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
         deliver_frame_cb) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 
   RenderWidgetHost* rwh = tracker_->GetTargetRenderWidgetHost();
   RenderWidgetHostViewBase* view =
       rwh ? static_cast<RenderWidgetHostViewBase*>(rwh->GetView()) : NULL;
   if (!view) {
-    deliver_frame_cb.Run(base::TimeTicks(), false);
+    deliver_frame_cb.Run(base::TimeTicks(), gfx::Rect(), false);
     return;
   }
 
   gfx::Size view_size = view->GetViewBounds().size();
   if (view_size != last_view_size_) {
     last_view_size_ = view_size;
 
     // Measure the number of kilopixels.
     UMA_HISTOGRAM_COUNTS_10000(
         "TabCapture.ViewChangeKiloPixels",
@@ skipping 104 matching lines @@
   if (response == READBACK_SUCCESS) {
     UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeBitmap", now - start_time);
     TRACE_EVENT_ASYNC_STEP_INTO0("gpu.capture", "Capture", target.get(),
                                  "Render");
     render_thread_->task_runner()->PostTask(
         FROM_HERE, base::Bind(&RenderVideoFrame, bitmap, target,
                               base::Bind(deliver_frame_cb, start_time)));
   } else {
     // Capture can fail due to transient issues, so just skip this frame.
     DVLOG(1) << "CopyFromBackingStore failed; skipping frame.";
-    deliver_frame_cb.Run(start_time, false);
+    deliver_frame_cb.Run(start_time, gfx::Rect(), false);
   }
 }
 
 void WebContentsCaptureMachine::DidCopyFromCompositingSurfaceToVideoFrame(
     const base::TimeTicks& start_time,
     const RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback&
         deliver_frame_cb,
+    gfx::Rect region_in_frame,
     bool success) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
   base::TimeTicks now = base::TimeTicks::Now();
 
   if (success) {
     UMA_HISTOGRAM_TIMES("TabCapture.CopyTimeVideoFrame", now - start_time);
   } else {
     // Capture can fail due to transient issues, so just skip this frame.
     DVLOG(1) << "CopyFromCompositingSurface failed; skipping frame.";
   }
-  deliver_frame_cb.Run(start_time, success);
+  deliver_frame_cb.Run(start_time, region_in_frame, success);
 }
 
 void WebContentsCaptureMachine::RenewFrameSubscription(bool had_target) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 
   RenderWidgetHost* const rwh =
       had_target ? tracker_->GetTargetRenderWidgetHost() : nullptr;
 
   // Always destroy the old subscription before creating a new one.
   const bool had_subscription = !!subscription_;
@@ skipping 82 matching lines @@
     scoped_ptr<Client> client) {
   DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString();
   core_->AllocateAndStart(params, client.Pass());
 }
 
 void WebContentsVideoCaptureDevice::StopAndDeAllocate() {
   core_->StopAndDeAllocate();
 }
 
 }  // namespace content