cast: Support for low-latency interactive mode

content/browser/media/capture/web_contents_video_capture_device.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // Implementation notes: This needs to work on a variety of hardware
 // configurations where the speed of the CPU and GPU greatly affect overall
 // performance. Spanning several threads, the process of capturing has been
 // split up into four conceptual stages:
 //
 //   1. Reserve Buffer: Before a frame can be captured, a slot in the client's
@@ skipping 50 matching lines @@
 #include "base/memory/weak_ptr.h"
 #include "base/metrics/histogram.h"
 #include "base/sequenced_task_runner.h"
 #include "base/single_thread_task_runner.h"
 #include "base/threading/thread.h"
 #include "base/threading/thread_checker.h"
 #include "base/time/time.h"
 #include "content/browser/media/capture/cursor_renderer.h"
 #include "content/browser/media/capture/web_contents_capture_util.h"
 #include "content/browser/media/capture/web_contents_tracker.h"
+#include "content/browser/media/capture/window_activity_tracker.h"
 #include "content/browser/renderer_host/render_widget_host_impl.h"
 #include "content/browser/renderer_host/render_widget_host_view_base.h"
 #include "content/public/browser/browser_thread.h"
 #include "content/public/browser/render_process_host.h"
 #include "content/public/browser/render_widget_host_view.h"
 #include "content/public/browser/render_widget_host_view_frame_subscriber.h"
 #include "content/public/browser/web_contents.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/video_capture_types.h"
+#include "media/base/video_frame_metadata.h"
 #include "media/base/video_util.h"
 #include "media/capture/content/screen_capture_device_core.h"
 #include "media/capture/content/thread_safe_capture_oracle.h"
 #include "media/capture/content/video_capture_oracle.h"
 #include "skia/ext/image_operations.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 #include "third_party/skia/include/core/SkColor.h"
 #include "ui/base/layout.h"
 #include "ui/gfx/geometry/dip_util.h"
 #include "ui/gfx/geometry/size_conversions.h"
 
 #if defined(USE_AURA)
 #include "content/browser/media/capture/cursor_renderer_aura.h"
+#include "content/browser/media/capture/window_activity_tracker_aura.h"
 #endif
 
 namespace content {
 
 namespace {
 
+enum InteractiveModeSettings {
+  // Minimum amount of time for which there should be no animation detected
+  // to consider interactive mode being active. This is to prevent very brief
+  // periods of animated content not being detected (due to CPU fluctations for
+  // example) from causing a flip-flop on interactive mode.
+  kMinPeriodNoAnimationMillis = 3000
+};
+
 void DeleteOnWorkerThread(scoped_ptr<base::Thread> render_thread,
                           const base::Closure& callback) {
   render_thread.reset();
 
   // After thread join call the callback on UI thread.
   BrowserThread::PostTask(BrowserThread::UI, FROM_HERE, callback);
 }
 
 // Responsible for logging the effective frame rate.
 class VideoFrameDeliveryLog {
@@ skipping 12 matching lines @@
   DISALLOW_COPY_AND_ASSIGN(VideoFrameDeliveryLog);
 };
 
 // FrameSubscriber is a proxy to the ThreadSafeCaptureOracle that's compatible
 // with RenderWidgetHostViewFrameSubscriber. We create one per event type.
 class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber {
  public:
   FrameSubscriber(media::VideoCaptureOracle::Event event_type,
                   const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle,
                   VideoFrameDeliveryLog* delivery_log,
-                  base::WeakPtr<content::CursorRenderer> cursor_renderer)
+                  base::WeakPtr<content::CursorRenderer> cursor_renderer,
+                  base::WeakPtr<content::WindowActivityTracker> tracker)
       : event_type_(event_type),
         oracle_proxy_(oracle),
         delivery_log_(delivery_log),
         cursor_renderer_(cursor_renderer),
+        window_activity_tracker_(tracker),
         weak_ptr_factory_(this) {}
 
   bool ShouldCaptureFrame(
       const gfx::Rect& damage_rect,
       base::TimeTicks present_time,
       scoped_refptr<media::VideoFrame>* storage,
       RenderWidgetHostViewFrameSubscriber::DeliverFrameCallback*
           deliver_frame_cb) override;
 
   static void DidCaptureFrame(
       base::WeakPtr<FrameSubscriber> frame_subscriber_,
       const media::ThreadSafeCaptureOracle::CaptureFrameCallback&
           capture_frame_cb,
       const scoped_refptr<media::VideoFrame>& frame,
       base::TimeTicks timestamp,
       const gfx::Rect& region_in_frame,
       bool success);
 
+  bool IsUserInteractingWithContent();
+
  private:
   const media::VideoCaptureOracle::Event event_type_;
   scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_;
   VideoFrameDeliveryLog* const delivery_log_;
   // We need a weak pointer since FrameSubscriber is owned externally and
   // may outlive the cursor renderer.
   base::WeakPtr<CursorRenderer> cursor_renderer_;
+  // We need a weak pointer since FrameSubscriber is owned externally and
+  // may outlive the ui activity tracker.
+  base::WeakPtr<WindowActivityTracker> window_activity_tracker_;
   base::WeakPtrFactory<FrameSubscriber> weak_ptr_factory_;
 };
 
 // ContentCaptureSubscription is the relationship between a RenderWidgetHost
 // whose content is updating, a subscriber that is deciding which of these
 // updates to capture (and where to deliver them to), and a callback that
 // knows how to do the capture and prepare the result for delivery.
 //
 // In practice, this means (a) installing a RenderWidgetHostFrameSubscriber in
 // the RenderWidgetHostView, to process compositor updates, and (b) running a
@@ skipping 32 matching lines @@
   VideoFrameDeliveryLog delivery_log_;
   scoped_ptr<FrameSubscriber> timer_subscriber_;
   CaptureCallback capture_callback_;
   base::Timer timer_;
 
   // Responsible for tracking the cursor state and input events to make
   // decisions and then render the mouse cursor on the video frame after
   // capture is completed.
   scoped_ptr<content::CursorRenderer> cursor_renderer_;
 
+  // Responsible for tracking the UI events and making a decision on whether
+  // user is actively interacting with content.
+  scoped_ptr<content::WindowActivityTracker> window_activity_tracker_;
+
   DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription);
 };
 
 // Render the SkBitmap |input| into the given VideoFrame buffer |output|, then
 // invoke |done_cb| to indicate success or failure. |input| is expected to be
 // ARGB. |output| must be YV12 or I420. Colorspace conversion is always done.
 // Scaling and letterboxing will be done as needed.
 //
 // This software implementation should be used only when GPU acceleration of
 // these activities is not possible. This operation may be expensive (tens to
@@ skipping 135 matching lines @@
     bool success) {
   // We can get a callback in the shutdown sequence for the browser main loop
   // and this can result in a DCHECK failure. Avoid this by doing DCHECK only
   // on success.
   if (success) {
     DCHECK_CURRENTLY_ON(BrowserThread::UI);
     // TODO(isheriff): Unclear if taking a snapshot of cursor here affects user
     // experience in any particular scenarios. Doing it prior to capture may
     // require evaluating region_in_frame in this file.
     if (frame_subscriber_ && frame_subscriber_->cursor_renderer_) {
-      if (frame_subscriber_->cursor_renderer_->SnapshotCursorState(
-              region_in_frame))
-        frame_subscriber_->cursor_renderer_->RenderOnVideoFrame(frame);
+      CursorRenderer* cursor_renderer =
+          frame_subscriber_->cursor_renderer_.get();
+      if (cursor_renderer->SnapshotCursorState(region_in_frame))
+        cursor_renderer->RenderOnVideoFrame(frame);
+      frame->metadata()->SetBoolean(
+          media::VideoFrameMetadata::INTERACTIVE_CONTENT,
+          frame_subscriber_->IsUserInteractingWithContent());
     }
   }
   capture_frame_cb.Run(frame, timestamp, success);
 }
 
+bool FrameSubscriber::IsUserInteractingWithContent() {
+  bool interactive_mode = false;
+  bool ui_activity = false;
+  if (window_activity_tracker_.get()) {
+    ui_activity = window_activity_tracker_->IsUiInteractionActive();
+  }
+  if (cursor_renderer_.get()) {
+    bool animation_active =
+        (base::TimeTicks::Now() -
+         oracle_proxy_->last_time_animation_was_detected()) <
+        base::TimeDelta::FromMilliseconds(kMinPeriodNoAnimationMillis);
+    if (ui_activity && !animation_active) {
+      interactive_mode = true;
+    } else if (animation_active) {
+      window_activity_tracker_->Reset();
+    }
+  }
+  return interactive_mode;
+}
+
 ContentCaptureSubscription::ContentCaptureSubscription(
     const RenderWidgetHost& source,
     const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy,
     const CaptureCallback& capture_callback)
     : render_process_id_(source.GetProcess()->GetID()),
       render_widget_id_(source.GetRoutingID()),
       delivery_log_(),
       capture_callback_(capture_callback),
       timer_(true, true) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 
   RenderWidgetHostView* const view = source.GetView();
 // TODO(isheriff): Cursor resources currently only available on linux. Remove
 // this once we add the necessary resources for windows.
 #if defined(USE_AURA) && defined(OS_LINUX)
-  if (view)
+  if (view) {
     cursor_renderer_.reset(
         new content::CursorRendererAura(view->GetNativeView()));
+    window_activity_tracker_.reset(

[miu, 2015/12/08 00:16:16]

This looks like it'll work for all Aura platforms, not just Linux.  Do you want
to move this out?

[Irfan, 2015/12/08 01:30:18]

Done.

+        new content::WindowActivityTrackerAura(view->GetNativeView()));
+  }
 #endif
   timer_subscriber_.reset(new FrameSubscriber(
       media::VideoCaptureOracle::kTimerPoll, oracle_proxy, &delivery_log_,
       cursor_renderer_ ? cursor_renderer_->GetWeakPtr()
-                       : base::WeakPtr<CursorRenderer>()));
+                       : base::WeakPtr<CursorRenderer>(),
+      window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr()
+                               : base::WeakPtr<WindowActivityTracker>()));
 
   // Subscribe to compositor updates. These will be serviced directly by the
   // oracle.
   if (view) {
     scoped_ptr<RenderWidgetHostViewFrameSubscriber> subscriber(
-        new FrameSubscriber(media::VideoCaptureOracle::kCompositorUpdate,
-                            oracle_proxy, &delivery_log_,
-                            cursor_renderer_
-                                ? cursor_renderer_->GetWeakPtr()
-                                : base::WeakPtr<CursorRenderer>()));
+        new FrameSubscriber(
+            media::VideoCaptureOracle::kCompositorUpdate, oracle_proxy,
+            &delivery_log_, cursor_renderer_ ? cursor_renderer_->GetWeakPtr()
+                                             : base::WeakPtr<CursorRenderer>(),
+            window_activity_tracker_ ? window_activity_tracker_->GetWeakPtr()
+                                     : base::WeakPtr<WindowActivityTracker>()));
     view->BeginFrameSubscription(subscriber.Pass());
   }
 
   // Subscribe to timer events. This instance will service these as well.
   timer_.Start(FROM_HERE,
                std::max(oracle_proxy->min_capture_period(),
                         base::TimeDelta::FromMilliseconds(media
                             ::VideoCaptureOracle::kMinTimerPollPeriodMillis)),
                base::Bind(&ContentCaptureSubscription::OnTimer,
                           base::Unretained(this)));
@@ skipping 490 matching lines @@
     scoped_ptr<Client> client) {
   DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString();
   core_->AllocateAndStart(params, client.Pass());
 }
 
 void WebContentsVideoCaptureDevice::StopAndDeAllocate() {
   core_->StopAndDeAllocate();
 }
 
 }  // namespace content