cast: Support for low-latency interactive mode

content/browser/media/capture/web_contents_video_capture_device.cc

Index: content/browser/media/capture/web_contents_video_capture_device.cc
diff --git a/content/browser/media/capture/web_contents_video_capture_device.cc b/content/browser/media/capture/web_contents_video_capture_device.cc
index 3a432d856c068ed7f638913321e573c37922f0ea..2db20539752512d555bdec9a2176010c9060be83 100644
--- a/content/browser/media/capture/web_contents_video_capture_device.cc
+++ b/content/browser/media/capture/web_contents_video_capture_device.cc
@@ -66,6 +66,7 @@
 #include "base/threading/thread_checker.h"
 #include "base/time/time.h"
 #include "content/browser/media/capture/cursor_renderer.h"
+#include "content/browser/media/capture/ui_activity_tracker.h"
 #include "content/browser/media/capture/web_contents_capture_util.h"
 #include "content/browser/media/capture/web_contents_tracker.h"
 #include "content/browser/renderer_host/render_widget_host_impl.h"
@@ -77,6 +78,7 @@
 #include "content/public/browser/web_contents.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/video_capture_types.h"
+#include "media/base/video_frame_metadata.h"
 #include "media/base/video_util.h"
 #include "media/capture/content/screen_capture_device_core.h"
 #include "media/capture/content/thread_safe_capture_oracle.h"
@@ -90,12 +92,21 @@
 
 #if defined(USE_AURA)
 #include "content/browser/media/capture/cursor_renderer_aura.h"
+#include "content/browser/media/capture/ui_activity_tracker_aura.h"
 #endif
 
 namespace content {
 
 namespace {
 
+enum InteractiveModeSettings {
+  // Minimum amount of time for which there should be no animation detected
+  // to consider interactive mode being active. This is to prevent very brief
+  // periods of animated content not being detected (due to CPU fluctations for
+  // example) from causing a flip-flop on interactive mode.
+  kMinPeriodNoAnimationMillis = 3000
+};
+
 void DeleteOnWorkerThread(scoped_ptr<base::Thread> render_thread,
                           const base::Closure& callback) {
   render_thread.reset();
@@ -128,11 +139,13 @@ class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber {
   FrameSubscriber(media::VideoCaptureOracle::Event event_type,
                   const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle,
                   VideoFrameDeliveryLog* delivery_log,
-                  base::WeakPtr<content::CursorRenderer> cursor_renderer)
+                  base::WeakPtr<content::CursorRenderer> cursor_renderer,
+                  base::WeakPtr<content::UiActivityTracker> ui_activity_tracker)
       : event_type_(event_type),
         oracle_proxy_(oracle),
         delivery_log_(delivery_log),
         cursor_renderer_(cursor_renderer),
+        ui_activity_tracker_(ui_activity_tracker),
         weak_ptr_factory_(this) {}
 
   bool ShouldCaptureFrame(
@@ -151,6 +164,8 @@ class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber {
       const gfx::Rect& region_in_frame,
       bool success);
 
+  bool IsInteractiveModeActive();
+
  private:
   const media::VideoCaptureOracle::Event event_type_;
   scoped_refptr<media::ThreadSafeCaptureOracle> oracle_proxy_;
@@ -158,6 +173,9 @@ class FrameSubscriber : public RenderWidgetHostViewFrameSubscriber {
   // We need a weak pointer since FrameSubscriber is owned externally and
   // may outlive the cursor renderer.
   base::WeakPtr<CursorRenderer> cursor_renderer_;
+  // We need a weak pointer since FrameSubscriber is owned externally and
+  // may outlive the ui activity tracker.
+  base::WeakPtr<UiActivityTracker> ui_activity_tracker_;
   base::WeakPtrFactory<FrameSubscriber> weak_ptr_factory_;
 };
 
@@ -210,6 +228,10 @@ class ContentCaptureSubscription {
   // capture is completed.
   scoped_ptr<content::CursorRenderer> cursor_renderer_;
 
+  // Responsible for tracking the UI events and making a decision on whether
+  // user is actively interacting with content.
+  scoped_ptr<content::UiActivityTracker> ui_activity_tracker_;
+
   DISALLOW_COPY_AND_ASSIGN(ContentCaptureSubscription);
 };
 
@@ -365,14 +387,38 @@ void FrameSubscriber::DidCaptureFrame(
     // experience in any particular scenarios. Doing it prior to capture may
     // require evaluating region_in_frame in this file.
     if (frame_subscriber_ && frame_subscriber_->cursor_renderer_) {
-      if (frame_subscriber_->cursor_renderer_->SnapshotCursorState(
-              region_in_frame))
-        frame_subscriber_->cursor_renderer_->RenderOnVideoFrame(frame);
+      CursorRenderer* cursor_renderer =
+          frame_subscriber_->cursor_renderer_.get();
+      if (cursor_renderer->SnapshotCursorState(region_in_frame))
+        cursor_renderer->RenderOnVideoFrame(frame);
+      frame->metadata()->SetBoolean(
+          media::VideoFrameMetadata::INTERACTIVE_MODE,
+          frame_subscriber_->IsInteractiveModeActive());
     }
   }
   capture_frame_cb.Run(frame, timestamp, success);
 }
 
+bool FrameSubscriber::IsInteractiveModeActive() {

[miu, 2015/12/03 21:26:23]

naming nit: To aid those unfamiliar with this code, how about something like:
IsUserInteractingWithContent()

[Irfan, 2015/12/04 22:45:37]

Done.

+  bool interactive_mode = false;
+  bool ui_activity = false;
+  if (ui_activity_tracker_.get()) {
+    ui_activity = ui_activity_tracker_->uiInteractionActive();
+  }
+  if (cursor_renderer_.get()) {
+    bool animation_active =
+        (base::TimeTicks::Now() -
+         oracle_proxy_->last_time_animation_was_detected()) <
+        base::TimeDelta::FromMilliseconds(kMinPeriodNoAnimationMillis);
+    if (ui_activity && !animation_active) {
+      interactive_mode = true;
+    } else if (animation_active) {
+      ui_activity_tracker_->reset();
+    }
+  }
+  return interactive_mode;
+}
+
 ContentCaptureSubscription::ContentCaptureSubscription(
     const RenderWidgetHost& source,
     const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy,
@@ -388,24 +434,30 @@ ContentCaptureSubscription::ContentCaptureSubscription(
 // TODO(isheriff): Cursor resources currently only available on linux. Remove
 // this once we add the necessary resources for windows.
 #if defined(USE_AURA) && defined(OS_LINUX)
-  if (view)
+  if (view) {
     cursor_renderer_.reset(
         new content::CursorRendererAura(view->GetNativeView()));
+    ui_activity_tracker_.reset(
+        new content::UiActivityTrackerAura(view->GetNativeView()));
+  }
 #endif
   timer_subscriber_.reset(new FrameSubscriber(
       media::VideoCaptureOracle::kTimerPoll, oracle_proxy, &delivery_log_,
       cursor_renderer_ ? cursor_renderer_->GetWeakPtr()
-                       : base::WeakPtr<CursorRenderer>()));
+                       : base::WeakPtr<CursorRenderer>(),
+      ui_activity_tracker_ ? ui_activity_tracker_->GetWeakPtr()
+                           : base::WeakPtr<UiActivityTracker>()));
 
   // Subscribe to compositor updates. These will be serviced directly by the
   // oracle.
   if (view) {
     scoped_ptr<RenderWidgetHostViewFrameSubscriber> subscriber(
-        new FrameSubscriber(media::VideoCaptureOracle::kCompositorUpdate,
-                            oracle_proxy, &delivery_log_,
-                            cursor_renderer_
-                                ? cursor_renderer_->GetWeakPtr()
-                                : base::WeakPtr<CursorRenderer>()));
+        new FrameSubscriber(
+            media::VideoCaptureOracle::kCompositorUpdate, oracle_proxy,
+            &delivery_log_, cursor_renderer_ ? cursor_renderer_->GetWeakPtr()
+                                             : base::WeakPtr<CursorRenderer>(),
+            ui_activity_tracker_ ? ui_activity_tracker_->GetWeakPtr()
+                                 : base::WeakPtr<UiActivityTracker>()));
     view->BeginFrameSubscription(subscriber.Pass());
   }