cast: Support for low-latency interactive mode

content/browser/media/capture/web_contents_video_capture_device.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // Implementation notes: This needs to work on a variety of hardware
 // configurations where the speed of the CPU and GPU greatly affect overall
 // performance. Spanning several threads, the process of capturing has been
 // split up into four conceptual stages:
 //
 //   1. Reserve Buffer: Before a frame can be captured, a slot in the client's
@@ skipping 59 matching lines @@
 #include "content/browser/media/capture/web_contents_tracker.h"
 #include "content/browser/renderer_host/render_widget_host_impl.h"
 #include "content/browser/renderer_host/render_widget_host_view_base.h"
 #include "content/public/browser/browser_thread.h"
 #include "content/public/browser/render_process_host.h"
 #include "content/public/browser/render_widget_host_view.h"
 #include "content/public/browser/render_widget_host_view_frame_subscriber.h"
 #include "content/public/browser/web_contents.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/video_capture_types.h"
+#include "media/base/video_frame_metadata.h"
 #include "media/base/video_util.h"
 #include "media/capture/content/screen_capture_device_core.h"
 #include "media/capture/content/thread_safe_capture_oracle.h"
 #include "media/capture/content/video_capture_oracle.h"
 #include "skia/ext/image_operations.h"
 #include "third_party/skia/include/core/SkBitmap.h"
 #include "third_party/skia/include/core/SkColor.h"
 #include "ui/base/layout.h"
 #include "ui/gfx/geometry/dip_util.h"
 #include "ui/gfx/geometry/size_conversions.h"
@@ skipping 264 matching lines @@
         capture_frame_cb,
     const scoped_refptr<media::VideoFrame>& frame,
     base::TimeTicks timestamp,
     const gfx::Rect& region_in_frame,
     bool success) {
   // We can get a callback in the shutdown sequence for the browser main loop
   // and this can result in a DCHECK failure. Avoid this by doing DCHECK only
   // on success.
   if (success) {
     DCHECK_CURRENTLY_ON(BrowserThread::UI);
+    bool ui_event_detected = false;
+    bool animation_content = true;
     // TODO(isheriff): Unclear if taking a snapshot of cursor here affects user
     // experience in any particular scenarios. Doing it prior to capture may
     // require evaluating region_in_frame in this file.
     if (frame_subscriber_ && frame_subscriber_->cursor_renderer_) {
-      if (frame_subscriber_->cursor_renderer_->SnapshotCursorState(
-              region_in_frame))
-        frame_subscriber_->cursor_renderer_->RenderOnVideoFrame(frame);
+      CursorRenderer* cursor_renderer =
+          frame_subscriber_->cursor_renderer_.get();
+      if (cursor_renderer->SnapshotCursorState(region_in_frame))
+        cursor_renderer->RenderOnVideoFrame(frame);
+      ui_event_detected = cursor_renderer->ui_event_detected();
+      animation_content = frame_subscriber_->oracle_proxy_->IsAnimationActive();
     }
+    frame->metadata()->SetBoolean(media::VideoFrameMetadata::ANIMATION_CONTENT,

[miu, 2015/12/01 21:15:27]

In-line with other comments (video_sender.cc), I think it'd be more powerful to
provide timestamps in the metadata instead of booleans.  Specifically:

1. They should be TimeDeltas on the same timeline as the VideoFrame timestamps
themselves.

2. It seems simplest/sufficient to just make these the timestamp of the last
frame where the condition was true.

3. I'd suggest moving all time-comparison logic into this method so it's clear
they are specific to WCVCD capture.  So, for example, instead of a
ui_event_detected() method, I'd suggest a time_of_last_ui_event() accessor that
returns a TimeTicks.  Likewise, instead of IsAnimationActive(), a
time_animation_was_last_detected() accessor.

4. They metadata entries should not be set if state is unknown (i.e.,
|cursor_renderer| is null).

See other comments...

[Irfan, 2015/12/02 22:32:44]

As discussed will look at moving to a single boolean instead

+                                  animation_content);
+    frame->metadata()->SetBoolean(media::VideoFrameMetadata::USER_INTERACTION,
+                                  ui_event_detected);
   }
   capture_frame_cb.Run(frame, timestamp, success);
 }
 
 ContentCaptureSubscription::ContentCaptureSubscription(
     const RenderWidgetHost& source,
     const scoped_refptr<media::ThreadSafeCaptureOracle>& oracle_proxy,
     const CaptureCallback& capture_callback)
     : render_process_id_(source.GetProcess()->GetID()),
       render_widget_id_(source.GetRoutingID()),
@@ skipping 527 matching lines @@
     scoped_ptr<Client> client) {
   DVLOG(1) << "Allocating " << params.requested_format.frame_size.ToString();
   core_->AllocateAndStart(params, client.Pass());
 }
 
 void WebContentsVideoCaptureDevice::StopAndDeAllocate() {
   core_->StopAndDeAllocate();
 }
 
 }  // namespace content