cast: Support for low-latency interactive mode

media/cast/sender/video_sender.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/cast/sender/video_sender.h"
 
 #include <algorithm>
 #include <cmath>
 #include <cstring>
 
@@ skipping 21 matching lines @@
 // network quality (e.g., additional time that accounts for encode and decode
 // time).
 const int kConstantTimeMs = 75;
 
 // The target maximum utilization of the encoder and network resources.  This is
 // used to attenuate the actual measured utilization values in order to provide
 // "breathing room" (i.e., to ensure there will be sufficient CPU and bandwidth
 // available to handle the occasional more-complex frames).
 const int kTargetUtilizationPercentage = 75;
 
+// Minimum number of user interactions before we consider the user to be in
+// interactive mode. The goal is to prevent user interactions to launch
+// animated content from causing target playout time flip-flop.
+const int kMinUserInteractionsForInteractiveMode = 5;
+
 // Extract capture begin/end timestamps from |video_frame|'s metadata and log
 // it.
 void LogVideoCaptureTimestamps(CastEnvironment* cast_environment,
                                const media::VideoFrame& video_frame,
                                RtpTimestamp rtp_timestamp) {
   scoped_ptr<FrameEvent> capture_begin_event(new FrameEvent());
   capture_begin_event->type = FRAME_CAPTURE_BEGIN;
   capture_begin_event->media_type = VIDEO_EVENT;
   capture_begin_event->rtp_timestamp = rtp_timestamp;
 
@@ skipping 46 matching lines @@
           video_config.use_external_encoder
               ? NewFixedCongestionControl(
                     (video_config.min_bitrate + video_config.max_bitrate) / 2)
               : NewAdaptiveCongestionControl(cast_environment->Clock(),
                                              video_config.max_bitrate,
                                              video_config.min_bitrate,
                                              video_config.max_frame_rate)),
       frames_in_encoder_(0),
       last_bitrate_(0),
       playout_delay_change_cb_(playout_delay_change_cb),
+      animation_content_(true),
+      user_interaction_(false),
+      user_interaction_count_(0),
+      interactive_mode_(false),
       last_reported_deadline_utilization_(-1.0),
       last_reported_lossy_utilization_(-1.0),
       weak_factory_(this) {
   video_encoder_ = VideoEncoder::Create(
       cast_environment_,
       video_config,
       status_change_cb,
       create_vea_cb,
       create_video_encode_mem_cb);
   if (!video_encoder_) {
@@ skipping 36 matching lines @@
   LogVideoCaptureTimestamps(cast_environment_.get(), *video_frame,
                             rtp_timestamp);
 
   // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
   TRACE_EVENT_INSTANT2(
       "cast_perf_test", "InsertRawVideoFrame",
       TRACE_EVENT_SCOPE_THREAD,
       "timestamp", reference_time.ToInternalValue(),
       "rtp_timestamp", rtp_timestamp);
 
+  bool prev_user_interaction = user_interaction_;
+  if (video_frame->metadata()->GetBoolean(VideoFrameMetadata::ANIMATION_CONTENT,
+                                          &animation_content_) &&
+      video_frame->metadata()->GetBoolean(VideoFrameMetadata::USER_INTERACTION,
+                                          &user_interaction_)) {
+    if (user_interaction_ && !prev_user_interaction) {
+      user_interaction_count_++;

[miu, 2015/12/01 21:15:27]

IIUC, this is counting the number of frames where USER_INTERACTION was true. 
However, frame rate can fluctuate wildly, anywhere from zero (for long periods
of time) to the max frame rate.

I think it'd be better to use time-based thresholds.  If the VideoFrameMetadata
provided timestamps, it would be easy to determine how long to wait before
toggling between the lower or higher playout delay, by comparing the current
VideoFrame's timestamp to the two metadata timestamps.  Something like:

  base::TimeDelta last_time_animation_detected;
  base::TimeDelta last_time_user_interaction_detected;
  video_frame->metadata()->Get...
  video_frame->metadata()->Get...
  const bool is_animating =
      (video_frame->timestamp() - last_time_animation_detected) < "3 seconds";
  const bool is_user_interacting = ...ditto...;
  // ...logic to determine whether to make playout change here...

[Irfan, 2015/12/02 22:32:44]

This has moved to capture side now.

+    }
+    if (user_interaction_count_ > kMinUserInteractionsForInteractiveMode &&
+        !animation_content_ && !interactive_mode_) {

[miu, 2015/12/01 21:15:27]

Perhaps we should only consider the "user interaction" signal, and snap to the
lower playout delay regardless of whether content is animating?  For example, if
the user is seeking a video with a mouse drag on the "scrub bar," they'd
probably want to be using low-latency mirroring.  Thoughts on this?

[Irfan, 2015/12/02 22:37:55]

As discussed f2f, I think we should be more conservative for user interaction on
animated content

+      interactive_mode_ = true;
+      VLOG(1) << "Interactive mode playout time "
+              << media::cast::kInteractiveModeStartPlayoutTimeMs;
+      playout_delay_change_cb_.Run(base::TimeDelta::FromMilliseconds(
+          media::cast::kInteractiveModeStartPlayoutTimeMs));
+    } else if (animation_content_) {
+      // Reset user interactions when animating content is playing.
+      user_interaction_count_ = 0;
+      interactive_mode_ = false;
+    }
+  }
+

[miu, 2015/12/01 21:15:27]

BTW--What about audio?  If we change video's playout delay, we'll lose A/V sync
unless audio is also changed.

OTOH, this will require some refactoring: Something like a
SetTargetPlayoutDelay() method added to CastSenderImpl, which then propagates
the playout delay changes to both AudioSender and VideoSender.

[Irfan, 2015/12/02 22:32:44]

This already works as you describe :-)

   // Drop the frame if either its RTP or reference timestamp is not an increase
   // over the last frame's.  This protects: 1) the duration calculations that
   // assume timestamps are monotonically non-decreasing, and 2) assumptions made
   // deeper in the implementation where each frame's RTP timestamp needs to be
   // unique.
   if (!last_enqueued_frame_reference_time_.is_null() &&
       (!IsNewerRtpTimestamp(rtp_timestamp,
                             last_enqueued_frame_rtp_timestamp_) ||
        reference_time <= last_enqueued_frame_reference_time_)) {
     VLOG(1) << "Dropping video frame: RTP or reference time did not increase.";
@@ skipping 11 matching lines @@
   // OnEncodedVideoFrame().
   const base::TimeDelta duration_added_by_next_frame = frames_in_encoder_ > 0 ?
       reference_time - last_enqueued_frame_reference_time_ :
       base::TimeDelta::FromSecondsD(1.0 / max_frame_rate_);
 
   if (ShouldDropNextFrame(duration_added_by_next_frame)) {
     base::TimeDelta new_target_delay = std::min(
         current_round_trip_time_ * kRoundTripsNeeded +
         base::TimeDelta::FromMilliseconds(kConstantTimeMs),
         max_playout_delay_);
-    if (new_target_delay > target_playout_delay_) {
+    // In case of interactive mode, we prefer frame drops over increasing
+    // playout time.
+    if (!interactive_mode_ && new_target_delay > target_playout_delay_) {
+      // In case we detect user is no more in an interactive mode and there is
+      // a need to drop a frame, we ensure the playout delay is at-least the
+      // the starting value that is known to work well.
+      // This is intentended to minimize freeze when moving from an interactive
+      // session to watching animating content while being limited by end-to-end
+      // delay.
+      VLOG(1) << "Ensure playout time is at least "
+              << media::cast::kNonInteractiveModeStartPlayoutTimeMs;
+      if (new_target_delay.InMilliseconds() <
+          media::cast::kNonInteractiveModeStartPlayoutTimeMs)
+        new_target_delay = base::TimeDelta::FromMilliseconds(
+            kNonInteractiveModeStartPlayoutTimeMs);
       VLOG(1) << "New target delay: " << new_target_delay.InMilliseconds();
       playout_delay_change_cb_.Run(new_target_delay);
     }
 
     // Some encoder implementations have a frame window for analysis. Since we
     // are dropping this frame, unless we instruct the encoder to flush all the
     // frames that have been enqueued for encoding, frames_in_encoder_ and
     // last_enqueued_frame_reference_time_ will never be updated and we will
     // drop every subsequent frame for the rest of the session.
     video_encoder_->EmitFrames();
@@ skipping 150 matching lines @@
         media::VideoFrameMetadata::RESOURCE_UTILIZATION,
         encoded_frame->dependency == EncodedFrame::KEY ?
             std::min(1.0, attenuated_utilization) : attenuated_utilization);
   }
 
   SendEncodedFrame(encoder_bitrate, encoded_frame.Pass());
 }
 
 }  // namespace cast
 }  // namespace media