Add video frame refresh to MediaStream and VideoCapture stacks.

content/renderer/media/webrtc/media_stream_video_webrtc_sink.cc

Index: content/renderer/media/webrtc/media_stream_video_webrtc_sink.cc
diff --git a/content/renderer/media/webrtc/media_stream_video_webrtc_sink.cc b/content/renderer/media/webrtc/media_stream_video_webrtc_sink.cc
index c767c41dda8add8cdc698424308eef28210d8a8f..eda9ad257f2ee5c3d47590f5a14c07c3e1f56bba 100644
--- a/content/renderer/media/webrtc/media_stream_video_webrtc_sink.cc
+++ b/content/renderer/media/webrtc/media_stream_video_webrtc_sink.cc
@@ -5,17 +5,33 @@
 #include "content/renderer/media/webrtc/media_stream_video_webrtc_sink.h"
 
 #include "base/location.h"
+#include "base/numerics/safe_conversions.h"
 #include "base/single_thread_task_runner.h"
 #include "base/strings/utf_string_conversions.h"
 #include "base/synchronization/lock.h"
 #include "base/thread_task_runner_handle.h"
+#include "base/timer/timer.h"
 #include "content/common/media/media_stream_options.h"
 #include "content/renderer/media/media_stream_constraints_util.h"
 #include "content/renderer/media/media_stream_video_track.h"
 #include "content/renderer/media/webrtc/peer_connection_dependency_factory.h"
+#include "media/base/limits.h"
 
 namespace content {
 
+namespace {
+
+// The default number of microseconds that should elapse since the last video
+// frame was received, before requesting a refresh frame.
+const int64_t kDefaultRefreshIntervalMicros =
+    base::Time::kMicrosecondsPerSecond;
+
+// A lower-bound for the refresh interval.
+const int64_t kLowerBoundRefreshIntervalMicros =
+    base::Time::kMicrosecondsPerSecond / media::limits::kMaxFramesPerSecond;
+
+}  // namespace
+
 // Simple help class used for receiving video frames on the IO-thread from a
 // MediaStreamVideoTrack and forward the frames to a WebRtcVideoCapturerAdapter
 // on libjingle's worker thread. WebRtcVideoCapturerAdapter implements a video
@@ -27,7 +43,9 @@ class MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter
       const scoped_refptr<base::SingleThreadTaskRunner>&
           libjingle_worker_thread,
       const scoped_refptr<webrtc::VideoTrackSourceInterface>& source,
-      WebRtcVideoCapturerAdapter* capture_adapter);
+      WebRtcVideoCapturerAdapter* capture_adapter,
+      base::TimeDelta refresh_interval,
+      base::WeakPtr<MediaStreamVideoWebRtcSink> sink);
 
   // MediaStreamVideoWebRtcSink can be destroyed on the main render thread or
   // libjingles worker thread since it posts video frames on that thread. But
@@ -46,6 +64,11 @@ class MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter
   friend class base::RefCountedThreadSafe<WebRtcVideoSourceAdapter>;
   virtual ~WebRtcVideoSourceAdapter();
 
+  // Called whenever a video frame was just delivered on the IO thread. This
+  // restarts the delay period before the |refresh_timer_| will fire the next
+  // time.
+  void ResetRefreshTimerOnMainThread();
+
   scoped_refptr<base::SingleThreadTaskRunner> render_thread_task_runner_;
 
   // |render_thread_checker_| is bound to the main render thread.
@@ -66,17 +89,40 @@ class MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter
   base::Lock capture_adapter_stop_lock_;
   // |capture_adapter_| is owned by |video_source_|
   WebRtcVideoCapturerAdapter* capture_adapter_;
+
+  // Requests a refresh frame at regular intervals. The delay on this timer is
+  // reset each time a frame is received so that it will not fire for at least
+  // an additional period. This means refresh frames will only be requested when
+  // the source has halted delivery (e.g., a screen capturer stops sending
+  // frames because the screen is not being updated).
+  //
+  // This mechanism solves a number of problems. First, it will ensure that
+  // remote clients that join a distributed session receive a first video frame
+  // in a timely manner. Second, it will allow WebRTC's internal bandwidth
+  // estimation logic to maintain a more optimal state, since sending a video
+  // frame will "prime it." Third, it allows lossy encoders to clean up
+  // artifacts in a still image.  http://crbug.com/486274
+  base::RepeatingTimer refresh_timer_;
 };
 
 MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::WebRtcVideoSourceAdapter(
     const scoped_refptr<base::SingleThreadTaskRunner>& libjingle_worker_thread,
     const scoped_refptr<webrtc::VideoTrackSourceInterface>& source,
-    WebRtcVideoCapturerAdapter* capture_adapter)
+    WebRtcVideoCapturerAdapter* capture_adapter,
+    base::TimeDelta refresh_interval,
+    base::WeakPtr<MediaStreamVideoWebRtcSink> sink)
     : render_thread_task_runner_(base::ThreadTaskRunnerHandle::Get()),
       libjingle_worker_thread_(libjingle_worker_thread),
       video_source_(source),
       capture_adapter_(capture_adapter) {
   io_thread_checker_.DetachFromThread();
+  if (!refresh_interval.is_zero()) {
+    VLOG(1) << "Starting frame refresh timer with interval "

[mcasas, 2016/04/05 23:12:52]

s/VLOG/DVLOG/, right?

[miu, 2016/04/06 00:03:18]

For the time being, I wanted to leave it as a VLOG().  We may need it to debug
external-to-Chromium use cases (such as Hangouts integration).

+            << refresh_interval.InMillisecondsF() << " ms.";
+    refresh_timer_.Start(
+        FROM_HERE, refresh_interval,
+        base::Bind(&MediaStreamVideoWebRtcSink::RequestRefreshFrame, sink));
+  }
 }
 
 MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
@@ -91,6 +137,13 @@ MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
 }
 
 void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
+ResetRefreshTimerOnMainThread() {
+  DCHECK(render_thread_checker_.CalledOnValidThread());
+  if (refresh_timer_.IsRunning())
+    refresh_timer_.Reset();
+}
+
+void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
 ReleaseSourceOnMainThread() {
   DCHECK(render_thread_checker_.CalledOnValidThread());
   // Since frames are posted to the worker thread, this object might be deleted
@@ -106,6 +159,10 @@ void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::OnVideoFrameOnIO(
     const scoped_refptr<media::VideoFrame>& frame,
     base::TimeTicks estimated_capture_time) {
   DCHECK(io_thread_checker_.CalledOnValidThread());
+  render_thread_task_runner_->PostTask(
+      FROM_HERE,
+      base::Bind(&WebRtcVideoSourceAdapter::ResetRefreshTimerOnMainThread,
+                 this));
   libjingle_worker_thread_->PostTask(
       FROM_HERE,
       base::Bind(&WebRtcVideoSourceAdapter::OnVideoFrameOnWorkerThread,
@@ -124,7 +181,7 @@ void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
 MediaStreamVideoWebRtcSink::MediaStreamVideoWebRtcSink(
     const blink::WebMediaStreamTrack& track,
     PeerConnectionDependencyFactory* factory)
-    : web_track_(track) {
+    : weak_factory_(this) {
   const blink::WebMediaConstraints& constraints =
       MediaStreamVideoTrack::GetVideoTrack(track)->constraints();
 
@@ -133,6 +190,41 @@ MediaStreamVideoWebRtcSink::MediaStreamVideoWebRtcSink(
   bool is_screencast = GetConstraintValueAsString(
       constraints, &blink::WebMediaTrackConstraintSet::mediaStreamSource,
       &value);
+
+  // Enable automatic frame refreshes for the screen capture sources, which will
+  // stop producing frames whenever screen content is not changing. Check the
+  // frameRate constraint to determine the rate of refreshes. If a minimum
+  // frameRate is provided, use that. Otherwise, use the maximum frameRate if it
+  // happens to be less than the default.
+  base::TimeDelta refresh_interval = base::TimeDelta::FromMicroseconds(0);
+  if (is_screencast) {

[mcasas, 2016/04/05 23:12:52]

if (!is_screencast)
  return;
// ...

[miu, 2016/04/06 00:03:18]

I can't return early here.  All the code after L226 should unconditionally
execute.

+    // Start with the default refresh interval, and refine based on constraints.
+    refresh_interval =
+        base::TimeDelta::FromMicroseconds(kDefaultRefreshIntervalMicros);
+    double value = 0.0;
+    if (GetConstraintMinAsDouble(
+            constraints, &blink::WebMediaTrackConstraintSet::frameRate,
+            &value) &&
+        value > 0.0) {
+      refresh_interval =
+          base::TimeDelta::FromMicroseconds(base::saturated_cast<int64_t>(
+              base::Time::kMicrosecondsPerSecond / value));
+    }
+    if (GetConstraintMaxAsDouble(
+            constraints, &blink::WebMediaTrackConstraintSet::frameRate,
+            &value) &&
+        value > 0.0) {
+      const base::TimeDelta alternate_refresh_interval =
+          base::TimeDelta::FromMicroseconds(base::saturated_cast<int64_t>(
+              base::Time::kMicrosecondsPerSecond / value));
+      refresh_interval = std::max(refresh_interval, alternate_refresh_interval);
+    }
+    if (refresh_interval.InMicroseconds() < kLowerBoundRefreshIntervalMicros) {
+      refresh_interval =
+          base::TimeDelta::FromMicroseconds(kLowerBoundRefreshIntervalMicros);
+    }
+  }
+
   WebRtcVideoCapturerAdapter* capture_adapter =
       factory->CreateVideoCapturer(is_screencast);
 
@@ -140,20 +232,21 @@ MediaStreamVideoWebRtcSink::MediaStreamVideoWebRtcSink(
   scoped_refptr<webrtc::VideoTrackSourceInterface> video_source(
       factory->CreateVideoSource(capture_adapter));
 
-  video_track_ = factory->CreateLocalVideoTrack(web_track_.id().utf8(),
+  video_track_ = factory->CreateLocalVideoTrack(track.id().utf8(),
                                                 video_source.get());
 
-  video_track_->set_enabled(web_track_.isEnabled());
+  video_track_->set_enabled(track.isEnabled());
 
   source_adapter_ = new WebRtcVideoSourceAdapter(
       factory->GetWebRtcWorkerThread(),
       video_source,
-      capture_adapter);
+      capture_adapter,
+      refresh_interval,
+      weak_factory_.GetWeakPtr());
 
-  AddToVideoTrack(
-      this,
-      base::Bind(&WebRtcVideoSourceAdapter::OnVideoFrameOnIO, source_adapter_),
-      web_track_);
+  MediaStreamVideoSink::ConnectToTrack(
+      track,
+      base::Bind(&WebRtcVideoSourceAdapter::OnVideoFrameOnIO, source_adapter_));
 
   DVLOG(3) << "MediaStreamVideoWebRtcSink ctor() : is_screencast "
            << is_screencast;
@@ -162,7 +255,7 @@ MediaStreamVideoWebRtcSink::MediaStreamVideoWebRtcSink(
 MediaStreamVideoWebRtcSink::~MediaStreamVideoWebRtcSink() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DVLOG(3) << "MediaStreamVideoWebRtcSink dtor().";
-  RemoveFromVideoTrack(this, web_track_);
+  MediaStreamVideoSink::DisconnectFromTrack();
   source_adapter_->ReleaseSourceOnMainThread();
 }