Add video frame refresh to MediaStream and VideoCapture stacks.

content/renderer/media/webrtc/media_stream_video_webrtc_sink.cc

Index: content/renderer/media/webrtc/media_stream_video_webrtc_sink.cc
diff --git a/content/renderer/media/webrtc/media_stream_video_webrtc_sink.cc b/content/renderer/media/webrtc/media_stream_video_webrtc_sink.cc
index c767c41dda8add8cdc698424308eef28210d8a8f..410a16b12fa24adc4a6ae4448f8798dec5d2d08a 100644
--- a/content/renderer/media/webrtc/media_stream_video_webrtc_sink.cc
+++ b/content/renderer/media/webrtc/media_stream_video_webrtc_sink.cc
@@ -9,6 +9,7 @@
 #include "base/strings/utf_string_conversions.h"
 #include "base/synchronization/lock.h"
 #include "base/thread_task_runner_handle.h"
+#include "base/timer/timer.h"
 #include "content/common/media/media_stream_options.h"
 #include "content/renderer/media/media_stream_constraints_util.h"
 #include "content/renderer/media/media_stream_video_track.h"
@@ -16,6 +17,12 @@
 
 namespace content {
 
+namespace {
+// The maximum number of milliseconds that should elapse since the last video
+// frame was received, before requesting a refresh frame.
+const int kRefreshIntervalMilliseconds = 1000;
+}  // namespace
+
 // Simple help class used for receiving video frames on the IO-thread from a
 // MediaStreamVideoTrack and forward the frames to a WebRtcVideoCapturerAdapter
 // on libjingle's worker thread. WebRtcVideoCapturerAdapter implements a video
@@ -27,7 +34,8 @@ class MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter
       const scoped_refptr<base::SingleThreadTaskRunner>&
           libjingle_worker_thread,
       const scoped_refptr<webrtc::VideoTrackSourceInterface>& source,
-      WebRtcVideoCapturerAdapter* capture_adapter);
+      WebRtcVideoCapturerAdapter* capture_adapter,
+      base::WeakPtr<MediaStreamVideoWebRtcSink> sink);
 
   // MediaStreamVideoWebRtcSink can be destroyed on the main render thread or
   // libjingles worker thread since it posts video frames on that thread. But
@@ -46,6 +54,11 @@ class MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter
   friend class base::RefCountedThreadSafe<WebRtcVideoSourceAdapter>;
   virtual ~WebRtcVideoSourceAdapter();
 
+  // Called whenever a video frame was just delivered on the IO thread. This
+  // restarts the delay period before the |refresh_timer_| will fire the next
+  // time.
+  void ResetRefreshTimerOnMainThread();
+
   scoped_refptr<base::SingleThreadTaskRunner> render_thread_task_runner_;
 
   // |render_thread_checker_| is bound to the main render thread.
@@ -66,17 +79,36 @@ class MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter
   base::Lock capture_adapter_stop_lock_;
   // |capture_adapter_| is owned by |video_source_|
   WebRtcVideoCapturerAdapter* capture_adapter_;
+
+  // Requests a refresh frame once per second. The delay on this timer is reset
+  // each time a frame is received so that it will not fire for at least an
+  // additional second. This means refresh frames will only be requested when
+  // the source has halted delivery (e.g., a screen capturer stops sending
+  // frames because the screen is not being updated).
+  //
+  // This mechanism solves a number of problems. First, it will ensure that
+  // remote clients that join a distributed session receive a first video frame
+  // in a timely manner. Second, it will allow WebRTC's internal bandwidth
+  // estimation logic to maintain a more optimal state, since sending a video
+  // frame will "prime it." Third, it allows lossy encoders to clean up
+  // artifacts in a still image.  http://crbug.com/486274
+  base::RepeatingTimer refresh_timer_;
 };
 
 MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::WebRtcVideoSourceAdapter(
     const scoped_refptr<base::SingleThreadTaskRunner>& libjingle_worker_thread,
     const scoped_refptr<webrtc::VideoTrackSourceInterface>& source,
-    WebRtcVideoCapturerAdapter* capture_adapter)
+    WebRtcVideoCapturerAdapter* capture_adapter,
+    base::WeakPtr<MediaStreamVideoWebRtcSink> sink)
     : render_thread_task_runner_(base::ThreadTaskRunnerHandle::Get()),
       libjingle_worker_thread_(libjingle_worker_thread),
       video_source_(source),
       capture_adapter_(capture_adapter) {
   io_thread_checker_.DetachFromThread();
+  refresh_timer_.Start(
+      FROM_HERE,
+      base::TimeDelta::FromMilliseconds(kRefreshIntervalMilliseconds),
+      base::Bind(&MediaStreamVideoWebRtcSink::RequestRefreshFrame, sink));

[emircan, 2016/04/01 00:06:47]

This logic doesn't fit to the spec descriptions of <canvas> and <video> capture.
These APIs are supposed to provide a MediaStream respecting a frame rate
constraint, including frameRate(0) where users manually request frames. This
suggests frame rate >=1 by default. Can we apply this logic to only tab capture
somehow? Maybe split the API such that RequestInitialFrame()
RequestRefreshFrame()? Canvas capture would provide only the first, and tab
capture would provide both? WDYT?

w3.org/TR/mediacapture-fromelement

[miu, 2016/04/01 23:24:27]

Sorry, I had not considered the other video sources.  So, yes, I changed this so
that it only applies to the screen capture cases, and I've also added some code
to respect minimum frameRate constraints, if any.

 }
 
 MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
@@ -91,6 +123,12 @@ MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
 }
 
 void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
+ResetRefreshTimerOnMainThread() {
+  DCHECK(render_thread_checker_.CalledOnValidThread());
+  refresh_timer_.Reset();
+}
+
+void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
 ReleaseSourceOnMainThread() {
   DCHECK(render_thread_checker_.CalledOnValidThread());
   // Since frames are posted to the worker thread, this object might be deleted
@@ -106,6 +144,10 @@ void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::OnVideoFrameOnIO(
     const scoped_refptr<media::VideoFrame>& frame,
     base::TimeTicks estimated_capture_time) {
   DCHECK(io_thread_checker_.CalledOnValidThread());
+  render_thread_task_runner_->PostTask(
+      FROM_HERE,
+      base::Bind(&WebRtcVideoSourceAdapter::ResetRefreshTimerOnMainThread,
+                 this));
   libjingle_worker_thread_->PostTask(
       FROM_HERE,
       base::Bind(&WebRtcVideoSourceAdapter::OnVideoFrameOnWorkerThread,
@@ -124,7 +166,7 @@ void MediaStreamVideoWebRtcSink::WebRtcVideoSourceAdapter::
 MediaStreamVideoWebRtcSink::MediaStreamVideoWebRtcSink(
     const blink::WebMediaStreamTrack& track,
     PeerConnectionDependencyFactory* factory)
-    : web_track_(track) {
+    : weak_factory_(this) {
   const blink::WebMediaConstraints& constraints =
       MediaStreamVideoTrack::GetVideoTrack(track)->constraints();
 
@@ -140,20 +182,20 @@ MediaStreamVideoWebRtcSink::MediaStreamVideoWebRtcSink(
   scoped_refptr<webrtc::VideoTrackSourceInterface> video_source(
       factory->CreateVideoSource(capture_adapter));
 
-  video_track_ = factory->CreateLocalVideoTrack(web_track_.id().utf8(),
+  video_track_ = factory->CreateLocalVideoTrack(track.id().utf8(),
                                                 video_source.get());
 
-  video_track_->set_enabled(web_track_.isEnabled());
+  video_track_->set_enabled(track.isEnabled());
 
   source_adapter_ = new WebRtcVideoSourceAdapter(
       factory->GetWebRtcWorkerThread(),
       video_source,
-      capture_adapter);
+      capture_adapter,
+      weak_factory_.GetWeakPtr());
 
-  AddToVideoTrack(
-      this,
-      base::Bind(&WebRtcVideoSourceAdapter::OnVideoFrameOnIO, source_adapter_),
-      web_track_);
+  MediaStreamVideoSink::ConnectToTrack(
+      track,
+      base::Bind(&WebRtcVideoSourceAdapter::OnVideoFrameOnIO, source_adapter_));
 
   DVLOG(3) << "MediaStreamVideoWebRtcSink ctor() : is_screencast "
            << is_screencast;
@@ -162,7 +204,7 @@ MediaStreamVideoWebRtcSink::MediaStreamVideoWebRtcSink(
 MediaStreamVideoWebRtcSink::~MediaStreamVideoWebRtcSink() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DVLOG(3) << "MediaStreamVideoWebRtcSink dtor().";
-  RemoveFromVideoTrack(this, web_track_);
+  MediaStreamVideoSink::DisconnectFromTrack();
   source_adapter_->ReleaseSourceOnMainThread();
 }