Cast: deliver video frames on the IO thread

chrome/renderer/media/cast_rtp_stream.cc

Index: chrome/renderer/media/cast_rtp_stream.cc
diff --git a/chrome/renderer/media/cast_rtp_stream.cc b/chrome/renderer/media/cast_rtp_stream.cc
index 99a8543ae21c9205b57561bfacf037fbf2e984de..31dd2a75bfd4ed064a5a0c3fcc052140b40a4baf 100644
--- a/chrome/renderer/media/cast_rtp_stream.cc
+++ b/chrome/renderer/media/cast_rtp_stream.cc
@@ -200,9 +200,11 @@ bool ToVideoSenderConfig(const CastRtpParams& params,
 }  // namespace
 
 // This class receives MediaStreamTrack events and video frames from a
-// MediaStreamTrack. Video frames are submitted to media::cast::FrameInput.
+// MediaStreamTrack.
 //
-// Threading: Video frames are received on the render thread.
+// Threading: Video frames are received on the IO thread and then
+// forwarded to media::cast::VideoFrameInput through a static method.
+// Member variables of this class are only accessed on the render thread.
 class CastVideoSink : public base::SupportsWeakPtr<CastVideoSink>,
                       public content::MediaStreamVideoSink {
  public:
@@ -222,32 +224,29 @@ class CastVideoSink : public base::SupportsWeakPtr<CastVideoSink>,
       RemoveFromVideoTrack(this, track_);
   }
 
-  // content::MediaStreamVideoSink implementation.
-  virtual void OnVideoFrame(const scoped_refptr<media::VideoFrame>& frame)
-      OVERRIDE {
-    if (frame->coded_size() != expected_coded_size_) {
-      error_callback_.Run("Video frame resolution does not match config.");
+  // This static method is used to forward video frames to |frame_input|.
+  static void OnVideoFrame(
+      // These parameters are already bound when callback is created.
+      const gfx::Size& expected_coded_size,
+      const CastRtpStream::ErrorCallback& error_callback,
+      const scoped_refptr<media::cast::VideoFrameInput> frame_input,
+      // These parameters are passed for each frame.
+      const scoped_refptr<media::VideoFrame>& frame,
+      const media::VideoCaptureFormat& format) {
+    if (frame->coded_size() != expected_coded_size) {

[perkj_chrome, 2014/05/08 08:15:08]

FYI- coded_size() is the size from the source before any resolution constraints
are applied for scaling /cropping . So if you want track constraints to have
effect - you must look at the natural_size and visible_rect. 

[Alpha Left Google, 2014/05/08 18:45:51]

I see. I think this is good for now. We're not applying any constraints to crop.

+      error_callback.Run("Video frame resolution does not match config.");
       return;
     }
 
-    // Capture time is calculated using the time when the first frame
-    // is delivered. Doing so has less jitter because each frame has
-    // a TimeDelta from the first frame. However there is a delay between
-    // capture and delivery here for the first frame. We do not account
-    // for this delay.
-    if (first_frame_timestamp_.is_null())
-      first_frame_timestamp_ = base::TimeTicks::Now() - frame->timestamp();
+    const base::TimeTicks now = base::TimeTicks::Now();
 
     // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
     TRACE_EVENT_INSTANT2(
-        "mirroring", "MediaStreamVideoSink::OnVideoFrame",
+        "cast_perf_test", "MediaStreamVideoSink::OnVideoFrame",
         TRACE_EVENT_SCOPE_THREAD,
-        "timestamp",
-        (first_frame_timestamp_ + frame->timestamp()).ToInternalValue(),
+        "timestamp",  now.ToInternalValue(),
         "time_delta", frame->timestamp().ToInternalValue());
-
-    frame_input_->InsertRawVideoFrame(
-        frame, first_frame_timestamp_ + frame->timestamp());
+    frame_input->InsertRawVideoFrame(frame, now);
   }
 
   // Attach this sink to a video track represented by |track_|.
@@ -256,18 +255,21 @@ class CastVideoSink : public base::SupportsWeakPtr<CastVideoSink>,
       const scoped_refptr<media::cast::VideoFrameInput>& frame_input) {
     DCHECK(!sink_added_);
     sink_added_ = true;
-
-    frame_input_ = frame_input;
-    AddToVideoTrack(this, track_);
+    AddToVideoTrack(
+        this,
+        base::Bind(
+            &CastVideoSink::OnVideoFrame,
+            expected_coded_size_,
+            error_callback_,
+            frame_input),
+        track_);
   }
 
  private:
   blink::WebMediaStreamTrack track_;
-  scoped_refptr<media::cast::VideoFrameInput> frame_input_;
   bool sink_added_;
   gfx::Size expected_coded_size_;
   CastRtpStream::ErrorCallback error_callback_;
-  base::TimeTicks first_frame_timestamp_;
 
   DISALLOW_COPY_AND_ASSIGN(CastVideoSink);
 };
@@ -276,6 +278,8 @@ class CastVideoSink : public base::SupportsWeakPtr<CastVideoSink>,
 // media::cast::FrameInput.
 //
 // Threading: Audio frames are received on the real-time audio thread.
+// Note that RemoveFromAudioTrack() is synchronous and we have
+// gurantee that there will be no more audio data after calling it.
 class CastAudioSink : public base::SupportsWeakPtr<CastAudioSink>,
                       public content::MediaStreamAudioSink {
  public:
@@ -289,9 +293,9 @@ class CastAudioSink : public base::SupportsWeakPtr<CastAudioSink>,
         sink_added_(false),
         error_callback_(error_callback),
         weak_factory_(this),
-        input_preroll_(0),
         output_channels_(output_channels),
-        output_sample_rate_(output_sample_rate) {}
+        output_sample_rate_(output_sample_rate),
+        input_preroll_(0) {}
 
   virtual ~CastAudioSink() {
     if (sink_added_)
@@ -399,15 +403,15 @@ class CastAudioSink : public base::SupportsWeakPtr<CastAudioSink>,
   CastRtpStream::ErrorCallback error_callback_;
   base::WeakPtrFactory<CastAudioSink> weak_factory_;
 
-  scoped_ptr<media::MultiChannelResampler> resampler_;
-  scoped_ptr<media::AudioFifo> fifo_;
-  scoped_ptr<media::AudioBus> fifo_input_bus_;
-  int input_preroll_;
   const int output_channels_;
   const int output_sample_rate_;
 
-  // This member is accessed on the real-time audio time.
+  // These member are accessed on the real-time audio time only.
   scoped_refptr<media::cast::AudioFrameInput> frame_input_;
+  scoped_ptr<media::MultiChannelResampler> resampler_;
+  scoped_ptr<media::AudioFifo> fifo_;
+  scoped_ptr<media::AudioBus> fifo_input_bus_;
+  int input_preroll_;
 
   DISALLOW_COPY_AND_ASSIGN(CastAudioSink);
 };