Cast: deliver video frames on the IO thread

chrome/renderer/media/cast_rtp_stream.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/renderer/media/cast_rtp_stream.h"
 
 #include "base/bind.h"
 #include "base/debug/trace_event.h"
 #include "base/logging.h"
 #include "base/memory/weak_ptr.h"
@@ skipping 182 matching lines @@
   }
   if (!config->use_external_encoder) {
     config->number_of_encode_threads = NumberOfEncodeThreads();
   }
   return true;
 }
 
 }  // namespace
 
 // This class receives MediaStreamTrack events and video frames from a
-// MediaStreamTrack. Video frames are submitted to media::cast::FrameInput.
+// MediaStreamTrack.
 //
-// Threading: Video frames are received on the render thread.
+// Threading: Video frames are received on the IO thread and then
+// forwarded to media::cast::VideoFrameInput through a static method.
+// Member variables of this class are only accessed on the render thread.
 class CastVideoSink : public base::SupportsWeakPtr<CastVideoSink>,
                       public content::MediaStreamVideoSink {
  public:
   // |track| provides data for this sink.
   // |expected_coded_size| is the expected dimension of the video frame.
   // |error_callback| is called if video formats don't match.
   CastVideoSink(const blink::WebMediaStreamTrack& track,
                 const gfx::Size& expected_coded_size,
                 const CastRtpStream::ErrorCallback& error_callback)
       : track_(track),
         sink_added_(false),
         expected_coded_size_(expected_coded_size),
         error_callback_(error_callback) {}
 
   virtual ~CastVideoSink() {
     if (sink_added_)
       RemoveFromVideoTrack(this, track_);
   }
 
-  // content::MediaStreamVideoSink implementation.
-  virtual void OnVideoFrame(const scoped_refptr<media::VideoFrame>& frame)
-      OVERRIDE {
-    if (frame->coded_size() != expected_coded_size_) {
-      error_callback_.Run("Video frame resolution does not match config.");
+  // This static method is used to forward video frames to |frame_input|.
+  static void OnVideoFrame(
+      // These parameters are already bound when callback is created.
+      const gfx::Size& expected_coded_size,
+      const CastRtpStream::ErrorCallback& error_callback,
+      const scoped_refptr<media::cast::VideoFrameInput> frame_input,
+      // These parameters are passed for each frame.
+      const scoped_refptr<media::VideoFrame>& frame,
+      const media::VideoCaptureFormat& format) {
+    if (frame->coded_size() != expected_coded_size) {

[perkj_chrome, 2014/05/08 08:15:08]

FYI- coded_size() is the size from the source before any resolution constraints
are applied for scaling /cropping . So if you want track constraints to have
effect - you must look at the natural_size and visible_rect. 

[Alpha Left Google, 2014/05/08 18:45:51]

I see. I think this is good for now. We're not applying any constraints to crop.

+      error_callback.Run("Video frame resolution does not match config.");
       return;
     }
 
-    // Capture time is calculated using the time when the first frame
-    // is delivered. Doing so has less jitter because each frame has
-    // a TimeDelta from the first frame. However there is a delay between
-    // capture and delivery here for the first frame. We do not account
-    // for this delay.
-    if (first_frame_timestamp_.is_null())
-      first_frame_timestamp_ = base::TimeTicks::Now() - frame->timestamp();
+    const base::TimeTicks now = base::TimeTicks::Now();
 
     // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
     TRACE_EVENT_INSTANT2(
-        "mirroring", "MediaStreamVideoSink::OnVideoFrame",
+        "cast_perf_test", "MediaStreamVideoSink::OnVideoFrame",
         TRACE_EVENT_SCOPE_THREAD,
-        "timestamp",
-        (first_frame_timestamp_ + frame->timestamp()).ToInternalValue(),
+        "timestamp",  now.ToInternalValue(),
         "time_delta", frame->timestamp().ToInternalValue());
-
-    frame_input_->InsertRawVideoFrame(
-        frame, first_frame_timestamp_ + frame->timestamp());
+    frame_input->InsertRawVideoFrame(frame, now);
   }
 
   // Attach this sink to a video track represented by |track_|.
   // Data received from the track will be submitted to |frame_input|.
   void AddToTrack(
       const scoped_refptr<media::cast::VideoFrameInput>& frame_input) {
     DCHECK(!sink_added_);
     sink_added_ = true;
-
-    frame_input_ = frame_input;
-    AddToVideoTrack(this, track_);
+    AddToVideoTrack(
+        this,
+        base::Bind(
+            &CastVideoSink::OnVideoFrame,
+            expected_coded_size_,
+            error_callback_,
+            frame_input),
+        track_);
   }
 
  private:
   blink::WebMediaStreamTrack track_;
-  scoped_refptr<media::cast::VideoFrameInput> frame_input_;
   bool sink_added_;
   gfx::Size expected_coded_size_;
   CastRtpStream::ErrorCallback error_callback_;
-  base::TimeTicks first_frame_timestamp_;
 
   DISALLOW_COPY_AND_ASSIGN(CastVideoSink);
 };
 
 // Receives audio data from a MediaStreamTrack. Data is submitted to
 // media::cast::FrameInput.
 //
 // Threading: Audio frames are received on the real-time audio thread.
+// Note that RemoveFromAudioTrack() is synchronous and we have
+// gurantee that there will be no more audio data after calling it.
 class CastAudioSink : public base::SupportsWeakPtr<CastAudioSink>,
                       public content::MediaStreamAudioSink {
  public:
   // |track| provides data for this sink.
   // |error_callback| is called if audio formats don't match.
   CastAudioSink(const blink::WebMediaStreamTrack& track,
                 const CastRtpStream::ErrorCallback& error_callback,
                 int output_channels,
                 int output_sample_rate)
       : track_(track),
         sink_added_(false),
         error_callback_(error_callback),
         weak_factory_(this),
-        input_preroll_(0),
         output_channels_(output_channels),
-        output_sample_rate_(output_sample_rate) {}
+        output_sample_rate_(output_sample_rate),
+        input_preroll_(0) {}
 
   virtual ~CastAudioSink() {
     if (sink_added_)
       RemoveFromAudioTrack(this, track_);
   }
 
   // Called on real-time audio thread.
   // content::MediaStreamAudioSink implementation.
   virtual void OnData(const int16* audio_data,
                       int sample_rate,
@@ skipping 87 matching lines @@
   void ProvideData(int frame_delay, media::AudioBus* output_bus) {
     fifo_->Consume(output_bus, 0, output_bus->frames());
   }
 
  private:
   blink::WebMediaStreamTrack track_;
   bool sink_added_;
   CastRtpStream::ErrorCallback error_callback_;
   base::WeakPtrFactory<CastAudioSink> weak_factory_;
 
+  const int output_channels_;
+  const int output_sample_rate_;
+
+  // These member are accessed on the real-time audio time only.
+  scoped_refptr<media::cast::AudioFrameInput> frame_input_;
   scoped_ptr<media::MultiChannelResampler> resampler_;
   scoped_ptr<media::AudioFifo> fifo_;
   scoped_ptr<media::AudioBus> fifo_input_bus_;
   int input_preroll_;
-  const int output_channels_;
-  const int output_sample_rate_;
-
-  // This member is accessed on the real-time audio time.
-  scoped_refptr<media::cast::AudioFrameInput> frame_input_;
 
   DISALLOW_COPY_AND_ASSIGN(CastAudioSink);
 };
 
 CastRtpParams::CastRtpParams(const CastRtpPayloadParams& payload_params)
     : payload(payload_params) {}
 
 CastCodecSpecificParams::CastCodecSpecificParams() {}
 
 CastCodecSpecificParams::~CastCodecSpecificParams() {}
@@ skipping 107 matching lines @@
 }
 
 void CastRtpStream::DidEncounterError(const std::string& message) {
   // Save the WeakPtr first because the error callback might delete this object.
   base::WeakPtr<CastRtpStream> ptr = weak_factory_.GetWeakPtr();
   error_callback_.Run(message);
   content::RenderThread::Get()->GetMessageLoop()->PostTask(
       FROM_HERE,
       base::Bind(&CastRtpStream::Stop, ptr));
 }