Cast: Resample input audio to the configured sample rate

chrome/renderer/media/cast_rtp_stream.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/renderer/media/cast_rtp_stream.h"
 
 #include "base/bind.h"
 #include "base/logging.h"
 #include "base/memory/weak_ptr.h"
 #include "chrome/renderer/media/cast_session.h"
 #include "chrome/renderer/media/cast_udp_transport.h"
 #include "content/public/renderer/media_stream_audio_sink.h"
 #include "content/public/renderer/media_stream_video_sink.h"
 #include "content/public/renderer/render_thread.h"
+#include "media/audio/audio_parameters.h"
 #include "media/base/audio_bus.h"
 #include "media/base/bind_to_current_loop.h"
+#include "media/base/multi_channel_resampler.h"
 #include "media/cast/cast_config.h"
 #include "media/cast/cast_defines.h"
 #include "media/cast/cast_sender.h"
 #include "media/cast/transport/cast_transport_config.h"
 #include "third_party/WebKit/public/platform/WebMediaStreamSource.h"
 
 using media::cast::AudioSenderConfig;
 using media::cast::VideoSenderConfig;
 
 namespace {
@@ skipping 140 matching lines @@
 // Receives audio data from a MediaStreamTrack. Data is submitted to
 // media::cast::FrameInput.
 //
 // Threading: Audio frames are received on the real-time audio thread.
 class CastAudioSink : public base::SupportsWeakPtr<CastAudioSink>,
                       public content::MediaStreamAudioSink {
  public:
   // |track| provides data for this sink.
   // |error_callback| is called if audio formats don't match.
   CastAudioSink(const blink::WebMediaStreamTrack& track,
-                const CastRtpStream::ErrorCallback& error_callback)
+                const CastRtpStream::ErrorCallback& error_callback,
+                int output_channels,
+                int output_sample_rate)
       : track_(track),
         sink_added_(false),
         error_callback_(error_callback),
         weak_factory_(this),
         render_thread_task_runner_(content::RenderThread::Get()
-                                       ->GetMessageLoop()
-                                       ->message_loop_proxy()) {}
+                                   ->GetMessageLoop()
+                                   ->message_loop_proxy()),
+        output_channels_(output_channels),
+        output_sample_rate_(output_sample_rate),
+        input_data_(NULL),
+        input_frames_(0),
+        input_bytes_per_frame_(0) {}
 
   virtual ~CastAudioSink() {
     if (sink_added_)
       RemoveFromAudioTrack(this, track_);
   }
 
   // Called on real-time audio thread.
   // content::MediaStreamAudioSink implementation.
   virtual void OnData(const int16* audio_data,
                       int sample_rate,
                       int number_of_channels,
                       int number_of_frames) OVERRIDE {
-    scoped_ptr<media::AudioBus> audio_bus(
-        media::AudioBus::Create(number_of_channels, number_of_frames));
-    audio_bus->FromInterleaved(audio_data, number_of_frames, 2);
+    input_data_ = audio_data;
+    input_frames_ = number_of_frames;
+
+    DCHECK_EQ(number_of_channels, output_channels_);
+    scoped_ptr<media::AudioBus> output_bus(
+        media::AudioBus::Create(
+            output_channels_,
+            output_sample_rate_ * number_of_frames / sample_rate));
+
+    // Resampler will then call ProvideData() below to fetch data from
+    // |input_data_|.
+    resampler_->Resample(output_bus->frames(), output_bus.get());

[DaleCurtis, 2014/03/06 20:57:16]

While this guarantees all input data is consumed it doesn't mean you've gotten
all available output frames.  There may still be frames in the resampler. 
You'll need to do some math based on the input and output ratios to figure out
how much.  It'd be nice to make that calculation available on SincResampler.  We
have some upcoming changes from rileya@ which would use that information.

An easier solution would be to use an AudioPullFifo you push the input data into
and then pull from during Resample().

Note: Depending on the input sample rate you may be called back less than you
expect here.  I.e. the input data is 44KhZ and you want 48kHz, you're only going
to get data delivered at 44kHz which may lead to stalls in your pipeline (I
haven't dug into your code further to see if this is the case).

+    input_data_ = NULL;
+    input_frames_ = 0;
 
     // TODO(hclam): Pass in the accurate capture time to have good
     // audio / video sync.
 
     // TODO(hclam): We shouldn't hop through the render thread.
     // Bounce the call from the real-time audio thread to the render thread.
     // Needed since frame_input_ can be changed runtime by the render thread.
-    media::AudioBus* const audio_bus_ptr = audio_bus.get();
+    media::AudioBus* const output_bus_ptr = output_bus.get();
     render_thread_task_runner_->PostTask(
         FROM_HERE,
         base::Bind(&CastAudioSink::SendAudio,
                    weak_factory_.GetWeakPtr(),
-                   audio_bus_ptr,
+                   output_bus_ptr,
                    base::TimeTicks::Now(),
-                   base::Bind(&DeleteAudioBus, base::Passed(&audio_bus))));
+                   base::Bind(&DeleteAudioBus, base::Passed(&output_bus))));
   }
 
-  void SendAudio(const media::AudioBus* audio_bus_ptr,
+  void SendAudio(const media::AudioBus* audio_bus,
                  const base::TimeTicks& recorded_time,
                  const base::Closure& done_callback) {
     DCHECK(render_thread_task_runner_->BelongsToCurrentThread());
     DCHECK(frame_input_);
-    frame_input_->InsertAudio(audio_bus_ptr, recorded_time, done_callback);
+    frame_input_->InsertAudio(audio_bus, recorded_time, done_callback);
   }
 
   // Called on real-time audio thread.
   virtual void OnSetFormat(const media::AudioParameters& params) OVERRIDE {
-    NOTIMPLEMENTED();
+    resampler_.reset(new media::MultiChannelResampler(
+        output_channels_,
+        static_cast<double>(params.sample_rate()) / output_sample_rate_,
+        params.frames_per_buffer(),
+        base::Bind(&CastAudioSink::ProvideData, base::Unretained(this))));
+    input_bytes_per_frame_ = params.bits_per_sample() / 8;

[DaleCurtis, 2014/03/06 20:57:16]

This is calculating bytes per channel, not bytes per frame? Rename?

   }
 
   // See CastVideoSink for details.
   void AddToTrack(const scoped_refptr<media::cast::FrameInput>& frame_input) {
     DCHECK(render_thread_task_runner_->BelongsToCurrentThread());
     frame_input_ = frame_input;
     if (!sink_added_) {
       AddToAudioTrack(this, track_);
       sink_added_ = true;
     }
   }
 
+  void ProvideData(int frame_delay, media::AudioBus* output_bus) {
+    DCHECK_EQ(input_frames_, output_bus->frames());
+    output_bus->FromInterleaved(input_data_, input_frames_,
+                                input_bytes_per_frame_);
+  }
+
  private:
   blink::WebMediaStreamTrack track_;
   scoped_refptr<media::cast::FrameInput> frame_input_;
   bool sink_added_;
   CastRtpStream::ErrorCallback error_callback_;
   base::WeakPtrFactory<CastAudioSink> weak_factory_;
   scoped_refptr<base::SingleThreadTaskRunner> render_thread_task_runner_;
 
+  scoped_ptr<media::MultiChannelResampler> resampler_;
+  const int output_channels_;
+  const int output_sample_rate_;
+  const void* input_data_;
+  int input_frames_;
+  int input_bytes_per_frame_;
+
   DISALLOW_COPY_AND_ASSIGN(CastAudioSink);
 };
 
 CastRtpParams::CastRtpParams(const CastRtpPayloadParams& payload_params)
     : payload(payload_params) {}
 
 CastCodecSpecificParams::CastCodecSpecificParams() {}
 
 CastCodecSpecificParams::~CastCodecSpecificParams() {}
 
@@ skipping 44 matching lines @@
                           const ErrorCallback& error_callback) {
   stop_callback_ = stop_callback;
   error_callback_ = error_callback;
 
   if (IsAudio()) {
     AudioSenderConfig config;
     if (!ToAudioSenderConfig(params, &config)) {
       DidEncounterError("Invalid parameters for audio.");
       return;
     }
+
     // In case of error we have to go through DidEncounterError() to stop
     // the streaming after reporting the error.
     audio_sink_.reset(new CastAudioSink(
         track_,
         media::BindToCurrentLoop(base::Bind(&CastRtpStream::DidEncounterError,
-                                            weak_factory_.GetWeakPtr()))));
+                                            weak_factory_.GetWeakPtr())),
+        params.payload.channels,
+        params.payload.clock_rate));
     cast_session_->StartAudio(
         config,
         base::Bind(&CastAudioSink::AddToTrack,
                    audio_sink_->AsWeakPtr()));
     start_callback.Run();
   } else {
     VideoSenderConfig config;
     if (!ToVideoSenderConfig(params, &config)) {
       DidEncounterError("Invalid parameters for video.");
       return;
@@ skipping 27 matching lines @@
 }
 
 bool CastRtpStream::IsAudio() const {
   return track_.source().type() == blink::WebMediaStreamSource::TypeAudio;
 }
 
 void CastRtpStream::DidEncounterError(const std::string& message) {
   error_callback_.Run(message);
   Stop();
 }