Cast: Resample input audio to the configured sample rate

chrome/renderer/media/cast_rtp_stream.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/renderer/media/cast_rtp_stream.h"
 
 #include "base/bind.h"
 #include "base/logging.h"
 #include "base/memory/weak_ptr.h"
 #include "chrome/renderer/media/cast_session.h"
 #include "chrome/renderer/media/cast_udp_transport.h"
 #include "content/public/renderer/media_stream_audio_sink.h"
 #include "content/public/renderer/media_stream_video_sink.h"
 #include "content/public/renderer/render_thread.h"
+#include "media/audio/audio_parameters.h"
 #include "media/base/audio_bus.h"
+#include "media/base/audio_fifo.h"
 #include "media/base/bind_to_current_loop.h"
+#include "media/base/multi_channel_resampler.h"
 #include "media/cast/cast_config.h"
 #include "media/cast/cast_defines.h"
 #include "media/cast/cast_sender.h"
 #include "media/cast/transport/cast_transport_config.h"
 #include "third_party/WebKit/public/platform/WebMediaStreamSource.h"
 
 using media::cast::AudioSenderConfig;
 using media::cast::VideoSenderConfig;
 
 namespace {
+
 const char kCodecNameOpus[] = "OPUS";
 const char kCodecNameVp8[] = "VP8";
 
+// This constant defines the number of sets of audio data to buffer
+// in the FIFO. If input audio and output data have different resampling
+// rates then buffer is necessary to avoid audio glitches.
+// See CastAudioSink::ResampleData() and CastAudioSink::OnSetFormat()
+// for more defaults.
+const int kBufferAudioData = 2;
+
 CastRtpPayloadParams DefaultOpusPayload() {
   CastRtpPayloadParams payload;
   payload.ssrc = 1;
   payload.feedback_ssrc = 1;
   payload.payload_type = 127;
   payload.codec_name = kCodecNameOpus;
   payload.clock_rate = 48000;
   payload.channels = 2;
   payload.min_bitrate = payload.max_bitrate =
       media::cast::kDefaultAudioEncoderBitrate;
@@ skipping 127 matching lines @@
 // Receives audio data from a MediaStreamTrack. Data is submitted to
 // media::cast::FrameInput.
 //
 // Threading: Audio frames are received on the real-time audio thread.
 class CastAudioSink : public base::SupportsWeakPtr<CastAudioSink>,
                       public content::MediaStreamAudioSink {
  public:
   // |track| provides data for this sink.
   // |error_callback| is called if audio formats don't match.
   CastAudioSink(const blink::WebMediaStreamTrack& track,
-                const CastRtpStream::ErrorCallback& error_callback)
+                const CastRtpStream::ErrorCallback& error_callback,
+                int output_channels,
+                int output_sample_rate)
       : track_(track),
         sink_added_(false),
         error_callback_(error_callback),
         weak_factory_(this),
         render_thread_task_runner_(content::RenderThread::Get()
-                                       ->GetMessageLoop()
-                                       ->message_loop_proxy()) {}
+                                   ->GetMessageLoop()
+                                   ->message_loop_proxy()),
+        output_channels_(output_channels),
+        output_sample_rate_(output_sample_rate) {}
 
   virtual ~CastAudioSink() {
     if (sink_added_)
       RemoveFromAudioTrack(this, track_);
   }
 
   // Called on real-time audio thread.
   // content::MediaStreamAudioSink implementation.
   virtual void OnData(const int16* audio_data,
                       int sample_rate,
                       int number_of_channels,
                       int number_of_frames) OVERRIDE {
-    scoped_ptr<media::AudioBus> audio_bus(
-        media::AudioBus::Create(number_of_channels, number_of_frames));
-    audio_bus->FromInterleaved(audio_data, number_of_frames, 2);
+    scoped_ptr<media::AudioBus> input_bus =

[DaleCurtis, 2014/03/07 01:14:03]

Don't create this every time, it can be quite expensive.  |number_of_frames|
should be constant, instead create it in SetFormat() below and just have a
FromInterleaved here.

I see you need at least one to pass to SendAudio, which kind of stinks, so
choose whether it's easier avoid the allocation on output or input.

[Alpha Left Google, 2014/03/07 01:34:01]

I think the only way is to save on the allocation for the input. But then the
code will be different to support having resampling or not. I actually prefer
just doing the allocation and fix later if this turns out to be a problem for
performance.

[DaleCurtis, 2014/03/07 01:49:00]

How about reflowing this block so that ResampleData() takes |audio_data| instead
of an input_bus.  Then you can allocate a resampler_input_ bus during
SetFormat().  These callbacks happen a lot and allocating a large aligned memory
block everytime in expensive.  In testing elsewhere the slowdown was ~2.4x (on a
z620 no less) vs not allocating everytime.

+        media::AudioBus::Create(
+            number_of_channels, number_of_frames).Pass();
+    input_bus->FromInterleaved(
+        audio_data, number_of_frames, number_of_channels);
+
+    if (resampler_) {
+      DCHECK_EQ(number_of_channels, output_channels_);
+      input_bus = ResampleData(input_bus.Pass(), sample_rate);
+      if (!input_bus)
+        return;
+    }
 
     // TODO(hclam): Pass in the accurate capture time to have good
     // audio / video sync.
 
     // TODO(hclam): We shouldn't hop through the render thread.
     // Bounce the call from the real-time audio thread to the render thread.
     // Needed since frame_input_ can be changed runtime by the render thread.
-    media::AudioBus* const audio_bus_ptr = audio_bus.get();
+    media::AudioBus* const input_bus_ptr = input_bus.get();
     render_thread_task_runner_->PostTask(
         FROM_HERE,
         base::Bind(&CastAudioSink::SendAudio,
                    weak_factory_.GetWeakPtr(),
-                   audio_bus_ptr,
+                   input_bus_ptr,
                    base::TimeTicks::Now(),
-                   base::Bind(&DeleteAudioBus, base::Passed(&audio_bus))));
+                   base::Bind(&DeleteAudioBus, base::Passed(&input_bus))));
   }
 
-  void SendAudio(const media::AudioBus* audio_bus_ptr,
+  // Return a resampled audio data from input. This is called when the
+  // input sample rate doesn't match the output.
+  // The flow of data is as follows:
+  // |audio_data| ->
+  //     AudioFifo |fifo_| ->
+  //         MultiChannelResampler |resampler|.
+  //
+  // The resampler pulls data out of the FIFO and resample the data in
+  // frequency domain. It might call |fifo_| for more than once. But no more
+  // than |kBufferAudioData| times. That's why we need to ensure |fifo_| has
+  // at least |kBufferAudioData| sets of audio data. Each set of audio data is
+  // 10ms worth.
+  scoped_ptr<media::AudioBus> ResampleData(

[DaleCurtis, 2014/03/07 01:14:03]

Seems unnecessary to extract this to a new function given how little OnData()
does.  Up to you though.

[Alpha Left Google, 2014/03/07 01:34:01]

It looks cleaner to me to have a separate method for resample logic.

+      scoped_ptr<media::AudioBus> input_bus,
+      int input_sample_rate) {
+    // Make sure FIFO is completely full.
+    fifo_->Push(input_bus.get());
+    if (fifo_->frames() != fifo_->max_frames())

[DaleCurtis, 2014/03/07 01:14:03]

I think you just want to make sure you don't respond to the first callback. 
Like I said it's possible you get two callbacks in one request, you just won't
get one at all on the next call to Resample().

[Alpha Left Google, 2014/03/07 01:34:01]

I changed it to a preroll.

+      return scoped_ptr<media::AudioBus>();
+
+    scoped_ptr<media::AudioBus> output_bus(

[DaleCurtis, 2014/03/07 01:14:03]

Ditto.  Just return a bool if there's no data available.

[Alpha Left Google, 2014/03/07 01:34:01]

Not sure I understand the suggestion here.

+        media::AudioBus::Create(
+            output_channels_,
+            output_sample_rate_ * input_bus->frames() / input_sample_rate));
+
+    // Resampler will then call ProvideData() below to fetch data from
+    // |input_data_|.
+    resampler_->Resample(output_bus->frames(), output_bus.get());
+    return output_bus.Pass();
+  }
+
+  void SendAudio(const media::AudioBus* audio_bus,
                  const base::TimeTicks& recorded_time,
                  const base::Closure& done_callback) {
     DCHECK(render_thread_task_runner_->BelongsToCurrentThread());
     DCHECK(frame_input_);
-    frame_input_->InsertAudio(audio_bus_ptr, recorded_time, done_callback);
+    frame_input_->InsertAudio(audio_bus, recorded_time, done_callback);
   }
 
   // Called on real-time audio thread.
   virtual void OnSetFormat(const media::AudioParameters& params) OVERRIDE {
-    NOTIMPLEMENTED();
+    if (params.sample_rate() == output_sample_rate_)

[DaleCurtis, 2014/03/07 01:14:03]

When and who calls this? Do you need to worry about blowing away the fifo or
resampler while a callback is in progress above?

[Alpha Left Google, 2014/03/07 01:34:01]

This is called once before there audio data arrives by the provider of audio
data. I don't think we need to worry about parameters changed or this method
called a couple times.

+      return;
+    fifo_.reset(new media::AudioFifo(
+        output_channels_,
+        kBufferAudioData * params.frames_per_buffer()));
+    resampler_.reset(new media::MultiChannelResampler(
+        output_channels_,
+        static_cast<double>(params.sample_rate()) / output_sample_rate_,
+        params.frames_per_buffer(),
+        base::Bind(&CastAudioSink::ProvideData, base::Unretained(this))));
   }
 
   // See CastVideoSink for details.
   void AddToTrack(const scoped_refptr<media::cast::FrameInput>& frame_input) {
     DCHECK(render_thread_task_runner_->BelongsToCurrentThread());
     frame_input_ = frame_input;
     if (!sink_added_) {
       AddToAudioTrack(this, track_);
       sink_added_ = true;
     }
   }
 
+  void ProvideData(int frame_delay, media::AudioBus* output_bus) {
+    fifo_->Consume(output_bus, 0, output_bus->frames());
+  }
+
  private:
   blink::WebMediaStreamTrack track_;
   scoped_refptr<media::cast::FrameInput> frame_input_;
   bool sink_added_;
   CastRtpStream::ErrorCallback error_callback_;
   base::WeakPtrFactory<CastAudioSink> weak_factory_;
   scoped_refptr<base::SingleThreadTaskRunner> render_thread_task_runner_;
 
+  scoped_ptr<media::MultiChannelResampler> resampler_;
+  scoped_ptr<media::AudioFifo> fifo_;
+  const int output_channels_;
+  const int output_sample_rate_;
+
   DISALLOW_COPY_AND_ASSIGN(CastAudioSink);
 };
 
 CastRtpParams::CastRtpParams(const CastRtpPayloadParams& payload_params)
     : payload(payload_params) {}
 
 CastCodecSpecificParams::CastCodecSpecificParams() {}
 
 CastCodecSpecificParams::~CastCodecSpecificParams() {}
 
@@ skipping 44 matching lines @@
                           const ErrorCallback& error_callback) {
   stop_callback_ = stop_callback;
   error_callback_ = error_callback;
 
   if (IsAudio()) {
     AudioSenderConfig config;
     if (!ToAudioSenderConfig(params, &config)) {
       DidEncounterError("Invalid parameters for audio.");
       return;
     }
+
     // In case of error we have to go through DidEncounterError() to stop
     // the streaming after reporting the error.
     audio_sink_.reset(new CastAudioSink(
         track_,
         media::BindToCurrentLoop(base::Bind(&CastRtpStream::DidEncounterError,
-                                            weak_factory_.GetWeakPtr()))));
+                                            weak_factory_.GetWeakPtr())),
+        params.payload.channels,
+        params.payload.clock_rate));
     cast_session_->StartAudio(
         config,
         base::Bind(&CastAudioSink::AddToTrack,
                    audio_sink_->AsWeakPtr()));
     start_callback.Run();
   } else {
     VideoSenderConfig config;
     if (!ToVideoSenderConfig(params, &config)) {
       DidEncounterError("Invalid parameters for video.");
       return;
@@ skipping 27 matching lines @@
 }
 
 bool CastRtpStream::IsAudio() const {
   return track_.source().type() == blink::WebMediaStreamSource::TypeAudio;
 }
 
 void CastRtpStream::DidEncounterError(const std::string& message) {
   error_callback_.Run(message);
   Stop();
 }