Collapse AudioRendererMixer and OnMoreDataResampler into AudioTransform.

media/audio/audio_output_resampler.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/audio/audio_output_resampler.h"
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/command_line.h"
 #include "base/compiler_specific.h"
 #include "base/message_loop.h"
 #include "base/metrics/histogram.h"
 #include "base/time.h"
 #include "media/audio/audio_io.h"
 #include "media/audio/audio_output_dispatcher_impl.h"
 #include "media/audio/audio_output_proxy.h"
 #include "media/audio/audio_util.h"
 #include "media/audio/sample_rates.h"
-#include "media/base/audio_pull_fifo.h"
-#include "media/base/channel_mixer.h"
+#include "media/base/audio_transform.h"
 #include "media/base/limits.h"
 #include "media/base/media_switches.h"
-#include "media/base/multi_channel_resampler.h"
 
 namespace media {
 
-class OnMoreDataResampler : public AudioOutputStream::AudioSourceCallback {
+class OnMoreDataConverter
+    : public AudioOutputStream::AudioSourceCallback,
+      public AudioConverter::InputCallback {
  public:
-  OnMoreDataResampler(double io_ratio,
-                      const AudioParameters& input_params,
+  OnMoreDataConverter(const AudioParameters& input_params,
                       const AudioParameters& output_params);
-  virtual ~OnMoreDataResampler();
+  virtual ~OnMoreDataConverter();
 
   // AudioSourceCallback interface.
   virtual int OnMoreData(AudioBus* dest,
                          AudioBuffersState buffers_state) OVERRIDE;
   virtual int OnMoreIOData(AudioBus* source,
                            AudioBus* dest,
                            AudioBuffersState buffers_state) OVERRIDE;
   virtual void OnError(AudioOutputStream* stream, int code) OVERRIDE;
   virtual void WaitTillDataReady() OVERRIDE;
 
   // Sets |source_callback_|.  If this is not a new object, then Stop() must be
   // called before Start().
   void Start(AudioOutputStream::AudioSourceCallback* callback);
 
   // Clears |source_callback_| and flushes the resampler.
   void Stop();
 
  private:
-  // Called by MultiChannelResampler when more data is necessary.
-  void ProvideInput(AudioBus* audio_bus);
-
-  // Called by AudioPullFifo when more data is necessary.  Requires
-  // |source_lock_| to have been acquired.
-  void SourceCallback_Locked(AudioBus* audio_bus);
-
-  // Passes through |source| to the |source_callback_| OnMoreIOData() call.
-  void SourceIOCallback_Locked(AudioBus* source, AudioBus* dest);
+  // AudioConverter::InputCallback implementation.
+  virtual double ProvideInput(AudioBus* audio_bus,
+                              base::TimeDelta buffer_delay) OVERRIDE;
 
   // Ratio of input bytes to output bytes used to correct playback delay with
   // regard to buffering and resampling.
   double io_ratio_;
 
   // Source callback and associated lock.
   base::Lock source_lock_;
   AudioOutputStream::AudioSourceCallback* source_callback_;
 
+  // |source| passed to OnMoreIOData() which should be passed downstream.
+  AudioBus* source_bus_;
+
   // Last AudioBuffersState object received via OnMoreData(), used to correct
   // playback delay by ProvideInput() and passed on to |source_callback_|.
   AudioBuffersState current_buffers_state_;
 
-  // Total number of bytes (in terms of output parameters) stored in resampler
-  // or FIFO buffers which have not been sent to the audio device.
-  int outstanding_audio_bytes_;
+  const int input_bytes_per_second_;
 
-  // Used to buffer data between the client and the output device in cases where
-  // the client buffer size is not the same as the output device buffer size.
-  // Bound to SourceCallback_Locked() so must only be used when |source_lock_|
-  // has already been acquired.
-  scoped_ptr<AudioPullFifo> audio_fifo_;
+  // Handles resampling, buffering, and channel mixing between input and output
+  // parameters.
+  AudioConverter audio_converter_;
 
-  // Handles resampling.
-  scoped_ptr<MultiChannelResampler> resampler_;
-
-  // Handles channel transforms.  |unmixed_audio_| is a temporary destination
-  // for audio data before it goes into the channel mixer.
-  scoped_ptr<ChannelMixer> channel_mixer_;
-  scoped_ptr<AudioBus> unmixed_audio_;
-
-  int output_bytes_per_frame_;
-  int input_bytes_per_frame_;
-
-  // Since resampling is expensive, figure out if we should downmix channels
-  // before resampling.
-  bool downmix_early_;
-
-  DISALLOW_COPY_AND_ASSIGN(OnMoreDataResampler);
+  DISALLOW_COPY_AND_ASSIGN(OnMoreDataConverter);
 };
 
 // Record UMA statistics for hardware output configuration.
 static void RecordStats(const AudioParameters& output_params) {
   UMA_HISTOGRAM_ENUMERATION(
       "Media.HardwareAudioBitsPerChannel", output_params.bits_per_sample(),
       limits::kMaxBitsPerSample);
   UMA_HISTOGRAM_ENUMERATION(
       "Media.HardwareAudioChannelLayout", output_params.channel_layout(),
       CHANNEL_LAYOUT_MAX);
@@ skipping 54 matching lines @@
       AudioParameters::AUDIO_PCM_LINEAR, input_params.channel_layout(),
       input_params.sample_rate(), input_params.bits_per_sample(),
       frames_per_buffer);
 }
 
 AudioOutputResampler::AudioOutputResampler(AudioManager* audio_manager,
                                            const AudioParameters& input_params,
                                            const AudioParameters& output_params,
                                            const base::TimeDelta& close_delay)
     : AudioOutputDispatcher(audio_manager, input_params),
-      io_ratio_(1),
       close_delay_(close_delay),
       output_params_(output_params),
       streams_opened_(false) {
   DCHECK(input_params.IsValid());
   DCHECK(output_params.IsValid());
   DCHECK_EQ(output_params_.format(), AudioParameters::AUDIO_PCM_LOW_LATENCY);
 
   // Record UMA statistics for the hardware configuration.
   RecordStats(output_params);
 
   Initialize();
 }
 
 AudioOutputResampler::~AudioOutputResampler() {
   DCHECK(callbacks_.empty());
 }
 
 void AudioOutputResampler::Initialize() {
   DCHECK(!streams_opened_);
   DCHECK(callbacks_.empty());
-
-  io_ratio_ = 1;
-
-  // Only resample or rebuffer if the input parameters don't match the output
-  // parameters to avoid any unnecessary work.
-  if (params_.channels() != output_params_.channels() ||
-      params_.sample_rate() != output_params_.sample_rate() ||
-      params_.bits_per_sample() != output_params_.bits_per_sample() ||
-      params_.frames_per_buffer() != output_params_.frames_per_buffer()) {
-    if (params_.sample_rate() != output_params_.sample_rate()) {
-      double io_sample_rate_ratio = params_.sample_rate() /
-          static_cast<double>(output_params_.sample_rate());
-      // Include the I/O resampling ratio in our global I/O ratio.
-      io_ratio_ *= io_sample_rate_ratio;
-    }
-
-    // Include bits per channel differences.
-    io_ratio_ *= static_cast<double>(params_.bits_per_sample()) /
-        output_params_.bits_per_sample();
-
-    // Include channel count differences.
-    io_ratio_ *= static_cast<double>(params_.channels()) /
-        output_params_.channels();
-
-    DVLOG(1) << "I/O ratio is " << io_ratio_;
-  } else {
-    DVLOG(1) << "Input and output params are the same; in pass-through mode.";
-  }
-
-  // TODO(dalecurtis): All this code should be merged into AudioOutputMixer once
-  // we've stabilized the issues there.
   dispatcher_ = new AudioOutputDispatcherImpl(
       audio_manager_, output_params_, close_delay_);
 }
 
 bool AudioOutputResampler::OpenStream() {
   DCHECK_EQ(MessageLoop::current(), message_loop_);
 
   if (dispatcher_->OpenStream()) {
     // Only record the UMA statistic if we didn't fallback during construction
     // and only for the first stream we open.
@@ skipping 32 matching lines @@
 
   // Retry, if this fails, there's nothing left to do but report the error back.
   return dispatcher_->OpenStream();
 }
 
 bool AudioOutputResampler::StartStream(
     AudioOutputStream::AudioSourceCallback* callback,
     AudioOutputProxy* stream_proxy) {
   DCHECK_EQ(MessageLoop::current(), message_loop_);
 
-  OnMoreDataResampler* resampler_callback = NULL;
+  OnMoreDataConverter* resampler_callback = NULL;
   CallbackMap::iterator it = callbacks_.find(stream_proxy);
   if (it == callbacks_.end()) {
-    resampler_callback = new OnMoreDataResampler(
-        io_ratio_, params_, output_params_);
+    resampler_callback = new OnMoreDataConverter(params_, output_params_);
     callbacks_[stream_proxy] = resampler_callback;
   } else {
     resampler_callback = it->second;
   }
   resampler_callback->Start(callback);
   return dispatcher_->StartStream(resampler_callback, stream_proxy);
 }
 
 void AudioOutputResampler::StreamVolumeSet(AudioOutputProxy* stream_proxy,
                                            double volume) {
   DCHECK_EQ(MessageLoop::current(), message_loop_);
   dispatcher_->StreamVolumeSet(stream_proxy, volume);
 }
 
 void AudioOutputResampler::StopStream(AudioOutputProxy* stream_proxy) {
   DCHECK_EQ(MessageLoop::current(), message_loop_);
   dispatcher_->StopStream(stream_proxy);
 
   // Now that StopStream() has completed the underlying physical stream should
   // be stopped and no longer calling OnMoreData(), making it safe to Stop() the
-  // OnMoreDataResampler.
+  // OnMoreDataConverter.
   CallbackMap::iterator it = callbacks_.find(stream_proxy);
   if (it != callbacks_.end())
     it->second->Stop();
 }
 
 void AudioOutputResampler::CloseStream(AudioOutputProxy* stream_proxy) {
   DCHECK_EQ(MessageLoop::current(), message_loop_);
   dispatcher_->CloseStream(stream_proxy);
 
   // We assume that StopStream() is always called prior to CloseStream(), so
-  // that it is safe to delete the OnMoreDataResampler here.
+  // that it is safe to delete the OnMoreDataConverter here.
   CallbackMap::iterator it = callbacks_.find(stream_proxy);
   if (it != callbacks_.end()) {
     delete it->second;
     callbacks_.erase(it);
   }
 }
 
 void AudioOutputResampler::Shutdown() {
   DCHECK_EQ(MessageLoop::current(), message_loop_);
 
   // No AudioOutputProxy objects should hold a reference to us when we get
   // to this stage.
   DCHECK(HasOneRef()) << "Only the AudioManager should hold a reference";
 
   dispatcher_->Shutdown();
   DCHECK(callbacks_.empty());
 }
 
-OnMoreDataResampler::OnMoreDataResampler(
-    double io_ratio, const AudioParameters& input_params,
-    const AudioParameters& output_params)
-    : io_ratio_(io_ratio),
-      source_callback_(NULL),
-      outstanding_audio_bytes_(0),
-      output_bytes_per_frame_(output_params.GetBytesPerFrame()),
-      input_bytes_per_frame_(input_params.GetBytesPerFrame()),
-      downmix_early_(false) {
-  // Handle different input and output channel layouts.
-  if (input_params.channel_layout() != output_params.channel_layout()) {
-    DVLOG(1) << "Remixing channel layout from " << input_params.channel_layout()
-             << " to " << output_params.channel_layout() << "; from "
-             << input_params.channels() << " channels to "
-             << output_params.channels() << " channels.";
-    channel_mixer_.reset(new ChannelMixer(
-        input_params.channel_layout(), output_params.channel_layout()));
-
-    // Pare off data as early as we can for efficiency.
-    downmix_early_ = input_params.channels() > output_params.channels();
-    if (downmix_early_) {
-      DVLOG(1) << "Remixing channel layout prior to resampling.";
-      // If we're downmixing early we need a temporary AudioBus which matches
-      // the the input channel count and input frame size since we're passing
-      // |unmixed_audio_| directly to the |source_callback_|.
-      unmixed_audio_ = AudioBus::Create(input_params);
-    } else {
-      // Instead, if we're not downmixing early we need a temporary AudioBus
-      // which matches the input channel count but uses the output frame size
-      // since we'll mix into the AudioBus from the output stream.
-      unmixed_audio_ = AudioBus::Create(
-          input_params.channels(), output_params.frames_per_buffer());
-    }
-  }
-
-  // Only resample if necessary since it's expensive.
-  if (input_params.sample_rate() != output_params.sample_rate()) {
-    DVLOG(1) << "Resampling from " << input_params.sample_rate() << " to "
-             << output_params.sample_rate();
-    double io_sample_rate_ratio = input_params.sample_rate() /
-        static_cast<double>(output_params.sample_rate());
-    resampler_.reset(new MultiChannelResampler(
-        downmix_early_ ? output_params.channels() :
-            input_params.channels(),
-        io_sample_rate_ratio, base::Bind(
-            &OnMoreDataResampler::ProvideInput, base::Unretained(this))));
-  }
-
-  // Since the resampler / output device may want a different buffer size than
-  // the caller asked for, we need to use a FIFO to ensure that both sides
-  // read in chunk sizes they're configured for.
-  if (input_params.sample_rate() != output_params.sample_rate() ||
-      input_params.frames_per_buffer() != output_params.frames_per_buffer()) {
-    DVLOG(1) << "Rebuffering from " << input_params.frames_per_buffer()
-             << " to " << output_params.frames_per_buffer();
-    audio_fifo_.reset(new AudioPullFifo(
-        downmix_early_ ? output_params.channels() :
-            input_params.channels(),
-        input_params.frames_per_buffer(), base::Bind(
-            &OnMoreDataResampler::SourceCallback_Locked,
-            base::Unretained(this))));
-  }
+OnMoreDataConverter::OnMoreDataConverter(const AudioParameters& input_params,
+                                         const AudioParameters& output_params)
+    : source_callback_(NULL),
+      source_bus_(NULL),
+      input_bytes_per_second_(input_params.GetBytesPerSecond()),
+      audio_converter_(input_params, output_params, false) {

[scherkus (not reviewing), 2012/11/17 01:04:37]

who calls disable_fifo w/ true?

[DaleCurtis, 2012/11/17 01:07:14]

AudioRendererMixer.

+  io_ratio_ =
+      static_cast<double>(input_params.GetBytesPerSecond()) /
+      output_params.GetBytesPerSecond();
 }
 
-OnMoreDataResampler::~OnMoreDataResampler() {}
+OnMoreDataConverter::~OnMoreDataConverter() {}
 
-void OnMoreDataResampler::Start(
+void OnMoreDataConverter::Start(
     AudioOutputStream::AudioSourceCallback* callback) {
   base::AutoLock auto_lock(source_lock_);
   DCHECK(!source_callback_);
   source_callback_ = callback;
+
+  // While AudioConverter can handle multiple inputs, we're using it only with
+  // a single input currently.  Eventually this may be the basis for a browser
+  // side mixer.
+  audio_converter_.AddInput(this);
 }
 
-void OnMoreDataResampler::Stop() {
+void OnMoreDataConverter::Stop() {
   base::AutoLock auto_lock(source_lock_);
   source_callback_ = NULL;
-  outstanding_audio_bytes_ = 0;
-  if (audio_fifo_)
-    audio_fifo_->Clear();
-  if (resampler_)
-    resampler_->Flush();
+  audio_converter_.RemoveInput(this);
 }
 
-int OnMoreDataResampler::OnMoreData(AudioBus* dest,
+int OnMoreDataConverter::OnMoreData(AudioBus* dest,
                                     AudioBuffersState buffers_state) {
   return OnMoreIOData(NULL, dest, buffers_state);
 }
 
-int OnMoreDataResampler::OnMoreIOData(AudioBus* source,
+int OnMoreDataConverter::OnMoreIOData(AudioBus* source,
                                       AudioBus* dest,
                                       AudioBuffersState buffers_state) {
   base::AutoLock auto_lock(source_lock_);
   // While we waited for |source_lock_| the callback might have been cleared.
   if (!source_callback_) {
     dest->Zero();
     return dest->frames();
   }
 
+  source_bus_ = source;
   current_buffers_state_ = buffers_state;
+  audio_converter_.Convert(dest);
 
-  bool needs_mixing = channel_mixer_ && !downmix_early_;
-  AudioBus* temp_dest = needs_mixing ? unmixed_audio_.get() : dest;
-
-  if (!resampler_ && !audio_fifo_) {
-    // We have no internal buffers, so clear any outstanding audio data.
-    outstanding_audio_bytes_ = 0;
-    SourceIOCallback_Locked(source, temp_dest);
-  } else {
-    if (resampler_)
-      resampler_->Resample(temp_dest, temp_dest->frames());
-    else
-      ProvideInput(temp_dest);
-
-    // Calculate how much data is left in the internal FIFO and resampler.
-    outstanding_audio_bytes_ -= temp_dest->frames() * output_bytes_per_frame_;
-  }
-
-  if (needs_mixing) {
-    DCHECK_EQ(temp_dest->frames(), dest->frames());
-    channel_mixer_->Transform(temp_dest, dest);
-  }
-
-  // Due to rounding errors while multiplying against |io_ratio_|,
-  // |outstanding_audio_bytes_| might (rarely) slip below zero.
-  if (outstanding_audio_bytes_ < 0) {
-    DLOG(ERROR) << "Outstanding audio bytes went negative! Value: "
-                << outstanding_audio_bytes_;
-    outstanding_audio_bytes_ = 0;
-  }
-
-  // Always return the full number of frames requested, ProvideInput() will pad
-  // with silence if it wasn't able to acquire enough data.
+  // Always return the full number of frames requested, ProvideInput_Locked()
+  // will pad with silence if it wasn't able to acquire enough data.
   return dest->frames();
 }
 
-void OnMoreDataResampler::SourceCallback_Locked(AudioBus* dest) {
-  SourceIOCallback_Locked(NULL, dest);
+double OnMoreDataConverter::ProvideInput(AudioBus* dest,
+                                         base::TimeDelta buffer_delay) {
+  source_lock_.AssertAcquired();
+
+  // Adjust playback delay to include |buffer_delay|.
+  AudioBuffersState new_buffers_state;

[Chris Rogers, 2012/11/20 20:01:24]

TODO to stop dealing in calculations in units of bytes and switch to frames

[DaleCurtis, 2012/11/21 00:33:30]

Done.

+  new_buffers_state.pending_bytes =
+      io_ratio_ * (current_buffers_state_.total_bytes() +
+                   buffer_delay.InSecondsF() * input_bytes_per_second_);
+
+  // Retrieve data from the original callback.
+  int frames = source_callback_->OnMoreIOData(
+      source_bus_, dest, new_buffers_state);
+
+  // |source_bus_| should only be provided once.

[Chris Rogers, 2012/11/20 20:01:24]

Setting to NULL doesn't really solve the problem (which we don't need to solve
in this CL).  We should add a TODO describing that *if* we're called back more
than once then we currently don't handle |source_bus_| correctly.  That's ok for
now, but just need to make a note...

[DaleCurtis, 2012/11/21 00:33:30]

Done.

+  source_bus_ = NULL;
+
+  // Zero any unfilled frames if anything was filled, otherwise we'll just
+  // return a volume of zero and let AudioConverter drop the output.
+  if (frames > 0 && frames < dest->frames())
+    dest->ZeroFramesPartial(frames, dest->frames() - frames);
+
+  // TODO(dalecurtis): Return the correct volume here.
+  return frames > 0 ? 1 : 0;
 }
 
-void OnMoreDataResampler::SourceIOCallback_Locked(AudioBus* source,
-                                                  AudioBus* dest) {
-  source_lock_.AssertAcquired();
-
-  // Adjust playback delay to include the state of the internal buffers used by
-  // the resampler and/or the FIFO.  Since the sample rate and bits per channel
-  // may be different, we need to scale this value appropriately.
-  AudioBuffersState new_buffers_state;
-  new_buffers_state.pending_bytes = io_ratio_ *
-      (current_buffers_state_.total_bytes() + outstanding_audio_bytes_);
-
-  bool needs_downmix = channel_mixer_ && downmix_early_;
-  AudioBus* temp_dest = needs_downmix ? unmixed_audio_.get() : dest;
-
-  // Retrieve data from the original callback.  Zero any unfilled frames.
-  int frames = source_callback_->OnMoreIOData(
-      source, temp_dest, new_buffers_state);
-  if (frames < temp_dest->frames())
-    temp_dest->ZeroFramesPartial(frames, temp_dest->frames() - frames);
-
-  // Scale the number of frames we got back in terms of input bytes to output
-  // bytes accordingly.
-  outstanding_audio_bytes_ +=
-      (temp_dest->frames() * input_bytes_per_frame_) / io_ratio_;
-
-  if (needs_downmix) {
-    DCHECK_EQ(temp_dest->frames(), dest->frames());
-    channel_mixer_->Transform(temp_dest, dest);
-  }
-}
-
-void OnMoreDataResampler::ProvideInput(AudioBus* audio_bus) {
-  audio_fifo_->Consume(audio_bus, audio_bus->frames());
-}
-
-void OnMoreDataResampler::OnError(AudioOutputStream* stream, int code) {
+void OnMoreDataConverter::OnError(AudioOutputStream* stream, int code) {
   base::AutoLock auto_lock(source_lock_);
   if (source_callback_)
     source_callback_->OnError(stream, code);
 }
 
-void OnMoreDataResampler::WaitTillDataReady() {
+void OnMoreDataConverter::WaitTillDataReady() {
   base::AutoLock auto_lock(source_lock_);
-  if (source_callback_ && !outstanding_audio_bytes_)
+  if (source_callback_)
     source_callback_->WaitTillDataReady();
 }
 
 }  // namespace media