[Chromecast] Add support for different playback rates to ALSA backend

chromecast/media/cma/backend/alsa/audio_decoder_alsa.cc

Index: chromecast/media/cma/backend/alsa/audio_decoder_alsa.cc
diff --git a/chromecast/media/cma/backend/alsa/audio_decoder_alsa.cc b/chromecast/media/cma/backend/alsa/audio_decoder_alsa.cc
index cb6014f4b641d5e6b795a3083f63fb2a522f3046..6eff4920e4e48d62fe94c0d8b0c0772409941b45 100644
--- a/chromecast/media/cma/backend/alsa/audio_decoder_alsa.cc
+++ b/chromecast/media/cma/backend/alsa/audio_decoder_alsa.cc
@@ -15,8 +15,15 @@
 #include "base/trace_event/trace_event.h"
 #include "chromecast/base/task_runner_impl.h"
 #include "chromecast/media/cma/backend/alsa/media_pipeline_backend_alsa.h"
+#include "chromecast/media/cma/base/decoder_buffer_adapter.h"
 #include "chromecast/media/cma/base/decoder_buffer_base.h"
 #include "chromecast/public/media/cast_decoder_buffer.h"
+#include "media/base/audio_buffer.h"
+#include "media/base/audio_bus.h"
+#include "media/base/channel_layout.h"
+#include "media/base/decoder_buffer.h"
+#include "media/base/sample_format.h"
+#include "media/filters/audio_renderer_algorithm.h"
 
 #define TRACE_FUNCTION_ENTRY0() TRACE_EVENT0("cma", __FUNCTION__)
 
@@ -31,26 +38,51 @@ namespace media {
 
 namespace {
 
+const int kNumChannels = 2;
+const int kBitsPerSample = 32;
+const int kDefaultFramesPerBuffer = 1024;
+const int kSilenceBufferFrames = 2048;
+const int kMaxOutputMs = 20;
+const int kMillisecondsPerSecond = 1000;
+
+const double kPlaybackRateEpsilon = 0.001;
+
 const CastAudioDecoder::OutputFormat kDecoderSampleFormat =
     CastAudioDecoder::kOutputPlanarFloat;
 
-const int64_t kInvalidDelayTimestamp = std::numeric_limits<int64_t>::min();
+const int64_t kInvalidTimestamp = std::numeric_limits<int64_t>::min();
 
-AudioDecoderAlsa::RenderingDelay kInvalidRenderingDelay() {
-  AudioDecoderAlsa::RenderingDelay delay;
-  delay.timestamp_microseconds = kInvalidDelayTimestamp;
-  delay.delay_microseconds = 0;
-  return delay;
+const int64_t kNoPendingOutput = -1;
+
+scoped_refptr<::media::AudioBuffer> CreateSilenceBuffer(int sample_rate) {

[DaleCurtis, 2016/12/13 20:13:35]

Just use AudioBuffer::CreateEmptyBuffer() it will create a silent buffer w/ zero
size and N frames.

[kmackay, 2016/12/15 17:29:40]

Done.

+  scoped_refptr<::media::AudioBuffer> silence_buffer =
+      ::media::AudioBuffer::CreateBuffer(
+          ::media::kSampleFormatPlanarF32, ::media::CHANNEL_LAYOUT_STEREO,
+          kNumChannels, sample_rate, kSilenceBufferFrames);
+  for (uint8_t* channel : silence_buffer->channel_data()) {
+    float* real_channel = reinterpret_cast<float*>(channel);
+    std::fill_n(real_channel, kSilenceBufferFrames, 0.0f);
+  }
+  return silence_buffer;
 }
 
 }  // namespace
 
+AudioDecoderAlsa::RateShifterInfo::RateShifterInfo(float playback_rate)
+    : rate(playback_rate), input_frames(0), output_frames(0) {}
+
 AudioDecoderAlsa::AudioDecoderAlsa(MediaPipelineBackendAlsa* backend)
     : backend_(backend),
       task_runner_(backend->GetTaskRunner()),
       delegate_(nullptr),
-      is_eos_(false),
-      error_(false),
+      pending_buffer_complete_(false),
+      got_eos_(false),
+      pushed_eos_(false),
+      mixer_error_(false),
+      rate_shifter_output_(
+          ::media::AudioBus::Create(kNumChannels, kDefaultFramesPerBuffer)),
+      current_pts_(kInvalidTimestamp),
+      pending_output_frames_(kNoPendingOutput),
       volume_multiplier_(1.0f),
       weak_factory_(this) {
   TRACE_FUNCTION_ENTRY0();
@@ -74,11 +106,14 @@ void AudioDecoderAlsa::Initialize() {
   TRACE_FUNCTION_ENTRY0();
   DCHECK(delegate_);
   stats_ = Statistics();
-  is_eos_ = false;
-  last_buffer_pts_ = std::numeric_limits<int64_t>::min();
-
-  last_known_delay_.timestamp_microseconds = kInvalidDelayTimestamp;
-  last_known_delay_.delay_microseconds = 0;
+  pending_buffer_complete_ = false;
+  got_eos_ = false;
+  pushed_eos_ = false;
+  current_pts_ = kInvalidTimestamp;
+  pending_output_frames_ = kNoPendingOutput;
+
+  last_mixer_delay_.timestamp_microseconds = kInvalidTimestamp;
+  last_mixer_delay_.delay_microseconds = 0;
 }
 
 bool AudioDecoderAlsa::Start(int64_t start_pts) {
@@ -90,8 +125,12 @@ bool AudioDecoderAlsa::Start(int64_t start_pts) {
   mixer_input_->SetVolumeMultiplier(volume_multiplier_);
   // Create decoder_ if necessary. This can happen if Stop() was called, and
   // SetConfig() was not called since then.
-  if (!decoder_)
+  if (!decoder_) {
     CreateDecoder();
+  }
+  if (!rate_shifter_) {
+    CreateRateShifter(config_.samples_per_second);
+  }
   return true;
 }
 
@@ -99,6 +138,8 @@ void AudioDecoderAlsa::Stop() {
   TRACE_FUNCTION_ENTRY0();
   decoder_.reset();
   mixer_input_.reset();
+  rate_shifter_.reset();
+  weak_factory_.InvalidateWeakPtrs();
 
   Initialize();
 }
@@ -117,20 +158,35 @@ bool AudioDecoderAlsa::Resume() {
   return true;
 }
 
+bool AudioDecoderAlsa::SetPlaybackRate(float rate) {

[DaleCurtis, 2016/12/13 20:13:35]

Hmmm, are you using this to actually change playback rates or is this for
varispeed playback? The AudioRendererAlgorithm uses WSOLA which isn't
appropriate for this task (compute heavy and may dramatically change the
signal), so for simple varispeed at rates near ~1.0 you probably just want
MultiChannelResampler and to change the io ratio.

https://cs.chromium.org/chromium/src/media/base/audio_shifter.h

Does this pretty much.

[kmackay, 2016/12/13 20:39:47]

We want to allow playback at eg 2x normal rate or 0.5x normal rate without pitch
shifting.

+  if (std::abs(rate - 1.0) < kPlaybackRateEpsilon) {
+    // AudioRendererAlgorithm treats values close to 1 as exactly 1.
+    rate = 1.0f;
+  }
+  LOG(INFO) << "SetPlaybackRate to " << rate;
+
+  while (!rate_shifter_info_.empty() &&
+         rate_shifter_info_.back().input_frames == 0) {
+    rate_shifter_info_.pop_back();
+  }
+  rate_shifter_info_.push_back(RateShifterInfo(rate));
+  return true;
+}
+
 AudioDecoderAlsa::BufferStatus AudioDecoderAlsa::PushBuffer(
     CastDecoderBuffer* buffer) {
   TRACE_FUNCTION_ENTRY0();
   DCHECK(task_runner_->BelongsToCurrentThread());
   DCHECK(buffer);
-  DCHECK(!is_eos_);
-  DCHECK(!error_);
+  DCHECK(!got_eos_);
+  DCHECK(!mixer_error_);
+  DCHECK(!pending_buffer_complete_);
 
   uint64_t input_bytes = buffer->end_of_stream() ? 0 : buffer->data_size();
   scoped_refptr<DecoderBufferBase> buffer_base(
       static_cast<DecoderBufferBase*>(buffer));
   if (!buffer->end_of_stream()) {
-    last_buffer_pts_ = buffer->timestamp();
-    current_pts_ = std::min(current_pts_, last_buffer_pts_);
+    current_pts_ = buffer->timestamp();
   }
 
   // If the buffer is already decoded, do not attempt to decode. Call
@@ -174,6 +230,11 @@ bool AudioDecoderAlsa::SetConfig(const AudioConfig& config) {
     return false;
   }
 
+  if (!rate_shifter_ ||
+      config.samples_per_second != config_.samples_per_second) {
+    CreateRateShifter(config.samples_per_second);
+  }
+
   if (mixer_input_ && config.samples_per_second != config_.samples_per_second) {
     // Destroy the old input first to ensure that the mixer output sample rate
     // is updated.
@@ -181,11 +242,17 @@ bool AudioDecoderAlsa::SetConfig(const AudioConfig& config) {
     mixer_input_.reset(new StreamMixerAlsaInput(
         this, config.samples_per_second, backend_->Primary()));
     mixer_input_->SetVolumeMultiplier(volume_multiplier_);
+    pending_output_frames_ = kNoPendingOutput;
   }
 
   config_ = config;
   decoder_.reset();
   CreateDecoder();
+
+  if (pending_buffer_complete_ && !rate_shifter_->IsQueueFull()) {
+    pending_buffer_complete_ = false;
+    delegate_->OnPushBufferComplete(MediaPipelineBackendAlsa::kBufferSuccess);
+  }
   return true;
 }
 
@@ -208,6 +275,17 @@ void AudioDecoderAlsa::CreateDecoder() {
                  weak_factory_.GetWeakPtr()));
 }
 
+void AudioDecoderAlsa::CreateRateShifter(int samples_per_second) {
+  rate_shifter_info_.clear();
+  rate_shifter_info_.push_back(RateShifterInfo(1.0f));
+
+  rate_shifter_.reset(new ::media::AudioRendererAlgorithm());
+  rate_shifter_->Initialize(::media::AudioParameters(
+      ::media::AudioParameters::AUDIO_PCM_LINEAR,
+      ::media::CHANNEL_LAYOUT_STEREO, samples_per_second, kBitsPerSample,
+      kDefaultFramesPerBuffer));
+}
+
 bool AudioDecoderAlsa::SetVolume(float multiplier) {
   TRACE_FUNCTION_ENTRY1(multiplier);
   DCHECK(task_runner_->BelongsToCurrentThread());
@@ -219,7 +297,24 @@ bool AudioDecoderAlsa::SetVolume(float multiplier) {
 
 AudioDecoderAlsa::RenderingDelay AudioDecoderAlsa::GetRenderingDelay() {
   TRACE_FUNCTION_ENTRY0();
-  return last_known_delay_;
+  AudioDecoderAlsa::RenderingDelay delay = last_mixer_delay_;
+  if (delay.timestamp_microseconds != kInvalidTimestamp) {
+    double usec_per_sample = 1000000.0 / config_.samples_per_second;
+
+    // Account for data that has been queued in the rate shifters.
+    for (const RateShifterInfo& info : rate_shifter_info_) {
+      double queued_output_frames =
+          (info.input_frames / info.rate) - info.output_frames;
+      delay.delay_microseconds += queued_output_frames * usec_per_sample;
+    }
+
+    // Account for data that is in the process of being pushed to the mixer.
+    if (pending_output_frames_ != kNoPendingOutput) {
+      delay.delay_microseconds += pending_output_frames_ * usec_per_sample;
+    }
+  }
+
+  return delay;
 }
 
 void AudioDecoderAlsa::OnDecoderInitialized(bool success) {
@@ -237,29 +332,160 @@ void AudioDecoderAlsa::OnBufferDecoded(
     const scoped_refptr<DecoderBufferBase>& decoded) {
   TRACE_FUNCTION_ENTRY0();
   DCHECK(task_runner_->BelongsToCurrentThread());
-  DCHECK(!is_eos_);
-
-  Statistics delta = Statistics();
+  DCHECK(!got_eos_);
+  DCHECK(!pending_buffer_complete_);
+  DCHECK(rate_shifter_);
 
   if (status == CastAudioDecoder::Status::kDecodeError) {
     LOG(ERROR) << "Decode error";
-    task_runner_->PostTask(FROM_HERE,
-                           base::Bind(&AudioDecoderAlsa::OnWritePcmCompletion,
-                                      weak_factory_.GetWeakPtr(),
-                                      MediaPipelineBackendAlsa::kBufferFailed,
-                                      kInvalidRenderingDelay()));
-    UpdateStatistics(delta);
+    delegate_->OnPushBufferComplete(MediaPipelineBackendAlsa::kBufferFailed);
+    return;
+  }
+  if (mixer_error_) {
+    delegate_->OnPushBufferComplete(MediaPipelineBackendAlsa::kBufferFailed);
     return;
   }
 
+  Statistics delta;
   delta.decoded_bytes = input_bytes;
   UpdateStatistics(delta);
 
-  if (decoded->end_of_stream())
-    is_eos_ = true;
+  if (decoded->end_of_stream()) {
+    got_eos_ = true;
+  } else {
+    int input_frames = decoded->data_size() / (kNumChannels * sizeof(float));
+
+    DCHECK(!rate_shifter_info_.empty());
+    RateShifterInfo* rate_info = &rate_shifter_info_.front();
+    // Bypass rate shifter if the rate is 1.0, and there are no frames queued
+    // in the rate shifter.
+    if (rate_info->rate == 1.0 && rate_shifter_->frames_buffered() == 0 &&
+        pending_output_frames_ == kNoPendingOutput) {
+      DCHECK_EQ(rate_info->output_frames, rate_info->input_frames);
+      pending_buffer_complete_ = true;
+      pending_output_frames_ = input_frames;
+      if (got_eos_) {
+        DCHECK(!pushed_eos_);
+        pushed_eos_ = true;
+      }
+      mixer_input_->WritePcm(decoded);
+      return;
+    }
+
+    // Otherwise, queue data into the rate shifter, and then try to push the
+    // rate-shifted data.
+    const uint8_t* channels[kNumChannels] = {
+        decoded->data(), decoded->data() + input_frames * sizeof(float)};
+    scoped_refptr<::media::AudioBuffer> buffer = ::media::AudioBuffer::CopyFrom(
+        ::media::kSampleFormatPlanarF32, ::media::CHANNEL_LAYOUT_STEREO,
+        kNumChannels, config_.samples_per_second, input_frames, channels,
+        base::TimeDelta());
+    rate_shifter_->EnqueueBuffer(buffer);
+    rate_shifter_info_.back().input_frames += input_frames;
+  }
 
+  PushRateShifted();
+  DCHECK(!rate_shifter_info_.empty());
+  // Can't check got_eos_ here, since it may have already been reset by a call
+  // to Stop().
+  if (decoded->end_of_stream() || (!rate_shifter_->IsQueueFull() &&
+                                   rate_shifter_info_.front().rate != 1.0)) {
+    delegate_->OnPushBufferComplete(MediaPipelineBackendAlsa::kBufferSuccess);
+  } else {
+    pending_buffer_complete_ = true;
+  }
+}
+
+void AudioDecoderAlsa::PushRateShifted() {
   DCHECK(mixer_input_);
-  mixer_input_->WritePcm(decoded);
+
+  if (pending_output_frames_ != kNoPendingOutput) {
+    return;
+  }
+
+  if (got_eos_) {
+    // Push some silence into the rate shifter so we can get out any remaining
+    // rate-shifted data.
+    rate_shifter_->EnqueueBuffer(
+        CreateSilenceBuffer(config_.samples_per_second));
+  }
+
+  DCHECK(!rate_shifter_info_.empty());
+  RateShifterInfo* rate_info = &rate_shifter_info_.front();
+  int64_t possible_output_frames = rate_info->input_frames / rate_info->rate;
+  DCHECK_GE(possible_output_frames, rate_info->output_frames);
+
+  int desired_output_frames = possible_output_frames - rate_info->output_frames;
+  if (desired_output_frames == 0) {
+    if (got_eos_) {
+      DCHECK(!pushed_eos_);
+      pending_output_frames_ = 0;
+      pushed_eos_ = true;
+
+      scoped_refptr<DecoderBufferBase> eos_buffer(
+          new DecoderBufferAdapter(::media::DecoderBuffer::CreateEOSBuffer()));
+      mixer_input_->WritePcm(eos_buffer);
+    }
+    return;
+  }
+  // Don't push too many frames at a time.
+  desired_output_frames = std::min(
+      desired_output_frames,
+      config_.samples_per_second * kMaxOutputMs / kMillisecondsPerSecond);
+
+  if (desired_output_frames > rate_shifter_output_->frames()) {
+    rate_shifter_output_ =
+        ::media::AudioBus::Create(kNumChannels, desired_output_frames);
+  }
+
+  int out_frames = rate_shifter_->FillBuffer(
+      rate_shifter_output_.get(), 0, desired_output_frames, rate_info->rate);
+  if (out_frames <= 0) {
+    return;
+  }
+
+  rate_info->output_frames += out_frames;
+  DCHECK_GE(possible_output_frames, rate_info->output_frames);
+
+  int channel_data_size = out_frames * sizeof(float);
+  scoped_refptr<DecoderBufferBase> output_buffer(new DecoderBufferAdapter(
+      new ::media::DecoderBuffer(channel_data_size * kNumChannels)));
+  for (int c = 0; c < kNumChannels; ++c) {
+    memcpy(output_buffer->writable_data() + c * channel_data_size,
+           rate_shifter_output_->channel(c), channel_data_size);
+  }
+  pending_output_frames_ = out_frames;
+  mixer_input_->WritePcm(output_buffer);
+
+  if (rate_shifter_info_.size() > 1 &&
+      possible_output_frames == rate_info->output_frames) {
+    double remaining_input_frames =
+        rate_info->input_frames - (rate_info->output_frames * rate_info->rate);
+    rate_shifter_info_.pop_front();
+
+    rate_info = &rate_shifter_info_.front();
+    LOG(INFO) << "New playback rate in effect: " << rate_info->rate;
+    rate_info->input_frames += remaining_input_frames;
+    DCHECK_EQ(0, rate_info->output_frames);
+
+    // If new playback rate is 1.0, clear out 'extra' data in the rate shifter.
+    // When doing rate shifting, the rate shifter queue holds data after it has
+    // been logically played; once we switch to passthrough mode (rate == 1.0),
+    // that old data needs to be cleared out.
+    if (rate_info->rate == 1.0) {
+      int extra_frames = rate_shifter_->frames_buffered() -
+                         static_cast<int>(rate_info->input_frames);
+      if (extra_frames > 0) {
+        // Clear out extra buffered data.
+        std::unique_ptr<::media::AudioBus> dropped =
+            ::media::AudioBus::Create(kNumChannels, extra_frames);
+        int cleared_frames =
+            rate_shifter_->FillBuffer(dropped.get(), 0, extra_frames, 1.0f);
+        DCHECK_EQ(extra_frames, cleared_frames);
+      }
+      rate_info->input_frames = rate_shifter_->frames_buffered();
+    }
+  }
 }
 
 bool AudioDecoderAlsa::BypassDecoder() const {
@@ -273,13 +499,34 @@ void AudioDecoderAlsa::OnWritePcmCompletion(BufferStatus status,
                                             const RenderingDelay& delay) {
   TRACE_FUNCTION_ENTRY0();
   DCHECK(task_runner_->BelongsToCurrentThread());
-  if (status == MediaPipelineBackendAlsa::kBufferSuccess && !is_eos_)
-    current_pts_ = last_buffer_pts_;
-  if (delay.timestamp_microseconds != kInvalidDelayTimestamp)
-    last_known_delay_ = delay;
-  delegate_->OnPushBufferComplete(status);
-  if (is_eos_)
+  DCHECK_EQ(MediaPipelineBackendAlsa::kBufferSuccess, status);
+  pending_output_frames_ = kNoPendingOutput;
+  last_mixer_delay_ = delay;
+
+  if (pushed_eos_) {
+    if (pending_buffer_complete_) {
+      pending_buffer_complete_ = false;
+      delegate_->OnPushBufferComplete(MediaPipelineBackendAlsa::kBufferSuccess);
+    }
     delegate_->OnEndOfStream();
+  } else {
+    task_runner_->PostTask(FROM_HERE, base::Bind(&AudioDecoderAlsa::PushMorePcm,
+                                                 weak_factory_.GetWeakPtr()));
+  }
+}
+
+void AudioDecoderAlsa::PushMorePcm() {
+  PushRateShifted();
+
+  DCHECK(!rate_shifter_info_.empty());
+  if (pending_buffer_complete_) {
+    double rate = rate_shifter_info_.front().rate;
+    if ((rate == 1.0 && pending_output_frames_ == kNoPendingOutput) ||
+        (rate != 1.0 && !rate_shifter_->IsQueueFull())) {
+      pending_buffer_complete_ = false;
+      delegate_->OnPushBufferComplete(MediaPipelineBackendAlsa::kBufferSuccess);
+    }
+  }
 }
 
 void AudioDecoderAlsa::OnMixerError(MixerError error) {
@@ -287,7 +534,7 @@ void AudioDecoderAlsa::OnMixerError(MixerError error) {
   DCHECK(task_runner_->BelongsToCurrentThread());
   if (error != MixerError::kInputIgnored)
     LOG(ERROR) << "Mixer error occurred.";
-  error_ = true;
+  mixer_error_ = true;
   delegate_->OnDecoderError();
 }