Support (E)AC3 passthrough

media/renderers/audio_renderer_impl.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/renderers/audio_renderer_impl.h"
 
 #include <math.h>
 #include <stddef.h>
 #include <algorithm>
 #include <utility>
 
 #include "base/bind.h"
 #include "base/callback.h"
 #include "base/callback_helpers.h"
 #include "base/command_line.h"
 #include "base/logging.h"
 #include "base/power_monitor/power_monitor.h"
 #include "base/single_thread_task_runner.h"
 #include "base/time/default_tick_clock.h"
 #include "build/build_config.h"
 #include "media/base/audio_buffer.h"
 #include "media/base/audio_buffer_converter.h"
 #include "media/base/audio_latency.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/channel_mixing_matrix.h"
 #include "media/base/demuxer_stream.h"
+#include "media/base/media_client.h"
 #include "media/base/media_log.h"
 #include "media/base/media_switches.h"
 #include "media/base/renderer_client.h"
 #include "media/base/timestamp_constants.h"
 #include "media/filters/audio_clock.h"
 #include "media/filters/decrypting_demuxer_stream.h"
 
 namespace media {
 
+static const int kMaxFramesPerCompressedAudioBuffer = 4096;

[DaleCurtis, 2017/06/15 21:46:33]

Needs explanation; possibly should be stored in AudioParameters.

[AndyWu, 2017/08/02 01:43:41]

Done.

+
 AudioRendererImpl::AudioRendererImpl(
     const scoped_refptr<base::SingleThreadTaskRunner>& task_runner,
     media::AudioRendererSink* sink,
     const CreateAudioDecodersCB& create_audio_decoders_cb,
     MediaLog* media_log)
     : task_runner_(task_runner),
       expecting_config_changes_(false),
       sink_(sink),
       media_log_(media_log),
       client_(nullptr),
       tick_clock_(new base::DefaultTickClock()),
       last_audio_memory_usage_(0),
       last_decoded_sample_rate_(0),
       last_decoded_channel_layout_(CHANNEL_LAYOUT_NONE),
       is_encrypted_(false),
       last_decoded_channels_(0),
       playback_rate_(0.0),
       state_(kUninitialized),
       create_audio_decoders_cb_(create_audio_decoders_cb),
       buffering_state_(BUFFERING_HAVE_NOTHING),
       rendering_(false),
       sink_playing_(false),
       pending_read_(false),
       received_end_of_stream_(false),
       rendered_end_of_stream_(false),
       is_suspending_(false),
+      is_passthrough_(false),
+      last_reported_media_time_(kNoTimestamp),
       weak_factory_(this) {
   DCHECK(create_audio_decoders_cb_);
   // Tests may not have a power monitor.
   base::PowerMonitor* monitor = base::PowerMonitor::Get();
   if (!monitor)
     return;
 
   // PowerObserver's must be added and removed from the same thread, but we
   // won't remove the observer until we're destructed on |task_runner_| so we
   // must post it here if we're on the wrong thread.
@@ skipping 100 matching lines @@
 
   base::AutoLock auto_lock(lock_);
   DCHECK(!rendering_);
   DCHECK_EQ(state_, kFlushed);
 
   start_timestamp_ = time;
   ended_timestamp_ = kInfiniteDuration;
   last_render_time_ = stop_rendering_time_ = base::TimeTicks();
   first_packet_timestamp_ = kNoTimestamp;
   audio_clock_.reset(new AudioClock(time, audio_parameters_.sample_rate()));
+  last_reported_media_time_ = kNoTimestamp;
 }
 
 base::TimeDelta AudioRendererImpl::CurrentMediaTime() {
   base::AutoLock auto_lock(lock_);
 
   // Return the current time based on the known extents of the rendered audio
   // data plus an estimate based on the last time those values were calculated.
   base::TimeDelta current_media_time = audio_clock_->front_timestamp();
   if (!last_render_time_.is_null()) {
     current_media_time +=
         (tick_clock_->NowTicks() - last_render_time_) * playback_rate_;
     if (current_media_time > audio_clock_->back_timestamp())
       current_media_time = audio_clock_->back_timestamp();
   }
 
-  return current_media_time;
+  // Clamp current media time to the last reported value, this prevents higher

[DaleCurtis, 2017/06/15 21:46:33]

We already do this in PipelineImpl.

[AndyWu, 2017/08/02 01:43:41]

Hi Chris, do you agree to remove this logic?

[chcunningham, 2017/08/04 19:26:40]

Yep, should be fine. It used to live here, but had to move for reasons I can't
recall. Basically, users of this renderer are already savvy about it moving time
backward.

I do appreciate the documentation though. You might add this snip about
compressed audio to the .h file comments for this method.

[AndyWu, 2017/08/04 21:45:52]

Thanks for your feedback.

+  // level clients from seeing time go backwards. This may happen when seeking
+  // a passthrough audio stream, since we are unable to trim a compressed audio
+  // buffer.
+  if (last_reported_media_time_ < current_media_time)
+    last_reported_media_time_ = current_media_time;
+
+  return last_reported_media_time_;
 }
 
 bool AudioRendererImpl::GetWallClockTimes(
     const std::vector<base::TimeDelta>& media_timestamps,
     std::vector<base::TimeTicks>* wall_clock_times) {
   base::AutoLock auto_lock(lock_);
   DCHECK(wall_clock_times->empty());
 
   // When playback is paused (rate is zero), assume a rate of 1.0.
   const double playback_rate = playback_rate_ ? playback_rate_ : 1.0;
@@ skipping 154 matching lines @@
 
   audio_buffer_stream_->set_config_change_observer(base::Bind(
       &AudioRendererImpl::OnConfigChange, weak_factory_.GetWeakPtr()));
 
   // Always post |init_cb_| because |this| could be destroyed if initialization
   // failed.
   init_cb_ = BindToCurrentLoop(init_cb);
 
   auto output_device_info = sink_->GetOutputDeviceInfo();
   const AudioParameters& hw_params = output_device_info.output_params();
+  AudioCodec codec = stream->audio_decoder_config().codec();
+  MediaClient* media_client = GetMediaClient();

[DaleCurtis, 2017/06/15 21:46:33]

if (auto* mc = GetMediaClient())
  is_passthrough_ = mc->IsSupportedBitstreamAudioCodec(codec);
else
  is_passthrough_ = false

[AndyWu, 2017/08/02 01:43:41]

Done.

+  is_passthrough_ =
+      media_client && media_client->IsSupportedBitstreamAudioCodec(codec);
   expecting_config_changes_ = stream->SupportsConfigChanges();
 
   bool use_stream_params = !expecting_config_changes_ || !hw_params.IsValid() ||
                            hw_params.format() == AudioParameters::AUDIO_FAKE ||
                            !sink_->IsOptimizedForHardwareParameters();
 
   if (stream->audio_decoder_config().channel_layout() ==
           CHANNEL_LAYOUT_DISCRETE &&
       sink_->IsOptimizedForHardwareParameters()) {
     use_stream_params = false;
   }
 
-  if (use_stream_params) {
+  if (is_passthrough_) {
+    AudioParameters::Format format = AudioParameters::AUDIO_FAKE;
+    if (codec == kCodecAC3) {
+      format = AudioParameters::AUDIO_BITSTREAM_AC3;
+    } else if (codec == kCodecEAC3) {
+      format = AudioParameters::AUDIO_BITSTREAM_EAC3;
+    } else {
+      NOTREACHED();
+    }
+
+    const int buffer_size = kMaxFramesPerCompressedAudioBuffer *
+                            stream->audio_decoder_config().bytes_per_frame();
+
+    audio_parameters_.Reset(
+        format, stream->audio_decoder_config().channel_layout(),
+        stream->audio_decoder_config().samples_per_second(),
+        stream->audio_decoder_config().bits_per_channel(), buffer_size);
+    buffer_converter_.reset();
+  } else if (use_stream_params) {
     // The actual buffer size is controlled via the size of the AudioBus
     // provided to Render(), but we should choose a value here based on hardware
     // parameters if possible since it affects the initial buffer size used by
     // the algorithm. Too little will cause underflow on Bluetooth devices.
     int buffer_size =
         std::max(stream->audio_decoder_config().samples_per_second() / 100,
                  hw_params.IsValid() ? hw_params.frames_per_buffer() : 0);
     audio_parameters_.Reset(AudioParameters::AUDIO_PCM_LOW_LATENCY,
                             stream->audio_decoder_config().channel_layout(),
                             stream->audio_decoder_config().samples_per_second(),
@@ skipping 267 matching lines @@
 
   AttemptRead_Locked();
 }
 
 bool AudioRendererImpl::HandleDecodedBuffer_Locked(
     const scoped_refptr<AudioBuffer>& buffer) {
   lock_.AssertAcquired();
   if (buffer->end_of_stream()) {
     received_end_of_stream_ = true;
   } else {
-    if (state_ == kPlaying) {
+    if (buffer->IsBitstreamFormat() && state_ == kPlaying) {
       if (IsBeforeStartTime(buffer))
         return true;
 
+      // Adjust the start time since we are unable to trim a compressed audio
+      // buffer.
+      if (buffer->timestamp() < start_timestamp_ &&

[DaleCurtis, 2017/06/15 21:46:33]

I'd just skip this and seek to the first full buffer after the existing start
time. I'm not sure this will work correctly in all cases.

[AndyWu, 2017/08/02 01:43:41]

Done.

+          (buffer->timestamp() + buffer->duration()) > start_timestamp_) {
+        start_timestamp_ = buffer->timestamp();
+        audio_clock_.reset(new AudioClock(buffer->timestamp(),
+                                          audio_parameters_.sample_rate()));
+      }
+    } else if (state_ == kPlaying) {
+      if (IsBeforeStartTime(buffer))
+        return true;
+
       // Trim off any additional time before the start timestamp.
       const base::TimeDelta trim_time = start_timestamp_ - buffer->timestamp();
       if (trim_time > base::TimeDelta()) {
         buffer->TrimStart(buffer->frame_count() *
                           (static_cast<double>(trim_time.InMicroseconds()) /
                            buffer->duration().InMicroseconds()));
         buffer->set_timestamp(start_timestamp_);
       }
       // If the entire buffer was trimmed, request a new one.
       if (!buffer->frame_count())
@@ skipping 75 matching lines @@
 }
 
 void AudioRendererImpl::SetPlaybackRate(double playback_rate) {
   DVLOG(1) << __func__ << "(" << playback_rate << ")";
   DCHECK(task_runner_->BelongsToCurrentThread());
   DCHECK_GE(playback_rate, 0);
   DCHECK(sink_.get());
 
   base::AutoLock auto_lock(lock_);
 
+  if (is_passthrough_ && playback_rate != 0 && playback_rate != 1) {
+    MEDIA_LOG(ERROR, media_log_)

[DaleCurtis, 2017/06/15 21:46:32]

WARNING or INFO, no need for ERROR.

[AndyWu, 2017/08/02 01:43:41]

Done.

+        << "Unsupported playback rate when outputing compressed bitstream."
+        << " Playback Rate: " << playback_rate;
+    return;
+  }
+
   // We have two cases here:
   // Play: current_playback_rate == 0 && playback_rate != 0
   // Pause: current_playback_rate != 0 && playback_rate == 0
   double current_playback_rate = playback_rate_;
   playback_rate_ = playback_rate;
 
   if (!rendering_)
     return;
 
   if (current_playback_rate == 0 && playback_rate != 0) {
@@ skipping 11 matching lines @@
     const scoped_refptr<AudioBuffer>& buffer) {
   DCHECK_EQ(state_, kPlaying);
   return buffer.get() && !buffer->end_of_stream() &&
          (buffer->timestamp() + buffer->duration()) < start_timestamp_;
 }
 
 int AudioRendererImpl::Render(base::TimeDelta delay,
                               base::TimeTicks delay_timestamp,
                               int prior_frames_skipped,
                               AudioBus* audio_bus) {
-  const int frames_requested = audio_bus->frames();
+  int frames_requested = audio_bus->frames();
   DVLOG(4) << __func__ << " delay:" << delay
            << " prior_frames_skipped:" << prior_frames_skipped
            << " frames_requested:" << frames_requested;
 
   int frames_written = 0;
   {
     base::AutoLock auto_lock(lock_);
     last_render_time_ = tick_clock_->NowTicks();
 
     int64_t frames_delayed = AudioTimestampHelper::TimeToFrames(
@@ skipping 18 matching lines @@
       return 0;
     }
 
     // Mute audio by returning 0 when not playing.
     if (state_ != kPlaying) {
       audio_clock_->WroteAudio(0, frames_requested, frames_delayed,
                                playback_rate_);
       return 0;
     }
 
-    // Delay playback by writing silence if we haven't reached the first
-    // timestamp yet; this can occur if the video starts before the audio.
-    if (algorithm_->frames_buffered() > 0) {
+    if (is_passthrough_ && algorithm_->frames_buffered() > 0) {
+      frames_written += algorithm_->FillBuffer(audio_bus, 0, frames_requested,

[chcunningham, 2017/06/14 20:03:08]

special passthrough logic needs "why" documentation

[DaleCurtis, 2017/06/15 21:46:33]

FYI, this is going to be wrong in cases where the first packet timestamp is far
into the future relative to the video. See lines l.900..l.921 -- I suspect this
will actually happen in practice and will lead to some AV sync delay, so you
should print a media_log warning in this case.

[AndyWu, 2017/08/02 01:43:41]

Done.

[AndyWu, 2017/08/02 01:43:41]

Done.

+                                               playback_rate_);
+      frames_requested = frames_written;
+    } else if (algorithm_->frames_buffered() > 0) {
+      // Delay playback by writing silence if we haven't reached the first
+      // timestamp yet; this can occur if the video starts before the audio.
       CHECK_NE(first_packet_timestamp_, kNoTimestamp);
       CHECK_GE(first_packet_timestamp_, base::TimeDelta());
       const base::TimeDelta play_delay =
           first_packet_timestamp_ - audio_clock_->back_timestamp();
       if (play_delay > base::TimeDelta()) {
         DCHECK_EQ(frames_written, 0);
 
         // Don't multiply |play_delay| out since it can be a huge value on
         // poorly encoded media and multiplying by the sample rate could cause
         // the value to overflow.
@@ skipping 169 matching lines @@
   // All channels with a zero mix are muted and can be ignored.
   std::vector<bool> channel_mask(audio_parameters_.channels(), false);
   for (size_t ch = 0; ch < matrix.size(); ++ch) {
     channel_mask[ch] = std::any_of(matrix[ch].begin(), matrix[ch].end(),
                                    [](float mix) { return !!mix; });
   }
   algorithm_->SetChannelMask(std::move(channel_mask));
 }
 
 }  // namespace media