Add MediaCodecAudioDecoder implementation

media/filters/android_audio_decoder.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/filters/android_audio_decoder.h"
+
+#include "base/android/build_info.h"
+#include "base/bind.h"
+#include "base/callback_helpers.h"
+#include "base/logging.h"
+#include "base/thread_task_runner_handle.h"
+
+#include "media/base/android/sdk_media_codec_bridge.h"
+#include "media/base/audio_buffer.h"
+#include "media/base/bind_to_current_loop.h"
+
+namespace media {
+
+// Android MediaCodec can only output 16bit PCM audio.
+const int kBytesPerOutputSample = 2;
+
+inline const base::TimeDelta DecodePollDelay() {
+  return base::TimeDelta::FromMilliseconds(10);
+}
+
+inline const base::TimeDelta NoWaitTimeOut() {
+  return base::TimeDelta::FromMicroseconds(0);
+}
+
+inline const base::TimeDelta IdleTimerTimeOut() {
+  return base::TimeDelta::FromSeconds(1);
+}
+
+AndroidAudioDecoder::AndroidAudioDecoder()
+    : task_runner_(base::ThreadTaskRunnerHandle::Get()),
+      state_(kStateUninitialized),
+      channel_count_(0),
+      bytes_per_frame_(0),
+      cdm_registration_id_(0),
+      pending_input_buf_index_(-1),
+      weak_factory_(this) {
+  DVLOG(1) << __FUNCTION__;
+}
+
+AndroidAudioDecoder::~AndroidAudioDecoder() {
+  DVLOG(1) << __FUNCTION__;
+
+  if (!set_cdm_ready_cb_.is_null())
+    base::ResetAndReturn(&set_cdm_ready_cb_).Run(CdmReadyCB());
+  if (!cdm_attached_cb_.is_null())
+    base::ResetAndReturn(&cdm_attached_cb_).Run(false);
+  if (!init_cb_.is_null())
+    base::ResetAndReturn(&init_cb_).Run(false);
+
+  media_codec_.reset();
+
+  // Report kAborted status for pending EOS and all frames in the input queue.
+  if (!eos_decode_cb_.is_null())
+    base::ResetAndReturn(&eos_decode_cb_).Run(kAborted);
+
+  for (const auto& entry : input_queue_) {
+    entry.second.Run(kAborted);
+  }
+}
+
+std::string AndroidAudioDecoder::GetDisplayName() const {
+  return "AndroidAudioDecoder";
+}
+
+void AndroidAudioDecoder::Initialize(const AudioDecoderConfig& config,
+                                     const SetCdmReadyCB& set_cdm_ready_cb,
+                                     const InitCB& init_cb,
+                                     const OutputCB& output_cb) {
+  DVLOG(1) << __FUNCTION__ << ": " << config.AsHumanReadableString();
+
+  InitCB bound_init_cb = BindToCurrentLoop(init_cb);
+
+  // Keep this consistent with AudioCodecBridge.
+  const bool is_codec_supported = config.codec() == kCodecVorbis ||
+                                  config.codec() == kCodecAAC ||
+                                  config.codec() == kCodecOpus;
+  if (!is_codec_supported) {
+    DVLOG(1) << "Unsuported codec " << GetCodecName(config.codec());
+    bound_init_cb.Run(false);
+    return;
+  }
+
+  config_ = config;
+  init_cb_ = bound_init_cb;
+  output_cb_ = BindToCurrentLoop(output_cb);
+
+  // The following derived parameters are frequently used.
+  channel_count_ = ChannelLayoutToChannelCount(config_.channel_layout());
+  bytes_per_frame_ = kBytesPerOutputSample * channel_count_;
+
+  if (config.is_encrypted()) {
+    // Delay configuration until we get MediaCrypto object.
+    set_cdm_ready_cb_ = set_cdm_ready_cb;
+    set_cdm_ready_cb_.Run(BindToCurrentLoop(
+        base::Bind(&AndroidAudioDecoder::SetCdm, weak_factory_.GetWeakPtr())));
+    SetState(kStateWaitingForCDM);
+    return;
+  }
+
+  const bool success = ConfigureMediaCodec();
+
+  SetState(success ? kStateReady : kStateUninitialized);
+  base::ResetAndReturn(&init_cb_).Run(success);
+}
+
+void AndroidAudioDecoder::Decode(const scoped_refptr<DecoderBuffer>& buffer,
+                                 const DecodeCB& decode_cb) {
+  DecodeCB bound_decode_cb = BindToCurrentLoop(decode_cb);
+
+  if (state_ == kStateError) {
+    DVLOG(2) << __FUNCTION__ << " " << buffer->AsShortString()
+             << ": Error state, dropping buffer";
+    bound_decode_cb.Run(kDecodeError);
+    return;
+  }
+
+  DVLOG(2) << __FUNCTION__ << " " << buffer->AsShortString();
+
+  input_queue_.push_back(std::make_pair(buffer, bound_decode_cb));
+
+  DoIOTask();
+}
+
+void AndroidAudioDecoder::Reset(const base::Closure& closure) {
+  DVLOG(1) << __FUNCTION__;
+
+  io_timer_.Stop();
+
+  if (!eos_decode_cb_.is_null())
+    base::ResetAndReturn(&eos_decode_cb_).Run(kAborted);
+
+  // Report kAborted status for all frames in the input queue.
+  for (const auto& entry : input_queue_) {
+    entry.second.Run(kAborted);
+  }
+  input_queue_.clear();
+
+  // Flush if we can, otherwise completely recreate and reconfigure the codec.
+  // Prior to JB-MR2, flush() had several bugs (b/8125974, b/8347958).
+  bool success = false;
+  if (state_ != kStateError && state_ != kStateDrained &&
+      base::android::BuildInfo::GetInstance()->sdk_int() < 18) {
+    // media_codec_->Reset() calls MediaCodec.flush().
+    success = (media_codec_->Reset() == MEDIA_CODEC_OK);
+  }
+
+  if (!success) {
+    media_codec_.reset();
+    success = ConfigureMediaCodec();
+  }
+
+  SetState(success ? kStateReady : kStateError);
+
+  task_runner_->PostTask(FROM_HERE, closure);
+}
+
+void AndroidAudioDecoder::SetCdm(CdmContext* cdm_context,
+                                 const CdmAttachedCB& cdm_attached_cb) {
+  DVLOG(1) << __FUNCTION__;
+
+  DCHECK(!init_cb_.is_null());
+  DCHECK(!set_cdm_ready_cb_.is_null());
+  set_cdm_ready_cb_.Reset();
+
+  if (!cdm_context || cdm_context->GetCdmId() == CdmContext::kInvalidCdmId) {
+    DVLOG(1) << __FUNCTION__ << ": CDM ID not available.";
+    cdm_attached_cb.Run(false);
+    base::ResetAndReturn(&init_cb_).Run(false);
+    SetState(kStateUninitialized);
+    return;
+  }
+
+  if (cdm_) {
+    NOTREACHED() << "We do not support resetting CDM.";
+    cdm_attached_cb.Run(false);
+    base::ResetAndReturn(&init_cb_).Run(false);
+    SetState(kStateUninitialized);
+    return;
+  }
+
+// TODO(timav): get |cdm_| from cdm_context->GetCdmId().
+
+// #define CDM_IN_ANDROID_AUDIO_DECODE
+#if !defined(CDM_IN_ANDROID_AUDIO_DECODER)
+  NOTIMPLEMENTED();
+  cdm_attached_cb.Run(false);
+  base::ResetAndReturn(&init_cb_).Run(false);
+  SetState(kStateUninitialized);
+#else
+  // cdm_ = media::MojoCdmService::GetCdm(cdm_id);
+  // DCHECK(cdm_);
+
+  // On Android platform the MediaKeys will be its subclass MediaDrmBridge.
+  MediaDrmBridge* drm_bridge = static_cast<MediaDrmBridge*>(cdm_.get());
+
+  // Register CDM callbacks. The callbacks registered will be posted back to
+  // this thread via BindToCurrentLoop.
+
+  // Since |this| holds a reference to the |cdm_|, by the time the CDM is
+  // destructed, UnregisterPlayer() must have been called and |this| has been
+  // destructed as well. So the |cdm_unset_cb| will never have a chance to be
+  // called.
+  // TODO(xhwang): Remove |cdm_unset_cb| after it's not used on all platforms.
+  cdm_registration_id_ = drm_bridge->RegisterPlayer(
+      BindToCurrentLoop(base::Bind(&AndroidAudioDecoder::OnKeyAdded,
+                                   weak_factory_.GetWeakPtr())),
+      base::Bind(&base::DoNothing));
+
+  drm_bridge->SetMediaCryptoReadyCB(BindToCurrentLoop(base::Bind(
+      &AndroidAudioDecoder::OnMediaCryptoReady, weak_factory_.GetWeakPtr())));
+
+  // Postpone cdm_attached_cb.Run() call till CreateMediaCodec() which we can do
+  // after OnMediaCryptoReady().
+  cdm_attached_cb_ = cdm_attached_cb;
+  SetState(kStateWaitingForCrypto);
+#endif
+}
+
+void AndroidAudioDecoder::OnMediaCryptoReady(
+    media::MediaDrmBridge::JavaObjectPtr media_crypto) {
+  DVLOG(1) << __FUNCTION__;
+
+  if (!media_crypto) {
+    LOG(ERROR) << "MediaCrypto is not available, can't play encrypted stream.";
+    base::ResetAndReturn(&cdm_attached_cb_).Run(false);
+    base::ResetAndReturn(&init_cb_).Run(false);
+    SetState(kStateUninitialized);
+    return;
+  }
+
+  DCHECK(!media_crypto->is_null());
+
+  // We assume this is a part of the initialization process, thus MediaCodec
+  // is not created yet.
+  DCHECK(!media_codec_);
+
+  media_crypto_ = std::move(media_crypto);
+
+  // After receiving |media_crypto_| we can configure MediaCodec.
+  const bool success = ConfigureMediaCodec();
+
+  SetState(success ? kStateReady : kStateUninitialized);
+
+  base::ResetAndReturn(&cdm_attached_cb_).Run(success);
+  base::ResetAndReturn(&init_cb_).Run(success);
+}
+
+void AndroidAudioDecoder::OnKeyAdded() {
+  DVLOG(1) << __FUNCTION__;
+
+  if (state_ == kStateWaitingForKey)
+    SetState(kStateReady);
+
+  DoIOTask();
+}
+
+void AndroidAudioDecoder::DoIOTask() {
+  if (state_ == kStateError)
+    return;
+
+  const bool did_input = QueueInput();
+  const bool did_output = DequeueOutput();
+
+  ManageTimer(did_input || did_output);
+}
+
+bool AndroidAudioDecoder::QueueInput() {
+  DVLOG(2) << __FUNCTION__;
+
+  if (input_queue_.empty())
+    return false;
+
+  if (state_ == kStateWaitingForKey)
+    return false;
+
+  if (state_ == kStateDraining)
+    return false;
+
+  scoped_refptr<DecoderBuffer> decoder_buffer = input_queue_.front().first;
+  const DecodeCB& decode_cb = input_queue_.front().second;
+
+  int input_buf_index = pending_input_buf_index_;
+
+  // Do not dequeue a new input buffer if we failed with MEDIA_CODEC_NO_KEY.
+  // That status does not return the input buffer back to the pool of
+  // available input buffers. We have to reuse it in QueueSecureInputBuffer().
+  if (input_buf_index == -1) {
+    media::MediaCodecStatus status =
+        media_codec_->DequeueInputBuffer(NoWaitTimeOut(), &input_buf_index);
+    switch (status) {
+      case media::MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER:
+        return false;
+      case media::MEDIA_CODEC_ERROR:
+        DVLOG(1) << __FUNCTION__ << ": MEDIA_CODEC_ERROR";
+        SetState(kStateError);
+
+        // Report an error to the pipeline.
+        decode_cb.Run(kDecodeError);
+        return false;
+      case media::MEDIA_CODEC_OK:
+        break;
+      default:
+        NOTREACHED() << "Unknown DequeueInputBuffer status " << status;
+        return false;
+    }
+  }
+
+  DCHECK_NE(input_buf_index, -1);
+
+  if (decoder_buffer->end_of_stream()) {
+    media_codec_->QueueEOS(input_buf_index);
+
+    // After queueing EOS we need to flush decoder, i.e. receive this EOS at
+    // the output, and then call corresponding decoder_cb.
+    eos_decode_cb_ = decode_cb;
+    input_queue_.pop_front();
+    SetState(kStateDraining);
+    return true;
+  }
+
+  // Make sure our DecoderBuffer is not EOS, otherwise we can't call any method.
+  DCHECK(!decoder_buffer->end_of_stream());
+
+  media::MediaCodecStatus status;
+  const DecryptConfig* decrypt_config = decoder_buffer->decrypt_config();
+  if (decrypt_config) {
+    // If pending_input_buf_index_ != -1 the input buffer is already filled up,
+    // no need to copy it again.
+    const uint8_t* memory =
+        (pending_input_buf_index_ == -1) ? decoder_buffer->data() : nullptr;
+
+    status = media_codec_->QueueSecureInputBuffer(
+        input_buf_index, memory, decoder_buffer->data_size(),
+        decrypt_config->key_id(), decrypt_config->iv(),
+        decrypt_config->subsamples(), decoder_buffer->timestamp());
+
+    DVLOG(2) << __FUNCTION__
+             << ": QueueInputBuffer: pts:" << decoder_buffer->timestamp()
+             << " status:" << status;
+  } else {
+    status = media_codec_->QueueInputBuffer(
+        input_buf_index, decoder_buffer->data(), decoder_buffer->data_size(),
+        decoder_buffer->timestamp());
+    DVLOG(2) << __FUNCTION__
+             << ": QueueSecureInputBuffer: pts:" << decoder_buffer->timestamp()
+             << " status:" << status;
+  }
+
+  if (status == media::MEDIA_CODEC_NO_KEY) {
+    // Keep trying to enqueue the same input buffer.
+    // The buffer is owned by us (not the MediaCodec) and is filled with data.
+    DVLOG(1) << "QueueSecureInputBuffer failed: MEDIA_CODEC_NO_KEY";
+    pending_input_buf_index_ = input_buf_index;
+    SetState(kStateWaitingForKey);
+    return false;
+  }
+
+  pending_input_buf_index_ = -1;
+
+  // Although audio_decoder.h says "Once the buffer is decoded the decoder calls
+  // |decode_cb|", we call |decode_cb| when the buffer is accepted by
+  // MediaCodec, not when it is completely decoded. It seems consistent to what
+  // other decoders do.
+  decode_cb.Run(kOk);
+  input_queue_.pop_front();
+  return true;
+}
+
+bool AndroidAudioDecoder::DequeueOutput() {

[liberato (no reviews please), 2016/02/01 15:18:06]

DequeueOutput looks like it's an improvement over AVDA's implementation.  it
also seems to have little or no dependencies on any types that aren't common to
both use-cases.

i particularly like the On* callbacks and SetState.

+  DVLOG(2) << __FUNCTION__;
+
+  DCHECK(media_codec_);
+
+  MediaCodecStatus status;
+  OutputBufferInfo out;

[liberato (no reviews please), 2016/02/01 15:18:06]

OutputBufferInfo: good idea for avda.

+  bool work_done = false;
+  do {
+    status = media_codec_->DequeueOutputBuffer(NoWaitTimeOut(), &out.buf_index,
+                                               &out.offset, &out.size, &out.pts,
+                                               &out.is_eos, &out.is_key_frame);
+
+    switch (status) {
+      case MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED:
+        // Output buffers are replaced in MediaCodecBridge, nothing to do.
+        DVLOG(2) << __FUNCTION__ << " MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED";
+        work_done = true;
+        break;
+
+      case MEDIA_CODEC_OUTPUT_FORMAT_CHANGED:
+        DVLOG(2) << __FUNCTION__ << " MEDIA_CODEC_OUTPUT_FORMAT_CHANGED";
+        OnOutputFormatChanged();
+        work_done = true;
+        break;
+
+      case MEDIA_CODEC_OK:
+        // We got the decoded frame.
+        if (out.is_eos) {
+          media_codec_->ReleaseOutputBuffer(out.buf_index, false);
+
+          DCHECK_EQ(state_, kStateDraining);
+          DCHECK(!eos_decode_cb_.is_null());
+
+          // Report the end of decoding for EOS DecoderBuffer.
+          base::ResetAndReturn(&eos_decode_cb_).Run(kOk);
+
+          // media_decoder_job.cc says: once output EOS has occurred, we should
+          // not be asked to decode again.
+          // Have a separate state for this case.
+          SetState(kStateDrained);
+        } else {
+          // Process the real decoded frame.
+          OnDecodedFrame(&out);
+        }
+
+        work_done = true;
+        break;
+
+      case MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER:
+        // Nothing to do.
+        DVLOG(2) << __FUNCTION__ << " MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER";
+        break;
+
+      case MEDIA_CODEC_ERROR:
+        DVLOG(0) << __FUNCTION__
+                 << ": MEDIA_CODEC_ERROR from DequeueOutputBuffer";
+
+        // Next Decode() will report the error to the pipeline.
+        SetState(kStateError);
+        break;
+
+      default:
+        NOTREACHED();
+        break;
+    }
+  } while (status != MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER &&
+           status != MEDIA_CODEC_ERROR && !out.is_eos);
+
+  return work_done;
+}
+
+void AndroidAudioDecoder::ManageTimer(bool did_work) {
+  bool should_be_running = true;
+
+  base::TimeTicks now = base::TimeTicks::Now();
+  if (!did_work) {
+    // Make sure that we have done work recently enough, else stop the timer.
+    if (now - most_recent_work_ > IdleTimerTimeOut())
+      should_be_running = false;
+  } else {
+    most_recent_work_ = now;
+  }
+
+  if (should_be_running && !io_timer_.IsRunning()) {
+    io_timer_.Start(FROM_HERE, DecodePollDelay(), this,
+                    &AndroidAudioDecoder::DoIOTask);
+  } else if (!should_be_running && io_timer_.IsRunning()) {
+    io_timer_.Stop();
+  }
+}
+
+void AndroidAudioDecoder::SetState(State new_state) {

[liberato (no reviews please), 2016/02/01 15:18:06]

this is an improvement over avda.

+  DVLOG(1) << __FUNCTION__ << ": " << AsString(state_) << "->"
+           << AsString(new_state);
+  state_ = new_state;
+}
+
+bool AndroidAudioDecoder::ConfigureMediaCodec() {
+  DVLOG(1) << __FUNCTION__;
+
+  media_codec_.reset(AudioCodecBridge::Create(config_.codec()));
+  if (!media_codec_) {
+    DVLOG(0) << __FUNCTION__ << " failed: cannot create AudioCodecBridge";
+    return false;
+  }
+
+  AudioCodecBridge* audio_codec_bridge =
+      static_cast<AudioCodecBridge*>(media_codec_.get());
+  DCHECK(audio_codec_bridge);
+
+  jobject media_crypto = media_crypto_ ? media_crypto_->obj() : nullptr;
+
+  const bool play_audio = false;  // Do not create AudioTrack object.
+  if (!audio_codec_bridge->ConfigureAndStart(
+          config_.codec(), config_.samples_per_second(), channel_count_,
+          &config_.extra_data()[0], config_.extra_data().size(),
+          config_.codec_delay(), config_.seek_preroll().InMicroseconds() * 1000,
+          play_audio, media_crypto)) {
+    DVLOG(0) << __FUNCTION__ << " failed: cannot start audio codec";
+    media_codec_.reset();
+    return false;
+  }
+
+  return true;
+}
+
+void AndroidAudioDecoder::OnDecodedFrame(const OutputBufferInfo* out) {
+  DCHECK(out);
+  DCHECK_NE(out->size, 0U);
+  DCHECK(media_codec_);
+
+  DVLOG(2) << __FUNCTION__ << " pts:" << out->pts;
+
+  // Create AudioOutput buffer based on configuration.
+  const size_t frame_count = out->size / bytes_per_frame_;
+
+  scoped_refptr<AudioBuffer> audio_buffer = AudioBuffer::CreateBuffer(
+      kSampleFormatS16, config_.channel_layout(), channel_count_,
+      config_.samples_per_second(), frame_count);
+
+  // Copy data into AudioBuffer.
+  media_codec_->CopyFromOutputBuffer(out->buf_index, out->offset,
+                                     audio_buffer->interleaved_data(),
+                                     audio_buffer->interleaved_data_size());
+
+  // Release MediaCodec output buffer.
+  media_codec_->ReleaseOutputBuffer(out->buf_index, false);
+
+  // Call the output_cb_.
+  output_cb_.Run(audio_buffer);
+}
+
+void AndroidAudioDecoder::OnOutputFormatChanged() {
+  DVLOG(0) << __FUNCTION__ << ": not implemented, going to error state";
+
+  SetState(kStateError);
+}
+
+#undef RETURN_STRING
+#define RETURN_STRING(x) \
+  case x:                \
+    return #x;
+
+// static
+const char* AndroidAudioDecoder::AsString(State state) {
+  switch (state) {
+    RETURN_STRING(kStateUninitialized);
+    RETURN_STRING(kStateWaitingForCDM);
+    RETURN_STRING(kStateWaitingForCrypto);
+    RETURN_STRING(kStateReady);
+    RETURN_STRING(kStateWaitingForKey);
+    RETURN_STRING(kStateDraining);
+    RETURN_STRING(kStateDrained);
+    RETURN_STRING(kStateError);
+  }
+  return nullptr;  // crash early
+}
+
+#undef RETURN_STRING
+
+}  // namespace media