Separate MediaCodecLoop from MediaCodecAudioDecoder

media/base/android/media_codec_loop.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/base/android/media_codec_loop.h"
+
+#include "base/android/build_info.h"
+#include "base/bind.h"
+#include "base/callback_helpers.h"
+#include "base/logging.h"
+#include "base/threading/thread_task_runner_handle.h"
+#include "media/base/android/sdk_media_codec_bridge.h"
+#include "media/base/audio_buffer.h"
+#include "media/base/audio_timestamp_helper.h"
+#include "media/base/bind_to_current_loop.h"
+#include "media/base/timestamp_constants.h"
+
+namespace media {
+
+namespace {
+
+// TODO(liberato): the client should choose these.
+inline const base::TimeDelta DecodePollDelay() {
+  return base::TimeDelta::FromMilliseconds(10);
+}
+
+inline const base::TimeDelta NoWaitTimeout() {
+  return base::TimeDelta::FromMicroseconds(0);
+}
+
+inline const base::TimeDelta IdleTimerTimeout() {
+  return base::TimeDelta::FromSeconds(1);
+}
+
+}  // namespace (anonymous)
+
+MediaCodecLoop::InputBufferData::InputBufferData()
+    : memory(nullptr), length(0), is_eos(false), is_encrypted(false) {}
+// TODO(liberato): does anybody call this?  if not, maybe they should.
+MediaCodecLoop::InputBufferData::InputBufferData(const uint8_t* m,
+                                                 size_t l,
+                                                 const DecodeCB& cb)
+    : memory(m),
+      length(l),
+      completion_cb(cb),
+      is_eos(false),
+      is_encrypted(false) {}
+
+MediaCodecLoop::InputBufferData::InputBufferData(const InputBufferData& other)
+    : memory(other.memory),
+      length(other.length),
+      key_id(other.key_id),
+      iv(other.iv),
+      subsamples(other.subsamples),
+      presentation_time(other.presentation_time),
+      completion_cb(other.completion_cb),
+      is_eos(other.is_eos),
+      is_encrypted(other.is_encrypted) {}
+
+MediaCodecLoop::InputBufferData::~InputBufferData() {}
+
+MediaCodecLoop::MediaCodecLoop(Client* client,
+                               std::unique_ptr<MediaCodecBridge>&& media_codec)
+    : state_(STATE_READY),
+      client_(client),
+      media_codec_(std::move(media_codec)),
+      pending_input_buf_index_(kInvalidBufferIndex),
+      weak_factory_(this) {
+  DVLOG(1) << __FUNCTION__;
+}
+
+MediaCodecLoop::~MediaCodecLoop() {
+  DVLOG(1) << __FUNCTION__;
+
+  media_codec_.reset();
+}
+
+void MediaCodecLoop::OnKeyAdded() {
+  DVLOG(1) << __FUNCTION__;
+
+  if (state_ == STATE_WAITING_FOR_KEY)
+    SetState(STATE_READY);
+
+  DoIOTask();
+}
+
+bool MediaCodecLoop::TryFlush() {
+  DVLOG(1) << __FUNCTION__;
+
+  // We do not clear the input queue here.  It depends on the caller.
+  // For decoder reset, then it is appropriate.  Otherwise, the requests might
+  // simply be sent to us later, such as on a format change.
+
+  // Flush if we can, otherwise completely recreate and reconfigure the codec.
+  // Prior to JellyBean-MR2, flush() had several bugs (b/8125974, b/8347958) so
+  // we have to completely destroy and recreate the codec there.
+  bool success = false;
+  if (state_ != STATE_ERROR && state_ != STATE_DRAINED &&
+      base::android::BuildInfo::GetInstance()->sdk_int() >= 18) {
+    io_timer_.Stop();
+
+    // media_codec_->Reset() calls MediaCodec.flush().
+    success = (media_codec_->Reset() == MEDIA_CODEC_OK);
+
+    // Transition to the error state if the flush failed.
+    SetState(success ? STATE_READY : STATE_ERROR);
+  }
+
+  return success;
+}
+
+void MediaCodecLoop::DoIOTask() {
+  if (state_ == STATE_ERROR)
+    return;
+
+  const bool did_input = QueueInput();
+  const bool did_output = DequeueOutput();
+
+  ManageTimer(did_input || did_output);
+}
+
+bool MediaCodecLoop::QueueInput() {
+  DVLOG(2) << __FUNCTION__;
+
+  bool did_work = false;
+  while (QueueOneInputBuffer())
+    did_work = true;
+
+  return did_work;
+}
+
+bool MediaCodecLoop::QueueOneInputBuffer() {
+  DVLOG(2) << __FUNCTION__;
+
+  if (!client_->IsAnyInputPending())

[Tima Vaisburd, 2016/06/01 00:52:40]

To be equivalent to the old code this method should return true after we got
NO_KEY so that we continue with this pending buffer even if there is no more
input. 

[liberato (no reviews please), 2016/06/01 21:47:54]

great catch, thanks!

+    return false;
+
+  if (state_ == STATE_WAITING_FOR_KEY || state_ == STATE_DRAINING ||
+      state_ == STATE_DRAINED)
+    return false;
+
+  // DequeueInputBuffer() may set STATE_ERROR.
+  InputBufferInfo input_info = DequeueInputBuffer();
+
+  if (input_info.buf_index == kInvalidBufferIndex) {
+    if (state_ == STATE_ERROR)
+      client_->ClearInputQueue(DecodeStatus::DECODE_ERROR);
+
+    return false;
+  }
+
+  // EnqueueInputBuffer() may set STATE_DRAINING, STATE_WAITING_FOR_KEY or
+  // STATE_ERROR.
+  EnqueueInputBuffer(input_info);
+  return state_ == STATE_READY;
+}
+
+MediaCodecLoop::InputBufferInfo MediaCodecLoop::DequeueInputBuffer() {
+  DVLOG(2) << __FUNCTION__;
+
+  // Do not dequeue a new input buffer if we failed with MEDIA_CODEC_NO_KEY.
+  // That status does not return the input buffer back to the pool of
+  // available input buffers. We have to reuse it in QueueSecureInputBuffer().
+  if (pending_input_buf_index_ != kInvalidBufferIndex) {
+    InputBufferInfo result(pending_input_buf_index_, true);
+    pending_input_buf_index_ = kInvalidBufferIndex;
+    return result;
+  }
+
+  int input_buf_index = kInvalidBufferIndex;
+
+  media::MediaCodecStatus status =
+      media_codec_->DequeueInputBuffer(NoWaitTimeout(), &input_buf_index);
+  switch (status) {
+    case media::MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER:
+      DVLOG(2) << __FUNCTION__ << ": MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER";
+      break;
+
+    case media::MEDIA_CODEC_ERROR:
+      DVLOG(1) << __FUNCTION__ << ": MEDIA_CODEC_ERROR from DequeInputBuffer";
+      SetState(STATE_ERROR);
+      break;
+
+    case media::MEDIA_CODEC_OK:
+      break;
+
+    default:
+      NOTREACHED() << "Unknown DequeueInputBuffer status " << status;
+      SetState(STATE_ERROR);
+      break;
+  }
+
+  return InputBufferInfo(input_buf_index, false);
+}
+
+void MediaCodecLoop::EnqueueInputBuffer(const InputBufferInfo& input_info) {
+  DVLOG(2) << __FUNCTION__ << ": index:" << input_info.buf_index;
+
+  DCHECK_NE(input_info.buf_index, kInvalidBufferIndex);
+
+  InputBufferData input_data;
+
+  if (input_info.is_pending) {
+    // A pending buffer is already filled with data, no need to copy it again.
+    DVLOG(2) << __FUNCTION__ << ": QueueSecureInputBuffer (pending):"
+             << " index:" << input_info.buf_index
+             << " pts:" << pending_input_buf_data_.presentation_time
+             << " size:" << pending_input_buf_data_.length;
+
+    input_data = pending_input_buf_data_;
+  } else {
+    input_data = client_->ProvideInputData();
+  }
+
+  // Process this buffer.
+
+  if (input_data.is_eos) {
+    media_codec_->QueueEOS(input_info.buf_index);
+    SetState(STATE_DRAINING);
+    return;
+  }
+
+  media::MediaCodecStatus status = MEDIA_CODEC_OK;
+
+  if (input_data.is_encrypted) {
+    // Note that input_data might not have a valid memory ptr if this is a
+    // re-send of a buffer that was sent before decryption keys arrived.
+
+    DVLOG(2) << __FUNCTION__ << ": QueueSecureInputBuffer:"
+             << " index:" << input_info.buf_index
+             << " pts:" << input_data.presentation_time
+             << " size:" << input_data.length;
+
+    status = media_codec_->QueueSecureInputBuffer(
+        input_info.buf_index, input_data.memory, input_data.length,
+        input_data.key_id, input_data.iv, input_data.subsamples,
+        input_data.presentation_time);
+
+  } else {
+    DVLOG(2) << __FUNCTION__ << ": QueueInputBuffer:"
+             << " index:" << input_info.buf_index
+             << " pts:" << input_data.presentation_time
+             << " size:" << input_data.length;
+
+    status = media_codec_->QueueInputBuffer(
+        input_info.buf_index, input_data.memory, input_data.length,
+        input_data.presentation_time);
+  }
+
+  switch (status) {
+    case MEDIA_CODEC_ERROR:
+      DVLOG(0) << __FUNCTION__ << ": MEDIA_CODEC_ERROR from QueueInputBuffer";
+      SetState(STATE_ERROR);
+      // TODO(liberato): why do we run the completion cb and clear the queue
+      // here (and also in some of the other cases)?  did the audio decoder do
+      // this before?

[Tima Vaisburd, 2016/06/01 00:52:40]

MCAD is the first decoder that uses MediaCodec and implements AudioDecoder
interface, in that sense we did not run completion cb before. Completion cb is
required by the protocol, the caller waits for it for the next move. In other
audio decoders it is more straightforward, there is no queue there.

If you look for the way to get rid of ClearInputQueue() here, I think we can
call something like client_->OnError() and let the client decide. 

[liberato (no reviews please), 2016/06/01 21:47:54]

done.

+      // (i think that it was done in EnqueueInputBuffer based on the state.
+      // however, it also handled popping stuff from the input queue, which
+      // is now done before we queue it by the client.  that's different; we
+      // used to not dequeue on error.)
+      input_data.completion_cb.Run(DecodeStatus::DECODE_ERROR);
+      // TODO(liberato): do we want to do this for AVDA?
+      client_->ClearInputQueue(DecodeStatus::DECODE_ERROR);
+      break;
+
+    case MEDIA_CODEC_NO_KEY:
+      DVLOG(1) << "QueueSecureInputBuffer failed: MEDIA_CODEC_NO_KEY";
+      // Do not call the completion cb here.  It will be called when we retry
+      // after getting the key.
+      pending_input_buf_index_ = input_info.buf_index;
+      pending_input_buf_data_ = input_data;
+      // These aren't needed, and aren't guaranted to be valid after we return.

[Tima Vaisburd, 2016/06/01 00:52:40]

I do not quite understand, do you plan to use these fields on l.210 ?

Just in case: in MCAD I set |memory = nullptr| to avoid a call to
FillInputBuffer() here:
https://code.google.com/p/chromium/codesearch#chromium/src/media/base/android...

|length| is used.

[liberato (no reviews please), 2016/06/01 21:47:54]

i just meant that the ptr provided by the client isn't guaranteed to remain
valid after we return.  the nullptr will be re-used @210.

i'll update the comment.

+      pending_input_buf_data_.memory = nullptr;
+      pending_input_buf_data_.length = 0;
+      SetState(STATE_WAITING_FOR_KEY);
+      break;
+
+    case MEDIA_CODEC_OK:
+      input_data.completion_cb.Run(DecodeStatus::OK);
+      break;
+
+    default:
+      NOTREACHED() << "Unknown Queue(Secure)InputBuffer status " << status;
+      SetState(STATE_ERROR);
+      input_data.completion_cb.Run(DecodeStatus::DECODE_ERROR);
+      // TODO(liberato): do we want to do this for AVDA?
+      client_->ClearInputQueue(DecodeStatus::DECODE_ERROR);
+      break;
+  }
+}
+
+bool MediaCodecLoop::DequeueOutput() {
+  DVLOG(2) << __FUNCTION__;
+
+  MediaCodecStatus status;
+  OutputBufferInfo out;
+  bool did_work = false;
+  do {
+    status = media_codec_->DequeueOutputBuffer(NoWaitTimeout(), &out.buf_index,
+                                               &out.offset, &out.size, &out.pts,
+                                               &out.is_eos, &out.is_key_frame);
+
+    switch (status) {
+      case MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED:
+        // Output buffers are replaced in MediaCodecBridge, nothing to do.
+        DVLOG(2) << __FUNCTION__ << " MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED";
+        did_work = true;
+        break;
+
+      case MEDIA_CODEC_OUTPUT_FORMAT_CHANGED:
+        DVLOG(2) << __FUNCTION__ << " MEDIA_CODEC_OUTPUT_FORMAT_CHANGED";
+        if (!client_->OnOutputFormatChanged())
+          SetState(STATE_ERROR);
+        did_work = (state_ != STATE_ERROR);
+        break;
+
+      case MEDIA_CODEC_OK:
+        // We got the decoded frame.
+        if (out.is_eos) {
+          // Set state STATE_DRAINED after we have received EOS frame at the
+          // output. media_decoder_job.cc says: once output EOS has occurred, we
+          // should not be asked to decode again.
+          DCHECK_EQ(state_, STATE_DRAINING);
+          SetState(STATE_DRAINED);
+
+          DCHECK_NE(out.buf_index, kInvalidBufferIndex);
+          DCHECK(media_codec_);
+
+          media_codec_->ReleaseOutputBuffer(out.buf_index, false);
+
+          client_->OnDecodedEos(out);
+        } else {
+          client_->OnDecodedFrame(out);
+        }
+
+        did_work = true;
+        break;
+
+      case MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER:
+        // Nothing to do.
+        DVLOG(2) << __FUNCTION__ << " MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER";
+        break;
+
+      case MEDIA_CODEC_ERROR:
+        DVLOG(0) << __FUNCTION__
+                 << ": MEDIA_CODEC_ERROR from DequeueOutputBuffer";
+
+        // Next Decode() will report the error to the pipeline.
+        SetState(STATE_ERROR);
+        break;
+
+      default:
+        NOTREACHED() << "Unknown DequeueOutputBuffer status " << status;
+        // Next Decode() will report the error to the pipeline.
+        SetState(STATE_ERROR);
+        break;
+    }
+  } while (status != MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER &&
+           status != MEDIA_CODEC_ERROR && !out.is_eos);
+
+  return did_work;
+}
+
+void MediaCodecLoop::ManageTimer(bool did_work) {
+  bool should_be_running = true;
+
+  base::TimeTicks now = base::TimeTicks::Now();
+  if (did_work || idle_time_begin_ == base::TimeTicks()) {
+    idle_time_begin_ = now;
+  } else {
+    // Make sure that we have done work recently enough, else stop the timer.
+    if (now - idle_time_begin_ > IdleTimerTimeout())
+      should_be_running = false;
+  }
+
+  if (should_be_running && !io_timer_.IsRunning()) {
+    io_timer_.Start(FROM_HERE, DecodePollDelay(), this,
+                    &MediaCodecLoop::DoIOTask);
+  } else if (!should_be_running && io_timer_.IsRunning()) {
+    io_timer_.Stop();
+  }
+}
+
+MediaCodecLoop::State MediaCodecLoop::GetState() const {
+  return state_;
+}
+
+void MediaCodecLoop::SetState(State new_state) {
+  DVLOG(1) << __FUNCTION__ << ": " << AsString(state_) << "->"
+           << AsString(new_state);
+  state_ = new_state;
+  client_->OnStateChanged(new_state);
+}
+
+MediaCodecBridge* MediaCodecLoop::GetCodec() const {
+  return media_codec_.get();
+}
+
+#undef RETURN_STRING
+#define RETURN_STRING(x) \
+  case x:                \
+    return #x;
+
+// static
+const char* MediaCodecLoop::AsString(State state) {
+  switch (state) {
+    RETURN_STRING(STATE_READY);
+    RETURN_STRING(STATE_WAITING_FOR_KEY);
+    RETURN_STRING(STATE_DRAINING);
+    RETURN_STRING(STATE_DRAINED);
+    RETURN_STRING(STATE_ERROR);
+  }
+  NOTREACHED() << "Unknown state " << state;
+  return nullptr;
+}
+
+#undef RETURN_STRING
+
+}  // namespace media