Implementation of MediaCodecPlayer stage 1

media/base/android/media_codec_decoder.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/base/android/media_codec_decoder.h"
+
+#include "base/bind.h"
+#include "base/bind_helpers.h"
+#include "base/callback_helpers.h"
+#include "base/logging.h"
+#include "media/base/android/media_codec_bridge.h"
+
+namespace media {
+
+namespace {
+
+// Stop requesting new data in the PREFETCH state when the queue size reached

[watk, 2015/06/04 20:56:43]

s/reached/reaches

[Tima Vaisburd, 2015/06/05 04:17:47]

Done.

+// this limit.
+const int PREFETCH_LIMIT = 8;

[watk, 2015/06/04 20:56:43]

The chromium convention is kConstantNaming style

[Tima Vaisburd, 2015/06/05 04:17:47]

Done.

+
+// Request new data in the RUNNING state if the queue size is less than this.
+const int PLAYBACK_LOW_LIMIT = 4;
+
+// Posting delay of the next frame processing, in milliseconds
+const int NEXT_FRAME_DELAY_MS = 2;
+
+// Timeout for dequeuing an input buffer from MediaCodec in milliseconds.
+const int INPUT_BUFFER_TIMEOUT_MS = 20;
+
+// Timeout for dequeuing an output buffer from MediaCodec in milliseconds.
+const int OUTPUT_BUFFER_TIMEOUT_MS = 20;
+}
+
+MediaCodecDecoder::MediaCodecDecoder(
+    const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner,
+    const base::Closure& request_data_cb,
+    const base::Closure& starvation_cb,
+    const base::Closure& stop_done_cb,
+    const base::Closure& error_cb,
+    const char* decoder_thread_name)
+    : media_task_runner_(media_task_runner),
+      decoder_thread_(decoder_thread_name),
+      request_data_cb_(request_data_cb),
+      starvation_cb_(starvation_cb),
+      stop_done_cb_(stop_done_cb),
+      error_cb_(error_cb),
+      state_(STOPPED),
+      eos_enqueued_(false),
+      completed_(false),
+      last_frame_posted_(false),
+      weak_factory_(this) {
+  // Media thread
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << "Decoder::Decoder() " << decoder_thread_name;
+
+  weak_this_ = weak_factory_.GetWeakPtr();
+}
+
+MediaCodecDecoder::~MediaCodecDecoder() {
+  // Media thread

[watk, 2015/06/04 20:56:43]

I would remove these comments because they're redundant with the DHCECKs below
them.

[Tima Vaisburd, 2015/06/05 04:17:46]

Done.

+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << "Decoder::~Decoder()";
+
+  // NB: ReleaseDecoderResources() is virtual
+  ReleaseDecoderResources();
+}
+
+void MediaCodecDecoder::ReleaseDecoderResources() {
+  // Media thread
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  decoder_thread_.Stop(); // synchronous
+  state_ = STOPPED;
+  media_codec_bridge_.reset();
+}
+
+void MediaCodecDecoder::Flush() {
+  // Media thread
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  DCHECK_EQ(GetState(), STOPPED);
+
+  eos_enqueued_ = false;
+  completed_ = false;
+  au_queue_.Flush();
+
+  if (media_codec_bridge_) {
+    // MediaCodecBridge::Reset() performs MediaCodecBridge.flush()
+    MediaCodecStatus flush_status = media_codec_bridge_->Reset();
+    if (flush_status != MEDIA_CODEC_OK)
+      media_task_runner_->PostTask(FROM_HERE, error_cb_);
+  }
+}
+
+void MediaCodecDecoder::ReleaseMediaCodec() {
+  // Media thread
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  media_codec_bridge_.reset();
+}
+
+bool MediaCodecDecoder::IsPrefetchingOrPlaying() const {
+  // Media thread
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  base::AutoLock lock(state_lock_);
+  return state_ == PREFETCHING || state_ == RUNNING;
+}
+
+bool MediaCodecDecoder::IsStopped() const {
+  // Media thread
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  return GetState() == STOPPED;
+}
+
+bool MediaCodecDecoder::IsCompleted() const {
+  // Media thread
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  return completed_;
+}
+
+base::android::ScopedJavaLocalRef<jobject>
+MediaCodecDecoder::GetMediaCrypto() {
+  base::android::ScopedJavaLocalRef<jobject> media_crypto;
+
+  // drm_bridge_ is not implemented
+  // if (drm_bridge_)
+  //   media_crypto = drm_bridge_->GetMediaCrypto();
+  return media_crypto;
+}
+
+void MediaCodecDecoder::Prefetch(const base::Closure& prefetch_done_cb) {
+  // Media thread
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  DCHECK(GetState() == STOPPED);
+
+  prefetch_done_cb_ = prefetch_done_cb;
+
+  SetState(PREFETCHING);
+  PrefetchNextChunk();
+}
+
+MediaCodecDecoder::ConfigStatus MediaCodecDecoder::Configure() {
+  // Media thread
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  // Here I assume that OnDemuxerConfigsAvailable won't come
+  // in the middle of demuxer data.
+
+  if (media_codec_bridge_) {
+    DVLOG(1) << class_name() << "::" << __FUNCTION__
+             << ": reconfiguration is not required, ignoring";
+    return CONFIG_OK;
+  }
+
+  return ConfigureInternal();
+}
+
+bool MediaCodecDecoder::Start(base::TimeDelta current_time) {
+  // Media thread
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__
+           << " current_time:" << current_time;
+
+  if (state_ == RUNNING) {
+    DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": already started";
+    return true; // already started
+  }
+
+  if (state_ == STOPPING) {
+    DVLOG(0) << class_name() << "::" << __FUNCTION__
+             << ": wrong state STOPPING, ignoring";
+    return false;
+  }
+
+  DCHECK(!decoder_thread_.IsRunning());
+
+  // We only synchronize video stream.
+  // When audio is present, the |current_time| is audio time.
+  SynchronizePTSWithTime(current_time);
+
+  last_frame_posted_ = false;
+
+  // Start the decoder thread
+  if (!decoder_thread_.Start()) {
+    DVLOG(1) << class_name() << "::" << __FUNCTION__
+             << ": cannot start decoder thread";
+    return false;
+  }
+
+  SetState(RUNNING);
+
+  decoder_thread_.task_runner()->PostTask(
+      FROM_HERE,
+      base::Bind(&MediaCodecDecoder::ProcessNextFrame, base::Unretained(this)));
+
+  return true;
+}
+
+void MediaCodecDecoder::SyncStop() {
+  // Media thread
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  // After this method returns, decoder thread will not be running.
+
+  decoder_thread_.Stop(); // synchronous
+  state_ = STOPPED;
+
+  // Shall we move |delayed_buffers_| from VideoDecoder to Decoder class?
+  ReleaseDelayedBuffers();
+}
+
+void MediaCodecDecoder::RequestToStop() {
+  // Media thread
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  DCHECK(GetState() == RUNNING);
+  SetState(STOPPING);
+}
+
+void MediaCodecDecoder::OnLastFrameRendered(bool completed) {
+  // Media thread
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__
+           << " completed:" << completed;
+
+  decoder_thread_.Stop(); // synchronous
+  state_ = STOPPED;
+  completed_ = completed;
+
+  media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
+}
+
+void MediaCodecDecoder::OnDemuxerDataAvailable(const DemuxerData& data) {
+  // Media thread
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(2) << class_name() << "::" << __FUNCTION__
+           << " #AUs:" << data.access_units.size()
+           << " #Configs:" << data.demuxer_configs.size();
+#if 0

[watk, 2015/06/05 01:00:33]

Delete this?

[Tima Vaisburd, 2015/06/05 04:17:47]

It is useful for debugging, I did #if !defined(NDEBUG)

+    for (const auto& unit : data.access_units)
+      DVLOG(2) << class_name() << "::" << __FUNCTION__ << " au: " << unit;
+#endif
+
+  au_queue_.PushBack(data);
+
+  if (state_ == PREFETCHING)
+    PrefetchNextChunk();
+}
+
+void MediaCodecDecoder::CheckLastFrame(bool eos_encountered,
+                                       bool has_delayed_tasks) {
+  // Decoder thread
+  DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread());
+
+  bool last_frame_when_stopping =
+      GetState() == STOPPING && !has_delayed_tasks;
+
+  if (last_frame_when_stopping || eos_encountered) {
+    media_task_runner_->PostTask(
+        FROM_HERE,
+        base::Bind(&MediaCodecDecoder::OnLastFrameRendered,
+                   weak_this_, eos_encountered));
+    last_frame_posted_ = true;
+  }
+}
+
+void MediaCodecDecoder::PrefetchNextChunk() {
+  // Media thread
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  AccessUnitQueue::Info au_info;
+  au_queue_.GetInfo(&au_info);
+
+  if (eos_enqueued_ || au_info.length >= PREFETCH_LIMIT || au_info.has_eos) {
+    // We are done prefetching
+    DVLOG(1) << class_name() << "::" << __FUNCTION__ << " posting PrefetchDone";
+    media_task_runner_->PostTask(
+        FROM_HERE, base::ResetAndReturn(&prefetch_done_cb_));
+    return;
+  }
+
+  request_data_cb_.Run();
+}
+
+void MediaCodecDecoder::ProcessNextFrame() {
+  // Decoder thread
+  DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread());
+
+  DVLOG(2) << class_name() << "::" << __FUNCTION__;
+
+  DecoderState state = GetState();
+
+  if (state != RUNNING && state != STOPPING) {
+    DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": not running";
+    return;
+  }
+
+  if (state == STOPPING) {
+    if (NumDelayedRenderTasks() == 0 && !last_frame_posted_) {
+      DVLOG(1) << class_name() << "::" << __FUNCTION__
+               << ": STOPPING, posting OnLastFrameRendered";
+      media_task_runner_->PostTask(
+          FROM_HERE,
+          base::Bind(&MediaCodecDecoder::OnLastFrameRendered,
+                     weak_this_, false));
+      last_frame_posted_ = true;
+    }
+
+    // We can stop processing, the |au_queue_| and MediaCodec queues can freeze.
+    // We only need to let finish the delayed rendering tasks.
+    return;
+  }
+
+  DCHECK(state == RUNNING);
+
+  // Keep the number pending video frames low, ideally maintaining
+  // the same audio and video duration after stop request
+
+  if (NumDelayedRenderTasks() <= 1) {
+    if (!EnqueueInputBuffer())
+      return;
+  }
+
+  bool eos_encountered = false;
+  if (!DepleteOutputBufferQueue(&eos_encountered))
+    return;
+
+  if (eos_encountered) {
+    DVLOG(1) << class_name() << "::" << __FUNCTION__
+             << " EOS dequeued, stopping frame processing";
+    return;
+  }
+
+  // We need a small delay if we want to stop this thread by
+  // decoder_thread_.Stop() reliably.
+  // The decoder thread message loop processes all pending
+  // (but not delayed) tasks before it can quit; without a delay
+  // the message loop might be forever processing the pendng tasks.
+  decoder_thread_.task_runner()->PostDelayedTask(
+      FROM_HERE,
+      base::Bind(&MediaCodecDecoder::ProcessNextFrame,
+                 base::Unretained(this)),
+      base::TimeDelta::FromMilliseconds(NEXT_FRAME_DELAY_MS));
+}
+
+// Returns false if there was MediaCodec error.
+bool MediaCodecDecoder::EnqueueInputBuffer() {
+  // Decoder thread
+  DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread());
+
+  DVLOG(2) << class_name() << "::" << __FUNCTION__;
+
+  if (eos_enqueued_) {
+    DVLOG(1) << class_name() << "::" << __FUNCTION__
+             << ": eos_enqueued, returning";
+    return true; // Nothing to do
+  }
+
+  // Get the next frame from the queue and the queue info
+
+  AccessUnitQueue::Info au_info;
+  au_queue_.GetInfo(&au_info);
+
+  // Request the data from Demuxer
+  if (au_info.length <= PLAYBACK_LOW_LIMIT && !au_info.has_eos)
+    media_task_runner_->PostTask(FROM_HERE, request_data_cb_);
+
+  // Get the next frame from the queue
+
+  if (!au_info.length) {
+    // Report starvation and return, Start() will be called again later.
+    DVLOG(1) << class_name() << "::" << __FUNCTION__
+             << ": starvation detected";
+    media_task_runner_->PostTask(FROM_HERE, starvation_cb_);
+    return true;
+  }
+
+  if (au_info.configs) {
+    DVLOG(1) << class_name() << "::" << __FUNCTION__
+             << ": received new configs, not implemented";
+    // post an error for now?
+    media_task_runner_->PostTask(FROM_HERE, error_cb_);
+    return true;
+  }
+
+  // Dequeue input buffer
+
+  base::TimeDelta timeout =
+      base::TimeDelta::FromMilliseconds(INPUT_BUFFER_TIMEOUT_MS);
+  int index = -1;
+  MediaCodecStatus status =
+      media_codec_bridge_->DequeueInputBuffer(timeout, &index);
+
+  DVLOG(2) << class_name() << ":: DequeueInputBuffer index:" << index;
+
+  switch (status) {
+    case MEDIA_CODEC_ERROR:
+      DVLOG(0) << class_name() << "::" << __FUNCTION__
+               << ": MEDIA_CODEC_ERROR DequeueInputBuffer failed";
+      media_task_runner_->PostTask(FROM_HERE, error_cb_);
+      return false;
+
+    case MEDIA_CODEC_DEQUEUE_INPUT_AGAIN_LATER:
+      return true;
+
+    default:
+      break;
+  }
+
+  // We got the buffer
+  DCHECK_EQ(status, MEDIA_CODEC_OK);
+  DCHECK_GE(index, 0);
+
+  const AccessUnit* unit = au_info.front_unit;
+  DCHECK(unit);
+
+  if (unit->is_end_of_stream) {
+    DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": QueueEOS";
+    media_codec_bridge_->QueueEOS(index);
+    eos_enqueued_ = true;
+    return true;
+  }
+
+  status = media_codec_bridge_->QueueInputBuffer(
+      index, &unit->data[0], unit->data.size(), unit->timestamp);
+
+  if (status == MEDIA_CODEC_ERROR) {
+    DVLOG(1) << class_name() << "::" << __FUNCTION__
+             << ": MEDIA_CODEC_ERROR: QueueInputBuffer failed";
+    media_task_runner_->PostTask(FROM_HERE, error_cb_);
+    return false;
+  }
+
+  // Have successfully queued input buffer, go to next access unit.
+  au_queue_.Advance();
+  return true;
+}
+
+// Returns false if there was MediaCodec error.
+bool MediaCodecDecoder::DepleteOutputBufferQueue(bool* eos_encountered) {
+  // Decoder thread
+  DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread());
+
+  DVLOG(2) << class_name() << "::" << __FUNCTION__;
+
+  int buffer_index = 0;
+  size_t offset = 0;
+  size_t size = 0;
+  base::TimeDelta pts;
+  MediaCodecStatus status;
+
+  base::TimeDelta timeout[2];
+  timeout[0] = base::TimeDelta::FromMilliseconds(OUTPUT_BUFFER_TIMEOUT_MS);
+  timeout[1] = base::TimeDelta::FromMilliseconds(0);
+
+  int timeout_index = 0;

[watk, 2015/06/05 01:00:32]

I think it would be simpler to just initialize "timeout" to
base::TimeDelta::FromMilliseconds(OUTPUT_BUFFER_TIMEOUT_MS); and later set it to
base::TimeDelta::FromMilliseconds(0);. 

Can you also add a comment on why we allow a timeout on the first one but not
subsequent ones? It's not obvious to me

[Tima Vaisburd, 2015/06/05 04:17:47]

Yes, it's obviously simpler to just replace the timeout, thank you.

I tried to explain in the comments why there are two timeouts, not sure how
clear it went. The idea here is that we do want some timeout (you give 20 ms,
normally you get the buffer in 1-5 ms), but after you get the first buffer
usually there is nothing else, and you do not want to spend another 20 ms to
figure this out.

+
+  do {
+    status = media_codec_bridge_->DequeueOutputBuffer(
+        timeout[timeout_index],
+        &buffer_index,
+        &offset,
+        &size,
+        &pts,
+        eos_encountered,
+        nullptr);
+
+    timeout_index = 1;
+
+    switch (status) {
+      case  MEDIA_CODEC_OUTPUT_FORMAT_CHANGED:
+        DVLOG(2) << class_name() << "::" << __FUNCTION__
+                 << " MEDIA_CODEC_OUTPUT_FORMAT_CHANGED";
+        OnOutputFormatChanged();
+        break;
+
+      case MEDIA_CODEC_OK:
+        // We got the decoded frame
+        Render(buffer_index, size, true, pts, *eos_encountered);
+        break;
+
+      case MEDIA_CODEC_ERROR:
+        DVLOG(0) << class_name() << "::" << __FUNCTION__
+                 << ": MEDIA_CODEC_ERROR from DequeueOutputBuffer";
+        media_task_runner_->PostTask(FROM_HERE, error_cb_);
+        break;
+
+      default:
+        break;

[watk, 2015/06/05 01:00:32]

Could you add a comment on why it's ok to ignore other statuses? It's not
obvious to me.

[Tima Vaisburd, 2015/06/05 04:17:47]

Well, there are just two more: MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED which is taken
care of in the bridge and MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER which breaks
the loop. I added them both.

+    }
+
+  } while (status != MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER &&
+           status != MEDIA_CODEC_ERROR &&
+           !*eos_encountered);
+
+  return status != MEDIA_CODEC_ERROR;
+}
+
+MediaCodecDecoder::DecoderState MediaCodecDecoder::GetState() const {
+  base::AutoLock lock(state_lock_);
+  return state_;
+}
+
+void MediaCodecDecoder::SetState(DecoderState state) {
+  DVLOG(1) << class_name() << "::" << __FUNCTION__ << " " << state;
+
+  base::AutoLock lock(state_lock_);
+  state_ = state;
+}
+
+}  // namespace media