Audio and video decoders for MediaCodecPlayer

media/base/android/media_codec_video_decoder.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/base/android/media_codec_video_decoder.h"
+
+#include "base/bind.h"
+#include "base/logging.h"
+#include "media/base/android/media_codec_bridge.h"
+#include "media/base/buffers.h"
+#include "media/base/demuxer_stream.h"
+
+namespace media {
+
+namespace {
+const int kDelayForStandAloneEOS = 2;  // milliseconds
+}
+
+MediaCodecVideoDecoder::MediaCodecVideoDecoder(
+    const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner,
+    const base::Closure& request_data_cb,
+    const base::Closure& starvation_cb,
+    const base::Closure& stop_done_cb,
+    const base::Closure& error_cb,
+    const SetTimeCallback& update_current_time_cb,
+    const VideoSizeChangedCallback& video_size_changed_cb,
+    const base::Closure& codec_created_cb)
+    : MediaCodecDecoder(media_task_runner,
+                        request_data_cb,
+                        starvation_cb,
+                        stop_done_cb,
+                        error_cb,
+                        "VideoDecoder"),
+      update_current_time_cb_(update_current_time_cb),
+      video_size_changed_cb_(video_size_changed_cb),
+      codec_created_cb_(codec_created_cb) {
+}
+
+MediaCodecVideoDecoder::~MediaCodecVideoDecoder() {
+  DVLOG(1) << "VideoDecoder::~VideoDecoder()";
+}
+
+bool MediaCodecVideoDecoder::HasStream() const {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  return configs_.video_codec != kUnknownVideoCodec;
+}
+
+void MediaCodecVideoDecoder::SetDemuxerConfigs(const DemuxerConfigs& configs) {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  configs_ = configs;
+
+  if (video_size_.IsEmpty()) {
+    video_size_ = configs_.video_size;
+    media_task_runner_->PostTask(
+        FROM_HERE, base::Bind(video_size_changed_cb_, video_size_));
+  }
+}
+
+void MediaCodecVideoDecoder::ReleaseDecoderResources() {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  MediaCodecDecoder::ReleaseDecoderResources();
+  surface_ = gfx::ScopedJavaSurface();
+  delayed_buffers_.clear();
+}
+
+void MediaCodecVideoDecoder::SetPendingSurface(gfx::ScopedJavaSurface surface) {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  surface_ = surface.Pass();
+
+  if (surface_.IsEmpty()) {
+    // Synchronously stop decoder thread and release MediaCodec
+    ReleaseDecoderResources();
+  }
+}
+
+bool MediaCodecVideoDecoder::HasPendingSurface() const {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  return !surface_.IsEmpty();
+}
+
+bool MediaCodecVideoDecoder::IsCodecReconfigureNeeded(
+    const DemuxerConfigs& curr,
+    const DemuxerConfigs& next) const {
+  if (curr.video_codec != next.video_codec ||
+      curr.is_video_encrypted != next.is_video_encrypted) {
+    return true;
+  }
+
+  // Only size changes below this point
+
+  if (curr.video_size.width() == next.video_size.width() &&
+      curr.video_size.height() == next.video_size.height()) {
+    return false;  // i.e. curr == next
+  }
+
+  return !static_cast<VideoCodecBridge*>(media_codec_bridge_.get())
+              ->IsAdaptivePlaybackSupported(next.video_size.width(),

[qinmin, 2015/06/19 06:57:40]

nit: fix the indent

[Tima Vaisburd, 2015/06/20 02:32:31]

I did although this was against |git cl format|.

+                                            next.video_size.height());
+}
+
+MediaCodecDecoder::ConfigStatus MediaCodecVideoDecoder::ConfigureInternal() {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  // If we cannot find a key frame in cache, the browser seek is needed.
+  if (!au_queue_.RewindToLastKeyFrame()) {
+    DVLOG(1) << class_name() << "::" << __FUNCTION__ << " key frame required";
+
+    // The processing of CONFIG_KEY_FRAME_REQUIRED is not implemented yet,

[wolenetz, 2015/06/19 23:28:41]

nit: add TODO / bug? (is this something the player would handle?)

[Tima Vaisburd, 2015/06/20 02:32:31]

Added TODO. This should be a part of the browser seek.

+    // return error for now
+    // return CONFIG_KEY_FRAME_REQUIRED;
+    return CONFIG_FAILURE;
+  }
+
+  // TODO(timav): implement DRM.
+  // bool is_secure = is_content_encrypted() && drm_bridge() &&
+  //    drm_bridge()->IsProtectedSurfaceRequired();
+
+  bool is_secure = false;  // DRM is not implemented
+
+  if (surface_.IsEmpty()) {
+    DVLOG(0) << class_name() << "::" << __FUNCTION__ << " surface required";
+    return CONFIG_FAILURE;
+  }
+
+  media_codec_bridge_.reset(VideoCodecBridge::CreateDecoder(
+      configs_.video_codec,
+      is_secure,
+      configs_.video_size,
+      surface_.j_surface().obj(),
+      GetMediaCrypto().obj()));
+
+  if (!media_codec_bridge_) {
+    DVLOG(1) << class_name() << "::" << __FUNCTION__ << " failed";
+    return CONFIG_FAILURE;
+  }
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__ << " succeeded";
+
+  media_task_runner_->PostTask(FROM_HERE, codec_created_cb_);
+
+  return CONFIG_OK;
+}
+
+void MediaCodecVideoDecoder::SynchronizePTSWithTime(
+    base::TimeDelta current_time) {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  start_time_ticks_ = base::TimeTicks::Now();
+  start_pts_ = current_time;
+  last_seen_pts_ = current_time;
+}
+
+void MediaCodecVideoDecoder::OnOutputFormatChanged() {
+  DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread());
+
+  gfx::Size prev_size = video_size_;
+
+  // See b/18224769. The values reported from MediaCodecBridge::GetOutputFormat
+  // correspond to the actual video frame size, but this is not necessarily the
+  // size that should be output.
+  video_size_ = configs_.video_size;
+  if (video_size_ != prev_size) {
+    media_task_runner_->PostTask(
+        FROM_HERE, base::Bind(video_size_changed_cb_, video_size_));
+  }
+}
+
+void MediaCodecVideoDecoder::Render(int buffer_index,
+                                    size_t size,
+                                    bool render_output,
+                                    base::TimeDelta pts,
+                                    bool eos_encountered) {
+  DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread());
+
+  DVLOG(2) << class_name() << "::" << __FUNCTION__ << " pts:" << pts
+           << " index:" << buffer_index << (eos_encountered ? " EOS" : "");
+
+  // EOS might come as a stand-alone frame with zero size

[wolenetz, 2015/06/19 23:28:41]

in what case is the media_source_delegate (or chunk_demuxer) emitting such a
strange EOS? If that's happening, seems it might be a bug there on the demuxer
to detect this EOS condition sooner. If not, then this workaround isn't needed.

[Tima Vaisburd, 2015/06/20 02:32:31]

MediaCodecBridge::QueueEOS generates these frames itself:
https://code.google.com/p/chromium/codesearch#chromium/src/media/base/android...
We use it now and will use for one extra case to drain decoder as a part of
dynamic reconfiguration process.

[Tima Vaisburd, 2015/06/21 22:35:20]

I checked again, this kind of EOS seems to always come from chunk demuxer. Maybe
I did not formulate it properly: by stand-alone I meant the access unit with
flag(s), but no data. From what I've seen so far, the access unit that holds EOS
has neither data nor PTS. The android MediaCodec always assigns PTS to the
output buffer though such that the next PTS is equal or greater than previous.
I changed the comment, hope it became clearer.

[wolenetz, 2015/06/22 21:41:21]

Thanks for the detail. I had confused myself: ::Render() is on the output of a
decode, not on the queuing of input (of course), hence this is really the only
way currently of detecting EOS within Render. Sorry for the confusion, and
thanks for clarifying in comment.

+  if (!size && eos_encountered) {
+    // Stand-alone EOS
+    // Discard the PTS that comes with it and ensure it is released last.
+    pts = last_seen_pts_ +
+          base::TimeDelta::FromMilliseconds(kDelayForStandAloneEOS);
+  } else {
+    // Keep track of last seen PTS
+    last_seen_pts_ = pts;
+  }
+
+  if (!render_output) {
+    ReleaseOutputBuffer(buffer_index, pts, false, eos_encountered);
+    return;
+  }
+
+  base::TimeDelta time_to_render =
+      pts - (base::TimeTicks::Now() - start_time_ticks_ + start_pts_);
+
+  if (time_to_render < base::TimeDelta()) {
+    // Skip late frames
+    ReleaseOutputBuffer(buffer_index, pts, false, eos_encountered);
+    return;
+  }
+
+  delayed_buffers_.insert(buffer_index);
+
+  bool do_render = size > 0;
+  decoder_thread_.task_runner()->PostDelayedTask(
+      FROM_HERE, base::Bind(&MediaCodecVideoDecoder::ReleaseOutputBuffer,
+                            base::Unretained(this), buffer_index, pts,
+                            do_render, eos_encountered),
+      time_to_render);
+}
+
+int MediaCodecVideoDecoder::NumDelayedRenderTasks() const {
+  DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread());
+
+  return delayed_buffers_.size();
+}
+
+void MediaCodecVideoDecoder::ReleaseDelayedBuffers() {
+  // Media thread
+  // Called when there is no decoder thread
+  for (int index : delayed_buffers_)
+    media_codec_bridge_->ReleaseOutputBuffer(index, false);
+  delayed_buffers_.clear();
+}
+
+void MediaCodecVideoDecoder::ReleaseOutputBuffer(int buffer_index,
+                                                 base::TimeDelta pts,
+                                                 bool render,
+                                                 bool eos_encountered) {
+  DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread());
+
+  DVLOG(2) << class_name() << "::" << __FUNCTION__ << " pts:" << pts;
+
+  media_codec_bridge_->ReleaseOutputBuffer(buffer_index, render);
+
+  delayed_buffers_.erase(buffer_index);
+
+  CheckLastFrame(eos_encountered, !delayed_buffers_.empty());
+
+  // |update_current_time_cb_| might be null if there is audio stream.
+  // Do not update current time for EOS frames.
+  if (!update_current_time_cb_.is_null() && !eos_encountered) {
+    media_task_runner_->PostTask(FROM_HERE,
+                                 base::Bind(update_current_time_cb_, pts, pts));
+  }
+}
+
+}  // namespace media