MediaCodecPlayer implementation (stage 4 - preroll)

media/base/android/media_codec_decoder.cc

Index: media/base/android/media_codec_decoder.cc
diff --git a/media/base/android/media_codec_decoder.cc b/media/base/android/media_codec_decoder.cc
index 67425c6c3bba0759dffddb0050e58e2db7b59a2d..afb20e5062ea39bd347203d58082e46636c9e1f6 100644
--- a/media/base/android/media_codec_decoder.cc
+++ b/media/base/android/media_codec_decoder.cc
@@ -35,6 +35,7 @@ MediaCodecDecoder::MediaCodecDecoder(
     const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner,
     const base::Closure& external_request_data_cb,
     const base::Closure& starvation_cb,
+    const base::Closure& preroll_done_cb,
     const base::Closure& stop_done_cb,
     const base::Closure& error_cb,
     const char* decoder_thread_name)
@@ -43,6 +44,7 @@ MediaCodecDecoder::MediaCodecDecoder(
       needs_reconfigure_(false),
       external_request_data_cb_(external_request_data_cb),
       starvation_cb_(starvation_cb),
+      preroll_done_cb_(preroll_done_cb),
       stop_done_cb_(stop_done_cb),
       error_cb_(error_cb),
       state_(kStopped),
@@ -61,6 +63,8 @@ MediaCodecDecoder::MediaCodecDecoder(
 
   internal_error_cb_ =
       base::Bind(&MediaCodecDecoder::OnCodecError, weak_factory_.GetWeakPtr());
+  internal_preroll_done_cb_ =
+      base::Bind(&MediaCodecDecoder::OnPrerollDone, weak_factory_.GetWeakPtr());
   request_data_cb_ =
       base::Bind(&MediaCodecDecoder::RequestData, weak_factory_.GetWeakPtr());
 }
@@ -89,7 +93,7 @@ void MediaCodecDecoder::ReleaseDecoderResources() {
   decoder_thread_.Stop();  // synchronous
 
   SetState(kStopped);
-  media_codec_bridge_.reset();
+  ReleaseMediaCodec();
 }
 
 void MediaCodecDecoder::Flush() {
@@ -131,16 +135,35 @@ void MediaCodecDecoder::Flush() {
 void MediaCodecDecoder::ReleaseMediaCodec() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
-  DVLOG(1) << class_name() << "::" << __FUNCTION__;
-
   media_codec_bridge_.reset();
 }
 
 bool MediaCodecDecoder::IsPrefetchingOrPlaying() const {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
+  // Whether decoder needs to be stopped.
   base::AutoLock lock(state_lock_);
-  return state_ == kPrefetching || state_ == kRunning;
+  switch (state_) {
+    case kPrefetching:
+    case kPrefetched:
+    case kPrerolling:
+    case kPrerolled:
+    case kRunning:
+      return true;
+    case kStopped:
+    case kStopping:
+    case kInEmergencyStop:
+    case kError:
+      return false;
+  }
+  NOTREACHED();
+  return false;
+}
+
+bool MediaCodecDecoder::IsPrerollDone() const {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  return !HasStream() || IsCompleted() || GetState() == kPrerolled;
 }
 
 bool MediaCodecDecoder::IsStopped() const {
@@ -192,12 +215,7 @@ MediaCodecDecoder::ConfigStatus MediaCodecDecoder::Configure() {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << ": needs reconfigure, deleting MediaCodec";
     needs_reconfigure_ = false;
-    media_codec_bridge_.reset();
-
-    // No need to release these buffers since the MediaCodec is deleted, just
-    // remove their indexes from |delayed_buffers_|.
-
-    ClearDelayedBuffers(false);
+    ReleaseMediaCodec();
   }
 
   MediaCodecDecoder::ConfigStatus result;
@@ -222,14 +240,24 @@ MediaCodecDecoder::ConfigStatus MediaCodecDecoder::Configure() {
   return result;
 }
 
-bool MediaCodecDecoder::Start(base::TimeDelta current_time) {
+void MediaCodecDecoder::SetPrerollTimestamp(base::TimeDelta preroll_timestamp) {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+  DCHECK(!decoder_thread_.IsRunning());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": " << preroll_timestamp;
+
+  // Do not set preroll timestamp if it's too close to zero.
+  preroll_timestamp_ = preroll_timestamp;
+}
+
+bool MediaCodecDecoder::Start(base::TimeDelta start_timestamp) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__
-           << " current_time:" << current_time;
+           << " start_timestamp:" << start_timestamp;
 
   DecoderState state = GetState();
-  if (state == kRunning) {
+  if (state == kPrerolling || state == kRunning) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": already started";
     return true;  // already started
   }
@@ -248,9 +276,13 @@ bool MediaCodecDecoder::Start(base::TimeDelta current_time) {
 
   DCHECK(!decoder_thread_.IsRunning());
 
+  const bool needs_preroll = (preroll_timestamp_ != base::TimeDelta());

[qinmin, 2015/07/31 18:43:10]

why not preroll if timestamp is 0? it will help us sync audio and video by
removing the decoder initialization delay.

[Tima Vaisburd, 2015/07/31 19:26:42]

I think decoder initialization delay has been already taken care of because
StartPlaybackDecoders()
(https://code.google.com/p/chromium/codesearch#chromium/src/media/base/android...)
at first configures both audio and video, and then starts both. If I'm not
missing anything, of course.
My thinking here was that with start_timestamp == 0 there could not be any prior
timestamp and so there is nothing to preroll.

[Tima Vaisburd, 2015/08/01 04:34:43]

Min explained that there is internal codec initialization that happens between
the moment we queue first input buffer till the moment we dequeue first output
buffer and for video codec it can be significant. Therefore after flush and
reconfiguration we need to do preroll event at pts == 0.
Done.

+
   // We only synchronize video stream.
-  // When audio is present, the |current_time| is audio time.
-  SynchronizePTSWithTime(current_time);
+  if (needs_preroll)
+    DissociatePTSFromTime();  // associaton will happen after preroll is done.
+  else
+    AssociateCurrentTimeWithPTS(start_timestamp);
 
   last_frame_posted_ = false;
 
@@ -261,7 +293,7 @@ bool MediaCodecDecoder::Start(base::TimeDelta current_time) {
     return false;
   }
 
-  SetState(kRunning);
+  SetState(needs_preroll ? kPrerolling : kRunning);
 
   decoder_thread_.task_runner()->PostTask(
       FROM_HERE,
@@ -270,6 +302,24 @@ bool MediaCodecDecoder::Start(base::TimeDelta current_time) {
   return true;
 }
 
+void MediaCodecDecoder::ResumeAfterPreroll() {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  DCHECK(GetState() == kPrerolled);
+  DCHECK(decoder_thread_.IsRunning());
+
+  SetState(kRunning);
+
+  AssociateCurrentTimeWithPTS(preroll_timestamp_);
+  preroll_timestamp_ = base::TimeDelta();
+
+  decoder_thread_.task_runner()->PostTask(
+      FROM_HERE,
+      base::Bind(&MediaCodecDecoder::ProcessNextFrame, base::Unretained(this)));
+}
+
 void MediaCodecDecoder::SyncStop() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
@@ -290,7 +340,7 @@ void MediaCodecDecoder::SyncStop() {
 
   SetState(kStopped);
 
-  ClearDelayedBuffers(true);  // release prior to clearing  |delayed_buffers_|.
+  ReleaseDelayedBuffers();
 }
 
 void MediaCodecDecoder::RequestToStop() {
@@ -307,6 +357,14 @@ void MediaCodecDecoder::RequestToStop() {
     case kRunning:
       SetState(kStopping);
       break;
+    case kPrerolling:
+    case kPrerolled:
+      DCHECK(decoder_thread_.IsRunning());
+      // Synchronous stop.
+      decoder_thread_.Stop();
+      SetState(kStopped);
+      media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
+      break;
     case kStopping:
       break;  // ignore
     case kStopped:
@@ -337,6 +395,17 @@ void MediaCodecDecoder::OnLastFrameRendered(bool completed) {
   media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
 }
 
+void MediaCodecDecoder::OnPrerollDone() {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  if (GetState() == kPrerolling) {
+    SetState(kPrerolled);
+    media_task_runner_->PostTask(FROM_HERE, preroll_done_cb_);
+  }
+}
+
 void MediaCodecDecoder::OnDemuxerDataAvailable(const DemuxerData& data) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
@@ -438,7 +507,7 @@ void MediaCodecDecoder::ProcessNextFrame() {
 
   DecoderState state = GetState();
 
-  if (state != kRunning && state != kStopping) {
+  if (state != kPrerolling && state != kRunning && state != kStopping) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": not running";
     return;
   }
@@ -458,7 +527,7 @@ void MediaCodecDecoder::ProcessNextFrame() {
     return;
   }
 
-  DCHECK(state == kRunning);
+  DCHECK(state == kPrerolling || state == kRunning);
 
   if (!EnqueueInputBuffer())
     return;
@@ -600,6 +669,9 @@ bool MediaCodecDecoder::DepleteOutputBufferQueue() {
   MediaCodecStatus status;
   bool eos_encountered = false;
 
+  RenderMode render_mode;
+  bool preroll_done = false;
+
   base::TimeDelta timeout =
       base::TimeDelta::FromMilliseconds(kOutputBufferTimeout);
 
@@ -626,10 +698,23 @@ bool MediaCodecDecoder::DepleteOutputBufferQueue() {
         OnOutputFormatChanged();
         break;
 
-      case MEDIA_CODEC_OK:
-        // We got the decoded frame
-        Render(buffer_index, size, true, pts, eos_encountered);
-        break;
+      case MEDIA_CODEC_OK: {
+        // We got the decoded frame.
+
+        if (pts < preroll_timestamp_)
+          render_mode = kRenderSkip;
+        else if (GetState() == kPrerolling)
+          render_mode = kRenderAfterPreroll;
+        else
+          render_mode = kRenderNow;
+
+        Render(buffer_index, size, render_mode, pts, eos_encountered);
+
+        if (render_mode == kRenderAfterPreroll) {
+          media_task_runner_->PostTask(FROM_HERE, internal_preroll_done_cb_);
+          preroll_done = true;

[qinmin, 2015/07/31 18:43:10]

why not just return false here.

[Tima Vaisburd, 2015/07/31 19:26:42]

Done.

+        }
+      } break;

[qinmin, 2015/07/31 18:43:10]

move the break to a new line

[Tima Vaisburd, 2015/07/31 19:26:42]

Done.

 
       case MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER:
         // Nothing to do.
@@ -647,7 +732,7 @@ bool MediaCodecDecoder::DepleteOutputBufferQueue() {
     }
 
   } while (status != MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER &&
-           status != MEDIA_CODEC_ERROR && !eos_encountered);
+           status != MEDIA_CODEC_ERROR && !eos_encountered && !preroll_done);
 
   if (eos_encountered) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
@@ -655,6 +740,12 @@ bool MediaCodecDecoder::DepleteOutputBufferQueue() {
     return false;
   }
 
+  if (preroll_done) {
+    DVLOG(1) << class_name() << "::" << __FUNCTION__
+             << " preroll done, stopping frame processing";
+    return false;
+  }
+
   if (status == MEDIA_CODEC_ERROR) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << " MediaCodec error, stopping frame processing";
@@ -681,18 +772,28 @@ void MediaCodecDecoder::SetState(DecoderState state) {
   case x:                \
     return #x;
 
+const char* MediaCodecDecoder::AsString(RenderMode render_mode) {
+  switch (render_mode) {
+    RETURN_STRING(kRenderSkip);
+    RETURN_STRING(kRenderAfterPreroll);
+    RETURN_STRING(kRenderNow);
+  }
+  return nullptr;  // crash early
+}
+
 const char* MediaCodecDecoder::AsString(DecoderState state) {
   switch (state) {
     RETURN_STRING(kStopped);
     RETURN_STRING(kPrefetching);
     RETURN_STRING(kPrefetched);
+    RETURN_STRING(kPrerolling);
+    RETURN_STRING(kPrerolled);
     RETURN_STRING(kRunning);
     RETURN_STRING(kStopping);
     RETURN_STRING(kInEmergencyStop);
     RETURN_STRING(kError);
-    default:
-      return "Unknown DecoderState";
   }
+  return nullptr;  // crash early
 }
 
 #undef RETURN_STRING