MediaCodecPlayer implementation (stage 4 - preroll)

media/base/android/media_codec_decoder.cc

Index: media/base/android/media_codec_decoder.cc
diff --git a/media/base/android/media_codec_decoder.cc b/media/base/android/media_codec_decoder.cc
index 534a8414b8eaab6c205a383fbab14f53f231ae8b..2bd3fef30967a99f872c9ee4864b7bd2c7625fb5 100644
--- a/media/base/android/media_codec_decoder.cc
+++ b/media/base/android/media_codec_decoder.cc
@@ -29,12 +29,16 @@ const int kInputBufferTimeout = 20;
 
 // Timeout for dequeuing an output buffer from MediaCodec in milliseconds.
 const int kOutputBufferTimeout = 20;
+
+// Estimated frame period in milliseconds
+const int kEstimatedFramePeriod = 20;

[qinmin, 2015/07/28 18:09:46]

why we need this variable? If we allow preroll for at pts 0, then we can ask
audio and video decoder to start decoding until they start generating frames. 

 }
 
 MediaCodecDecoder::MediaCodecDecoder(
     const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner,
     const base::Closure& external_request_data_cb,
     const base::Closure& starvation_cb,
+    const base::Closure& preroll_done_cb,
     const base::Closure& stop_done_cb,
     const base::Closure& error_cb,
     const char* decoder_thread_name)
@@ -43,6 +47,7 @@ MediaCodecDecoder::MediaCodecDecoder(
       needs_reconfigure_(false),
       external_request_data_cb_(external_request_data_cb),
       starvation_cb_(starvation_cb),
+      preroll_done_cb_(preroll_done_cb),
       stop_done_cb_(stop_done_cb),
       error_cb_(error_cb),
       state_(kStopped),
@@ -59,8 +64,16 @@ MediaCodecDecoder::MediaCodecDecoder(
 
   DVLOG(1) << "Decoder::Decoder() " << decoder_thread_name;
 
+  // For simplicity we use constant value instead of truly measuring the frame
+  // period. As long as it is used for determination of the
+  // kPrerolling -> kPrerolled switch only this simplification seems ok.
+  estimated_frame_period_ =
+      base::TimeDelta::FromMilliseconds(kEstimatedFramePeriod);
+
   internal_error_cb_ =
       base::Bind(&MediaCodecDecoder::OnCodecError, weak_factory_.GetWeakPtr());
+  internal_preroll_done_cb_ =
+      base::Bind(&MediaCodecDecoder::OnPrerollDone, weak_factory_.GetWeakPtr());
   request_data_cb_ =
       base::Bind(&MediaCodecDecoder::RequestData, weak_factory_.GetWeakPtr());
 }
@@ -139,8 +152,29 @@ void MediaCodecDecoder::ReleaseMediaCodec() {
 bool MediaCodecDecoder::IsPrefetchingOrPlaying() const {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
+  // Whether decoder needs to be stopped.
   base::AutoLock lock(state_lock_);
-  return state_ == kPrefetching || state_ == kRunning;
+  switch (state_) {
+    case kPrefetching:
+    case kPrefetched:
+    case kPrerolling:
+    case kPrerolled:
+    case kRunning:
+      return true;
+    case kStopped:
+    case kStopping:
+    case kInEmergencyStop:
+    case kError:
+      return false;
+  }
+  NOTREACHED();
+  return false;
+}
+
+bool MediaCodecDecoder::IsPrerollDone() const {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  return !HasStream() || IsCompleted() || GetState() == kPrerolled;
 }
 
 bool MediaCodecDecoder::IsStopped() const {
@@ -223,14 +257,26 @@ MediaCodecDecoder::ConfigStatus MediaCodecDecoder::Configure() {
   return result;
 }
 
-bool MediaCodecDecoder::Start(base::TimeDelta current_time) {
+void MediaCodecDecoder::SetPrerollTimestamp(base::TimeDelta preroll_timestamp) {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+  DCHECK(!decoder_thread_.IsRunning());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": " << preroll_timestamp;
+
+  // Do not set preroll timestamp if it's too close to zero.
+  preroll_timestamp_ = (preroll_timestamp < estimated_frame_period_)
+                           ? base::TimeDelta()
+                           : preroll_timestamp;
+}
+
+bool MediaCodecDecoder::Start(base::TimeDelta start_timestamp) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__
-           << " current_time:" << current_time;
+           << " start_timestamp:" << start_timestamp;
 
   DecoderState state = GetState();
-  if (state == kRunning) {
+  if (state == kPrerolling || state == kRunning) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": already started";
     return true;  // already started
   }
@@ -249,9 +295,13 @@ bool MediaCodecDecoder::Start(base::TimeDelta current_time) {
 
   DCHECK(!decoder_thread_.IsRunning());
 
+  const bool needs_preroll = (preroll_timestamp_ != base::TimeDelta());
+
   // We only synchronize video stream.
-  // When audio is present, the |current_time| is audio time.
-  SynchronizePTSWithTime(current_time);
+  if (needs_preroll)
+    DissociatePTSFromTime();  // associaton will happen after preroll is done.
+  else
+    AssociateCurrentTimeWithPTS(start_timestamp);
 
   last_frame_posted_ = false;
 
@@ -262,7 +312,7 @@ bool MediaCodecDecoder::Start(base::TimeDelta current_time) {
     return false;
   }
 
-  SetState(kRunning);
+  SetState(needs_preroll ? kPrerolling : kRunning);
 
   decoder_thread_.task_runner()->PostTask(
       FROM_HERE,
@@ -271,6 +321,24 @@ bool MediaCodecDecoder::Start(base::TimeDelta current_time) {
   return true;
 }
 
+void MediaCodecDecoder::ResumeAfterPreroll() {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  DCHECK(GetState() == kPrerolled);
+  DCHECK(decoder_thread_.IsRunning());
+
+  SetState(kRunning);
+
+  AssociateCurrentTimeWithPTS(preroll_timestamp_);
+  preroll_timestamp_ = base::TimeDelta();
+
+  decoder_thread_.task_runner()->PostTask(
+      FROM_HERE,
+      base::Bind(&MediaCodecDecoder::ProcessNextFrame, base::Unretained(this)));
+}
+
 void MediaCodecDecoder::SyncStop() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
@@ -309,6 +377,14 @@ void MediaCodecDecoder::RequestToStop() {
     case kRunning:
       SetState(kStopping);
       break;
+    case kPrerolling:
+    case kPrerolled:
+      DCHECK(decoder_thread_.IsRunning());
+      // Synchronous stop.
+      decoder_thread_.Stop();
+      SetState(kStopped);
+      media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
+      break;
     case kStopping:
       break;  // ignore
     case kStopped:
@@ -339,6 +415,17 @@ void MediaCodecDecoder::OnLastFrameRendered(bool completed) {
   media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
 }
 
+void MediaCodecDecoder::OnPrerollDone() {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  if (GetState() == kPrerolling) {
+    SetState(kPrerolled);
+    media_task_runner_->PostTask(FROM_HERE, preroll_done_cb_);
+  }
+}
+
 void MediaCodecDecoder::OnDemuxerDataAvailable(const DemuxerData& data) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
@@ -440,7 +527,7 @@ void MediaCodecDecoder::ProcessNextFrame() {
 
   DecoderState state = GetState();
 
-  if (state != kRunning && state != kStopping) {
+  if (state != kPrerolling && state != kRunning && state != kStopping) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": not running";
     return;
   }
@@ -460,7 +547,7 @@ void MediaCodecDecoder::ProcessNextFrame() {
     return;
   }
 
-  DCHECK(state == kRunning);
+  DCHECK(state == kPrerolling || state == kRunning);
 
   if (!EnqueueInputBuffer())
     return;
@@ -601,6 +688,7 @@ bool MediaCodecDecoder::DepleteOutputBufferQueue() {
   base::TimeDelta pts;
   MediaCodecStatus status;
   bool eos_encountered = false;
+  bool preroll_done = false;
 
   base::TimeDelta timeout =
       base::TimeDelta::FromMilliseconds(kOutputBufferTimeout);
@@ -628,10 +716,26 @@ bool MediaCodecDecoder::DepleteOutputBufferQueue() {
         OnOutputFormatChanged();
         break;
 
-      case MEDIA_CODEC_OK:
-        // We got the decoded frame
-        Render(buffer_index, size, true, pts, eos_encountered);
-        break;
+      case MEDIA_CODEC_OK: {
+        // We got the decoded frame.
+
+        // TODO(timav): this code won't render the very first frame in
+        // kPrerolling if it is already behind preroll_timestamp_. A more
+        // precise method would be to stop before Render() and resume after
+        // preroll with the Render(), but it would be more complicated.
+
+        DecoderState state = GetState();
+        const bool do_render = (state == kRunning || state == kStopping);

[qinmin, 2015/07/28 18:09:46]

I think it is ok to render if (state == kPrerolling && preroll_timestamp_ <=
pts)

[Tima Vaisburd, 2015/07/28 18:34:04]

I believe you mean "in addition to kRunning and kStopping" : after preroll is
done the state will switch to kRunning and we still need to render the frames.

All these hoops including the estimated frame period was intended to prevent the
situation where we render the first frame after preroll and then wait. According
to what I've seen this would be an audio frame in most cases. So, we will render
the first audio frame, then wait for video to preroll, then resume both. Do you
say that the first audio won't play anyway because audio needs to fill the audio
buffer and it would be ok? 

[qinmin, 2015/07/28 18:55:06]

You can modify the MediaCodecBridge.playOutputBuffer() call to take a param to
buffer this frame instead of playing it, and when the next frame comes,
playoutputbuffer() will automatically start playing.

+
+        Render(buffer_index, size, do_render, pts, eos_encountered);
+
+        // If next pts passes over |preroll_timestamp_| this frame is the last
+        // preroll frame.
+        if (!do_render && preroll_timestamp_ <= pts + estimated_frame_period_) {
+          media_task_runner_->PostTask(FROM_HERE, internal_preroll_done_cb_);
+          preroll_done = true;
+        }
+      } break;
 
       case MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER:
         // Nothing to do.
@@ -649,7 +753,7 @@ bool MediaCodecDecoder::DepleteOutputBufferQueue() {
     }
 
   } while (status != MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER &&
-           status != MEDIA_CODEC_ERROR && !eos_encountered);
+           status != MEDIA_CODEC_ERROR && !eos_encountered && !preroll_done);
 
   if (eos_encountered) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
@@ -657,6 +761,12 @@ bool MediaCodecDecoder::DepleteOutputBufferQueue() {
     return false;
   }
 
+  if (preroll_done) {
+    DVLOG(1) << class_name() << "::" << __FUNCTION__
+             << " preroll done, stopping frame processing";
+    return false;
+  }
+
   if (status == MEDIA_CODEC_ERROR) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << " MediaCodec error, stopping frame processing";
@@ -688,6 +798,8 @@ const char* MediaCodecDecoder::AsString(DecoderState state) {
     RETURN_STRING(kStopped);
     RETURN_STRING(kPrefetching);
     RETURN_STRING(kPrefetched);
+    RETURN_STRING(kPrerolling);
+    RETURN_STRING(kPrerolled);
     RETURN_STRING(kRunning);
     RETURN_STRING(kStopping);
     RETURN_STRING(kInEmergencyStop);