MediaCodecPlayer implementation (stage 4 - preroll)

media/base/android/media_codec_decoder.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/base/android/media_codec_decoder.h"
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/callback_helpers.h"
 #include "base/logging.h"
@@ skipping 11 matching lines @@
 const int kPlaybackLowLimit = 4;
 
 // Posting delay of the next frame processing, in milliseconds
 const int kNextFrameDelay = 1;
 
 // Timeout for dequeuing an input buffer from MediaCodec in milliseconds.
 const int kInputBufferTimeout = 20;
 
 // Timeout for dequeuing an output buffer from MediaCodec in milliseconds.
 const int kOutputBufferTimeout = 20;
+
+// Estimated frame period in milliseconds
+const int kEstimatedFramePeriod = 20;

[qinmin, 2015/07/28 18:09:46]

why we need this variable? If we allow preroll for at pts 0, then we can ask
audio and video decoder to start decoding until they start generating frames. 

 }
 
 MediaCodecDecoder::MediaCodecDecoder(
     const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner,
     const base::Closure& external_request_data_cb,
     const base::Closure& starvation_cb,
+    const base::Closure& preroll_done_cb,
     const base::Closure& stop_done_cb,
     const base::Closure& error_cb,
     const char* decoder_thread_name)
     : media_task_runner_(media_task_runner),
       decoder_thread_(decoder_thread_name),
       needs_reconfigure_(false),
       external_request_data_cb_(external_request_data_cb),
       starvation_cb_(starvation_cb),
+      preroll_done_cb_(preroll_done_cb),
       stop_done_cb_(stop_done_cb),
       error_cb_(error_cb),
       state_(kStopped),
       eos_enqueued_(false),
       completed_(false),
       last_frame_posted_(false),
       is_data_request_in_progress_(false),
       is_incoming_data_invalid_(false),
 #ifndef NDEBUG
       verify_next_frame_is_key_(false),
 #endif
       weak_factory_(this) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << "Decoder::Decoder() " << decoder_thread_name;
 
+  // For simplicity we use constant value instead of truly measuring the frame
+  // period. As long as it is used for determination of the
+  // kPrerolling -> kPrerolled switch only this simplification seems ok.
+  estimated_frame_period_ =
+      base::TimeDelta::FromMilliseconds(kEstimatedFramePeriod);
+
   internal_error_cb_ =
       base::Bind(&MediaCodecDecoder::OnCodecError, weak_factory_.GetWeakPtr());
+  internal_preroll_done_cb_ =
+      base::Bind(&MediaCodecDecoder::OnPrerollDone, weak_factory_.GetWeakPtr());
   request_data_cb_ =
       base::Bind(&MediaCodecDecoder::RequestData, weak_factory_.GetWeakPtr());
 }
 
 MediaCodecDecoder::~MediaCodecDecoder() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << "Decoder::~Decoder()";
 
   // NB: ReleaseDecoderResources() is virtual
@@ skipping 58 matching lines @@
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__;
 
   media_codec_bridge_.reset();
 }
 
 bool MediaCodecDecoder::IsPrefetchingOrPlaying() const {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
+  // Whether decoder needs to be stopped.
   base::AutoLock lock(state_lock_);
-  return state_ == kPrefetching || state_ == kRunning;
+  switch (state_) {
+    case kPrefetching:
+    case kPrefetched:
+    case kPrerolling:
+    case kPrerolled:
+    case kRunning:
+      return true;
+    case kStopped:
+    case kStopping:
+    case kInEmergencyStop:
+    case kError:
+      return false;
+  }
+  NOTREACHED();
+  return false;
+}
+
+bool MediaCodecDecoder::IsPrerollDone() const {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  return !HasStream() || IsCompleted() || GetState() == kPrerolled;
 }
 
 bool MediaCodecDecoder::IsStopped() const {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   return GetState() == kStopped;
 }
 
 bool MediaCodecDecoder::IsCompleted() const {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
@@ skipping 62 matching lines @@
 
     // For video the first frame after reconfiguration must be key frame.
     if (result == CONFIG_OK)
       verify_next_frame_is_key_ = true;
 #endif
   }
 
   return result;
 }
 
-bool MediaCodecDecoder::Start(base::TimeDelta current_time) {
+void MediaCodecDecoder::SetPrerollTimestamp(base::TimeDelta preroll_timestamp) {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+  DCHECK(!decoder_thread_.IsRunning());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": " << preroll_timestamp;
+
+  // Do not set preroll timestamp if it's too close to zero.
+  preroll_timestamp_ = (preroll_timestamp < estimated_frame_period_)
+                           ? base::TimeDelta()
+                           : preroll_timestamp;
+}
+
+bool MediaCodecDecoder::Start(base::TimeDelta start_timestamp) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__
-           << " current_time:" << current_time;
+           << " start_timestamp:" << start_timestamp;
 
   DecoderState state = GetState();
-  if (state == kRunning) {
+  if (state == kPrerolling || state == kRunning) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": already started";
     return true;  // already started
   }
 
   if (state != kPrefetched) {
     DVLOG(0) << class_name() << "::" << __FUNCTION__ << ": wrong state "
              << AsString(state) << " ignoring";
     return false;
   }
 
   if (!media_codec_bridge_) {
     DVLOG(0) << class_name() << "::" << __FUNCTION__
              << ": not configured, ignoring";
     return false;
   }
 
   DCHECK(!decoder_thread_.IsRunning());
 
+  const bool needs_preroll = (preroll_timestamp_ != base::TimeDelta());
+
   // We only synchronize video stream.
-  // When audio is present, the |current_time| is audio time.
-  SynchronizePTSWithTime(current_time);
+  if (needs_preroll)
+    DissociatePTSFromTime();  // associaton will happen after preroll is done.
+  else
+    AssociateCurrentTimeWithPTS(start_timestamp);
 
   last_frame_posted_ = false;
 
   // Start the decoder thread
   if (!decoder_thread_.Start()) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << ": cannot start decoder thread";
     return false;
   }
 
-  SetState(kRunning);
+  SetState(needs_preroll ? kPrerolling : kRunning);
 
   decoder_thread_.task_runner()->PostTask(
       FROM_HERE,
       base::Bind(&MediaCodecDecoder::ProcessNextFrame, base::Unretained(this)));
 
   return true;
 }
 
+void MediaCodecDecoder::ResumeAfterPreroll() {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  DCHECK(GetState() == kPrerolled);
+  DCHECK(decoder_thread_.IsRunning());
+
+  SetState(kRunning);
+
+  AssociateCurrentTimeWithPTS(preroll_timestamp_);
+  preroll_timestamp_ = base::TimeDelta();
+
+  decoder_thread_.task_runner()->PostTask(
+      FROM_HERE,
+      base::Bind(&MediaCodecDecoder::ProcessNextFrame, base::Unretained(this)));
+}
+
 void MediaCodecDecoder::SyncStop() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__;
 
   if (GetState() == kError) {
     DVLOG(0) << class_name() << "::" << __FUNCTION__
              << ": wrong state kError, ignoring";
     return;
   }
@@ skipping 18 matching lines @@
 
   DecoderState state = GetState();
   switch (state) {
     case kError:
       DVLOG(0) << class_name() << "::" << __FUNCTION__
                << ": wrong state kError, ignoring";
       break;
     case kRunning:
       SetState(kStopping);
       break;
+    case kPrerolling:
+    case kPrerolled:
+      DCHECK(decoder_thread_.IsRunning());
+      // Synchronous stop.
+      decoder_thread_.Stop();
+      SetState(kStopped);
+      media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
+      break;
     case kStopping:
       break;  // ignore
     case kStopped:
     case kPrefetching:
     case kPrefetched:
       // There is nothing to wait for, we can sent nofigication right away.
       DCHECK(!decoder_thread_.IsRunning());
       SetState(kStopped);
       media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
       break;
@@ skipping 10 matching lines @@
            << " completed:" << completed;
 
   decoder_thread_.Stop();  // synchronous
 
   SetState(kStopped);
   completed_ = completed;
 
   media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
 }
 
+void MediaCodecDecoder::OnPrerollDone() {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  if (GetState() == kPrerolling) {
+    SetState(kPrerolled);
+    media_task_runner_->PostTask(FROM_HERE, preroll_done_cb_);
+  }
+}
+
 void MediaCodecDecoder::OnDemuxerDataAvailable(const DemuxerData& data) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   // If |data| contains an aborted data, the last AU will have kAborted status.
   bool aborted_data =
       !data.access_units.empty() &&
       data.access_units.back().status == DemuxerStream::kAborted;
 
 #ifndef NDEBUG
   const char* explain_if_skipped =
@@ skipping 81 matching lines @@
   request_data_cb_.Run();
 }
 
 void MediaCodecDecoder::ProcessNextFrame() {
   DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread());
 
   DVLOG(2) << class_name() << "::" << __FUNCTION__;
 
   DecoderState state = GetState();
 
-  if (state != kRunning && state != kStopping) {
+  if (state != kPrerolling && state != kRunning && state != kStopping) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": not running";
     return;
   }
 
   if (state == kStopping) {
     if (NumDelayedRenderTasks() == 0 && !last_frame_posted_) {
       DVLOG(1) << class_name() << "::" << __FUNCTION__
                << ": kStopping, posting OnLastFrameRendered";
       media_task_runner_->PostTask(
           FROM_HERE, base::Bind(&MediaCodecDecoder::OnLastFrameRendered,
                                 weak_factory_.GetWeakPtr(), false));
       last_frame_posted_ = true;
     }
 
     // We can stop processing, the |au_queue_| and MediaCodec queues can freeze.
     // We only need to let finish the delayed rendering tasks.
     return;
   }
 
-  DCHECK(state == kRunning);
+  DCHECK(state == kPrerolling || state == kRunning);
 
   if (!EnqueueInputBuffer())
     return;
 
   if (!DepleteOutputBufferQueue())
     return;
 
   // We need a small delay if we want to stop this thread by
   // decoder_thread_.Stop() reliably.
   // The decoder thread message loop processes all pending
@@ skipping 120 matching lines @@
   DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread());
 
   DVLOG(2) << class_name() << "::" << __FUNCTION__;
 
   int buffer_index = 0;
   size_t offset = 0;
   size_t size = 0;
   base::TimeDelta pts;
   MediaCodecStatus status;
   bool eos_encountered = false;
+  bool preroll_done = false;
 
   base::TimeDelta timeout =
       base::TimeDelta::FromMilliseconds(kOutputBufferTimeout);
 
   // Extract all output buffers that are available.
   // Usually there will be only one, but sometimes it is preceeded by
   // MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED or MEDIA_CODEC_OUTPUT_FORMAT_CHANGED.
   do {
     status = media_codec_bridge_->DequeueOutputBuffer(
         timeout, &buffer_index, &offset, &size, &pts, &eos_encountered,
         nullptr);
 
     // Reset the timeout to 0 for the subsequent DequeueOutputBuffer() calls
     // to quickly break the loop after we got all currently available buffers.
     timeout = base::TimeDelta::FromMilliseconds(0);
 
     switch (status) {
       case MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED:
         // Output buffers are replaced in MediaCodecBridge, nothing to do.
         break;
 
       case MEDIA_CODEC_OUTPUT_FORMAT_CHANGED:
         DVLOG(2) << class_name() << "::" << __FUNCTION__
                  << " MEDIA_CODEC_OUTPUT_FORMAT_CHANGED";
         OnOutputFormatChanged();
         break;
 
-      case MEDIA_CODEC_OK:
-        // We got the decoded frame
-        Render(buffer_index, size, true, pts, eos_encountered);
-        break;
+      case MEDIA_CODEC_OK: {
+        // We got the decoded frame.
+
+        // TODO(timav): this code won't render the very first frame in
+        // kPrerolling if it is already behind preroll_timestamp_. A more
+        // precise method would be to stop before Render() and resume after
+        // preroll with the Render(), but it would be more complicated.
+
+        DecoderState state = GetState();
+        const bool do_render = (state == kRunning || state == kStopping);

[qinmin, 2015/07/28 18:09:46]

I think it is ok to render if (state == kPrerolling && preroll_timestamp_ <=
pts)

[Tima Vaisburd, 2015/07/28 18:34:04]

I believe you mean "in addition to kRunning and kStopping" : after preroll is
done the state will switch to kRunning and we still need to render the frames.

All these hoops including the estimated frame period was intended to prevent the
situation where we render the first frame after preroll and then wait. According
to what I've seen this would be an audio frame in most cases. So, we will render
the first audio frame, then wait for video to preroll, then resume both. Do you
say that the first audio won't play anyway because audio needs to fill the audio
buffer and it would be ok? 

[qinmin, 2015/07/28 18:55:06]

You can modify the MediaCodecBridge.playOutputBuffer() call to take a param to
buffer this frame instead of playing it, and when the next frame comes,
playoutputbuffer() will automatically start playing.

+
+        Render(buffer_index, size, do_render, pts, eos_encountered);
+
+        // If next pts passes over |preroll_timestamp_| this frame is the last
+        // preroll frame.
+        if (!do_render && preroll_timestamp_ <= pts + estimated_frame_period_) {
+          media_task_runner_->PostTask(FROM_HERE, internal_preroll_done_cb_);
+          preroll_done = true;
+        }
+      } break;
 
       case MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER:
         // Nothing to do.
         break;
 
       case MEDIA_CODEC_ERROR:
         DVLOG(0) << class_name() << "::" << __FUNCTION__
                  << ": MEDIA_CODEC_ERROR from DequeueOutputBuffer";
         media_task_runner_->PostTask(FROM_HERE, internal_error_cb_);
         break;
 
       default:
         NOTREACHED();
         break;
     }
 
   } while (status != MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER &&
-           status != MEDIA_CODEC_ERROR && !eos_encountered);
+           status != MEDIA_CODEC_ERROR && !eos_encountered && !preroll_done);
 
   if (eos_encountered) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << " EOS dequeued, stopping frame processing";
     return false;
   }
 
+  if (preroll_done) {
+    DVLOG(1) << class_name() << "::" << __FUNCTION__
+             << " preroll done, stopping frame processing";
+    return false;
+  }
+
   if (status == MEDIA_CODEC_ERROR) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << " MediaCodec error, stopping frame processing";
     return false;
   }
 
   return true;
 }
 
 MediaCodecDecoder::DecoderState MediaCodecDecoder::GetState() const {
@@ skipping 11 matching lines @@
 #undef RETURN_STRING
 #define RETURN_STRING(x) \
   case x:                \
     return #x;
 
 const char* MediaCodecDecoder::AsString(DecoderState state) {
   switch (state) {
     RETURN_STRING(kStopped);
     RETURN_STRING(kPrefetching);
     RETURN_STRING(kPrefetched);
+    RETURN_STRING(kPrerolling);
+    RETURN_STRING(kPrerolled);
     RETURN_STRING(kRunning);
     RETURN_STRING(kStopping);
     RETURN_STRING(kInEmergencyStop);
     RETURN_STRING(kError);
     default:
       return "Unknown DecoderState";
   }
 }
 
 #undef RETURN_STRING
 
 }  // namespace media