MediaCodecPlayer implementation (stage 4 - preroll)

media/base/android/media_codec_decoder.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/base/android/media_codec_decoder.h"
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/callback_helpers.h"
 #include "base/logging.h"
@@ skipping 10 matching lines @@
 // Request new data in the kRunning state if the queue size is less than this.
 const int kPlaybackLowLimit = 4;
 
 // Posting delay of the next frame processing, in milliseconds
 const int kNextFrameDelay = 1;
 
 // Timeout for dequeuing an input buffer from MediaCodec in milliseconds.
 const int kInputBufferTimeout = 20;
 
 // Timeout for dequeuing an output buffer from MediaCodec in milliseconds.
-const int kOutputBufferTimeout = 20;
+const int kOutputBufferTimeout = 10;

[Tima Vaisburd, 2015/08/03 19:44:34]

Upon more detailed examination of unit tests I found that 10 ms is better as it
eliminates the skipping of the frames. However in the tests the frame period is
20 ms (50 fps) which is more demanding than usual 24 or 30 fps.

[qinmin, 2015/08/03 21:25:15]

these values are too small, some devices will always timeout.

[Tima Vaisburd, 2015/08/03 22:49:05]

Restored to 20 ms

 }
 
 MediaCodecDecoder::MediaCodecDecoder(
     const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner,
     const base::Closure& external_request_data_cb,
     const base::Closure& starvation_cb,
+    const base::Closure& preroll_done_cb,
     const base::Closure& stop_done_cb,
     const base::Closure& error_cb,
     const char* decoder_thread_name)
     : media_task_runner_(media_task_runner),
       decoder_thread_(decoder_thread_name),
       needs_reconfigure_(false),
       external_request_data_cb_(external_request_data_cb),
       starvation_cb_(starvation_cb),
+      preroll_done_cb_(preroll_done_cb),
       stop_done_cb_(stop_done_cb),
       error_cb_(error_cb),
       state_(kStopped),
+      needs_preroll_(true),
       eos_enqueued_(false),
       completed_(false),
       last_frame_posted_(false),
       is_data_request_in_progress_(false),
       is_incoming_data_invalid_(false),
 #ifndef NDEBUG
       verify_next_frame_is_key_(false),
 #endif
       weak_factory_(this) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << "Decoder::Decoder() " << decoder_thread_name;
 
   internal_error_cb_ =
       base::Bind(&MediaCodecDecoder::OnCodecError, weak_factory_.GetWeakPtr());
+  internal_preroll_done_cb_ =
+      base::Bind(&MediaCodecDecoder::OnPrerollDone, weak_factory_.GetWeakPtr());
   request_data_cb_ =
       base::Bind(&MediaCodecDecoder::RequestData, weak_factory_.GetWeakPtr());
 }
 
 MediaCodecDecoder::~MediaCodecDecoder() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << "Decoder::~Decoder()";
 
   // NB: ReleaseDecoderResources() is virtual
   ReleaseDecoderResources();
 }
 
 const char* MediaCodecDecoder::class_name() const {
   return "Decoder";
 }
 
 void MediaCodecDecoder::ReleaseDecoderResources() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__;
 
   // Set [kInEmergencyStop| state to block already posted ProcessNextFrame().
   SetState(kInEmergencyStop);
 
   decoder_thread_.Stop();  // synchronous
 
   SetState(kStopped);
-  media_codec_bridge_.reset();
+  ReleaseMediaCodec();
+
+  needs_preroll_ = true;
 }
 
 void MediaCodecDecoder::Flush() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__;
 
   DCHECK_EQ(GetState(), kStopped);
 
   // Flush() is a part of the Seek request. Whenever we request a seek we need
@@ skipping 29 matching lines @@
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__;
 
   media_codec_bridge_.reset();
 }
 
 bool MediaCodecDecoder::IsPrefetchingOrPlaying() const {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
+  // Whether decoder needs to be stopped.
   base::AutoLock lock(state_lock_);
-  return state_ == kPrefetching || state_ == kRunning;
+  switch (state_) {
+    case kPrefetching:
+    case kPrefetched:
+    case kPrerolling:
+    case kPrerolled:
+    case kRunning:
+      return true;
+    case kStopped:
+    case kStopping:
+    case kInEmergencyStop:
+    case kError:
+      return false;
+  }
+  NOTREACHED();
+  return false;
+}
+
+bool MediaCodecDecoder::IsPrerollDone() const {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  return !HasStream() || IsCompleted() || GetState() == kPrerolled;
 }
 
 bool MediaCodecDecoder::IsStopped() const {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   return GetState() == kStopped;
 }
 
 bool MediaCodecDecoder::IsCompleted() const {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
@@ skipping 31 matching lines @@
 
   if (GetState() == kError) {
     DVLOG(0) << class_name() << "::" << __FUNCTION__ << ": wrong state kError";
     return kConfigFailure;
   }
 
   if (needs_reconfigure_) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << ": needs reconfigure, deleting MediaCodec";
     needs_reconfigure_ = false;
-    media_codec_bridge_.reset();
-
-    // No need to release these buffers since the MediaCodec is deleted, just
-    // remove their indexes from |delayed_buffers_|.
-
-    ClearDelayedBuffers(false);
+    ReleaseMediaCodec();
   }
 
   MediaCodecDecoder::ConfigStatus result;
   if (media_codec_bridge_) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << ": reconfiguration is not required, ignoring";
     result = kConfigOk;
   } else {
     result = ConfigureInternal();
 
 #ifndef NDEBUG
     // We check and reset |verify_next_frame_is_key_| on Decoder thread.
     // This DCHECK ensures we won't need to lock this variable.
     DCHECK(!decoder_thread_.IsRunning());
 
     // For video the first frame after reconfiguration must be key frame.
     if (result == kConfigOk)
       verify_next_frame_is_key_ = true;
 #endif
   }
 
   return result;
 }
 
-bool MediaCodecDecoder::Start(base::TimeDelta current_time) {
+void MediaCodecDecoder::SetNeedsPreroll(bool value) {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__ << ":" << value;
+
+  needs_preroll_ = value;
+}
+
+bool MediaCodecDecoder::Start(base::TimeDelta start_timestamp) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__
-           << " current_time:" << current_time;
+           << " start_timestamp:" << start_timestamp;
 
   DecoderState state = GetState();
-  if (state == kRunning) {
+  if (state == kPrerolling || state == kRunning) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": already started";
     return true;  // already started
   }
 
   if (state != kPrefetched) {
     DVLOG(0) << class_name() << "::" << __FUNCTION__ << ": wrong state "
              << AsString(state) << ", ignoring";
     return false;
   }
 
   if (!media_codec_bridge_) {
     DVLOG(0) << class_name() << "::" << __FUNCTION__
              << ": not configured, ignoring";
     return false;
   }
 
   DCHECK(!decoder_thread_.IsRunning());
 
   // We only synchronize video stream.
-  // When audio is present, the |current_time| is audio time.
-  SynchronizePTSWithTime(current_time);
+  if (needs_preroll_) {
+    DissociatePTSFromTime();  // associaton will happen after preroll is done.
+    preroll_timestamp_ = start_timestamp;
+  } else {
+    AssociateCurrentTimeWithPTS(start_timestamp);
+    preroll_timestamp_ = base::TimeDelta();
+  }
 
   last_frame_posted_ = false;
 
   // Start the decoder thread
   if (!decoder_thread_.Start()) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << ": cannot start decoder thread";
     return false;
   }
 
-  SetState(kRunning);
+  SetState(needs_preroll_ ? kPrerolling : kRunning);
+  needs_preroll_ = false;
 
   decoder_thread_.task_runner()->PostTask(
       FROM_HERE,
       base::Bind(&MediaCodecDecoder::ProcessNextFrame, base::Unretained(this)));
 
   return true;
 }
 
+void MediaCodecDecoder::ResumeAfterPreroll() {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  DCHECK(GetState() == kPrerolled);
+  DCHECK(decoder_thread_.IsRunning());
+
+  SetState(kRunning);
+
+  AssociateCurrentTimeWithPTS(preroll_timestamp_);
+  preroll_timestamp_ = base::TimeDelta();
+
+  decoder_thread_.task_runner()->PostTask(
+      FROM_HERE,
+      base::Bind(&MediaCodecDecoder::ProcessNextFrame, base::Unretained(this)));
+}
+
 void MediaCodecDecoder::SyncStop() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__;
 
   if (GetState() == kError) {
     DVLOG(0) << class_name() << "::" << __FUNCTION__
              << ": wrong state kError, ignoring";
     return;
   }
 
   // After this method returns, decoder thread will not be running.
 
   // Set [kInEmergencyStop| state to block already posted ProcessNextFrame().
   SetState(kInEmergencyStop);
 
   decoder_thread_.Stop();  // synchronous
 
   SetState(kStopped);
 
-  ClearDelayedBuffers(true);  // release prior to clearing  |delayed_buffers_|.
+  ReleaseDelayedBuffers();
 }
 
 void MediaCodecDecoder::RequestToStop() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__;
 
   DecoderState state = GetState();
   switch (state) {
     case kError:
       DVLOG(0) << class_name() << "::" << __FUNCTION__
                << ": wrong state kError, ignoring";
       break;
     case kRunning:
       SetState(kStopping);
       break;
+    case kPrerolling:
+    case kPrerolled:
+      DCHECK(decoder_thread_.IsRunning());
+      // Synchronous stop.
+      decoder_thread_.Stop();
+      SetState(kStopped);
+      media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
+      break;
     case kStopping:
       break;  // ignore
     case kStopped:
     case kPrefetching:
     case kPrefetched:
       // There is nothing to wait for, we can sent nofigication right away.
       DCHECK(!decoder_thread_.IsRunning());
       SetState(kStopped);
       media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
       break;
@@ skipping 10 matching lines @@
            << " completed:" << completed;
 
   decoder_thread_.Stop();  // synchronous
 
   SetState(kStopped);
   completed_ = completed;
 
   media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
 }
 
+void MediaCodecDecoder::OnPrerollDone() {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  if (GetState() == kPrerolling) {
+    SetState(kPrerolled);
+    media_task_runner_->PostTask(FROM_HERE, preroll_done_cb_);
+  }
+}
+
 void MediaCodecDecoder::OnDemuxerDataAvailable(const DemuxerData& data) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   // If |data| contains an aborted data, the last AU will have kAborted status.
   bool aborted_data =
       !data.access_units.empty() &&
       data.access_units.back().status == DemuxerStream::kAborted;
 
 #ifndef NDEBUG
   const char* explain_if_skipped =
@@ skipping 81 matching lines @@
   request_data_cb_.Run();
 }
 
 void MediaCodecDecoder::ProcessNextFrame() {
   DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread());
 
   DVLOG(2) << class_name() << "::" << __FUNCTION__;
 
   DecoderState state = GetState();
 
-  if (state != kRunning && state != kStopping) {
+  if (state != kPrerolling && state != kRunning && state != kStopping) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": not running";
     return;
   }
 
   if (state == kStopping) {
     if (NumDelayedRenderTasks() == 0 && !last_frame_posted_) {
       DVLOG(1) << class_name() << "::" << __FUNCTION__
                << ": kStopping, posting OnLastFrameRendered";
       media_task_runner_->PostTask(
           FROM_HERE, base::Bind(&MediaCodecDecoder::OnLastFrameRendered,
                                 weak_factory_.GetWeakPtr(), false));
       last_frame_posted_ = true;
     }
 
     // We can stop processing, the |au_queue_| and MediaCodec queues can freeze.
     // We only need to let finish the delayed rendering tasks.
     return;
   }
 
-  DCHECK(state == kRunning);
+  DCHECK(state == kPrerolling || state == kRunning);
 
   if (!EnqueueInputBuffer())
     return;
 
   if (!DepleteOutputBufferQueue())
     return;
 
   // We need a small delay if we want to stop this thread by
   // decoder_thread_.Stop() reliably.
   // The decoder thread message loop processes all pending
@@ skipping 121 matching lines @@
 
   DVLOG(2) << class_name() << "::" << __FUNCTION__;
 
   int buffer_index = 0;
   size_t offset = 0;
   size_t size = 0;
   base::TimeDelta pts;
   MediaCodecStatus status;
   bool eos_encountered = false;
 
+  RenderMode render_mode;
+
   base::TimeDelta timeout =
       base::TimeDelta::FromMilliseconds(kOutputBufferTimeout);
 
   // Extract all output buffers that are available.
   // Usually there will be only one, but sometimes it is preceeded by
   // MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED or MEDIA_CODEC_OUTPUT_FORMAT_CHANGED.
   do {
     status = media_codec_bridge_->DequeueOutputBuffer(
         timeout, &buffer_index, &offset, &size, &pts, &eos_encountered,
         nullptr);
 
     // Reset the timeout to 0 for the subsequent DequeueOutputBuffer() calls
     // to quickly break the loop after we got all currently available buffers.
     timeout = base::TimeDelta::FromMilliseconds(0);
 
     switch (status) {
       case MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED:
         // Output buffers are replaced in MediaCodecBridge, nothing to do.
         break;
 
       case MEDIA_CODEC_OUTPUT_FORMAT_CHANGED:
         DVLOG(2) << class_name() << "::" << __FUNCTION__
                  << " MEDIA_CODEC_OUTPUT_FORMAT_CHANGED";
         OnOutputFormatChanged();
         break;
 
       case MEDIA_CODEC_OK:
-        // We got the decoded frame
-        Render(buffer_index, size, true, pts, eos_encountered);
+        // We got the decoded frame.
+
+        if (pts < preroll_timestamp_)
+          render_mode = kRenderSkip;
+        else if (GetState() == kPrerolling)
+          render_mode = kRenderAfterPreroll;
+        else
+          render_mode = kRenderNow;
+
+        Render(buffer_index, size, render_mode, pts, eos_encountered);
+
+        if (render_mode == kRenderAfterPreroll) {
+          DVLOG(1) << class_name() << "::" << __FUNCTION__ << " pts:" << pts
+                   << " preroll done, stopping frame processing";
+          media_task_runner_->PostTask(FROM_HERE, internal_preroll_done_cb_);
+          return false;
+        }
         break;
 
       case MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER:
         // Nothing to do.
         break;
 
       case MEDIA_CODEC_ERROR:
         DVLOG(0) << class_name() << "::" << __FUNCTION__
                  << ": MEDIA_CODEC_ERROR from DequeueOutputBuffer";
         media_task_runner_->PostTask(FROM_HERE, internal_error_cb_);
@@ skipping 32 matching lines @@
 
   base::AutoLock lock(state_lock_);
   state_ = state;
 }
 
 #undef RETURN_STRING
 #define RETURN_STRING(x) \
   case x:                \
     return #x;
 
+const char* MediaCodecDecoder::AsString(RenderMode render_mode) {
+  switch (render_mode) {
+    RETURN_STRING(kRenderSkip);
+    RETURN_STRING(kRenderAfterPreroll);
+    RETURN_STRING(kRenderNow);
+  }
+  return nullptr;  // crash early
+}
+
 const char* MediaCodecDecoder::AsString(DecoderState state) {
   switch (state) {
     RETURN_STRING(kStopped);
     RETURN_STRING(kPrefetching);
     RETURN_STRING(kPrefetched);
+    RETURN_STRING(kPrerolling);
+    RETURN_STRING(kPrerolled);
     RETURN_STRING(kRunning);
     RETURN_STRING(kStopping);
     RETURN_STRING(kInEmergencyStop);
     RETURN_STRING(kError);
-    default:
-      return "Unknown DecoderState";
   }
+  return nullptr;  // crash early
 }
 
 #undef RETURN_STRING
 
 }  // namespace media