MediaCodecPlayer implementation (stage 4 - preroll)

media/base/android/media_codec_decoder.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/base/android/media_codec_decoder.h"
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/callback_helpers.h"
 #include "base/logging.h"
@@ skipping 17 matching lines @@
 const int kInputBufferTimeout = 20;
 
 // Timeout for dequeuing an output buffer from MediaCodec in milliseconds.
 const int kOutputBufferTimeout = 20;
 }
 
 MediaCodecDecoder::MediaCodecDecoder(
     const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner,
     const base::Closure& external_request_data_cb,
     const base::Closure& starvation_cb,
+    const base::Closure& preroll_done_cb,
     const base::Closure& stop_done_cb,
     const base::Closure& error_cb,
     const char* decoder_thread_name)
     : media_task_runner_(media_task_runner),
       decoder_thread_(decoder_thread_name),
       needs_reconfigure_(false),
       external_request_data_cb_(external_request_data_cb),
       starvation_cb_(starvation_cb),
+      preroll_done_cb_(preroll_done_cb),
       stop_done_cb_(stop_done_cb),
       error_cb_(error_cb),
       state_(kStopped),
       eos_enqueued_(false),
       completed_(false),
       last_frame_posted_(false),
       is_data_request_in_progress_(false),
       is_incoming_data_invalid_(false),
 #ifndef NDEBUG
       verify_next_frame_is_key_(false),
 #endif
       weak_factory_(this) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << "Decoder::Decoder() " << decoder_thread_name;
 
   internal_error_cb_ =
       base::Bind(&MediaCodecDecoder::OnCodecError, weak_factory_.GetWeakPtr());
+  internal_preroll_done_cb_ =
+      base::Bind(&MediaCodecDecoder::OnPrerollDone, weak_factory_.GetWeakPtr());
   request_data_cb_ =
       base::Bind(&MediaCodecDecoder::RequestData, weak_factory_.GetWeakPtr());
 }
 
 MediaCodecDecoder::~MediaCodecDecoder() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << "Decoder::~Decoder()";
 
   // NB: ReleaseDecoderResources() is virtual
   ReleaseDecoderResources();
 }
 
 const char* MediaCodecDecoder::class_name() const {
   return "Decoder";
 }
 
 void MediaCodecDecoder::ReleaseDecoderResources() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__;
 
   // Set [kInEmergencyStop| state to block already posted ProcessNextFrame().
   SetState(kInEmergencyStop);
 
   decoder_thread_.Stop();  // synchronous
 
   SetState(kStopped);
-  media_codec_bridge_.reset();
+  ReleaseMediaCodec();
 }
 
 void MediaCodecDecoder::Flush() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__;
 
   DCHECK_EQ(GetState(), kStopped);
 
   // Flush() is a part of the Seek request. Whenever we request a seek we need
@@ skipping 21 matching lines @@
       DVLOG(0) << class_name() << "::" << __FUNCTION__
                << "MediaCodecBridge::Reset() failed";
       media_task_runner_->PostTask(FROM_HERE, internal_error_cb_);
     }
   }
 }
 
 void MediaCodecDecoder::ReleaseMediaCodec() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
-  DVLOG(1) << class_name() << "::" << __FUNCTION__;
-
   media_codec_bridge_.reset();
 }
 
 bool MediaCodecDecoder::IsPrefetchingOrPlaying() const {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
+  // Whether decoder needs to be stopped.
   base::AutoLock lock(state_lock_);
-  return state_ == kPrefetching || state_ == kRunning;
+  switch (state_) {
+    case kPrefetching:
+    case kPrefetched:
+    case kPrerolling:
+    case kPrerolled:
+    case kRunning:
+      return true;
+    case kStopped:
+    case kStopping:
+    case kInEmergencyStop:
+    case kError:
+      return false;
+  }
+  NOTREACHED();
+  return false;
+}
+
+bool MediaCodecDecoder::IsPrerollDone() const {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  return !HasStream() || IsCompleted() || GetState() == kPrerolled;
 }
 
 bool MediaCodecDecoder::IsStopped() const {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   return GetState() == kStopped;
 }
 
 bool MediaCodecDecoder::IsCompleted() const {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
@@ skipping 31 matching lines @@
 
   if (GetState() == kError) {
     DVLOG(0) << class_name() << "::" << __FUNCTION__ << ": wrong state kError";
     return kConfigFailure;
   }
 
   if (needs_reconfigure_) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << ": needs reconfigure, deleting MediaCodec";
     needs_reconfigure_ = false;
-    media_codec_bridge_.reset();
-
-    // No need to release these buffers since the MediaCodec is deleted, just
-    // remove their indexes from |delayed_buffers_|.
-
-    ClearDelayedBuffers(false);
+    ReleaseMediaCodec();
   }
 
   MediaCodecDecoder::ConfigStatus result;
   if (media_codec_bridge_) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << ": reconfiguration is not required, ignoring";
     result = kConfigOk;
   } else {
     result = ConfigureInternal();
 
 #ifndef NDEBUG
     // We check and reset |verify_next_frame_is_key_| on Decoder thread.
     // This DCHECK ensures we won't need to lock this variable.
     DCHECK(!decoder_thread_.IsRunning());
 
     // For video the first frame after reconfiguration must be key frame.
     if (result == kConfigOk)
       verify_next_frame_is_key_ = true;
 #endif
   }
 
   return result;
 }
 
-bool MediaCodecDecoder::Start(base::TimeDelta current_time) {
+void MediaCodecDecoder::SetPrerollTimestamp(base::TimeDelta preroll_timestamp) {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+  DCHECK(!decoder_thread_.IsRunning());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": " << preroll_timestamp;
+
+  // Do not set preroll timestamp if it's too close to zero.
+  preroll_timestamp_ = preroll_timestamp;
+}
+
+bool MediaCodecDecoder::Start(base::TimeDelta start_timestamp) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__
-           << " current_time:" << current_time;
+           << " start_timestamp:" << start_timestamp;
 
   DecoderState state = GetState();
-  if (state == kRunning) {
+  if (state == kPrerolling || state == kRunning) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": already started";
     return true;  // already started
   }
 
   if (state != kPrefetched) {
     DVLOG(0) << class_name() << "::" << __FUNCTION__ << ": wrong state "
              << AsString(state) << ", ignoring";
     return false;
   }
 
   if (!media_codec_bridge_) {
     DVLOG(0) << class_name() << "::" << __FUNCTION__
              << ": not configured, ignoring";
     return false;
   }
 
   DCHECK(!decoder_thread_.IsRunning());
 
+  const bool needs_preroll = (preroll_timestamp_ != base::TimeDelta());

[qinmin, 2015/07/31 18:43:10]

why not preroll if timestamp is 0? it will help us sync audio and video by
removing the decoder initialization delay.

[Tima Vaisburd, 2015/07/31 19:26:42]

I think decoder initialization delay has been already taken care of because
StartPlaybackDecoders()
(https://code.google.com/p/chromium/codesearch#chromium/src/media/base/android...)
at first configures both audio and video, and then starts both. If I'm not
missing anything, of course.
My thinking here was that with start_timestamp == 0 there could not be any prior
timestamp and so there is nothing to preroll.

[Tima Vaisburd, 2015/08/01 04:34:43]

Min explained that there is internal codec initialization that happens between
the moment we queue first input buffer till the moment we dequeue first output
buffer and for video codec it can be significant. Therefore after flush and
reconfiguration we need to do preroll event at pts == 0.
Done.

+
   // We only synchronize video stream.
-  // When audio is present, the |current_time| is audio time.
-  SynchronizePTSWithTime(current_time);
+  if (needs_preroll)
+    DissociatePTSFromTime();  // associaton will happen after preroll is done.
+  else
+    AssociateCurrentTimeWithPTS(start_timestamp);
 
   last_frame_posted_ = false;
 
   // Start the decoder thread
   if (!decoder_thread_.Start()) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << ": cannot start decoder thread";
     return false;
   }
 
-  SetState(kRunning);
+  SetState(needs_preroll ? kPrerolling : kRunning);
 
   decoder_thread_.task_runner()->PostTask(
       FROM_HERE,
       base::Bind(&MediaCodecDecoder::ProcessNextFrame, base::Unretained(this)));
 
   return true;
 }
 
+void MediaCodecDecoder::ResumeAfterPreroll() {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  DCHECK(GetState() == kPrerolled);
+  DCHECK(decoder_thread_.IsRunning());
+
+  SetState(kRunning);
+
+  AssociateCurrentTimeWithPTS(preroll_timestamp_);
+  preroll_timestamp_ = base::TimeDelta();
+
+  decoder_thread_.task_runner()->PostTask(
+      FROM_HERE,
+      base::Bind(&MediaCodecDecoder::ProcessNextFrame, base::Unretained(this)));
+}
+
 void MediaCodecDecoder::SyncStop() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__;
 
   if (GetState() == kError) {
     DVLOG(0) << class_name() << "::" << __FUNCTION__
              << ": wrong state kError, ignoring";
     return;
   }
 
   // After this method returns, decoder thread will not be running.
 
   // Set [kInEmergencyStop| state to block already posted ProcessNextFrame().
   SetState(kInEmergencyStop);
 
   decoder_thread_.Stop();  // synchronous
 
   SetState(kStopped);
 
-  ClearDelayedBuffers(true);  // release prior to clearing  |delayed_buffers_|.
+  ReleaseDelayedBuffers();
 }
 
 void MediaCodecDecoder::RequestToStop() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__;
 
   DecoderState state = GetState();
   switch (state) {
     case kError:
       DVLOG(0) << class_name() << "::" << __FUNCTION__
                << ": wrong state kError, ignoring";
       break;
     case kRunning:
       SetState(kStopping);
       break;
+    case kPrerolling:
+    case kPrerolled:
+      DCHECK(decoder_thread_.IsRunning());
+      // Synchronous stop.
+      decoder_thread_.Stop();
+      SetState(kStopped);
+      media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
+      break;
     case kStopping:
       break;  // ignore
     case kStopped:
     case kPrefetching:
     case kPrefetched:
       // There is nothing to wait for, we can sent nofigication right away.
       DCHECK(!decoder_thread_.IsRunning());
       SetState(kStopped);
       media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
       break;
@@ skipping 10 matching lines @@
            << " completed:" << completed;
 
   decoder_thread_.Stop();  // synchronous
 
   SetState(kStopped);
   completed_ = completed;
 
   media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
 }
 
+void MediaCodecDecoder::OnPrerollDone() {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  if (GetState() == kPrerolling) {
+    SetState(kPrerolled);
+    media_task_runner_->PostTask(FROM_HERE, preroll_done_cb_);
+  }
+}
+
 void MediaCodecDecoder::OnDemuxerDataAvailable(const DemuxerData& data) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   // If |data| contains an aborted data, the last AU will have kAborted status.
   bool aborted_data =
       !data.access_units.empty() &&
       data.access_units.back().status == DemuxerStream::kAborted;
 
 #ifndef NDEBUG
   const char* explain_if_skipped =
@@ skipping 81 matching lines @@
   request_data_cb_.Run();
 }
 
 void MediaCodecDecoder::ProcessNextFrame() {
   DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread());
 
   DVLOG(2) << class_name() << "::" << __FUNCTION__;
 
   DecoderState state = GetState();
 
-  if (state != kRunning && state != kStopping) {
+  if (state != kPrerolling && state != kRunning && state != kStopping) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": not running";
     return;
   }
 
   if (state == kStopping) {
     if (NumDelayedRenderTasks() == 0 && !last_frame_posted_) {
       DVLOG(1) << class_name() << "::" << __FUNCTION__
                << ": kStopping, posting OnLastFrameRendered";
       media_task_runner_->PostTask(
           FROM_HERE, base::Bind(&MediaCodecDecoder::OnLastFrameRendered,
                                 weak_factory_.GetWeakPtr(), false));
       last_frame_posted_ = true;
     }
 
     // We can stop processing, the |au_queue_| and MediaCodec queues can freeze.
     // We only need to let finish the delayed rendering tasks.
     return;
   }
 
-  DCHECK(state == kRunning);
+  DCHECK(state == kPrerolling || state == kRunning);
 
   if (!EnqueueInputBuffer())
     return;
 
   if (!DepleteOutputBufferQueue())
     return;
 
   // We need a small delay if we want to stop this thread by
   // decoder_thread_.Stop() reliably.
   // The decoder thread message loop processes all pending
@@ skipping 121 matching lines @@
 
   DVLOG(2) << class_name() << "::" << __FUNCTION__;
 
   int buffer_index = 0;
   size_t offset = 0;
   size_t size = 0;
   base::TimeDelta pts;
   MediaCodecStatus status;
   bool eos_encountered = false;
 
+  RenderMode render_mode;
+  bool preroll_done = false;
+
   base::TimeDelta timeout =
       base::TimeDelta::FromMilliseconds(kOutputBufferTimeout);
 
   // Extract all output buffers that are available.
   // Usually there will be only one, but sometimes it is preceeded by
   // MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED or MEDIA_CODEC_OUTPUT_FORMAT_CHANGED.
   do {
     status = media_codec_bridge_->DequeueOutputBuffer(
         timeout, &buffer_index, &offset, &size, &pts, &eos_encountered,
         nullptr);
 
     // Reset the timeout to 0 for the subsequent DequeueOutputBuffer() calls
     // to quickly break the loop after we got all currently available buffers.
     timeout = base::TimeDelta::FromMilliseconds(0);
 
     switch (status) {
       case MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED:
         // Output buffers are replaced in MediaCodecBridge, nothing to do.
         break;
 
       case MEDIA_CODEC_OUTPUT_FORMAT_CHANGED:
         DVLOG(2) << class_name() << "::" << __FUNCTION__
                  << " MEDIA_CODEC_OUTPUT_FORMAT_CHANGED";
         OnOutputFormatChanged();
         break;
 
-      case MEDIA_CODEC_OK:
-        // We got the decoded frame
-        Render(buffer_index, size, true, pts, eos_encountered);
-        break;
+      case MEDIA_CODEC_OK: {
+        // We got the decoded frame.
+
+        if (pts < preroll_timestamp_)
+          render_mode = kRenderSkip;
+        else if (GetState() == kPrerolling)
+          render_mode = kRenderAfterPreroll;
+        else
+          render_mode = kRenderNow;
+
+        Render(buffer_index, size, render_mode, pts, eos_encountered);
+
+        if (render_mode == kRenderAfterPreroll) {
+          media_task_runner_->PostTask(FROM_HERE, internal_preroll_done_cb_);
+          preroll_done = true;

[qinmin, 2015/07/31 18:43:10]

why not just return false here.

[Tima Vaisburd, 2015/07/31 19:26:42]

Done.

+        }
+      } break;

[qinmin, 2015/07/31 18:43:10]

move the break to a new line

[Tima Vaisburd, 2015/07/31 19:26:42]

Done.

 
       case MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER:
         // Nothing to do.
         break;
 
       case MEDIA_CODEC_ERROR:
         DVLOG(0) << class_name() << "::" << __FUNCTION__
                  << ": MEDIA_CODEC_ERROR from DequeueOutputBuffer";
         media_task_runner_->PostTask(FROM_HERE, internal_error_cb_);
         break;
 
       default:
         NOTREACHED();
         break;
     }
 
   } while (status != MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER &&
-           status != MEDIA_CODEC_ERROR && !eos_encountered);
+           status != MEDIA_CODEC_ERROR && !eos_encountered && !preroll_done);
 
   if (eos_encountered) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << " EOS dequeued, stopping frame processing";
     return false;
   }
 
+  if (preroll_done) {
+    DVLOG(1) << class_name() << "::" << __FUNCTION__
+             << " preroll done, stopping frame processing";
+    return false;
+  }
+
   if (status == MEDIA_CODEC_ERROR) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << " MediaCodec error, stopping frame processing";
     return false;
   }
 
   return true;
 }
 
 MediaCodecDecoder::DecoderState MediaCodecDecoder::GetState() const {
   base::AutoLock lock(state_lock_);
   return state_;
 }
 
 void MediaCodecDecoder::SetState(DecoderState state) {
   DVLOG(1) << class_name() << "::" << __FUNCTION__ << " " << AsString(state);
 
   base::AutoLock lock(state_lock_);
   state_ = state;
 }
 
 #undef RETURN_STRING
 #define RETURN_STRING(x) \
   case x:                \
     return #x;
 
+const char* MediaCodecDecoder::AsString(RenderMode render_mode) {
+  switch (render_mode) {
+    RETURN_STRING(kRenderSkip);
+    RETURN_STRING(kRenderAfterPreroll);
+    RETURN_STRING(kRenderNow);
+  }
+  return nullptr;  // crash early
+}
+
 const char* MediaCodecDecoder::AsString(DecoderState state) {
   switch (state) {
     RETURN_STRING(kStopped);
     RETURN_STRING(kPrefetching);
     RETURN_STRING(kPrefetched);
+    RETURN_STRING(kPrerolling);
+    RETURN_STRING(kPrerolled);
     RETURN_STRING(kRunning);
     RETURN_STRING(kStopping);
     RETURN_STRING(kInEmergencyStop);
     RETURN_STRING(kError);
-    default:
-      return "Unknown DecoderState";
   }
+  return nullptr;  // crash early
 }
 
 #undef RETURN_STRING
 
 }  // namespace media