MediaCodecPlayer implementation (stage 5 - reconfiguration)

media/base/android/media_codec_decoder.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/base/android/media_codec_decoder.h"
 
 #include "base/bind.h"
 #include "base/bind_helpers.h"
 #include "base/callback_helpers.h"
 #include "base/logging.h"
@@ skipping 17 matching lines @@
 const int kInputBufferTimeout = 20;
 
 // Timeout for dequeuing an output buffer from MediaCodec in milliseconds.
 const int kOutputBufferTimeout = 20;
 }
 
 MediaCodecDecoder::MediaCodecDecoder(
     const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner,
     const base::Closure& external_request_data_cb,
     const base::Closure& starvation_cb,
+    const base::Closure& decoder_drained_cb,
     const base::Closure& stop_done_cb,
     const base::Closure& error_cb,
     const char* decoder_thread_name)
     : media_task_runner_(media_task_runner),
       decoder_thread_(decoder_thread_name),
       needs_reconfigure_(false),
+      drain_decoder_(false),
+      always_reconfigure_for_tests_(false),
       external_request_data_cb_(external_request_data_cb),
       starvation_cb_(starvation_cb),
+      decoder_drained_cb_(decoder_drained_cb),
       stop_done_cb_(stop_done_cb),
       error_cb_(error_cb),
       state_(kStopped),
-      needs_preroll_(true),
+      preroll_mode_(kPrerollTillPTS),
       eos_enqueued_(false),
       completed_(false),
       last_frame_posted_(false),
       is_data_request_in_progress_(false),
       is_incoming_data_invalid_(false),
 #ifndef NDEBUG
       verify_next_frame_is_key_(false),
 #endif
       weak_factory_(this) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
@@ skipping 21 matching lines @@
 
   DCHECK_EQ(GetState(), kStopped);
 
   // Flush() is a part of the Seek request. Whenever we request a seek we need
   // to invalidate the current data request.
   if (is_data_request_in_progress_)
     is_incoming_data_invalid_ = true;
 
   eos_enqueued_ = false;
   completed_ = false;
+  drain_decoder_ = false;
   au_queue_.Flush();
 
 #ifndef NDEBUG
   // We check and reset |verify_next_frame_is_key_| on Decoder thread.
   // This DCHECK ensures we won't need to lock this variable.
   DCHECK(!decoder_thread_.IsRunning());
 
   // For video the first frame after flush must be key frame.
   verify_next_frame_is_key_ = true;
 #endif
 
-  needs_preroll_ = true;
+  preroll_mode_ = kPrerollTillPTS;
 
   if (media_codec_bridge_) {
     // MediaCodecBridge::Reset() performs MediaCodecBridge.flush()
     MediaCodecStatus flush_status = media_codec_bridge_->Reset();
     if (flush_status != MEDIA_CODEC_OK) {
       DVLOG(0) << class_name() << "::" << __FUNCTION__
                << "MediaCodecBridge::Reset() failed";
       media_task_runner_->PostTask(FROM_HERE, internal_error_cb_);
     }
   }
 }
 
 void MediaCodecDecoder::ReleaseMediaCodec() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__;
 
   media_codec_bridge_.reset();
-  needs_preroll_ = true;
+  preroll_mode_ = kPrerollTillPTS;
 }
 
 bool MediaCodecDecoder::IsPrefetchingOrPlaying() const {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   // Whether decoder needs to be stopped.
   base::AutoLock lock(state_lock_);
   switch (state_) {
     case kPrefetching:
     case kPrefetched:
@@ skipping 19 matching lines @@
 
 bool MediaCodecDecoder::IsCompleted() const {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   return completed_;
 }
 
 bool MediaCodecDecoder::NotCompletedAndNeedsPreroll() const {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
-  return HasStream() && needs_preroll_ && !completed_;
+  return HasStream() && preroll_mode_ != kNoPreroll && !completed_;
+}
+
+void MediaCodecDecoder::SetDecodingUntilOutputIsPresent() {
+  DCHECK(media_task_runner_->BelongsToCurrentThread());
+  DVLOG(1) << class_name() << "::" << __FUNCTION__;
+
+  preroll_mode_ = kPrerollTillOutputIsPresent;
 }
 
 base::android::ScopedJavaLocalRef<jobject> MediaCodecDecoder::GetMediaCrypto() {
   base::android::ScopedJavaLocalRef<jobject> media_crypto;
 
   // TODO(timav): implement DRM.
   // drm_bridge_ is not implemented
   // if (drm_bridge_)
   //   media_crypto = drm_bridge_->GetMediaCrypto();
   return media_crypto;
@@ skipping 22 matching lines @@
     return kConfigFailure;
   }
 
   if (needs_reconfigure_) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << ": needs reconfigure, deleting MediaCodec";
     needs_reconfigure_ = false;
     ReleaseMediaCodec();
   }
 
-  MediaCodecDecoder::ConfigStatus result;
   if (media_codec_bridge_) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__
              << ": reconfiguration is not required, ignoring";
-    result = kConfigOk;
-  } else {
-    result = ConfigureInternal();
+    return kConfigOk;
+  }
+
+  // Read all |kConfigChanged| units preceding the data one.
+  AccessUnitQueue::Info au_info = au_queue_.GetInfo();
+  while (au_info.configs) {
+    SetDemuxerConfigs(*au_info.configs);
+    au_queue_.Advance();
+    au_info = au_queue_.GetInfo();
+  }
+
+  MediaCodecDecoder::ConfigStatus result = ConfigureInternal();
 
 #ifndef NDEBUG
-    // We check and reset |verify_next_frame_is_key_| on Decoder thread.
-    // This DCHECK ensures we won't need to lock this variable.
-    DCHECK(!decoder_thread_.IsRunning());
+  // We check and reset |verify_next_frame_is_key_| on Decoder thread.
+  // This DCHECK ensures we won't need to lock this variable.
+  DCHECK(!decoder_thread_.IsRunning());
 
-    // For video the first frame after reconfiguration must be key frame.
-    if (result == kConfigOk)
-      verify_next_frame_is_key_ = true;
+  // For video the first frame after reconfiguration must be key frame.
+  if (result == kConfigOk)
+    verify_next_frame_is_key_ = true;
 #endif
-  }
 
   return result;
 }
 
 bool MediaCodecDecoder::Preroll(base::TimeDelta preroll_timestamp,
                                 const base::Closure& preroll_done_cb) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__
            << " preroll_timestamp:" << preroll_timestamp;
 
   DecoderState state = GetState();
   if (state != kPrefetched) {
     DVLOG(0) << class_name() << "::" << __FUNCTION__ << ": wrong state "
              << AsString(state) << ", ignoring";
     return false;
   }
 
   if (!media_codec_bridge_) {
     DVLOG(0) << class_name() << "::" << __FUNCTION__
              << ": not configured, ignoring";
     return false;
   }
 
   DCHECK(!decoder_thread_.IsRunning());
-  DCHECK(needs_preroll_);
+  DCHECK(preroll_mode_ != kNoPreroll);
 
   preroll_done_cb_ = preroll_done_cb;
 
   // We only synchronize video stream.
   DissociatePTSFromTime();  // associaton will happen after preroll is done.
-  preroll_timestamp_ = preroll_timestamp;
+
+  preroll_timestamp_ = (preroll_mode_ == kPrerollTillPTS) ? preroll_timestamp
+                                                          : base::TimeDelta();
 
   last_frame_posted_ = false;
 
   // Start the decoder thread
   if (!decoder_thread_.Start()) {
     DVLOG(0) << class_name() << "::" << __FUNCTION__
              << ": cannot start decoder thread";
     return false;
   }
 
@@ skipping 85 matching lines @@
       // Synchronous stop.
       decoder_thread_.Stop();
       SetState(kStopped);
       media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
       break;
     case kStopping:
       break;  // ignore
     case kStopped:
     case kPrefetching:
     case kPrefetched:
-      // There is nothing to wait for, we can sent nofigication right away.
+      // There is nothing to wait for, we can sent notification right away.
       DCHECK(!decoder_thread_.IsRunning());
       SetState(kStopped);
       media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
       break;
     default:
       NOTREACHED();
       break;
   }
 }
 
-void MediaCodecDecoder::OnLastFrameRendered(bool completed) {
+void MediaCodecDecoder::OnLastFrameRendered(bool eos_encountered) {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
-  DVLOG(0) << class_name() << "::" << __FUNCTION__
-           << " completed:" << completed;
+  DVLOG(1) << class_name() << "::" << __FUNCTION__
+           << " eos_encountered:" << eos_encountered;
 
   decoder_thread_.Stop();  // synchronous
 
   SetState(kStopped);
-  completed_ = completed;
+  completed_ = (eos_encountered && !drain_decoder_);
 
-  if (completed_ && !preroll_done_cb_.is_null())
+  if (completed_ && !preroll_done_cb_.is_null()) {
     media_task_runner_->PostTask(FROM_HERE,
                                  base::ResetAndReturn(&preroll_done_cb_));
+  }
+
+  if (eos_encountered && drain_decoder_) {
+    ReleaseMediaCodec();
+    drain_decoder_ = false;
+    eos_enqueued_ = false;
+    preroll_mode_ = kPrerollTillOutputIsPresent;
+    media_task_runner_->PostTask(FROM_HERE, decoder_drained_cb_);
+  }
 
   media_task_runner_->PostTask(FROM_HERE, stop_done_cb_);
 }
 
 void MediaCodecDecoder::OnPrerollDone() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__;
 
-  needs_preroll_ = false;
+  preroll_mode_ = kNoPreroll;
 
   // The state might be kStopping.
   if (GetState() == kPrerolling) {
     SetState(kPrerolled);
     media_task_runner_->PostTask(FROM_HERE,
                                  base::ResetAndReturn(&preroll_done_cb_));
   }
 }
 
 void MediaCodecDecoder::OnDemuxerDataAvailable(const DemuxerData& data) {
@@ skipping 26 matching lines @@
   // data after kAborted and before the OnDemuxerSeekDone.
   if (GetState() == kPrefetching && !aborted_data)
     PrefetchNextChunk();
 }
 
 bool MediaCodecDecoder::IsPrerollingForTests() const {
   // UI task runner.
   return GetState() == kPrerolling;
 }
 
+void MediaCodecDecoder::SetAlwaysReconfigureForTests() {
+  // UI task runner.
+  always_reconfigure_for_tests_ = true;
+}
+
+void MediaCodecDecoder::SetCodecCreatedCallbackForTests(base::Closure cb) {
+  // UI task runner.
+  codec_created_for_tests_cb_ = cb;
+}
+
 int MediaCodecDecoder::NumDelayedRenderTasks() const {
   return 0;
 }
 
 void MediaCodecDecoder::DoEmergencyStop() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
   DVLOG(1) << class_name() << "::" << __FUNCTION__;
 
   // After this method returns, decoder thread will not be running.
 
@@ skipping 44 matching lines @@
   }
 }
 
 void MediaCodecDecoder::PrefetchNextChunk() {
   DCHECK(media_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << class_name() << "::" << __FUNCTION__;
 
   AccessUnitQueue::Info au_info = au_queue_.GetInfo();
 
-  if (eos_enqueued_ || au_info.length >= kPrefetchLimit || au_info.has_eos) {
+  if (eos_enqueued_ || au_info.data_length >= kPrefetchLimit ||
+      au_info.has_eos) {
     // We are done prefetching
     SetState(kPrefetched);
     DVLOG(1) << class_name() << "::" << __FUNCTION__ << " posting PrefetchDone";
     media_task_runner_->PostTask(FROM_HERE,
                                  base::ResetAndReturn(&prefetch_done_cb_));
     return;
   }
 
   request_data_cb_.Run();
 }
@@ skipping 58 matching lines @@
   }
 
   // Keep the number pending video frames low, ideally maintaining
   // the same audio and video duration after stop request
   if (NumDelayedRenderTasks() > 1) {
     DVLOG(2) << class_name() << "::" << __FUNCTION__ << ": # delayed buffers ("
              << NumDelayedRenderTasks() << ") exceeds 1, returning";
     return true;  // Nothing to do
   }
 
-  // Get the next frame from the queue and the queue info
+  // Get the next frame from the queue. As we go, request more data and
+  // consume |kConfigChanged| units.
 
-  AccessUnitQueue::Info au_info = au_queue_.GetInfo();
+  // |drain_decoder_| can be already set here if we could not dequeue the input
+  // buffer for it right away.
 
-  // Request the data from Demuxer
-  if (au_info.length <= kPlaybackLowLimit && !au_info.has_eos)
-    media_task_runner_->PostTask(FROM_HERE, request_data_cb_);
+  AccessUnitQueue::Info au_info;
+  if (!drain_decoder_) {
+    au_info = AdvanceAccessUnitQueue(&drain_decoder_);
+    if (!au_info.length) {
+      // Report starvation and return, Start() will be called again later.
+      DVLOG(1) << class_name() << "::" << __FUNCTION__
+               << ": starvation detected";

[liberato (no reviews please), 2015/08/28 21:42:47]

might want to mention |drain_decoder_| in the log.

[Tima Vaisburd, 2015/08/28 22:28:55]

Added the words about the drain inside AdvanceAccessUnitQueue() where I assign
|drain_decoder_|.

+      media_task_runner_->PostTask(FROM_HERE, starvation_cb_);
+      return true;
+    }
 
-  // Get the next frame from the queue
-
-  if (!au_info.length) {
-    // Report starvation and return, Start() will be called again later.
-    DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": starvation detected";
-    media_task_runner_->PostTask(FROM_HERE, starvation_cb_);
-    return true;
-  }
-
-  if (au_info.configs) {
-    DVLOG(1) << class_name() << "::" << __FUNCTION__
-             << ": received new configs, not implemented";
-    // post an error for now?
-    media_task_runner_->PostTask(FROM_HERE, internal_error_cb_);
-    return false;
-  }
-
-  // We are ready to enqueue the front unit.
+    DCHECK(au_info.front_unit);
 
 #ifndef NDEBUG
-  if (verify_next_frame_is_key_) {
-    verify_next_frame_is_key_ = false;
-    VerifyUnitIsKeyFrame(au_info.front_unit);
+    if (verify_next_frame_is_key_) {
+      verify_next_frame_is_key_ = false;
+      VerifyUnitIsKeyFrame(au_info.front_unit);
+    }
+#endif
   }
-#endif
 
   // Dequeue input buffer
 
   base::TimeDelta timeout =
       base::TimeDelta::FromMilliseconds(kInputBufferTimeout);
   int index = -1;
   MediaCodecStatus status =
       media_codec_bridge_->DequeueInputBuffer(timeout, &index);
 
   DVLOG(2) << class_name() << ":: DequeueInputBuffer index:" << index;
@@ skipping 13 matching lines @@
 
     default:
       break;
   }
 
   // We got the buffer
   DCHECK_EQ(status, MEDIA_CODEC_OK);
   DCHECK_GE(index, 0);
 
   const AccessUnit* unit = au_info.front_unit;
-  DCHECK(unit);
 
-  if (unit->is_end_of_stream) {
+  if (drain_decoder_ || unit->is_end_of_stream) {
     DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": QueueEOS";
     media_codec_bridge_->QueueEOS(index);
     eos_enqueued_ = true;
     return true;
   }
 
-  DVLOG(2) << class_name() << ":: QueueInputBuffer pts:" << unit->timestamp;
+  DCHECK(unit);
+
+  DVLOG(2) << class_name() << "::" << __FUNCTION__
+           << ": QueueInputBuffer pts:" << unit->timestamp;
 
   status = media_codec_bridge_->QueueInputBuffer(
       index, &unit->data[0], unit->data.size(), unit->timestamp);
 
   if (status == MEDIA_CODEC_ERROR) {
     DVLOG(0) << class_name() << "::" << __FUNCTION__
              << ": MEDIA_CODEC_ERROR: QueueInputBuffer failed";
     media_task_runner_->PostTask(FROM_HERE, internal_error_cb_);
     return false;
   }
 
   // Have successfully queued input buffer, go to next access unit.
   au_queue_.Advance();
   return true;
 }
 
+AccessUnitQueue::Info MediaCodecDecoder::AdvanceAccessUnitQueue(
+    bool* drain_decoder) {
+  DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread());
+  DVLOG(2) << class_name() << "::" << __FUNCTION__;
+
+  // Retrieve access units from the |au_queue_| in a loop until we either get
+  // a non-config front unit or until the queue is empty.
+
+  DCHECK(drain_decoder != nullptr);
+
+  AccessUnitQueue::Info au_info;
+
+  do {
+    // Get current frame

[liberato (no reviews please), 2015/08/28 21:42:47]

without the drain_decoder check here, the behavior changes if we have two config
change blocks in a row.  i don't think this matters, but i point it out anyway. 
before, we would break on the first and queue eos, now we'll consume both.

[Tima Vaisburd, 2015/08/28 22:28:55]

Yes, before we would break on first, and consume the other before
ConfiguerInternal(), there is another advancement there. I think both variants
are ok.

+    au_info = au_queue_.GetInfo();
+
+    // Request the data from Demuxer
+    if (au_info.data_length <= kPlaybackLowLimit && !au_info.has_eos)
+      media_task_runner_->PostTask(FROM_HERE, request_data_cb_);
+
+    if (!au_info.length)
+      break;  // Starvation
+
+    if (au_info.configs) {
+      DVLOG(1) << class_name() << "::" << __FUNCTION__ << ": received configs "
+               << (*au_info.configs);
+
+      // Compare the new and current configs.
+      if (IsCodecReconfigureNeeded(*au_info.configs)) {
+        DVLOG(1) << class_name() << "::" << __FUNCTION__
+                 << ": reconfiguration required";
+        drain_decoder_ = true;

[liberato (no reviews please), 2015/08/28 21:42:47]

(*drain_decoder)?

[Tima Vaisburd, 2015/08/28 22:28:55]

Yes, indeed. Thank you.

+      }
+
+      // Replace the current configs.
+      SetDemuxerConfigs(*au_info.configs);
+
+      // Move to the next frame
+      au_queue_.Advance();
+    }
+  } while (au_info.configs);
+
+  return au_info;
+}
+
 // Returns false if there was MediaCodec error.
 bool MediaCodecDecoder::DepleteOutputBufferQueue() {
   DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread());
 
   DVLOG(2) << class_name() << "::" << __FUNCTION__;
 
   int buffer_index = 0;
   size_t offset = 0;
   size_t size = 0;
   base::TimeDelta pts;
@@ skipping 34 matching lines @@
         if (pts < preroll_timestamp_)
           render_mode = kRenderSkip;
         else if (GetState() == kPrerolling)
           render_mode = kRenderAfterPreroll;
         else
           render_mode = kRenderNow;
 
         Render(buffer_index, offset, size, render_mode, pts, eos_encountered);
 
         if (render_mode == kRenderAfterPreroll) {
-          DVLOG(1) << class_name() << "::" << __FUNCTION__ << " pts:" << pts
+          DVLOG(1) << class_name() << "::" << __FUNCTION__ << " pts " << pts
+                   << " >= preroll timestamp " << preroll_timestamp_
                    << " preroll done, stopping frame processing";
           media_task_runner_->PostTask(FROM_HERE, internal_preroll_done_cb_);
           return false;
         }
         break;
 
       case MEDIA_CODEC_DEQUEUE_OUTPUT_AGAIN_LATER:
         // Nothing to do.
         break;
 
@@ skipping 63 matching lines @@
     RETURN_STRING(kStopping);
     RETURN_STRING(kInEmergencyStop);
     RETURN_STRING(kError);
   }
   return nullptr;  // crash early
 }
 
 #undef RETURN_STRING
 
 }  // namespace media