Reducing the IPC latency for MSE video decoding

media/base/android/media_source_player.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/base/android/media_source_player.h"
 
 #include <limits>
 
 #include "base/android/jni_android.h"
 #include "base/android/jni_string.h"
@@ skipping 43 matching lines @@
       is_video_encrypted_(false),
       volume_(-1.0),
       clock_(&default_tick_clock_),
       next_video_data_is_iframe_(true),
       doing_browser_seek_(false),
       pending_seek_(false),
       reconfig_audio_decoder_(false),
       reconfig_video_decoder_(false),
       weak_this_(this),
       drm_bridge_(NULL),
-      is_waiting_for_key_(false) {
+      is_waiting_for_key_(false),
+      has_pending_audio_data_request_(false),
+      has_pending_video_data_request_(false) {
   demuxer_->Initialize(this);
   clock_.SetMaxTime(base::TimeDelta());
 }
 
 MediaSourcePlayer::~MediaSourcePlayer() {
   Release();
 }
 
 void MediaSourcePlayer::SetVideoSurface(gfx::ScopedJavaSurface surface) {
   // For an empty surface, always pass it to the decoder job so that it
@@ skipping 140 matching lines @@
   // or drop data received across Release()+Start(). See http://crbug.com/306314
   // and http://crbug.com/304234.
   bool process_pending_events = false;
   process_pending_events = IsEventPending(PREFETCH_DONE_EVENT_PENDING) ||
       (audio_decoder_job_ && audio_decoder_job_->is_decoding()) ||
       (video_decoder_job_ && video_decoder_job_->is_decoding());
 
   // Clear all the pending events except seeks and config changes.
   pending_event_ &= (SEEK_EVENT_PENDING | CONFIG_CHANGE_EVENT_PENDING);
   is_surface_in_use_ = false;
-  audio_decoder_job_.reset();
+  ResetAudioDecoderJob();
   ResetVideoDecoderJob();
 
   // Prevent job re-creation attempts in OnDemuxerConfigsAvailable()
   reconfig_audio_decoder_ = false;
   reconfig_video_decoder_ = false;
 
   // Prevent player restart, including job re-creation attempts.
   playing_ = false;
 
   decoder_starvation_callback_.Cancel();
@@ skipping 102 matching lines @@
 
     // Resume decoding after the config change if we are still playing.
     if (playing_)
       StartInternal();
   }
 }
 
 void MediaSourcePlayer::OnDemuxerDataAvailable(const DemuxerData& data) {
   DVLOG(1) << __FUNCTION__ << "(" << data.type << ")";
   DCHECK_LT(0u, data.access_units.size());
+
+  if (has_pending_audio_data_request_ && data.type == DemuxerStream::AUDIO) {

[wolenetz, 2014/03/17 19:51:00]

Do I understand correctly that we drop on the floor the result of the pending
read if the player has been Release()'ed and not yet restarted when the video or
audio data arrives here? Is that the right thing to do, or should we hold onto
the data to feed newly created decoder, or do we always seek to take care of
this case on restart?

[qinmin, 2014/03/18 18:58:58]

Yes, the purpose of this is to drop the request that has been made by a released
player.
We cannot pass the data to a newly created decoder, because the data might be
requested for an inactive chunk, which is not the data the previous decoder is
currently decoding. Additionally, the data may not be an iframe and we cannot
feed it to a newly created decoder.

On 2014/03/17 19:51:00, wolenetz wrote:

+    has_pending_audio_data_request_ = false;
+    ProcessPendingEvents();
+    return;
+  }
+
+  if (has_pending_video_data_request_ && data.type == DemuxerStream::VIDEO) {
+    next_video_data_is_iframe_ = false;
+    has_pending_video_data_request_ = false;
+    ProcessPendingEvents();
+    return;
+  }
+
   if (data.type == DemuxerStream::AUDIO && audio_decoder_job_) {
     audio_decoder_job_->OnDataReceived(data);
   } else if (data.type == DemuxerStream::VIDEO) {
     next_video_data_is_iframe_ = false;
     if (video_decoder_job_)
       video_decoder_job_->OnDataReceived(data);
   }
 }
 
 void MediaSourcePlayer::OnDemuxerDurationChanged(base::TimeDelta duration) {
@@ skipping 53 matching lines @@
     DCHECK(doing_browser_seek_);
     pending_seek_ = false;
     SeekTo(pending_seek_time_);
     return;
   }
 
   // It is possible that a browser seek to I-frame had to seek to a buffered
   // I-frame later than the requested one due to data removal or GC. Update
   // player clock to the actual seek target.
   if (doing_browser_seek_) {
-    DCHECK(actual_browser_seek_time != kNoTimestamp());
+    base::TimeDelta seek_time = (actual_browser_seek_time == kNoTimestamp()) ?
+        base::TimeDelta() : actual_browser_seek_time;
     // A browser seek must not jump into the past. Ideally, it seeks to the
     // requested time, but it might jump into the future.
-    DCHECK(actual_browser_seek_time >= GetCurrentTime());
+    DCHECK(seek_time >= GetCurrentTime());
     DVLOG(1) << __FUNCTION__ << " : setting clock to actual browser seek time: "
-             << actual_browser_seek_time.InSecondsF();
-    clock_.SetTime(actual_browser_seek_time, actual_browser_seek_time);
+             << seek_time.InSecondsF();
+    clock_.SetTime(seek_time, seek_time);
     if (audio_timestamp_helper_)
-      audio_timestamp_helper_->SetBaseTimestamp(actual_browser_seek_time);
+      audio_timestamp_helper_->SetBaseTimestamp(seek_time);
   }
 
   reached_audio_eos_ = false;
   reached_video_eos_ = false;
 
   base::TimeDelta current_time = GetCurrentTime();
   // TODO(qinmin): Simplify the logic by using |start_presentation_timestamp_|
   // to preroll media decoder jobs. Currently |start_presentation_timestamp_|
   // is calculated from decoder output, while preroll relies on the access
   // unit's timestamp. There are some differences between the two.
@@ skipping 29 matching lines @@
   if (video_decoder_job_ && video_decoder_job_->is_decoding()) {
     DVLOG(1) << __FUNCTION__ << " : A video job is still decoding.";
     return;
   }
 
   if (audio_decoder_job_ && audio_decoder_job_->is_decoding()) {
     DVLOG(1) << __FUNCTION__ << " : An audio job is still decoding.";
     return;
   }
 
+  if (has_pending_audio_data_request_ || has_pending_video_data_request_) {
+    DVLOG(1) << __FUNCTION__ << " : has pending data request.";
+    return;
+  }
+
   if (IsEventPending(PREFETCH_DONE_EVENT_PENDING)) {
     DVLOG(1) << __FUNCTION__ << " : PREFETCH_DONE still pending.";
     return;
   }
 
   if (IsEventPending(SEEK_EVENT_PENDING)) {
     DVLOG(1) << __FUNCTION__ << " : Handling SEEK_EVENT";
     ClearDecodingData();
     demuxer_->RequestDemuxerSeek(GetCurrentTime(), doing_browser_seek_);
     return;
@@ skipping 14 matching lines @@
     ConfigureVideoDecoderJob();
 
     // Return early if we can't successfully configure a new video decoder job
     // yet.
     if (HasVideo() && !video_decoder_job_)
       return;
   }
 
   if (IsEventPending(PREFETCH_REQUEST_EVENT_PENDING)) {
     DVLOG(1) << __FUNCTION__ << " : Handling PREFETCH_REQUEST_EVENT.";
+    // If one of the decoder is not initialized, cancel this event as it will be

[wolenetz, 2014/03/17 19:51:00]

Were we hitting the DCHECK's below? In other words, is it possible for
PREFETCH_REQUEST_EVENT_PENDING to become true but the relevant decoder job(s) to
not yet be initialized?

[qinmin, 2014/03/18 18:58:58]

This is possible with the new implementation.
Considering the case that we pass an empty surface to the video decoder: In that
case, the old decoder will be destroyed while requesting some data. Since the
new surface is empty, we will not create the new video decoder. In our previous
implementation, we will early return when processing the SURFACE_CHANGE event.
Now after sometime the data requested by the old decoder arrives, and we start
to process pending events. Since SURFACE_CHANGE event has been processed, we
will hit the PREFETCH_REQUEST pending event and find that the video decoder is
actually null.

When decoder is null, there is no point to handle the prefetch request, that's
why we can clear the pending events here.

On 2014/03/17 19:51:00, wolenetz wrote:

+    // called later when Start() is called again.
+    if ((HasVideo() && !video_decoder_job_) ||
+        (HasAudio() && !audio_decoder_job_)) {
+      ClearPendingEvent(PREFETCH_REQUEST_EVENT_PENDING);
+      return;
+    }
+
     DCHECK(audio_decoder_job_ || AudioFinished());
     DCHECK(video_decoder_job_ || VideoFinished());
 
     int count = (AudioFinished() ? 0 : 1) + (VideoFinished() ? 0 : 1);
 
     // It is possible that all streams have finished decode, yet starvation
     // occurred during the last stream's EOS decode. In this case, prefetch is a
     // no-op.
     ClearPendingEvent(PREFETCH_REQUEST_EVENT_PENDING);
     if (count == 0)
@@ skipping 217 matching lines @@
 bool MediaSourcePlayer::AudioFinished() {
   return reached_audio_eos_ || !HasAudio();
 }
 
 bool MediaSourcePlayer::VideoFinished() {
   return reached_video_eos_ || !HasVideo();
 }
 
 void MediaSourcePlayer::ConfigureAudioDecoderJob() {
   if (!HasAudio()) {
-    audio_decoder_job_.reset();
+    ResetAudioDecoderJob();
     return;
   }
 
   // Create audio decoder job only if config changes.
   if (audio_decoder_job_ && !reconfig_audio_decoder_)
     return;
 
   base::android::ScopedJavaLocalRef<jobject> media_crypto = GetMediaCrypto();
   if (is_audio_encrypted_ && media_crypto.is_null())
     return;
 
   DCHECK(!audio_decoder_job_ || !audio_decoder_job_->is_decoding());
 
+  ResetAudioDecoderJob();
   DVLOG(1) << __FUNCTION__ << " : creating new audio decoder job";
-
   audio_decoder_job_.reset(AudioDecoderJob::Create(
       audio_codec_, sampling_rate_, num_channels_, &audio_extra_data_[0],
       audio_extra_data_.size(), media_crypto.obj(),
       base::Bind(&DemuxerAndroid::RequestDemuxerData,
                  base::Unretained(demuxer_.get()), DemuxerStream::AUDIO)));
 
   if (audio_decoder_job_) {
     SetVolumeInternal();
     audio_decoder_job_->BeginPrerolling(preroll_timestamp_);
     reconfig_audio_decoder_ =  false;
   }
 }
 
 void MediaSourcePlayer::ResetVideoDecoderJob() {
+  if (video_decoder_job_) {
+    has_pending_video_data_request_ =
+        video_decoder_job_->is_requesting_demuxer_data();
+  }
   video_decoder_job_.reset();
 
   // Any eventual video decoder job re-creation will use the current |surface_|.
   if (IsEventPending(SURFACE_CHANGE_EVENT_PENDING))
     ClearPendingEvent(SURFACE_CHANGE_EVENT_PENDING);
 }
 
+void MediaSourcePlayer::ResetAudioDecoderJob() {
+  if (audio_decoder_job_) {
+    has_pending_audio_data_request_ =
+        audio_decoder_job_->is_requesting_demuxer_data();
+  }
+  audio_decoder_job_.reset();
+}
+
 void MediaSourcePlayer::ConfigureVideoDecoderJob() {
   if (!HasVideo() || surface_.IsEmpty()) {
     ResetVideoDecoderJob();
     return;
   }
 
   // Create video decoder job only if config changes or we don't have a job.
   if (video_decoder_job_ && !reconfig_video_decoder_) {
     DCHECK(!IsEventPending(SURFACE_CHANGE_EVENT_PENDING));
     return;
@@ skipping 171 matching lines @@
 
 void MediaSourcePlayer::ClearPendingEvent(PendingEventFlags event) {
   DVLOG(1) << __FUNCTION__ << "(" << GetEventName(event) << ")";
   DCHECK_NE(event, NO_EVENT_PENDING);
   DCHECK(IsEventPending(event)) << GetEventName(event);
 
   pending_event_ &= ~event;
 }
 
 }  // namespace media