Reducing the IPC latency for MSE video decoding

media/base/android/media_source_player.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/base/android/media_source_player.h"
 
 #include <limits>
 
 #include "base/android/jni_android.h"
 #include "base/android/jni_string.h"
@@ skipping 43 matching lines @@
       is_video_encrypted_(false),
       volume_(-1.0),
       clock_(&default_tick_clock_),
       next_video_data_is_iframe_(true),
       doing_browser_seek_(false),
       pending_seek_(false),
       reconfig_audio_decoder_(false),
       reconfig_video_decoder_(false),
       weak_this_(this),
       drm_bridge_(NULL),
-      is_waiting_for_key_(false) {
+      is_waiting_for_key_(false),
+      has_pending_audio_data_request_(false),
+      has_pending_video_data_request_(false) {
   demuxer_->Initialize(this);
   clock_.SetMaxTime(base::TimeDelta());
 }
 
 MediaSourcePlayer::~MediaSourcePlayer() {
   Release();
 }
 
 void MediaSourcePlayer::SetVideoSurface(gfx::ScopedJavaSurface surface) {
   // For an empty surface, always pass it to the decoder job so that it
@@ skipping 24 matching lines @@
   // If video config change is already pending, processing of the pending
   // surface change event will occur in OnDemuxerConfigsAvailable().
   if (reconfig_video_decoder_ && IsEventPending(CONFIG_CHANGE_EVENT_PENDING))
     return;
 
   // Otherwise we need to trigger pending event processing now.
   ProcessPendingEvents();
 }
 
 void MediaSourcePlayer::ScheduleSeekEventAndStopDecoding(
-    const base::TimeDelta& seek_time) {
+    base::TimeDelta seek_time) {
   DVLOG(1) << __FUNCTION__ << "(" << seek_time.InSecondsF() << ")";
   DCHECK(!IsEventPending(SEEK_EVENT_PENDING));
 
   pending_seek_ = false;
 
   clock_.SetTime(seek_time, seek_time);
   if (audio_timestamp_helper_)
     audio_timestamp_helper_->SetBaseTimestamp(seek_time);
 
   if (audio_decoder_job_ && audio_decoder_job_->is_decoding())
@@ skipping 50 matching lines @@
 }
 
 int MediaSourcePlayer::GetVideoWidth() {
   return width_;
 }
 
 int MediaSourcePlayer::GetVideoHeight() {
   return height_;
 }
 
-void MediaSourcePlayer::SeekTo(const base::TimeDelta& timestamp) {
+void MediaSourcePlayer::SeekTo(base::TimeDelta timestamp) {
   DVLOG(1) << __FUNCTION__ << "(" << timestamp.InSecondsF() << ")";
 
   if (IsEventPending(SEEK_EVENT_PENDING)) {
     DCHECK(doing_browser_seek_) << "SeekTo while SeekTo in progress";
     DCHECK(!pending_seek_) << "SeekTo while SeekTo pending browser seek";
 
     // There is a browser seek currently in progress to obtain I-frame to feed
     // a newly constructed video decoder. Remember this real seek request so
     // it can be initiated once OnDemuxerSeekDone() occurs for the browser seek.
     pending_seek_ = true;
@@ skipping 24 matching lines @@
   // or drop data received across Release()+Start(). See http://crbug.com/306314
   // and http://crbug.com/304234.
   bool process_pending_events = false;
   process_pending_events = IsEventPending(PREFETCH_DONE_EVENT_PENDING) ||
       (audio_decoder_job_ && audio_decoder_job_->is_decoding()) ||
       (video_decoder_job_ && video_decoder_job_->is_decoding());
 
   // Clear all the pending events except seeks and config changes.
   pending_event_ &= (SEEK_EVENT_PENDING | CONFIG_CHANGE_EVENT_PENDING);
   is_surface_in_use_ = false;
-  audio_decoder_job_.reset();
+  ResetAudioDecoderJob();
   ResetVideoDecoderJob();
 
   // Prevent job re-creation attempts in OnDemuxerConfigsAvailable()
   reconfig_audio_decoder_ = false;
   reconfig_video_decoder_ = false;
 
   // Prevent player restart, including job re-creation attempts.
   playing_ = false;
 
   decoder_starvation_callback_.Cancel();
@@ skipping 102 matching lines @@
 
     // Resume decoding after the config change if we are still playing.
     if (playing_)
       StartInternal();
   }
 }
 
 void MediaSourcePlayer::OnDemuxerDataAvailable(const DemuxerData& data) {
   DVLOG(1) << __FUNCTION__ << "(" << data.type << ")";
   DCHECK_LT(0u, data.access_units.size());
+
+  if (has_pending_audio_data_request_ && data.type == DemuxerStream::AUDIO) {
+    has_pending_audio_data_request_ = false;
+    ProcessPendingEvents();
+    return;
+  }
+
+  if (has_pending_video_data_request_ && data.type == DemuxerStream::VIDEO) {
+    next_video_data_is_iframe_ = false;
+    has_pending_video_data_request_ = false;
+    ProcessPendingEvents();
+    return;
+  }
+
   if (data.type == DemuxerStream::AUDIO && audio_decoder_job_) {
     audio_decoder_job_->OnDataReceived(data);
+
+

[wolenetz, 2014/03/18 21:14:47]

nit: remove extra lines (both of these)

[qinmin, 2014/03/19 02:45:22]

Done.

   } else if (data.type == DemuxerStream::VIDEO) {
     next_video_data_is_iframe_ = false;
     if (video_decoder_job_)
       video_decoder_job_->OnDataReceived(data);
   }
 }
 
 void MediaSourcePlayer::OnDemuxerDurationChanged(base::TimeDelta duration) {
   duration_ = duration;
   clock_.SetDuration(duration_);
@@ skipping 30 matching lines @@
     drm_bridge_->SetMediaCryptoReadyCB(base::Bind(
         &MediaSourcePlayer::OnMediaCryptoReady, weak_this_.GetWeakPtr()));
     return;
   }
 
   if (playing_)
     StartInternal();
 }
 
 void MediaSourcePlayer::OnDemuxerSeekDone(
-    const base::TimeDelta& actual_browser_seek_time) {
+    base::TimeDelta actual_browser_seek_time) {
   DVLOG(1) << __FUNCTION__;
 
   ClearPendingEvent(SEEK_EVENT_PENDING);
   if (IsEventPending(PREFETCH_REQUEST_EVENT_PENDING))
     ClearPendingEvent(PREFETCH_REQUEST_EVENT_PENDING);
 
   next_video_data_is_iframe_ = true;
 
   if (pending_seek_) {
     DVLOG(1) << __FUNCTION__ << "processing pending seek";
     DCHECK(doing_browser_seek_);
     pending_seek_ = false;
     SeekTo(pending_seek_time_);
     return;
   }
 
   // It is possible that a browser seek to I-frame had to seek to a buffered
   // I-frame later than the requested one due to data removal or GC. Update
   // player clock to the actual seek target.
   if (doing_browser_seek_) {
-    DCHECK(actual_browser_seek_time != kNoTimestamp());
+    base::TimeDelta seek_time = (actual_browser_seek_time == kNoTimestamp()) ?

[wolenetz, 2014/03/18 21:14:47]

See also comment in unit tests. Why are we now allowing kNoTimestamp() for
browser seek actual seek time?

[qinmin, 2014/03/19 02:45:22]

I saw the kNoTimestamp() in unit tests and it makes me unsure about what the
chunk demuxer may return. Removed this if we can guarantee there will be no
kNoTimestamp from the demuxer.

On 2014/03/18 21:14:47, wolenetz wrote:

+        base::TimeDelta() : actual_browser_seek_time;
     // A browser seek must not jump into the past. Ideally, it seeks to the
     // requested time, but it might jump into the future.
-    DCHECK(actual_browser_seek_time >= GetCurrentTime());
+    DCHECK(seek_time >= GetCurrentTime());
     DVLOG(1) << __FUNCTION__ << " : setting clock to actual browser seek time: "
-             << actual_browser_seek_time.InSecondsF();
-    clock_.SetTime(actual_browser_seek_time, actual_browser_seek_time);
+             << seek_time.InSecondsF();
+    clock_.SetTime(seek_time, seek_time);
     if (audio_timestamp_helper_)
-      audio_timestamp_helper_->SetBaseTimestamp(actual_browser_seek_time);
+      audio_timestamp_helper_->SetBaseTimestamp(seek_time);
   }
 
   reached_audio_eos_ = false;
   reached_video_eos_ = false;
 
   base::TimeDelta current_time = GetCurrentTime();
   // TODO(qinmin): Simplify the logic by using |start_presentation_timestamp_|
   // to preroll media decoder jobs. Currently |start_presentation_timestamp_|
   // is calculated from decoder output, while preroll relies on the access
   // unit's timestamp. There are some differences between the two.
   preroll_timestamp_ = current_time;
   if (audio_decoder_job_)
     audio_decoder_job_->BeginPrerolling(preroll_timestamp_);
   if (video_decoder_job_)
     video_decoder_job_->BeginPrerolling(preroll_timestamp_);
 
   if (!doing_browser_seek_)
     manager()->OnSeekComplete(player_id(), current_time);
 
   ProcessPendingEvents();
 }
 
 void MediaSourcePlayer::UpdateTimestamps(
-    const base::TimeDelta& presentation_timestamp, size_t audio_output_bytes) {
+    base::TimeDelta presentation_timestamp, size_t audio_output_bytes) {
   base::TimeDelta new_max_time = presentation_timestamp;
 
   if (audio_output_bytes > 0) {
     audio_timestamp_helper_->AddFrames(
         audio_output_bytes / (kBytesPerAudioOutputSample * num_channels_));
     new_max_time = audio_timestamp_helper_->GetTimestamp();
   }
 
   clock_.SetMaxTime(new_max_time);
   manager()->OnTimeUpdate(player_id(), GetCurrentTime());
 }
 
 void MediaSourcePlayer::ProcessPendingEvents() {
   DVLOG(1) << __FUNCTION__ << " : 0x" << std::hex << pending_event_;
   // Wait for all the decoding jobs to finish before processing pending tasks.
   if (video_decoder_job_ && video_decoder_job_->is_decoding()) {
     DVLOG(1) << __FUNCTION__ << " : A video job is still decoding.";
     return;
   }
 
   if (audio_decoder_job_ && audio_decoder_job_->is_decoding()) {
     DVLOG(1) << __FUNCTION__ << " : An audio job is still decoding.";
     return;
   }
 
+  if (has_pending_audio_data_request_ || has_pending_video_data_request_) {
+    DVLOG(1) << __FUNCTION__ << " : has pending data request.";
+    return;
+  }
+
   if (IsEventPending(PREFETCH_DONE_EVENT_PENDING)) {
     DVLOG(1) << __FUNCTION__ << " : PREFETCH_DONE still pending.";
     return;
   }
 
   if (IsEventPending(SEEK_EVENT_PENDING)) {
     DVLOG(1) << __FUNCTION__ << " : Handling SEEK_EVENT";
     ClearDecodingData();
     demuxer_->RequestDemuxerSeek(GetCurrentTime(), doing_browser_seek_);
     return;
@@ skipping 14 matching lines @@
     ConfigureVideoDecoderJob();
 
     // Return early if we can't successfully configure a new video decoder job
     // yet.
     if (HasVideo() && !video_decoder_job_)
       return;
   }
 
   if (IsEventPending(PREFETCH_REQUEST_EVENT_PENDING)) {
     DVLOG(1) << __FUNCTION__ << " : Handling PREFETCH_REQUEST_EVENT.";
+    // If one of the decoder is not initialized, cancel this event as it will be
+    // called later when Start() is called again.
+    if ((HasVideo() && !video_decoder_job_) ||
+        (HasAudio() && !audio_decoder_job_)) {
+      ClearPendingEvent(PREFETCH_REQUEST_EVENT_PENDING);
+      return;
+    }
+
     DCHECK(audio_decoder_job_ || AudioFinished());
     DCHECK(video_decoder_job_ || VideoFinished());
 
     int count = (AudioFinished() ? 0 : 1) + (VideoFinished() ? 0 : 1);
 
     // It is possible that all streams have finished decode, yet starvation
     // occurred during the last stream's EOS decode. In this case, prefetch is a
     // no-op.
     ClearPendingEvent(PREFETCH_REQUEST_EVENT_PENDING);
     if (count == 0)
@@ skipping 15 matching lines @@
   DCHECK_EQ(pending_event_, NO_EVENT_PENDING);
 
   // Now that all pending events have been handled, resume decoding if we are
   // still playing.
   if (playing_)
     StartInternal();
 }
 
 void MediaSourcePlayer::MediaDecoderCallback(
     bool is_audio, MediaCodecStatus status,
-    const base::TimeDelta& presentation_timestamp, size_t audio_output_bytes) {
+    base::TimeDelta presentation_timestamp, size_t audio_output_bytes) {
   DVLOG(1) << __FUNCTION__ << ": " << is_audio << ", " << status;
 
   // TODO(xhwang): Drop IntToString() when http://crbug.com/303899 is fixed.
   if (is_audio) {
     TRACE_EVENT_ASYNC_END1("media",
                            "MediaSourcePlayer::DecodeMoreAudio",
                            audio_decoder_job_.get(),
                            "MediaCodecStatus",
                            base::IntToString(status));
   } else {
@@ skipping 181 matching lines @@
 bool MediaSourcePlayer::AudioFinished() {
   return reached_audio_eos_ || !HasAudio();
 }
 
 bool MediaSourcePlayer::VideoFinished() {
   return reached_video_eos_ || !HasVideo();
 }
 
 void MediaSourcePlayer::ConfigureAudioDecoderJob() {
   if (!HasAudio()) {
-    audio_decoder_job_.reset();
+    ResetAudioDecoderJob();
     return;
   }
 
   // Create audio decoder job only if config changes.
   if (audio_decoder_job_ && !reconfig_audio_decoder_)
     return;
 
   base::android::ScopedJavaLocalRef<jobject> media_crypto = GetMediaCrypto();
   if (is_audio_encrypted_ && media_crypto.is_null())
     return;
 
   DCHECK(!audio_decoder_job_ || !audio_decoder_job_->is_decoding());
 
+  ResetAudioDecoderJob();
   DVLOG(1) << __FUNCTION__ << " : creating new audio decoder job";
-
   audio_decoder_job_.reset(AudioDecoderJob::Create(
       audio_codec_, sampling_rate_, num_channels_, &audio_extra_data_[0],
       audio_extra_data_.size(), media_crypto.obj(),
       base::Bind(&DemuxerAndroid::RequestDemuxerData,
                  base::Unretained(demuxer_.get()), DemuxerStream::AUDIO)));
 
   if (audio_decoder_job_) {
     SetVolumeInternal();
     audio_decoder_job_->BeginPrerolling(preroll_timestamp_);
     reconfig_audio_decoder_ =  false;
   }
 }
 
 void MediaSourcePlayer::ResetVideoDecoderJob() {
+  if (video_decoder_job_) {
+    has_pending_video_data_request_ =
+        video_decoder_job_->is_requesting_demuxer_data();
+  }
   video_decoder_job_.reset();
 
   // Any eventual video decoder job re-creation will use the current |surface_|.
   if (IsEventPending(SURFACE_CHANGE_EVENT_PENDING))
     ClearPendingEvent(SURFACE_CHANGE_EVENT_PENDING);
 }
 
+void MediaSourcePlayer::ResetAudioDecoderJob() {
+  if (audio_decoder_job_) {
+    has_pending_audio_data_request_ =
+        audio_decoder_job_->is_requesting_demuxer_data();
+  }
+  audio_decoder_job_.reset();
+}
+
 void MediaSourcePlayer::ConfigureVideoDecoderJob() {
   if (!HasVideo() || surface_.IsEmpty()) {
     ResetVideoDecoderJob();
     return;
   }
 
   // Create video decoder job only if config changes or we don't have a job.
   if (video_decoder_job_ && !reconfig_video_decoder_) {
     DCHECK(!IsEventPending(SURFACE_CHANGE_EVENT_PENDING));
     return;
@@ skipping 56 matching lines @@
       player_id(), duration_, width_, height_, true);
 }
 
 void MediaSourcePlayer::OnDecoderStarved() {
   DVLOG(1) << __FUNCTION__;
   SetPendingEvent(PREFETCH_REQUEST_EVENT_PENDING);
   ProcessPendingEvents();
 }
 
 void MediaSourcePlayer::StartStarvationCallback(
-    const base::TimeDelta& presentation_timestamp) {
+    base::TimeDelta presentation_timestamp) {
   // 20ms was chosen because it is the typical size of a compressed audio frame.
   // Anything smaller than this would likely cause unnecessary cycling in and
   // out of the prefetch state.
   const base::TimeDelta kMinStarvationTimeout =
       base::TimeDelta::FromMilliseconds(20);
 
   base::TimeDelta current_timestamp = GetCurrentTime();
   base::TimeDelta timeout;
   if (HasAudio()) {
     timeout = audio_timestamp_helper_->GetTimestamp() - current_timestamp;
@@ skipping 94 matching lines @@
 
 void MediaSourcePlayer::ClearPendingEvent(PendingEventFlags event) {
   DVLOG(1) << __FUNCTION__ << "(" << GetEventName(event) << ")";
   DCHECK_NE(event, NO_EVENT_PENDING);
   DCHECK(IsEventPending(event)) << GetEventName(event);
 
   pending_event_ &= ~event;
 }
 
 }  // namespace media