Reducing the IPC latency for MSE video decoding

media/base/android/media_source_player.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/base/android/media_source_player.h"
 
 #include <limits>
 
 #include "base/android/jni_android.h"
 #include "base/android/jni_string.h"
@@ skipping 43 matching lines @@
       is_video_encrypted_(false),
       volume_(-1.0),
       clock_(&default_tick_clock_),
       next_video_data_is_iframe_(true),
       doing_browser_seek_(false),
       pending_seek_(false),
       reconfig_audio_decoder_(false),
       reconfig_video_decoder_(false),
       weak_this_(this),
       drm_bridge_(NULL),
-      is_waiting_for_key_(false) {
+      is_waiting_for_key_(false),
+      has_pending_audio_data_request_(false),
+      has_pending_video_data_request_(false) {
   demuxer_->Initialize(this);
   clock_.SetMaxTime(base::TimeDelta());
 }
 
 MediaSourcePlayer::~MediaSourcePlayer() {
   Release();
 }
 
 void MediaSourcePlayer::SetVideoSurface(gfx::ScopedJavaSurface surface) {
   // For an empty surface, always pass it to the decoder job so that it
@@ skipping 24 matching lines @@
   // If video config change is already pending, processing of the pending
   // surface change event will occur in OnDemuxerConfigsAvailable().
   if (reconfig_video_decoder_ && IsEventPending(CONFIG_CHANGE_EVENT_PENDING))
     return;
 
   // Otherwise we need to trigger pending event processing now.
   ProcessPendingEvents();
 }
 
 void MediaSourcePlayer::ScheduleSeekEventAndStopDecoding(
-    const base::TimeDelta& seek_time) {
+    base::TimeDelta seek_time) {
   DVLOG(1) << __FUNCTION__ << "(" << seek_time.InSecondsF() << ")";
   DCHECK(!IsEventPending(SEEK_EVENT_PENDING));
 
   pending_seek_ = false;
 
   clock_.SetTime(seek_time, seek_time);
   if (audio_timestamp_helper_)
     audio_timestamp_helper_->SetBaseTimestamp(seek_time);
 
   if (audio_decoder_job_ && audio_decoder_job_->is_decoding())
@@ skipping 50 matching lines @@
 }
 
 int MediaSourcePlayer::GetVideoWidth() {
   return width_;
 }
 
 int MediaSourcePlayer::GetVideoHeight() {
   return height_;
 }
 
-void MediaSourcePlayer::SeekTo(const base::TimeDelta& timestamp) {
+void MediaSourcePlayer::SeekTo(base::TimeDelta timestamp) {
   DVLOG(1) << __FUNCTION__ << "(" << timestamp.InSecondsF() << ")";
 
   if (IsEventPending(SEEK_EVENT_PENDING)) {
     DCHECK(doing_browser_seek_) << "SeekTo while SeekTo in progress";
     DCHECK(!pending_seek_) << "SeekTo while SeekTo pending browser seek";
 
     // There is a browser seek currently in progress to obtain I-frame to feed
     // a newly constructed video decoder. Remember this real seek request so
     // it can be initiated once OnDemuxerSeekDone() occurs for the browser seek.
     pending_seek_ = true;
@@ skipping 24 matching lines @@
   // or drop data received across Release()+Start(). See http://crbug.com/306314
   // and http://crbug.com/304234.
   bool process_pending_events = false;
   process_pending_events = IsEventPending(PREFETCH_DONE_EVENT_PENDING) ||
       (audio_decoder_job_ && audio_decoder_job_->is_decoding()) ||
       (video_decoder_job_ && video_decoder_job_->is_decoding());
 
   // Clear all the pending events except seeks and config changes.
   pending_event_ &= (SEEK_EVENT_PENDING | CONFIG_CHANGE_EVENT_PENDING);
   is_surface_in_use_ = false;
-  audio_decoder_job_.reset();
+  ResetAudioDecoderJob();
   ResetVideoDecoderJob();
 
   // Prevent job re-creation attempts in OnDemuxerConfigsAvailable()
   reconfig_audio_decoder_ = false;
   reconfig_video_decoder_ = false;
 
   // Prevent player restart, including job re-creation attempts.
   playing_ = false;
 
   decoder_starvation_callback_.Cancel();
@@ skipping 102 matching lines @@
 
     // Resume decoding after the config change if we are still playing.
     if (playing_)
       StartInternal();
   }
 }
 
 void MediaSourcePlayer::OnDemuxerDataAvailable(const DemuxerData& data) {
   DVLOG(1) << __FUNCTION__ << "(" << data.type << ")";
   DCHECK_LT(0u, data.access_units.size());
+
+  if (has_pending_audio_data_request_ && data.type == DemuxerStream::AUDIO) {
+    has_pending_audio_data_request_ = false;
+    ProcessPendingEvents();
+    return;
+  }
+
+  if (has_pending_video_data_request_ && data.type == DemuxerStream::VIDEO) {
+    next_video_data_is_iframe_ = false;
+    has_pending_video_data_request_ = false;
+    ProcessPendingEvents();
+    return;
+  }
+
   if (data.type == DemuxerStream::AUDIO && audio_decoder_job_) {
     audio_decoder_job_->OnDataReceived(data);
   } else if (data.type == DemuxerStream::VIDEO) {
     next_video_data_is_iframe_ = false;
     if (video_decoder_job_)
       video_decoder_job_->OnDataReceived(data);
   }
 }
 
 void MediaSourcePlayer::OnDemuxerDurationChanged(base::TimeDelta duration) {
@@ skipping 32 matching lines @@
     drm_bridge_->SetMediaCryptoReadyCB(base::Bind(
         &MediaSourcePlayer::OnMediaCryptoReady, weak_this_.GetWeakPtr()));
     return;
   }
 
   if (playing_)
     StartInternal();
 }
 
 void MediaSourcePlayer::OnDemuxerSeekDone(
-    const base::TimeDelta& actual_browser_seek_time) {
+    base::TimeDelta actual_browser_seek_time) {
   DVLOG(1) << __FUNCTION__;
 
   ClearPendingEvent(SEEK_EVENT_PENDING);
   if (IsEventPending(PREFETCH_REQUEST_EVENT_PENDING))
     ClearPendingEvent(PREFETCH_REQUEST_EVENT_PENDING);
 
   next_video_data_is_iframe_ = true;
 
   if (pending_seek_) {
     DVLOG(1) << __FUNCTION__ << "processing pending seek";
     DCHECK(doing_browser_seek_);
     pending_seek_ = false;
     SeekTo(pending_seek_time_);
     return;
   }
 
   // It is possible that a browser seek to I-frame had to seek to a buffered
   // I-frame later than the requested one due to data removal or GC. Update
   // player clock to the actual seek target.
   if (doing_browser_seek_) {
     DCHECK(actual_browser_seek_time != kNoTimestamp());
+    base::TimeDelta seek_time = actual_browser_seek_time;
     // A browser seek must not jump into the past. Ideally, it seeks to the
     // requested time, but it might jump into the future.
-    DCHECK(actual_browser_seek_time >= GetCurrentTime());
+    DCHECK(seek_time >= GetCurrentTime());
     DVLOG(1) << __FUNCTION__ << " : setting clock to actual browser seek time: "
-             << actual_browser_seek_time.InSecondsF();
-    clock_.SetTime(actual_browser_seek_time, actual_browser_seek_time);
+             << seek_time.InSecondsF();
+    clock_.SetTime(seek_time, seek_time);
     if (audio_timestamp_helper_)
-      audio_timestamp_helper_->SetBaseTimestamp(actual_browser_seek_time);
+      audio_timestamp_helper_->SetBaseTimestamp(seek_time);
   }
 

[wolenetz, 2014/03/19 17:26:05]

nit: Due to confusion in |actual_browser_seek_time| usage (see my current round
of unit test comments), perhaps add a DCHECK that enforces
actual_browser_seek_time == kNoTimestamp() if and only if NOT
|doing_browser_seek|?

See also
https://code.google.com/p/chromium/codesearch#chromium/src/media/base/android...

[qinmin, 2014/03/19 19:24:55]

Done.

   reached_audio_eos_ = false;
   reached_video_eos_ = false;
 
   base::TimeDelta current_time = GetCurrentTime();
   // TODO(qinmin): Simplify the logic by using |start_presentation_timestamp_|
   // to preroll media decoder jobs. Currently |start_presentation_timestamp_|
   // is calculated from decoder output, while preroll relies on the access
   // unit's timestamp. There are some differences between the two.
   preroll_timestamp_ = current_time;
   if (audio_decoder_job_)
     audio_decoder_job_->BeginPrerolling(preroll_timestamp_);
   if (video_decoder_job_)
     video_decoder_job_->BeginPrerolling(preroll_timestamp_);
 
   if (!doing_browser_seek_)
     manager()->OnSeekComplete(player_id(), current_time);
 
   ProcessPendingEvents();
 }
 
 void MediaSourcePlayer::UpdateTimestamps(
-    const base::TimeDelta& presentation_timestamp, size_t audio_output_bytes) {
+    base::TimeDelta presentation_timestamp, size_t audio_output_bytes) {
   base::TimeDelta new_max_time = presentation_timestamp;
 
   if (audio_output_bytes > 0) {
     audio_timestamp_helper_->AddFrames(
         audio_output_bytes / (kBytesPerAudioOutputSample * num_channels_));
     new_max_time = audio_timestamp_helper_->GetTimestamp();
   }
 
   clock_.SetMaxTime(new_max_time);
   manager()->OnTimeUpdate(player_id(), GetCurrentTime());
 }
 
 void MediaSourcePlayer::ProcessPendingEvents() {
   DVLOG(1) << __FUNCTION__ << " : 0x" << std::hex << pending_event_;
   // Wait for all the decoding jobs to finish before processing pending tasks.
   if (video_decoder_job_ && video_decoder_job_->is_decoding()) {
     DVLOG(1) << __FUNCTION__ << " : A video job is still decoding.";
     return;
   }
 
   if (audio_decoder_job_ && audio_decoder_job_->is_decoding()) {
     DVLOG(1) << __FUNCTION__ << " : An audio job is still decoding.";
     return;
   }
 
+  if (has_pending_audio_data_request_ || has_pending_video_data_request_) {
+    DVLOG(1) << __FUNCTION__ << " : has pending data request.";
+    return;
+  }
+
   if (IsEventPending(PREFETCH_DONE_EVENT_PENDING)) {
     DVLOG(1) << __FUNCTION__ << " : PREFETCH_DONE still pending.";
     return;
   }
 
   if (IsEventPending(SEEK_EVENT_PENDING)) {
     DVLOG(1) << __FUNCTION__ << " : Handling SEEK_EVENT";
     ClearDecodingData();
     demuxer_->RequestDemuxerSeek(GetCurrentTime(), doing_browser_seek_);
     return;
@@ skipping 14 matching lines @@
     ConfigureVideoDecoderJob();
 
     // Return early if we can't successfully configure a new video decoder job
     // yet.
     if (HasVideo() && !video_decoder_job_)
       return;
   }
 
   if (IsEventPending(PREFETCH_REQUEST_EVENT_PENDING)) {
     DVLOG(1) << __FUNCTION__ << " : Handling PREFETCH_REQUEST_EVENT.";
+    // If one of the decoder is not initialized, cancel this event as it will be
+    // called later when Start() is called again.
+    if ((HasVideo() && !video_decoder_job_) ||
+        (HasAudio() && !audio_decoder_job_)) {
+      ClearPendingEvent(PREFETCH_REQUEST_EVENT_PENDING);
+      return;
+    }
+
     DCHECK(audio_decoder_job_ || AudioFinished());
     DCHECK(video_decoder_job_ || VideoFinished());
 
     int count = (AudioFinished() ? 0 : 1) + (VideoFinished() ? 0 : 1);
 
     // It is possible that all streams have finished decode, yet starvation
     // occurred during the last stream's EOS decode. In this case, prefetch is a
     // no-op.
     ClearPendingEvent(PREFETCH_REQUEST_EVENT_PENDING);
     if (count == 0)
@@ skipping 15 matching lines @@
   DCHECK_EQ(pending_event_, NO_EVENT_PENDING);
 
   // Now that all pending events have been handled, resume decoding if we are
   // still playing.
   if (playing_)
     StartInternal();
 }
 
 void MediaSourcePlayer::MediaDecoderCallback(
     bool is_audio, MediaCodecStatus status,
-    const base::TimeDelta& presentation_timestamp, size_t audio_output_bytes) {
+    base::TimeDelta presentation_timestamp, size_t audio_output_bytes) {
   DVLOG(1) << __FUNCTION__ << ": " << is_audio << ", " << status;
 
   // TODO(xhwang): Drop IntToString() when http://crbug.com/303899 is fixed.
   if (is_audio) {
     TRACE_EVENT_ASYNC_END1("media",
                            "MediaSourcePlayer::DecodeMoreAudio",
                            audio_decoder_job_.get(),
                            "MediaCodecStatus",
                            base::IntToString(status));
   } else {
@@ skipping 181 matching lines @@
 bool MediaSourcePlayer::AudioFinished() {
   return reached_audio_eos_ || !HasAudio();
 }
 
 bool MediaSourcePlayer::VideoFinished() {
   return reached_video_eos_ || !HasVideo();
 }
 
 void MediaSourcePlayer::ConfigureAudioDecoderJob() {
   if (!HasAudio()) {
-    audio_decoder_job_.reset();
+    ResetAudioDecoderJob();
     return;
   }
 
   // Create audio decoder job only if config changes.
   if (audio_decoder_job_ && !reconfig_audio_decoder_)
     return;
 
   base::android::ScopedJavaLocalRef<jobject> media_crypto = GetMediaCrypto();
   if (is_audio_encrypted_ && media_crypto.is_null())
     return;
 
   DCHECK(!audio_decoder_job_ || !audio_decoder_job_->is_decoding());
 
+  ResetAudioDecoderJob();
   DVLOG(1) << __FUNCTION__ << " : creating new audio decoder job";
-
   audio_decoder_job_.reset(AudioDecoderJob::Create(
       audio_codec_, sampling_rate_, num_channels_, &audio_extra_data_[0],
       audio_extra_data_.size(), media_crypto.obj(),
       base::Bind(&DemuxerAndroid::RequestDemuxerData,
                  base::Unretained(demuxer_.get()), DemuxerStream::AUDIO)));
 
   if (audio_decoder_job_) {
     SetVolumeInternal();
     audio_decoder_job_->BeginPrerolling(preroll_timestamp_);
     reconfig_audio_decoder_ =  false;
   }
 }
 
 void MediaSourcePlayer::ResetVideoDecoderJob() {
+  if (video_decoder_job_) {
+    has_pending_video_data_request_ =
+        video_decoder_job_->is_requesting_demuxer_data();
+  }
   video_decoder_job_.reset();
 
   // Any eventual video decoder job re-creation will use the current |surface_|.
   if (IsEventPending(SURFACE_CHANGE_EVENT_PENDING))
     ClearPendingEvent(SURFACE_CHANGE_EVENT_PENDING);
 }
 
+void MediaSourcePlayer::ResetAudioDecoderJob() {
+  if (audio_decoder_job_) {
+    has_pending_audio_data_request_ =
+        audio_decoder_job_->is_requesting_demuxer_data();
+  }
+  audio_decoder_job_.reset();
+}
+
 void MediaSourcePlayer::ConfigureVideoDecoderJob() {
   if (!HasVideo() || surface_.IsEmpty()) {
     ResetVideoDecoderJob();
     return;
   }
 
   // Create video decoder job only if config changes or we don't have a job.
   if (video_decoder_job_ && !reconfig_video_decoder_) {
     DCHECK(!IsEventPending(SURFACE_CHANGE_EVENT_PENDING));
     return;
@@ skipping 56 matching lines @@
       player_id(), duration_, width_, height_, true);
 }
 
 void MediaSourcePlayer::OnDecoderStarved() {
   DVLOG(1) << __FUNCTION__;
   SetPendingEvent(PREFETCH_REQUEST_EVENT_PENDING);
   ProcessPendingEvents();
 }
 
 void MediaSourcePlayer::StartStarvationCallback(
-    const base::TimeDelta& presentation_timestamp) {
+    base::TimeDelta presentation_timestamp) {
   // 20ms was chosen because it is the typical size of a compressed audio frame.
   // Anything smaller than this would likely cause unnecessary cycling in and
   // out of the prefetch state.
   const base::TimeDelta kMinStarvationTimeout =
       base::TimeDelta::FromMilliseconds(20);
 
   base::TimeDelta current_timestamp = GetCurrentTime();
   base::TimeDelta timeout;
   if (HasAudio()) {
     timeout = audio_timestamp_helper_->GetTimestamp() - current_timestamp;
@@ skipping 94 matching lines @@
 
 void MediaSourcePlayer::ClearPendingEvent(PendingEventFlags event) {
   DVLOG(1) << __FUNCTION__ << "(" << GetEventName(event) << ")";
   DCHECK_NE(event, NO_EVENT_PENDING);
   DCHECK(IsEventPending(event)) << GetEventName(event);
 
   pending_event_ &= ~event;
 }
 
 }  // namespace media