Fix an issue that audio and video may run out of sync

media/base/android/media_source_player.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/base/android/media_source_player.h"
 
 #include <limits>
 
 #include "base/android/jni_android.h"
 #include "base/android/jni_string.h"
@@ skipping 298 matching lines @@
   DVLOG(1) << __FUNCTION__;
   duration_ = base::TimeDelta::FromMilliseconds(configs.duration_ms);
   clock_.SetDuration(duration_);
 
   audio_codec_ = configs.audio_codec;
   num_channels_ = configs.audio_channels;
   sampling_rate_ = configs.audio_sampling_rate;
   is_audio_encrypted_ = configs.is_audio_encrypted;
   audio_extra_data_ = configs.audio_extra_data;
   if (HasAudio()) {
-    DCHECK_GT(num_channels_, 0);
-    audio_timestamp_helper_.reset(new AudioTimestampHelper(sampling_rate_));
-    audio_timestamp_helper_->SetBaseTimestamp(GetCurrentTime());
+    if (reconfig_audio_decoder_ || !audio_decoder_job_) {
+      DCHECK_GT(num_channels_, 0);
+      audio_timestamp_helper_.reset(new AudioTimestampHelper(sampling_rate_));
+      audio_timestamp_helper_->SetBaseTimestamp(GetCurrentTime());
+    }
   } else {
     audio_timestamp_helper_.reset();
   }
 
   video_codec_ = configs.video_codec;
   width_ = configs.video_size.width();
   height_ = configs.video_size.height();
   is_video_encrypted_ = configs.is_video_encrypted;
 
   manager()->OnMediaMetadataChanged(
@@ skipping 132 matching lines @@
   if (video_decoder_job_)
     video_decoder_job_->BeginPrerolling(preroll_timestamp_);
 
   if (!doing_browser_seek_)
     manager()->OnSeekComplete(player_id(), current_time);
 
   ProcessPendingEvents();
 }
 
 void MediaSourcePlayer::UpdateTimestamps(
-    base::TimeDelta presentation_timestamp, size_t audio_output_bytes) {
-  base::TimeDelta new_max_time = presentation_timestamp;
-
+    base::TimeDelta presentation_timestamp, size_t audio_output_bytes,
+    int64 audio_head_position, base::TimeTicks audio_render_time) {
+  // If audio is present and there is no audio data passed to the AudioTrack,
+  // just let the clock progress by itself.
   if (audio_output_bytes > 0) {
     audio_timestamp_helper_->AddFrames(
         audio_output_bytes / (kBytesPerAudioOutputSample * num_channels_));
-    new_max_time = audio_timestamp_helper_->GetTimestamp();
+    int64 frames_to_play =
+        audio_timestamp_helper_->frame_count() - audio_head_position;
+    DCHECK_GE(frames_to_play, 0);
+    base::TimeDelta new_max_time = audio_timestamp_helper_->GetTimestamp();
+    // Do a rough estimation of the current presentation time, assuming
+    // playback is smooth since the last time send the data to the AudioTrack.
+    base::TimeDelta current_presentation_timestamp =
+        base::TimeTicks::Now() - audio_render_time + new_max_time -
+        audio_timestamp_helper_->GetFrameDuration(frames_to_play);
+    if (current_presentation_timestamp >= new_max_time)
+      current_presentation_timestamp = new_max_time;

[acolwell GONE FROM CHROMIUM, 2014/04/04 19:25:58]

It feels like the bulk of this logic should be in the AudioDecoderJob. It seems
like there would be a lot fewer changes if you moved the audio_timestamp_helper_
into AudioDecoderJob and simply posted a task to this thread with a new
current_presentation_timestamp and new_max_time. That would be more in line with
what the pipeline code does and as long as there aren't significant delays on
this thread, it should be pretty accurate.

Am I missing something key about why this couldn't work?

[qinmin, 2014/04/04 19:42:51]

ok， I will move the logic there. 
However, the received presentation timestamp is a past thingy due to the delays
introduced by the posted task. I think we can ignore the delay for now, unless
we found the delays to be too significant to ignore.

On 2014/04/04 19:25:58, acolwell wrote:

+    clock_.SetTime(current_presentation_timestamp, new_max_time);
+  } else if (!HasAudio()) {
+    clock_.SetMaxTime(presentation_timestamp);
   }
 
-  clock_.SetMaxTime(new_max_time);
   manager()->OnTimeUpdate(player_id(), GetCurrentTime());
 }
 
 void MediaSourcePlayer::ProcessPendingEvents() {
   DVLOG(1) << __FUNCTION__ << " : 0x" << std::hex << pending_event_;
   // Wait for all the decoding jobs to finish before processing pending tasks.
   if (video_decoder_job_ && video_decoder_job_->is_decoding()) {
     DVLOG(1) << __FUNCTION__ << " : A video job is still decoding.";
     return;
   }
@@ skipping 80 matching lines @@
   DCHECK_EQ(pending_event_, NO_EVENT_PENDING);
 
   // Now that all pending events have been handled, resume decoding if we are
   // still playing.
   if (playing_)
     StartInternal();
 }
 
 void MediaSourcePlayer::MediaDecoderCallback(
     bool is_audio, MediaCodecStatus status,
-    base::TimeDelta presentation_timestamp, size_t audio_output_bytes) {
+    base::TimeDelta presentation_timestamp, size_t audio_output_bytes,
+    int64 audio_head_position, base::TimeTicks audio_render_time) {
   DVLOG(1) << __FUNCTION__ << ": " << is_audio << ", " << status;
+  DCHECK(is_audio || (audio_output_bytes == 0 && audio_head_position == 0 &&
+      audio_render_time.is_null()));
 
   // TODO(xhwang): Drop IntToString() when http://crbug.com/303899 is fixed.
   if (is_audio) {
     TRACE_EVENT_ASYNC_END1("media",
                            "MediaSourcePlayer::DecodeMoreAudio",
                            audio_decoder_job_.get(),
                            "MediaCodecStatus",
                            base::IntToString(status));
   } else {
     TRACE_EVENT_ASYNC_END1("media",
@@ skipping 22 matching lines @@
   DCHECK(!IsEventPending(PREFETCH_DONE_EVENT_PENDING));
 
   // Let |SEEK_EVENT_PENDING| (the highest priority event outside of
   // |PREFETCH_DONE_EVENT_PENDING|) preempt output EOS detection here. Process
   // any other pending events only after handling EOS detection.
   if (IsEventPending(SEEK_EVENT_PENDING)) {
     ProcessPendingEvents();
     return;
   }
 
+  if (status == MEDIA_CODEC_OK && is_clock_manager &&
+      presentation_timestamp != kNoTimestamp()) {
+    UpdateTimestamps(
+        presentation_timestamp, audio_output_bytes, audio_head_position,
+        audio_render_time);
+  }
+
   if (status == MEDIA_CODEC_OUTPUT_END_OF_STREAM)
     PlaybackCompleted(is_audio);
 
   if (pending_event_ != NO_EVENT_PENDING) {
     ProcessPendingEvents();
     return;
   }
 
   if (status == MEDIA_CODEC_OUTPUT_END_OF_STREAM)
     return;
 
-  if (status == MEDIA_CODEC_OK && is_clock_manager &&
-      presentation_timestamp != kNoTimestamp()) {
-    UpdateTimestamps(presentation_timestamp, audio_output_bytes);
-  }
-
   if (!playing_) {
     if (is_clock_manager)
       clock_.Pause();
     return;
   }
 
   if (status == MEDIA_CODEC_NO_KEY) {
     is_waiting_for_key_ = true;
     return;
   }
@@ skipping 365 matching lines @@
 
 void MediaSourcePlayer::ClearPendingEvent(PendingEventFlags event) {
   DVLOG(1) << __FUNCTION__ << "(" << GetEventName(event) << ")";
   DCHECK_NE(event, NO_EVENT_PENDING);
   DCHECK(IsEventPending(event)) << GetEventName(event);
 
   pending_event_ &= ~event;
 }
 
 }  // namespace media