Handle config changes for MSE on android

media/base/android/media_source_player.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/base/android/media_source_player.h"
 
 #include "base/android/jni_android.h"
 #include "base/android/jni_string.h"
 #include "base/basictypes.h"
 #include "base/bind.h"
@@ skipping 37 matching lines @@
 
 }
 
 namespace media {
 
 MediaDecoderJob::MediaDecoderJob(base::Thread* thread, bool is_audio)
     : message_loop_(base::MessageLoopProxy::current()),
       thread_(thread),
       needs_flush_(false),
       is_audio_(is_audio),
-      weak_this_(this) {
+      weak_this_(this),
+      decoding_(false) {
 }
 
 MediaDecoderJob::~MediaDecoderJob() {}
 
 // Class for managing audio decoding jobs.
 class AudioDecoderJob : public MediaDecoderJob {
  public:
   AudioDecoderJob(
       const AudioCodec audio_codec, int sample_rate,
       int channel_count, const uint8* extra_data, size_t extra_data_size);
   virtual ~AudioDecoderJob() {}
 };
 
 // Class for managing video decoding jobs.
 class VideoDecoderJob : public MediaDecoderJob {
  public:
   VideoDecoderJob(
       const VideoCodec video_codec, const gfx::Size& size, jobject surface);
   virtual ~VideoDecoderJob() {}
 
   void Configure(
       const VideoCodec codec, const gfx::Size& size, jobject surface);
 };
 
 void MediaDecoderJob::Decode(
     const MediaPlayerHostMsg_ReadFromDemuxerAck_Params::AccessUnit& unit,
     const base::Time& start_wallclock_time,
     const base::TimeDelta& start_presentation_timestamp,
     const MediaDecoderJob::DecoderCallback& callback) {
+  DCHECK(!decoding_);
+  decoding_ = true;
   thread_->message_loop()->PostTask(FROM_HERE, base::Bind(
       &MediaDecoderJob::DecodeInternal, base::Unretained(this), unit,
       start_wallclock_time, start_presentation_timestamp, needs_flush_,
       callback));
   needs_flush_ = false;
 }
 
 void MediaDecoderJob::DecodeInternal(
     const MediaPlayerHostMsg_ReadFromDemuxerAck_Params::AccessUnit& unit,
     const base::Time& start_wallclock_time,
     const base::TimeDelta& start_presentation_timestamp,
     bool needs_flush,
     const MediaDecoderJob::DecoderCallback& callback) {
   if (needs_flush)
     media_codec_bridge_->Reset();
   base::TimeDelta timeout = base::TimeDelta::FromMicroseconds(
       kMediaCodecTimeoutInMicroseconds);
   int input_buf_index = media_codec_bridge_->DequeueInputBuffer(timeout);
+  if (input_buf_index == MediaCodecBridge::INFO_MEDIA_CODEC_ERROR) {
+    message_loop_->PostTask(FROM_HERE, base::Bind(
+        callback, false, start_presentation_timestamp, start_wallclock_time,
+        false));
+    return;
+  }
   // TODO(qinmin): skip frames if video is falling far behind.
   if (input_buf_index >= 0) {
     if (unit.end_of_stream) {
       media_codec_bridge_->QueueEOS(input_buf_index);
     } else {
       media_codec_bridge_->QueueInputBuffer(
           input_buf_index, &unit.data[0], unit.data.size(), unit.timestamp);
     }
   }
   size_t offset = 0;
@@ skipping 56 matching lines @@
   if (is_audio_) {
     static_cast<AudioCodecBridge*>(media_codec_bridge_.get())->PlayOutputBuffer(
         outputBufferIndex, size);
   }
   media_codec_bridge_->ReleaseOutputBuffer(outputBufferIndex, !is_audio_);
   message_loop_->PostTask(FROM_HERE, base::Bind(
       callback, true, presentation_timestamp, base::Time::Now(),
       end_of_stream));
 }
 
+void MediaDecoderJob::OnDecodeCompleted() {
+  decoding_ = false;
+}
+
 void MediaDecoderJob::Flush() {
   // Do nothing, flush when the next Decode() happens.
   needs_flush_ = true;
 }
 
 void MediaDecoderJob::Release() {
-  if (thread_->IsRunning() &&
-      thread_->message_loop() != base::MessageLoop::current()) {
+  // If |decoding_| is false, there is nothing running on the decoder thread.
+  // So it is safe to delete the MediaDecoderJob on the UI thread. However,
+  // if we post a task to the decoder thread to delete object, then we cannot
+  // immediately pass the surface to a new MediaDecoderJob instance because
+  // the java surface is still owned by the old object. New decoder creation
+  // will be blocked on the UI thread until the previous decoder gets deleted.
+  // This introduces extra latency during config changes, and makes the logic in
+  // MediaSourcePlayer more complicated.
+  //
+  // TODO(qinmin): Figure out the logic to passing the surface to a new
+  // MediaDecoderJob instance after the previous one gets deleted on the decoder
+  // thread.
+  if (decoding_ && thread_->message_loop() != base::MessageLoop::current())

[acolwell GONE FROM CHROMIUM, 2013/06/05 23:13:38]

This logic still doesn't seem safe to me, but I'm fine with defering the fix to
another CL.

[qinmin, 2013/06/06 01:17:25]

ok, i will follow up with another CL to address this. One workaround is that
when creating the MediaDecoderJob, we post a task onto the decoder thread to
configure the new decoder. So the old decoder will be gone first, and the new
decoder will be configured with the same surface.

On 2013/06/05 23:13:38, acolwell wrote:

     thread_->message_loop()->DeleteSoon(FROM_HERE, this);
-  } else {
+  else
     delete this;
-  }
 }
 
 VideoDecoderJob::VideoDecoderJob(
     const VideoCodec video_codec, const gfx::Size& size, jobject surface)
     : MediaDecoderJob(g_video_decoder_thread.Pointer(), false) {
   scoped_ptr<VideoCodecBridge> codec(VideoCodecBridge::Create(video_codec));
   codec->Start(video_codec, size, surface);
   media_codec_bridge_.reset(codec.release());
 }
 
 AudioDecoderJob::AudioDecoderJob(
     const AudioCodec audio_codec,
     int sample_rate,
     int channel_count,
     const uint8* extra_data,
     size_t extra_data_size)
     : MediaDecoderJob(g_audio_decoder_thread.Pointer(), true) {
   scoped_ptr<AudioCodecBridge> codec(AudioCodecBridge::Create(audio_codec));
   codec->Start(audio_codec, sample_rate, channel_count, extra_data,
                extra_data_size, true);
   media_codec_bridge_.reset(codec.release());
 }
 
 MediaSourcePlayer::MediaSourcePlayer(
     int player_id,
     MediaPlayerManager* manager)
     : MediaPlayerAndroid(player_id, manager),
       pending_event_(NO_EVENT_PENDING),
       active_decoding_tasks_(0),
+      seek_request_id_(0),
       width_(0),
       height_(0),
       audio_codec_(kUnknownAudioCodec),
       video_codec_(kUnknownVideoCodec),
       num_channels_(0),
       sampling_rate_(0),
       seekable_(true),
       audio_finished_(true),
       video_finished_(true),
       playing_(false),
+      reconfig_audio_decoder_(true),

[acolwell GONE FROM CHROMIUM, 2013/06/05 23:13:38]

Nit: I think these should default to false so the only reflect when actual
config changes occur.

[qinmin, 2013/06/06 01:17:25]

Done.

+      reconfig_video_decoder_(true),
       audio_access_unit_index_(0),
       video_access_unit_index_(0),
       waiting_for_audio_data_(false),
       waiting_for_video_data_(false),
-      use_empty_surface_(true),
       weak_this_(this) {
 }
 
 MediaSourcePlayer::~MediaSourcePlayer() {
   Release();
 }
 
 void MediaSourcePlayer::SetVideoSurface(gfx::ScopedJavaSurface surface) {
-  use_empty_surface_ =  surface.IsSurfaceEmpty();
+  surface_ =  surface.Pass();
 
-  // If we haven't processed a surface change event, do so now.
-  if (active_decoding_tasks_ > 0) {
-    pending_event_ |= SURFACE_CHANGE_EVENT_PENDING;
-    // Request a seek so that the next decoder will decode an I-frame first.
-    // Or otherwise, MediaCodec might crash. See b/8950387.
-    pending_event_ |= SEEK_EVENT_PENDING;
-    ProcessPendingEvents();
-    return;
-  }
-
-  if (HasVideo()) {
-    video_decoder_job_.reset(new VideoDecoderJob(
-        video_codec_, gfx::Size(width_, height_), surface.j_surface().obj()));
-  }
-
-  // Inform the fullscreen view the player is ready.
-  // TODO(qinmin): refactor MediaPlayerBridge so that we have a better way
-  // to inform ContentVideoView.
-  OnMediaMetadataChanged(duration_, width_, height_, true);
-
-  if (pending_event_ & SURFACE_CHANGE_EVENT_PENDING) {
-    // We should already requested a seek in this case.
-    pending_event_ &= ~SURFACE_CHANGE_EVENT_PENDING;
-  } else {
-    // Perform a seek so the new decoder can get the I-frame first.
-    pending_event_ |= SEEK_EVENT_PENDING;
-    ProcessPendingEvents();
-    return;
-  }
-
-  if (playing_)
-    StartInternal();
+  reconfig_video_decoder_ = true;

[acolwell GONE FROM CHROMIUM, 2013/06/05 23:13:38]

nit: Shouldn't this also set CONFIG_CHANGE_EVENT_PENDING?

[qinmin, 2013/06/06 01:17:25]

Surface change is a little different from config change, it requires recreation
of the decoder and perform a seek, But the player does not need to ask
chunk_demuxer for a new config. 
If we set CONFIG_CHANGE_EVENT here, then all the SURFACE_CHANGE_EVENT will be
accompanied with a CONFIG_CHANGE_EVENT, which will cause an extra round trip
IPCs to chunk demuxer.
Removed the "reconfig_video_decoder_ = true" statement here, so
reconfig_video_decoder_ only applies to real config changes in the stream, not
surface change

On 2013/06/05 23:13:38, acolwell wrote:

+  pending_event_ |= SURFACE_CHANGE_EVENT_PENDING;
+  // Setting a new surface will require a new MediaCodec to be created.
+  // Request a seek so that the new decoder will decode an I-frame first.
+  // Or otherwise, the new MediaCodec might crash. See b/8950387.
+  pending_event_ |= SEEK_EVENT_PENDING;
+  ProcessPendingEvents();
 }
 
 void MediaSourcePlayer::Start() {
   playing_ = true;
-  if (HasAudio() && !audio_decoder_job_) {
-    audio_decoder_job_.reset(new AudioDecoderJob(
-        audio_codec_, sampling_rate_, num_channels_,
-        &audio_extra_data_[0], audio_extra_data_.size()));
-  }
 
-  if (HasVideo() && !video_decoder_job_) {
-    // StartInternal() will be delayed until SetVideoSurface() gets called.
-    return;
-  }
+  CreateMediaDecoderJobs();
 
   StartInternal();
 }
 
 void MediaSourcePlayer::Pause() {
   playing_ = false;
   start_wallclock_time_ = base::Time();
 }
 
 bool MediaSourcePlayer::IsPlaying() {
@@ skipping 19 matching lines @@
 }
 
 base::TimeDelta MediaSourcePlayer::GetDuration() {
   return duration_;
 }
 
 void MediaSourcePlayer::Release() {
   ClearDecodingData();
   audio_decoder_job_.reset();
   video_decoder_job_.reset();
+  reconfig_audio_decoder_ = true;
+  reconfig_video_decoder_ = true;
   active_decoding_tasks_ = 0;
   playing_ = false;
   pending_event_ = NO_EVENT_PENDING;
+  surface_ = gfx::ScopedJavaSurface();
   ReleaseMediaResourcesFromManager();
 }
 
 void MediaSourcePlayer::SetVolume(float leftVolume, float rightVolume) {
 }
 
 bool MediaSourcePlayer::CanPause() {
   return seekable_;
 }
 
 bool MediaSourcePlayer::CanSeekForward() {
   return seekable_;
 }
 
 bool MediaSourcePlayer::CanSeekBackward() {
   return seekable_;
 }
 
 bool MediaSourcePlayer::IsPlayerReady() {
   return audio_decoder_job_ || video_decoder_job_;
 }
 
 void MediaSourcePlayer::StartInternal() {
   // Do nothing if the decoders are already running.
   if (active_decoding_tasks_ > 0 || pending_event_ != NO_EVENT_PENDING)
     return;
 
+  // If one of the decoder job is not ready, do nothing.
+  if ((HasAudio() && !audio_decoder_job_) ||
+      (HasVideo() && !video_decoder_job_)) {
+    return;
+  }
+
   if (HasAudio()) {
     audio_finished_ = false;
     DecodeMoreAudio();
   }
   if (HasVideo()) {
     video_finished_ = false;
     DecodeMoreVideo();
   }
 }
 
 void MediaSourcePlayer::DemuxerReady(
     const MediaPlayerHostMsg_DemuxerReady_Params& params) {
   if (params.duration_ms == std::numeric_limits<int>::max())
     seekable_ = false;
   duration_ = base::TimeDelta::FromMilliseconds(params.duration_ms);
   width_ = params.video_size.width();
   height_ = params.video_size.height();
   num_channels_ = params.audio_channels;
   sampling_rate_ = params.audio_sampling_rate;
   audio_codec_ = params.audio_codec;
   video_codec_ = params.video_codec;
   audio_extra_data_ = params.audio_extra_data;
   OnMediaMetadataChanged(duration_, width_, height_, true);
+  if (pending_event_ & CONFIG_CHANGE_EVENT_PENDING) {
+    CreateMediaDecoderJobs();
+    pending_event_ &= ~CONFIG_CHANGE_EVENT_PENDING;
+    // If there is a pending surface change, we can merge it with the config
+    // change.
+    pending_event_ &= ~SURFACE_CHANGE_EVENT_PENDING;

[acolwell GONE FROM CHROMIUM, 2013/06/05 23:13:38]

Shouldn't this only be done if a video config check was pending?

[qinmin, 2013/06/06 01:17:25]

good catch, fixed.

On 2013/06/05 23:13:38, acolwell wrote:

+    if (playing_ && pending_event_ == NO_EVENT_PENDING)
+      StartInternal();
+  }
 }
 
 void MediaSourcePlayer::ReadFromDemuxerAck(
     const MediaPlayerHostMsg_ReadFromDemuxerAck_Params& params) {
   if (params.type == DemuxerStream::AUDIO)
     waiting_for_audio_data_ = false;
   else
     waiting_for_video_data_ = false;
 
   // If there is a pending seek request, ignore the data from the chunk demuxer.
   // The data will be requested later when OnSeekRequestAck() is called.
   if (pending_event_ & SEEK_EVENT_PENDING)
     return;
 
   if (params.type == DemuxerStream::AUDIO) {
     DCHECK_EQ(0u, audio_access_unit_index_);
     received_audio_ = params;
     if (!pending_event_)
       DecodeMoreAudio();
   } else {
     DCHECK_EQ(0u, video_access_unit_index_);
     received_video_ = params;
     if (!pending_event_)
       DecodeMoreVideo();
   }
 }
 
-void MediaSourcePlayer::OnSeekRequestAck() {
+void MediaSourcePlayer::OnSeekRequestAck(unsigned seek_request_id) {
+  // Do nothing until the most recent seek request is processed.
+  if (seek_request_id_ != seek_request_id)
+    return;
   pending_event_ &= ~SEEK_EVENT_PENDING;
   OnSeekComplete();
-  if (playing_)
-    StartInternal();
+  ProcessPendingEvents();
 }
 
 void MediaSourcePlayer::UpdateTimestamps(
     const base::TimeDelta& presentation_timestamp,
     const base::Time& wallclock_time) {
   last_presentation_timestamp_ = presentation_timestamp;
   OnTimeUpdated();
   if (start_wallclock_time_.is_null() && playing_) {
     start_wallclock_time_ = wallclock_time;
     start_presentation_timestamp_ = last_presentation_timestamp_;
   }
 }
 
 void MediaSourcePlayer::ProcessPendingEvents() {
-  // Wait for all the decoding jobs to finish before sending a seek request.
+  // Wait for all the decoding jobs to finish before processing pending tasks.
   if (active_decoding_tasks_ > 0)
     return;
 
-  DCHECK(pending_event_ != NO_EVENT_PENDING);
-  if (use_empty_surface_ && (pending_event_ & SURFACE_CHANGE_EVENT_PENDING)) {
-    video_decoder_job_.reset();
+  if (pending_event_ & SEEK_EVENT_PENDING) {
+    ClearDecodingData();
+    manager()->OnMediaSeekRequest(
+        player_id(), last_presentation_timestamp_, ++seek_request_id_);
+    return;
+  }
+
+  start_wallclock_time_ = base::Time();
+  if (pending_event_ & CONFIG_CHANGE_EVENT_PENDING) {
+    manager()->OnMediaConfigRequest(player_id());
+    return;
+  }
+
+  if (pending_event_ & SURFACE_CHANGE_EVENT_PENDING) {
+    DCHECK(reconfig_audio_decoder_ || reconfig_video_decoder_);
+    CreateMediaDecoderJobs();

[acolwell GONE FROM CHROMIUM, 2013/06/05 23:13:38]

Shouldn't this only apply for a video reconfig? It seems like you should split
CreateMediaDecoderJobs() into audio and video methods so it is clear when only
one is really needed.

[qinmin, 2013/06/06 01:17:25]

Done, split the function into CreateVideoDecoderJob and CreateAudioDecoderJob.

On 2013/06/05 23:13:38, acolwell wrote:

     pending_event_ &= ~SURFACE_CHANGE_EVENT_PENDING;
   }
 
-  ClearDecodingData();
-  manager()->OnMediaSeekRequest(player_id(),
-                                last_presentation_timestamp_,
-                                pending_event_ & SURFACE_CHANGE_EVENT_PENDING);
+  if (playing_ && pending_event_ == NO_EVENT_PENDING)
+    StartInternal();
 }
 
 void MediaSourcePlayer::MediaDecoderCallback(
     bool is_audio, bool decode_succeeded,
     const base::TimeDelta& presentation_timestamp,
     const base::Time& wallclock_time, bool end_of_stream) {
   if (active_decoding_tasks_ > 0)
     active_decoding_tasks_--;
 
+  if (is_audio && audio_decoder_job_)
+    audio_decoder_job_->OnDecodeCompleted();
+  if (!is_audio && video_decoder_job_)
+    video_decoder_job_->OnDecodeCompleted();
+
   if (!decode_succeeded) {
     Release();
     OnMediaError(MEDIA_ERROR_DECODE);
     return;
   }
 
   if (pending_event_ != NO_EVENT_PENDING) {
     ProcessPendingEvents();
     return;
   }
@@ skipping 19 matching lines @@
   if (audio_access_unit_index_ >= received_audio_.access_units.size()) {
     if (!waiting_for_audio_data_) {
       manager()->OnReadFromDemuxer(player_id(), DemuxerStream::AUDIO, true);
       received_audio_ = MediaPlayerHostMsg_ReadFromDemuxerAck_Params();
       audio_access_unit_index_ = 0;
       waiting_for_audio_data_ = true;
     }
     return;
   }
 
+  if (DemuxerStream::kConfigChanged ==
+      received_audio_.access_units[audio_access_unit_index_].status) {
+    // Wait for demuxer ready message.
+    reconfig_audio_decoder_ = true;
+    pending_event_ |= CONFIG_CHANGE_EVENT_PENDING;
+    received_audio_ = MediaPlayerHostMsg_ReadFromDemuxerAck_Params();
+    audio_access_unit_index_ = 0;
+    ProcessPendingEvents();
+    return;
+  }
+
   audio_decoder_job_->Decode(
       received_audio_.access_units[audio_access_unit_index_],
       start_wallclock_time_, start_presentation_timestamp_,
       base::Bind(&MediaSourcePlayer::MediaDecoderCallback,
                  weak_this_.GetWeakPtr(), true));
   active_decoding_tasks_++;
   audio_access_unit_index_++;
 }
 
 void MediaSourcePlayer::DecodeMoreVideo() {
   if (video_access_unit_index_ >= received_video_.access_units.size()) {
     if (!waiting_for_video_data_) {
       manager()->OnReadFromDemuxer(player_id(), DemuxerStream::VIDEO, true);
       received_video_ = MediaPlayerHostMsg_ReadFromDemuxerAck_Params();
       video_access_unit_index_ = 0;
       waiting_for_video_data_ = true;
     }
     return;
   }
 
+  if (DemuxerStream::kConfigChanged ==
+      received_video_.access_units[video_access_unit_index_].status) {
+    // Wait for demuxer ready message.
+    reconfig_video_decoder_ = true;
+    pending_event_ |= CONFIG_CHANGE_EVENT_PENDING;
+    received_video_ = MediaPlayerHostMsg_ReadFromDemuxerAck_Params();
+    video_access_unit_index_ = 0;
+    ProcessPendingEvents();
+    return;
+  }
+
   video_decoder_job_->Decode(
       received_video_.access_units[video_access_unit_index_],
       start_wallclock_time_, start_presentation_timestamp_,
       base::Bind(&MediaSourcePlayer::MediaDecoderCallback,
                  weak_this_.GetWeakPtr(), false));
   active_decoding_tasks_++;
   video_access_unit_index_++;
 }
 
 
@@ skipping 23 matching lines @@
 }
 
 bool MediaSourcePlayer::HasVideo() {
   return kUnknownVideoCodec != video_codec_;
 }
 
 bool MediaSourcePlayer::HasAudio() {
   return kUnknownAudioCodec != audio_codec_;
 }
 
+void MediaSourcePlayer::CreateMediaDecoderJobs() {
+  DCHECK_EQ(0, active_decoding_tasks_);
+
+  // Create audio decoder job only if config changes.
+  if (HasAudio() && (reconfig_audio_decoder_ || !audio_decoder_job_)) {
+    audio_decoder_job_.reset(new AudioDecoderJob(
+        audio_codec_, sampling_rate_, num_channels_,
+        &audio_extra_data_[0], audio_extra_data_.size()));
+    reconfig_audio_decoder_ =  false;
+  }
+
+  if (!HasVideo() || surface_.IsSurfaceEmpty()) {
+    video_decoder_job_.reset();
+    return;
+  }
+
+  if (reconfig_video_decoder_ || !video_decoder_job_) {
+    video_decoder_job_.reset(new VideoDecoderJob(
+        video_codec_, gfx::Size(width_, height_), surface_.j_surface().obj()));
+    reconfig_video_decoder_ = false;
+  }
+
+  // Inform the fullscreen view the player is ready.
+  // TODO(qinmin): refactor MediaPlayerBridge so that we have a better way
+  // to inform ContentVideoView.
+  OnMediaMetadataChanged(duration_, width_, height_, true);
+}
+
 }  // namespace media