MediaCodecPlayer implementation (stage 3 - browser seek and surface change)

media/base/android/media_codec_player.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/base/android/media_codec_player.h"
 
 #include "base/barrier_closure.h"
 #include "base/bind.h"
 #include "base/lazy_instance.h"
 #include "base/logging.h"
@@ skipping 55 matching lines @@
   DCHECK(ui_task_runner_->BelongsToCurrentThread());
 
   DVLOG(1) << "MediaCodecPlayer::MediaCodecPlayer: player_id:" << player_id;
 
   request_resources_cb_ = base::Bind(request_media_resources_cb_, player_id);
 
   completion_cb_ =
       base::Bind(&MediaPlayerManager::OnPlaybackComplete, manager, player_id);
   seek_done_cb_ =
       base::Bind(&MediaPlayerManager::OnSeekComplete, manager, player_id);
+  error_cb_ = base::Bind(&MediaPlayerManager::OnError, manager, player_id);
   attach_listener_cb_ = base::Bind(&MediaPlayerAndroid::AttachListener,
                                    WeakPtrForUIThread(), nullptr);
   detach_listener_cb_ =
       base::Bind(&MediaPlayerAndroid::DetachListener, WeakPtrForUIThread());
   metadata_changed_cb_ = base::Bind(&MediaPlayerAndroid::OnMediaMetadataChanged,
                                     WeakPtrForUIThread());
   time_update_cb_ =
       base::Bind(&MediaPlayerAndroid::OnTimeUpdate, WeakPtrForUIThread());
 
   media_weak_this_ = media_weak_factory_.GetWeakPtr();
@@ skipping 37 matching lines @@
 
   // Post deletion onto Media thread
   GetMediaTaskRunner()->DeleteSoon(FROM_HERE, this);
 }
 
 void MediaCodecPlayer::SetVideoSurface(gfx::ScopedJavaSurface surface) {
   RUN_ON_MEDIA_THREAD(SetVideoSurface, base::Passed(&surface));
 
   DVLOG(1) << __FUNCTION__ << (surface.IsEmpty() ? " empty" : " non-empty");
 
-  // I assume that if video decoder already has the surface,
-  // there will be two calls:
-  // (1) SetVideoSurface(0)
-  // (2) SetVideoSurface(new_surface)
-  video_decoder_->SetPendingSurface(surface.Pass());
+  // Save the empty-ness before we pass the surface to the decoder.
+  bool surface_is_empty = surface.IsEmpty();
 
-  if (video_decoder_->HasPendingSurface() &&
-      state_ == STATE_WAITING_FOR_SURFACE) {
-    SetState(STATE_PLAYING);
-    StartPlaybackDecoders();
+  // Apparently RemoveVideoSurface() can be called several times in a row,
+  // ignore the second and subsequent calls.
+  if (surface_is_empty && !video_decoder_->HasVideoSurface()) {
+    DVLOG(1) << __FUNCTION__ << ": surface already removed, ignoring";
+    return;
+  }
+
+  video_decoder_->SetVideoSurface(surface.Pass());
+
+  if (surface_is_empty) {
+    // Remove video surface.
+    switch (state_) {
+      case STATE_PLAYING:
+        if (VideoFinished())
+          break;
+
+        DVLOG(1) << __FUNCTION__ << ": stopping and restarting";
+        // Stop decoders as quickly as possible.
+        StopDecoders();  // synchronous stop
+
+        // Prefetch or wait for initial configuration.
+        if (HasAudio() || HasVideo()) {
+          SetState(STATE_PREFETCHING);
+          StartPrefetchDecoders();
+        } else {
+          SetState(STATE_WAITING_FOR_CONFIG);
+        }
+        break;
+
+      default:
+        break;  // ignore
+    }
+  } else {
+    // Replace video surface.
+    switch (state_) {
+      case STATE_WAITING_FOR_SURFACE:
+        SetState(STATE_PLAYING);
+        StartPlaybackOrBrowserSeek();
+        break;
+
+      case STATE_PLAYING:
+        if (VideoFinished())
+          break;
+
+        DVLOG(1) << __FUNCTION__ << ": requesting to stop and restart";
+        SetState(STATE_STOPPING);
+        RequestToStopDecoders();
+        SetPendingStart(true);
+        break;
+
+      default:
+        break;  // ignore
+    }
   }
 }
 
 void MediaCodecPlayer::Start() {
   RUN_ON_MEDIA_THREAD(Start);
 
   DVLOG(1) << __FUNCTION__;
 
   switch (state_) {
     case STATE_PAUSED:
@@ skipping 86 matching lines @@
       NOTREACHED();
       break;
   }
 }
 
 void MediaCodecPlayer::Release() {
   RUN_ON_MEDIA_THREAD(Release);
 
   DVLOG(1) << __FUNCTION__;
 
-  SetState(STATE_PAUSED);
+  // Stop decoding threads and delete MediaCodecs, but keep IPC between browser

[Tima Vaisburd, 2015/07/28 22:56:44]

Changed this Release().

+  // and renderer processes going. Seek should work across and after Release().
+
   ReleaseDecoderResources();
+
+  SetPendingStart(false);
+
+  if (state_ != STATE_WAITING_FOR_SEEK)
+    SetState(STATE_PAUSED);
+
+  base::TimeDelta pending_seek_time = GetPendingSeek();
+  if (pending_seek_time != kNoTimestamp()) {
+    SetPendingSeek(kNoTimestamp());
+    SetState(STATE_WAITING_FOR_SEEK);
+    RequestDemuxerSeek(pending_seek_time);
+  }
 }
 
 void MediaCodecPlayer::SetVolume(double volume) {
   RUN_ON_MEDIA_THREAD(SetVolume, volume);
 
   DVLOG(1) << __FUNCTION__ << " " << volume;
   audio_decoder_->SetVolume(volume);
 }
 
 int MediaCodecPlayer::GetVideoWidth() {
@@ skipping 86 matching lines @@
   if (data.type == DemuxerStream::VIDEO)
     video_decoder_->OnDemuxerDataAvailable(data);
 }
 
 void MediaCodecPlayer::OnDemuxerSeekDone(
     base::TimeDelta actual_browser_seek_time) {
   DCHECK(GetMediaTaskRunner()->BelongsToCurrentThread());
 
   DVLOG(1) << __FUNCTION__ << " actual_time:" << actual_browser_seek_time;
 
-  if (state_ != STATE_WAITING_FOR_SEEK)
-    return;  // ignore
-
+  DCHECK(state_ == STATE_WAITING_FOR_SEEK);

[Tima Vaisburd, 2015/07/28 22:56:44]

Added this DCHECK instead of ignoring the signal if not in the expected state.

   DCHECK(seek_info_.get());
   DCHECK(seek_info_->seek_time != kNoTimestamp());
 
   // A browser seek must not jump into the past. Ideally, it seeks to the
   // requested time, but it might jump into the future.
   DCHECK(!seek_info_->is_browser_seek ||
          seek_info_->seek_time <= actual_browser_seek_time);
 
   // Restrict the current time to be equal to seek_time
   // for the next StartPlaybackDecoders() call.
@@ skipping 91 matching lines @@
   if (state_ != STATE_PREFETCHING) {
     DVLOG(1) << __FUNCTION__ << " wrong state " << AsString(state_)
              << " ignoring";
     return;  // Ignore
   }
 
   DVLOG(1) << __FUNCTION__;
 
   if (!HasAudio() && !HasVideo()) {
     // No configuration at all after prefetching.
     // This is an error, initial configuration is expected

[wolenetz, 2015/07/29 22:37:03]

nit: we might want to DCHECK that internal_error_cb_ is indeed set to something
(e.g. that CreateDecoders() has done that setup...)

[Tima Vaisburd, 2015/07/30 20:28:35]

Done.

     // before the first data chunk.
-    GetMediaTaskRunner()->PostTask(FROM_HERE, error_cb_);
+    GetMediaTaskRunner()->PostTask(FROM_HERE, internal_error_cb_);
     return;
   }
 
-  if (HasVideo() && !HasPendingSurface()) {
+  if (HasVideo() && !video_decoder_->HasVideoSurface()) {
     SetState(STATE_WAITING_FOR_SURFACE);
     return;
   }
 
   SetState(STATE_PLAYING);
-  StartPlaybackDecoders();
+  StartPlaybackOrBrowserSeek();
 }
 
 void MediaCodecPlayer::OnStopDone() {
   DCHECK(GetMediaTaskRunner()->BelongsToCurrentThread());
   DVLOG(1) << __FUNCTION__;
 
   if (!(audio_decoder_->IsStopped() && video_decoder_->IsStopped()))
     return;  // Wait until other stream is stopped
 
   // At this point decoder threads should not be running
@@ skipping 38 matching lines @@
 }
 
 void MediaCodecPlayer::OnError() {
   DCHECK(GetMediaTaskRunner()->BelongsToCurrentThread());
   DVLOG(1) << __FUNCTION__;
 
   // STATE_ERROR blocks all events
   SetState(STATE_ERROR);
 
   ReleaseDecoderResources();
+
+  ui_task_runner_->PostTask(FROM_HERE,
+                            base::Bind(error_cb_, MEDIA_ERROR_DECODE));
 }
 
 void MediaCodecPlayer::OnStarvation(DemuxerStream::Type type) {
   DCHECK(GetMediaTaskRunner()->BelongsToCurrentThread());
   DVLOG(1) << __FUNCTION__ << " stream type:" << type;
 
   if (state_ != STATE_PLAYING)
     return;  // Ignore
 
   SetState(STATE_STOPPING);
@@ skipping 46 matching lines @@
 
 // State machine operations, called on Media thread
 
 void MediaCodecPlayer::SetState(PlayerState new_state) {
   DCHECK(GetMediaTaskRunner()->BelongsToCurrentThread());
 
   DVLOG(1) << "SetState:" << AsString(state_) << " -> " << AsString(new_state);
   state_ = new_state;
 }
 
-void MediaCodecPlayer::SetPendingSurface(gfx::ScopedJavaSurface surface) {
-  DCHECK(GetMediaTaskRunner()->BelongsToCurrentThread());
-  DVLOG(1) << __FUNCTION__;
-
-  video_decoder_->SetPendingSurface(surface.Pass());
-}
-
-bool MediaCodecPlayer::HasPendingSurface() const {
-  return video_decoder_->HasPendingSurface();
-}
-
 void MediaCodecPlayer::SetPendingStart(bool need_to_start) {
   DCHECK(GetMediaTaskRunner()->BelongsToCurrentThread());
   DVLOG(1) << __FUNCTION__ << ": " << need_to_start;
   pending_start_ = need_to_start;
 }
 
 bool MediaCodecPlayer::HasPendingStart() const {
   DCHECK(GetMediaTaskRunner()->BelongsToCurrentThread());
   return pending_start_;
 }
@@ skipping 63 matching lines @@
   base::Closure prefetch_cb = base::BarrierClosure(
       count, base::Bind(&MediaCodecPlayer::OnPrefetchDone, media_weak_this_));
 
   if (do_audio)
     audio_decoder_->Prefetch(prefetch_cb);
 
   if (do_video)
     video_decoder_->Prefetch(prefetch_cb);
 }
 
-void MediaCodecPlayer::StartPlaybackDecoders() {
+void MediaCodecPlayer::StartPlaybackOrBrowserSeek() {
   DCHECK(GetMediaTaskRunner()->BelongsToCurrentThread());
   DVLOG(1) << __FUNCTION__;
 
-  // Configure all streams before the start since
-  // we may discover that browser seek is required.
+  // TODO(timav): consider replacing this method with posting a
+  // browser seek task (i.e. generate an event) from StartPlaybackDecoders().
+
+  StartStatus status = StartPlaybackDecoders();
+
+  switch (status) {
+    case START_BROWSER_SEEK_REQUIRED:
+      // Browser seek
+      SetState(STATE_WAITING_FOR_SEEK);
+      SetPendingStart(true);
+      StopDecoders();
+      RequestDemuxerSeek(GetInterpolatedTime(), true);
+      break;
+    case START_FAILED:
+      GetMediaTaskRunner()->PostTask(FROM_HERE, internal_error_cb_);
+      break;
+    case START_OK:
+      break;
+  }
+}
+
+MediaCodecPlayer::StartStatus MediaCodecPlayer::StartPlaybackDecoders() {
+  DCHECK(GetMediaTaskRunner()->BelongsToCurrentThread());
+  DVLOG(1) << __FUNCTION__;
 
   bool do_audio = !AudioFinished();
   bool do_video = !VideoFinished();
 
-  // If there is nothing to play, the state machine should determine
-  // this at the prefetch state and never call this method.
+  // If there is nothing to play, the state machine should determine this at the
+  // prefetch state and never call this method.
   DCHECK(do_audio || do_video);
 
+  // Configure all streams before the start since we may discover that browser
+  // seek is required. Start with video: if browser seek is required it would
+  // not make sense to configure audio.
+
+  if (do_video) {
+    MediaCodecDecoder::ConfigStatus status = video_decoder_->Configure();
+    switch (status) {
+      case MediaCodecDecoder::CONFIG_OK:
+        break;
+      case MediaCodecDecoder::CONFIG_KEY_FRAME_REQUIRED:
+        // TODO(timav): post a task or return the status?
+        return START_BROWSER_SEEK_REQUIRED;
+      case MediaCodecDecoder::CONFIG_FAILURE:
+        return START_FAILED;
+    }
+  }
+
   if (do_audio) {
     MediaCodecDecoder::ConfigStatus status = audio_decoder_->Configure();
     if (status != MediaCodecDecoder::CONFIG_OK) {
-      GetMediaTaskRunner()->PostTask(FROM_HERE, error_cb_);
-      return;
+      return START_FAILED;
     }
   }
 
-  if (do_video) {
-    MediaCodecDecoder::ConfigStatus status = video_decoder_->Configure();
-    if (status != MediaCodecDecoder::CONFIG_OK) {
-      GetMediaTaskRunner()->PostTask(FROM_HERE, error_cb_);
-      return;
-    }
-  }
-
   // At this point decoder threads should not be running.
   if (!interpolator_.interpolating())
     interpolator_.StartInterpolating();
 
   base::TimeDelta current_time = GetInterpolatedTime();
 
   if (do_audio) {
     if (!audio_decoder_->Start(current_time)) {
-      GetMediaTaskRunner()->PostTask(FROM_HERE, error_cb_);
-      return;
+      return START_FAILED;
     }
 
     // Attach listener on UI thread
     ui_task_runner_->PostTask(FROM_HERE, attach_listener_cb_);
   }
 
   if (do_video) {
     if (!video_decoder_->Start(current_time)) {
-      GetMediaTaskRunner()->PostTask(FROM_HERE, error_cb_);
-      return;
+      return START_FAILED;
     }
   }
+
+  return START_OK;
 }
 
 void MediaCodecPlayer::StopDecoders() {
   DCHECK(GetMediaTaskRunner()->BelongsToCurrentThread());
   DVLOG(1) << __FUNCTION__;
 
+  video_decoder_->SyncStop();
   audio_decoder_->SyncStop();
-  video_decoder_->SyncStop();
 }
 
 void MediaCodecPlayer::RequestToStopDecoders() {
   DCHECK(GetMediaTaskRunner()->BelongsToCurrentThread());
   DVLOG(1) << __FUNCTION__;
 
   bool do_audio = false;
   bool do_video = false;
 
   if (audio_decoder_->IsPrefetchingOrPlaying())
@@ skipping 10 matching lines @@
   if (do_audio)
     audio_decoder_->RequestToStop();
   if (do_video)
     video_decoder_->RequestToStop();
 }
 
 void MediaCodecPlayer::RequestDemuxerSeek(base::TimeDelta seek_time,
                                           bool is_browser_seek) {
   DCHECK(GetMediaTaskRunner()->BelongsToCurrentThread());
   DVLOG(1) << __FUNCTION__ << " " << seek_time
-           << (is_browser_seek ? " browser_seek" : "");
+           << (is_browser_seek ? " BROWSER_SEEK" : "");
 
   // Flush decoders before requesting demuxer.
   audio_decoder_->Flush();
   video_decoder_->Flush();
 
   // Save active seek data. Logically it is attached to STATE_WAITING_FOR_SEEK.
   DCHECK(state_ == STATE_WAITING_FOR_SEEK);
   seek_info_.reset(new SeekInfo(seek_time, is_browser_seek));
 
   demuxer_->RequestDemuxerSeek(seek_time, is_browser_seek);
@@ skipping 11 matching lines @@
 
   // At this point decoder threads should not be running
   if (interpolator_.interpolating())
     interpolator_.StopInterpolating();
 }
 
 void MediaCodecPlayer::CreateDecoders() {
   DCHECK(GetMediaTaskRunner()->BelongsToCurrentThread());
   DVLOG(1) << __FUNCTION__;
 
-  error_cb_ = base::Bind(&MediaCodecPlayer::OnError, media_weak_this_);
+  internal_error_cb_ = base::Bind(&MediaCodecPlayer::OnError, media_weak_this_);
 
   audio_decoder_.reset(new MediaCodecAudioDecoder(
       GetMediaTaskRunner(), base::Bind(&MediaCodecPlayer::RequestDemuxerData,
                                        media_weak_this_, DemuxerStream::AUDIO),
       base::Bind(&MediaCodecPlayer::OnStarvation, media_weak_this_,
                  DemuxerStream::AUDIO),
-      base::Bind(&MediaCodecPlayer::OnStopDone, media_weak_this_), error_cb_,
+      base::Bind(&MediaCodecPlayer::OnStopDone, media_weak_this_),
+      internal_error_cb_,
       base::Bind(&MediaCodecPlayer::OnTimeIntervalUpdate, media_weak_this_,
                  DemuxerStream::AUDIO)));
 
   video_decoder_.reset(new MediaCodecVideoDecoder(
       GetMediaTaskRunner(), base::Bind(&MediaCodecPlayer::RequestDemuxerData,
                                        media_weak_this_, DemuxerStream::VIDEO),
       base::Bind(&MediaCodecPlayer::OnStarvation, media_weak_this_,
                  DemuxerStream::VIDEO),
-      base::Bind(&MediaCodecPlayer::OnStopDone, media_weak_this_), error_cb_,
+      base::Bind(&MediaCodecPlayer::OnStopDone, media_weak_this_),
+      internal_error_cb_,
       base::Bind(&MediaCodecPlayer::OnTimeIntervalUpdate, media_weak_this_,
                  DemuxerStream::VIDEO),
       base::Bind(&MediaCodecPlayer::OnVideoResolutionChanged, media_weak_this_),
       base::Bind(&MediaCodecPlayer::OnVideoCodecCreated, media_weak_this_)));
 }
 
 bool MediaCodecPlayer::AudioFinished() const {
   return audio_decoder_->IsCompleted() || !audio_decoder_->HasStream();
 }
 
@@ skipping 23 matching lines @@
     RETURN_STRING(STATE_WAITING_FOR_SURFACE);
     RETURN_STRING(STATE_WAITING_FOR_SEEK);
     RETURN_STRING(STATE_ERROR);
   }
   return nullptr;  // crash early
 }
 
 #undef RETURN_STRING
 
 }  // namespace media