MediaCodecPlayer (stage 1 - play/pause only)

media/base/android/media_codec_player.h

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef MEDIA_BASE_ANDROID_MEDIA_CODEC_PLAYER_H_
 #define MEDIA_BASE_ANDROID_MEDIA_CODEC_PLAYER_H_
 
 #include "base/android/scoped_java_ref.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/memory/weak_ptr.h"
 #include "base/threading/thread.h"
+#include "base/time/default_tick_clock.h"
 #include "media/base/android/demuxer_android.h"
 #include "media/base/android/media_player_android.h"
+#include "media/base/demuxer_stream.h"
 #include "media/base/media_export.h"
+#include "media/base/time_delta_interpolator.h"
+#include "ui/gfx/geometry/size.h"
 #include "ui/gl/android/scoped_java_surface.h"
 
+// The MediaCodecPlayer class implements the media player by using Android's
+// MediaCodec. It differs from MediaSourcePlayer in that it removes most
+// processing away from the UI thread: it uses a dedicated Media thread to
+// receive the data and to handle the commands.
+
+// The player works as a state machine. Here are relationships between states:
+//
+//   [ Paused ] ------------------------                         (Any state)
+//        |                             |                             |
+//        |                             v                             v
+//        | <------------------[ WaitingForConfig ]               [ Error ]
+//        |
+//        |
+//        | <----------------------------------------------
+//        v                                                |
+//   [ Prefetching ] -------------------                   |
+//        |                             |                  |
+//        |                             v                  |
+//        | <-----------------[ WaitingForSurface ]        |
+//        v                                                |
+//   [ Playing ]                                           |
+//        |                                                |
+//        |                                                |
+//        v                                                |
+//   [ Stopping ] --------------------------------> [ WaitingForSeek ]
+
+
+//  Events and actions for pause/resume workflow.
+//  ---------------------------------------------
+//
+//                                           Start, no config:
+//      ------------------------> [ Paused ] -----------------> [   Waiting   ]
+//      |  StopDone:                                            [ for configs ]
+//      |                            ^  |                              /
+//      |                            |  |                             /
+//      |                    Pause:  |  | Start w/config:            /
+//      |                            |  |    dec.Prefetch           /
+//      |                            |  |                          /
+//      |                            |  |                         /
+//      |                            |  |                        /
+//      |                            |  |                       /
+// DemuxerConfigs:

[wolenetz, 2015/06/25 22:31:39]

On which transition(s) is DemuxerConfigs: to live? It looks wrong.

[Tima Vaisburd, 2015/06/26 00:02:06]

Indeed! Thank you. Fixed.

+//      |                            |  |                      /   dec.Prefetch
+//      |                            |  |                     /
+//      |                            |  |                    /
+//      |                            |  v                   /
+//      |                                                  /
+//      |   ------------------> [ Prefetching ]  <--------/      [   Waiting   ]
+//      |   |                   [             ] -------------->  [ for surface ]
+//      |   |                          |         PrefetchDone,         /
+//      |   |                          |          no surface:         /
+//      |   |                          |                             /
+//      |   |                          |                            /
+//      |   | StopDone w/              |                           /
+//      |   | pending start:           | PrefetchDone:            /
+//      |   |    dec.Prefetch          |   dec.Start             /
+//      |   |                          |                        / SetSurface:
+//      |   |                          |                       /     dec.Start
+//      |   |                          |                      /
+//      |   |                          v                     /
+//      |   |                                               /
+//      |   |                     [ Playing ]   <----------/
+//      |   |
+//      |   |                          |
+//      |   |                          |
+//      |   |                          | Pause: dec.RequestToStop
+//      |   |                          |
+//      |   |                          |
+//      |   |                          v
+//      |   |
+//      ------------------------- [ Stopping ]
+
+
+//   Events and actions for seek workflow.

[wolenetz, 2015/06/25 22:31:39]

nit: Seek isn't implemented in this CL, so perhaps this portion needs to be
added when Seek is implemented? No strong feeling; I'm fine if you include it.
Though adding it later would also help trigger code reviewers to validate the
seek implementation against the state diagram if added at same time...

[Tima Vaisburd, 2015/06/26 00:02:05]

Actually, I quite like this idea. Especially now as I have the seek CL that I'm
working on. I'll move it there.

+//   -------------------------------------
+//
+//                   Seek:                   --       --
+//                       demuxer.RequestSeek |         |
+//  [   Paused    ] -----------------------> |         |
+//  [             ] <----------------------- |         |--
+//                      SeekDone:            |         |  |
+//                                           |         |  |
+//                                           |         |  |
+//                                           |         |  |
+//                                           |         |  | Start:
+//                    Seek:                  |         |  |   SetPendingStart
+//                      dec.Stop             |         |  |
+//                      SetPendingSeek       |         |  |
+//                      demuxer.RequestSeek  |         |  |
+//  [ Prefetching ] -----------------------> |         |  |
+//  [             ] <----------------------  |         |  | Pause:
+//        |          SeekDone                |         |  |   RemovePendingStart
+//        |          w/pending start:        |         |  |
+//        |            dec.Prefetch          | Waiting |  |
+//        |                                  |   for   |  | Seek:
+//        |                                  |   seek  |  |   SetPendingSeek
+//        |                                  |         |  |
+//        | PrefetchDone: dec.Start          |         |  |
+//        |                                  |         |  | SeekDone
+//        v                                  |         |  | w/pending seek:
+//                                           |         |  | demuxer.RequestSeek
+//  [   Playing   ]                          |         |  |
+//                                           |         |  |
+//        |                                  |         |<-
+//        | Seek: SetPendingStart            |         |
+//        |       SetPendingSeek             |         |
+//        |       dec.RequestToStop          |         |
+//        |                                  |         |
+//        |                                  |         |
+//        v                                  |         |
+//                                           |         |
+//  [  Stopping   ] -----------------------> |         |
+//                  StopDone                 --       --
+//                    w/pending seek:
+//                      demuxer.RequestSeek
+
 namespace media {
 
 class BrowserCdm;
+class MediaCodecAudioDecoder;
+class MediaCodecVideoDecoder;
 
 // Returns the task runner for the media thread
 MEDIA_EXPORT scoped_refptr<base::SingleThreadTaskRunner> GetMediaTaskRunner();
 
-
-// This class implements the media player using Android's MediaCodec.
-// It differs from MediaSourcePlayer in that it removes most
-// processing away from UI thread: it uses a dedicated Media thread
-// to receive the data and to handle commands.
 class MEDIA_EXPORT MediaCodecPlayer : public MediaPlayerAndroid,
                                       public DemuxerAndroidClient {
  public:
+  // Typedefs for the notification callbacks
+  typedef base::Callback<void(base::TimeDelta, const gfx::Size&)>
+      MetadataChangedCallback;
+
+  typedef base::Callback<void(base::TimeDelta, base::TimeTicks)>
+      TimeUpdateCallback;
+
   // Constructs a player with the given ID and demuxer. |manager| must outlive
   // the lifetime of this object.
   MediaCodecPlayer(int player_id,
-                   MediaPlayerManager* manager,
+                   base::WeakPtr<MediaPlayerManager> manager,
                    const RequestMediaResourcesCB& request_media_resources_cb,
                    scoped_ptr<DemuxerAndroid> demuxer,
                    const GURL& frame_url);
   ~MediaCodecPlayer() override;
 
+  // A helper method that performs the media thread part of initialization.
+  void Initialize();
+
   // MediaPlayerAndroid implementation.
   void DeleteOnCorrectThread() override;
   void SetVideoSurface(gfx::ScopedJavaSurface surface) override;
   void Start() override;
   void Pause(bool is_media_related_action) override;
   void SeekTo(base::TimeDelta timestamp) override;
   void Release() override;
   void SetVolume(double volume) override;
   int GetVideoWidth() override;
   int GetVideoHeight() override;
   base::TimeDelta GetCurrentTime() override;
   base::TimeDelta GetDuration() override;
   bool IsPlaying() override;
   bool CanPause() override;
   bool CanSeekForward() override;
   bool CanSeekBackward() override;
   bool IsPlayerReady() override;
   void SetCdm(BrowserCdm* cdm) override;
 
   // DemuxerAndroidClient implementation.
   void OnDemuxerConfigsAvailable(const DemuxerConfigs& params) override;
   void OnDemuxerDataAvailable(const DemuxerData& params) override;
   void OnDemuxerSeekDone(base::TimeDelta actual_browser_seek_time) override;
   void OnDemuxerDurationChanged(base::TimeDelta duration) override;
 
-  // Helper methods
-  void Initialize();
-  void DestroySelf();
+ private:
+  // The state machine states.
+  enum PlayerState {
+    STATE_PAUSED,
+    STATE_WAITING_FOR_CONFIG,
+    STATE_PREFETCHING,
+    STATE_PLAYING,
+    STATE_STOPPING,
+    STATE_WAITING_FOR_SURFACE,
+    STATE_WAITING_FOR_SEEK,
+    STATE_ERROR,
+  };
 
- private:
+  // Cached values for the manager.
+  struct MediaMetadata {
+    base::TimeDelta duration;
+    gfx::Size video_size;
+  };
+
+  // MediaPlayerAndroid implementation.
+  // This method caches the data and calls manager's OnMediaMetadataChanged().
+  void OnMediaMetadataChanged(base::TimeDelta duration,
+                              const gfx::Size& video_size) override;
+
+  // This method cached the current time and calls manager's OnTimeUpdate().

[wolenetz, 2015/06/25 22:31:39]

nit: s/cached/caches/

[Tima Vaisburd, 2015/06/26 00:02:05]

Done.

+  void OnTimeUpdate(base::TimeDelta current_timestamp,
+                    base::TimeTicks current_time_ticks) override;
+
+  // Callbacks from decoders
+  void RequestDemuxerData(DemuxerStream::Type stream_type);
+  void OnPrefetchDone();

[wolenetz, 2015/06/25 22:31:39]

nit: why not parameterize (all of) these decoder callbacks by stream type?

[Tima Vaisburd, 2015/06/26 00:02:05]

OnPrefetchDone() is a barrier callback. OnError() is called not only from
decoders, but from the player itself. There is one case that OnStopDone() is
also generated by the player itself. I added the stream type to OnStarvation()
and OnTimeUpdate().

[wolenetz, 2015/06/26 00:59:25]

Acknowledged.

+  void OnStarvation();
+  void OnStopDone();
+  void OnError();
+  void OnTimeIntervalUpdate(base::TimeDelta now_playing,
+                            base::TimeDelta last_buffered);
+
+  // Callbacks from video decoder
+  void OnVideoCodecCreated();
+  void OnVideoSizeChanged(const gfx::Size& size);
+
+  // Operations called from the state machine.
+  void SetState(PlayerState new_state);
+  void SetPendingSurface(gfx::ScopedJavaSurface surface);
+  bool HasPendingSurface();
+  void SetPendingStart(bool need_to_start);
+  bool HasPendingStart();
+  bool HasVideo();
+  bool HasAudio();
+  void SetDemuxerConfigs(const DemuxerConfigs& configs);
+  void StartPrefetchDecoders();
+  void StartPlaybackDecoders();
+  void StopDecoders();
+  void RequestToStopDecoders();
+  void ReleaseDecoderResources();
+
+  // Helper methods.
+  void CreateDecoders();
+  bool AudioFinished();
+  bool VideoFinished();
+  base::TimeDelta GetInterpolatedTime();
+
+  static const char* AsString(PlayerState state);
+
+  // Data.
+
   // Object for posting tasks on UI thread.
   scoped_refptr<base::SingleThreadTaskRunner> ui_task_runner_;
 
+  // Major components: demuxer, audio and video decoders.
   scoped_ptr<DemuxerAndroid> demuxer_;
+  scoped_ptr<MediaCodecAudioDecoder> audio_decoder_;
+  scoped_ptr<MediaCodecVideoDecoder> video_decoder_;
 
-  base::WeakPtr<MediaCodecPlayer> weak_this_;
+  // The state of the state machine.
+  PlayerState state_;
+
+  // Notification callbacks, they call MediaPlayerManager.
+  base::Closure request_resources_cb_;
+  TimeUpdateCallback time_update_cb_;
+  base::Closure completion_cb_;
+
+  // A callback that updates metadata cache and calls the manager.
+  MetadataChangedCallback metadata_changed_cb_;
+
+  // We call the base class' AttachListener() and DetachListener() methods on UI
+  // thread with these callbacks.
+  base::Closure attach_listener_cb_;
+  base::Closure detach_listener_cb_;
+
+  // Error callback is posted by decoders if they get errors from MediaCodec

[wolenetz, 2015/06/25 22:31:39]

Does the player not intercept all error callbacks from the decoders? How else
can it correctly transition to Error state in those cases?

[Tima Vaisburd, 2015/06/26 00:02:06]

That's what I meant. I tried to also say why decoders post error callback, but
it turned out confusing. Removed "if they get..."

+  // operations of by this class itself if we cannot configure or start decoder.

[wolenetz, 2015/06/25 22:31:39]

nit: s/of/or/

[Tima Vaisburd, 2015/06/26 00:02:06]

Done.

+  base::Closure error_cb_;
+
+  // Total duration reported by demuxer.
+  base::TimeDelta duration_;
+
+  // base::TickClock used by |interpolator_|.
+  base::DefaultTickClock default_tick_clock_;
+
+  // Tracks the most recent media time update and provides interpolated values
+  // as playback progresses.
+  TimeDeltaInterpolator interpolator_;
+
+  // Pending data to be picked up by the upcoming state.
+  gfx::ScopedJavaSurface pending_surface_;
+  bool pending_start_;
+
+  // Configuration data for the manager, accessed on the UI thread.
+  MediaMetadata metadata_cache_;
+
+  // Cached current time, accessed on UI thread.
+  base::TimeDelta current_time_cache_;
+
+  base::WeakPtr<MediaCodecPlayer> media_weak_this_;
   // NOTE: Weak pointers must be invalidated before all other member variables.
-  base::WeakPtrFactory<MediaCodecPlayer> weak_factory_;
+  base::WeakPtrFactory<MediaCodecPlayer> media_weak_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(MediaCodecPlayer);
 };
 
 }  // namespace media
 
 #endif  // MEDIA_BASE_ANDROID_MEDIA_CODEC_PLAYER_H_