| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 #ifndef MEDIA_BASE_ANDROID_MEDIA_CODEC_DECODER_H_ | |
| 6 #define MEDIA_BASE_ANDROID_MEDIA_CODEC_DECODER_H_ | |
| 7 | |
| 8 #include <stddef.h> | |
| 9 | |
| 10 #include <memory> | |
| 11 | |
| 12 #include "base/android/scoped_java_ref.h" | |
| 13 #include "base/callback.h" | |
| 14 #include "base/macros.h" | |
| 15 #include "base/memory/ref_counted.h" | |
| 16 #include "base/single_thread_task_runner.h" | |
| 17 #include "base/synchronization/lock.h" | |
| 18 #include "base/threading/thread.h" | |
| 19 #include "base/time/time.h" | |
| 20 #include "media/base/android/access_unit_queue.h" | |
| 21 #include "media/base/android/demuxer_stream_player_params.h" | |
| 22 | |
| 23 namespace media { | |
| 24 | |
| 25 struct FrameStatistics; | |
| 26 class MediaCodecBridge; | |
| 27 | |
| 28 // The decoder for MediaCodecPlayer. | |
| 29 // This class accepts the incoming data into AccessUnitQueue and works with | |
| 30 // MediaCodecBridge for decoding and rendering the frames. The MediaCodecPlayer | |
| 31 // has two decoder objects: audio and video. | |
| 32 // | |
| 33 // The decoder works on two threads. The data from demuxer comes on Media | |
| 34 // thread. The commands from MediaCodecPlayer, such as Prefetch, Start, | |
| 35 // RequestToStop also come on the Media thread. The operations with MediaCodec | |
| 36 // buffers and rendering happen on a separate thread called Decoder thread. | |
| 37 // This class creates, starts and stops it as necessary. | |
| 38 // | |
| 39 // Decoder's internal state machine goes through the following states: | |
| 40 // | |
| 41 // [ Stopped ] <------------------- (any state except Error) | |
| 42 // | | | | |
| 43 // | Prefetch |--- internal ------| | |
| 44 // v | transition v | |
| 45 // [ Prefetching ] | [ Error ] | |
| 46 // | | | |
| 47 // | internal transition | | |
| 48 // v | Error recovery: | |
| 49 // [ Prefetched ] | | |
| 50 // | | (any state including Error) | |
| 51 // | Configure and Start | | | |
| 52 // v | | ReleaseDecoderResources | |
| 53 // [ Running ] | v | |
| 54 // | | [ InEmergencyStop ] | |
| 55 // | RequestToStop | | | |
| 56 // v | |(decoder thread stopped) | |
| 57 // [ Stopping ] ------------------- v | |
| 58 // [ Stopped ] | |
| 59 // | |
| 60 // [ Stopped ] -------------------- | |
| 61 // ^ | | |
| 62 // | Flush | | |
| 63 // --------------------------- | |
| 64 | |
| 65 // (any state except Error) | |
| 66 // | | |
| 67 // | SyncStop | |
| 68 // v | |
| 69 // [ InEmergencyStop ] | |
| 70 // | | |
| 71 // |(decoder thread stopped) | |
| 72 // v | |
| 73 // [ Stopped ] | |
| 74 | |
| 75 // Here is the workflow that is expected to be maintained by a caller, which is | |
| 76 // MediaCodecPlayer currently. | |
| 77 // | |
| 78 // [ Stopped ] | |
| 79 // | | |
| 80 // | Prefetch | |
| 81 // v | |
| 82 // [ Prefetching ] | |
| 83 // | | |
| 84 // | (Enough data received) | |
| 85 // v | |
| 86 // [ Prefetched ] | |
| 87 // | | |
| 88 // | <---------- SetDemuxerConfigs (*) | |
| 89 // | | |
| 90 // | <---------- SetVideoSurface (**) | |
| 91 // | | |
| 92 // | Configure --------------------------------------------+ | |
| 93 // | | | |
| 94 // v v | |
| 95 // ( Config Succeeded ) ( Key frame required ) | |
| 96 // | | | |
| 97 // | Start | | |
| 98 // v | | |
| 99 // [ Running ] ------------------------------+ | | |
| 100 // | | | | |
| 101 // | | | | |
| 102 // | RequestToStop | SyncStop | SyncStop | |
| 103 // | | | | |
| 104 // [ Stopping ] | | | |
| 105 // | | | | |
| 106 // | ( Last frame rendered ) | | | |
| 107 // | | | | |
| 108 // | | | | |
| 109 // v | | | |
| 110 // [ Stopped ] <-----------------------------+-----------------+ | |
| 111 // | |
| 112 // | |
| 113 // (*) Demuxer configs is a precondition to Configure(), but MediaCodecPlayer | |
| 114 // has stricter requirements and they are set before Prefetch(). | |
| 115 // | |
| 116 // (**) VideoSurface is a precondition to video decoder Configure(), can be set | |
| 117 // any time before Configure(). | |
| 118 | |
| 119 class MediaCodecDecoder { | |
| 120 public: | |
| 121 // The result of MediaCodec configuration, used by MediaCodecPlayer. | |
| 122 enum ConfigStatus { | |
| 123 kConfigFailure = 0, | |
| 124 kConfigOk, | |
| 125 kConfigKeyFrameRequired, | |
| 126 }; | |
| 127 | |
| 128 // The decoder reports current playback time to the MediaCodecPlayer. | |
| 129 // For audio, the parameters designate the beginning and end of a time | |
| 130 // interval. The beginning is the estimated time that is playing right now. | |
| 131 // The end is the playback time of the last buffered data. During normal | |
| 132 // playback the subsequent intervals overlap. | |
| 133 // For video both values are PTS of the corresponding frame, i.e. the interval | |
| 134 // has zero width. | |
| 135 // The third parameter means "postpone", it is set to true if the actual | |
| 136 // rendering will start in a later point in time. This only happens with | |
| 137 // audio after preroll. The MediaCodecPlayer might decide to update the | |
| 138 // current time but not pass it to the upper layer. | |
| 139 typedef base::Callback<void(base::TimeDelta, base::TimeDelta, bool)> | |
| 140 SetTimeCallback; | |
| 141 | |
| 142 // MediaCodecDecoder constructor. | |
| 143 // Parameters: | |
| 144 // decoder_thread_name: | |
| 145 // The thread name to be passed to decoder thread constructor. | |
| 146 // media_task_runner: | |
| 147 // A task runner for the controlling thread. All public methods should be | |
| 148 // called on this thread, and callbacks are delivered on this thread. | |
| 149 // The MediaCodecPlayer uses a dedicated (Media) thread for this. | |
| 150 // frame_statistics: | |
| 151 // A pointer to FrameStatistics object which gathers playback quality | |
| 152 // related data. | |
| 153 // external_request_data_cb: | |
| 154 // Called periodically as the amount of internally stored data decreases. | |
| 155 // The receiver should call OnDemuxerDataAvailable() with more data. | |
| 156 // starvation_cb: | |
| 157 // Called when starvation is detected. The decoder state does not change. | |
| 158 // The player is supposed to stop and then prefetch the decoder. | |
| 159 // decoder_drained_cb: | |
| 160 // Called when decoder is drained for reconfiguration. | |
| 161 // stop_done_cb: | |
| 162 // Called when async stop request is completed. | |
| 163 // waiting_for_decryption_key_cb: | |
| 164 // Will be executed whenever the key needed to decrypt the stream is not | |
| 165 // available. | |
| 166 // error_cb: | |
| 167 // Called when a MediaCodec error occurred. If this happens, a player has | |
| 168 // to either call ReleaseDecoderResources() or destroy the decoder object. | |
| 169 MediaCodecDecoder( | |
| 170 const char* decoder_thread_name, | |
| 171 const scoped_refptr<base::SingleThreadTaskRunner>& media_task_runner, | |
| 172 FrameStatistics* frame_statistics, | |
| 173 const base::Closure& external_request_data_cb, | |
| 174 const base::Closure& starvation_cb, | |
| 175 const base::Closure& decoder_drained_cb, | |
| 176 const base::Closure& stop_done_cb, | |
| 177 const base::Closure& waiting_for_decryption_key_cb, | |
| 178 const base::Closure& error_cb); | |
| 179 | |
| 180 virtual ~MediaCodecDecoder(); | |
| 181 | |
| 182 virtual const char* class_name() const; | |
| 183 | |
| 184 // MediaCodecDecoder exists through the whole lifetime of the player | |
| 185 // to support dynamic addition and removal of the streams. | |
| 186 // This method returns true if the current stream (audio or video) | |
| 187 // is currently active. | |
| 188 virtual bool HasStream() const = 0; | |
| 189 | |
| 190 // Stores configuration for the use of upcoming Configure() | |
| 191 virtual void SetDemuxerConfigs(const DemuxerConfigs& configs) = 0; | |
| 192 | |
| 193 // Returns true if the DemuxerConfigs announce that content is encrypted and | |
| 194 // that MediaCrypto is required for configuration. | |
| 195 virtual bool IsContentEncrypted() const = 0; | |
| 196 | |
| 197 // Stops decoder thread, releases the MediaCodecBridge and other resources. | |
| 198 virtual void ReleaseDecoderResources() = 0; | |
| 199 | |
| 200 // Flushes the MediaCodec, after that resets the AccessUnitQueue and blocks | |
| 201 // the input. Decoder thread should not be running. | |
| 202 virtual void Flush(); | |
| 203 | |
| 204 // Releases MediaCodecBridge and any related buffers or references. | |
| 205 virtual void ReleaseMediaCodec(); | |
| 206 | |
| 207 // Returns corresponding conditions. | |
| 208 bool IsPrefetchingOrPlaying() const; | |
| 209 bool IsStopped() const; | |
| 210 bool IsCompleted() const; | |
| 211 bool NotCompletedAndNeedsPreroll() const; | |
| 212 | |
| 213 // Forces reconfiguraton on the next Configure(). | |
| 214 void SetNeedsReconfigure(); | |
| 215 | |
| 216 // Sets preroll timestamp and requests preroll. | |
| 217 void SetPrerollTimestamp(base::TimeDelta preroll_ts); | |
| 218 | |
| 219 // Starts prefetching: accumulates enough data in AccessUnitQueue. | |
| 220 // Decoder thread is not running. | |
| 221 void Prefetch(const base::Closure& prefetch_done_cb); | |
| 222 | |
| 223 // Configures MediaCodec. | |
| 224 ConfigStatus Configure(jobject media_crypto); | |
| 225 | |
| 226 // Starts the decoder for prerolling. This method starts the decoder thread. | |
| 227 bool Preroll(const base::Closure& preroll_done_cb); | |
| 228 | |
| 229 // Starts the decoder after preroll is not needed, starting decoder thread | |
| 230 // if it has not started yet. | |
| 231 bool Start(base::TimeDelta start_timestamp); | |
| 232 | |
| 233 // Stops the playback process synchronously. This method stops the decoder | |
| 234 // thread synchronously, and then releases all MediaCodec buffers. | |
| 235 void SyncStop(); | |
| 236 | |
| 237 // Requests to stop the playback and returns. | |
| 238 // Decoder will stop asynchronously after all the dequeued output buffers | |
| 239 // are rendered. | |
| 240 void RequestToStop(); | |
| 241 | |
| 242 // Notification posted when asynchronous stop is done or playback completed. | |
| 243 void OnLastFrameRendered(bool eos_encountered); | |
| 244 | |
| 245 // Notification posted when last prerolled frame has been returned to codec. | |
| 246 void OnPrerollDone(); | |
| 247 | |
| 248 // Puts the incoming data into AccessUnitQueue. | |
| 249 void OnDemuxerDataAvailable(const DemuxerData& data); | |
| 250 | |
| 251 // For testing only. | |
| 252 | |
| 253 // Returns true if the decoder is in kPrerolling state. | |
| 254 bool IsPrerollingForTests() const; | |
| 255 | |
| 256 // Drains decoder and reconfigures for each |kConfigChanged|. | |
| 257 void SetAlwaysReconfigureForTests(); | |
| 258 | |
| 259 // Sets the notification to be called when MediaCodec is created. | |
| 260 void SetCodecCreatedCallbackForTests(base::Closure cb); | |
| 261 | |
| 262 protected: | |
| 263 enum RenderMode { | |
| 264 kRenderSkip = 0, | |
| 265 kRenderAfterPreroll, | |
| 266 kRenderNow, | |
| 267 }; | |
| 268 | |
| 269 // Returns true if the new DemuxerConfigs requires MediaCodec | |
| 270 // reconfiguration. | |
| 271 virtual bool IsCodecReconfigureNeeded(const DemuxerConfigs& next) const = 0; | |
| 272 | |
| 273 // Does the part of MediaCodecBridge configuration that is specific | |
| 274 // to audio or video. | |
| 275 virtual ConfigStatus ConfigureInternal(jobject media_crypto) = 0; | |
| 276 | |
| 277 // Associates PTS with device time so we can calculate delays. | |
| 278 // We use delays for video decoder only. | |
| 279 virtual void AssociateCurrentTimeWithPTS(base::TimeDelta current_time) {} | |
| 280 | |
| 281 // Invalidate delay calculation. We use delays for video decoder only. | |
| 282 virtual void DissociatePTSFromTime() {} | |
| 283 | |
| 284 // Processes the change of the output format, varies by stream. | |
| 285 // Returns true if this processing succeeded. | |
| 286 virtual bool OnOutputFormatChanged() = 0; | |
| 287 | |
| 288 // Renders the decoded frame and releases output buffer, or posts a delayed | |
| 289 // task to do it at a later time. Returnes false if an error occurred. | |
| 290 virtual bool Render(int buffer_index, | |
| 291 size_t offset, | |
| 292 size_t size, | |
| 293 RenderMode render_mode, | |
| 294 base::TimeDelta pts, | |
| 295 bool eos_encountered) = 0; | |
| 296 | |
| 297 // Returns the number of delayed task (we might have them for video). | |
| 298 virtual int NumDelayedRenderTasks() const; | |
| 299 | |
| 300 // Releases output buffers that are dequeued and not released yet (video). | |
| 301 virtual void ReleaseDelayedBuffers() {} | |
| 302 | |
| 303 #ifndef NDEBUG | |
| 304 // For video, checks that access unit is the key frame or stand-alone EOS. | |
| 305 virtual void VerifyUnitIsKeyFrame(const AccessUnit* unit) const {} | |
| 306 #endif | |
| 307 | |
| 308 // Helper methods. | |
| 309 | |
| 310 // Synchroniously stop decoder thread. | |
| 311 void DoEmergencyStop(); | |
| 312 | |
| 313 // Returns true if we are in the process of sync stop. | |
| 314 bool InEmergencyStop() const { return GetState() == kInEmergencyStop; } | |
| 315 | |
| 316 // Notifies the decoder if the frame is the last one. | |
| 317 void CheckLastFrame(bool eos_encountered, bool has_delayed_tasks); | |
| 318 | |
| 319 const char* AsString(RenderMode render_mode); | |
| 320 | |
| 321 // Protected data. | |
| 322 | |
| 323 // We call MediaCodecBridge on this thread for both input and output buffers. | |
| 324 base::Thread decoder_thread_; | |
| 325 | |
| 326 // Object for posting tasks on Media thread. | |
| 327 scoped_refptr<base::SingleThreadTaskRunner> media_task_runner_; | |
| 328 | |
| 329 // Statistics for UMA. | |
| 330 FrameStatistics* frame_statistics_; | |
| 331 | |
| 332 // Controls Android MediaCodec | |
| 333 std::unique_ptr<MediaCodecBridge> media_codec_bridge_; | |
| 334 | |
| 335 // The queue of access units. | |
| 336 AccessUnitQueue au_queue_; | |
| 337 | |
| 338 // Flag forces reconfiguration even if |media_codec_bridge_| exists. Currently | |
| 339 // is set by video decoder when the video surface changes. | |
| 340 bool needs_reconfigure_; | |
| 341 | |
| 342 // Flag forces to drain decoder in the process of dynamic reconfiguration. | |
| 343 bool drain_decoder_; | |
| 344 | |
| 345 // For tests only. Forces to always reconfigure for |kConfigChanged| unit. | |
| 346 bool always_reconfigure_for_tests_; | |
| 347 | |
| 348 // For tests only. Callback to be callned when MediaCodec is created. | |
| 349 base::Closure codec_created_for_tests_cb_; | |
| 350 | |
| 351 private: | |
| 352 enum DecoderState { | |
| 353 kStopped = 0, | |
| 354 kPrefetching, | |
| 355 kPrefetched, | |
| 356 kPrerolling, | |
| 357 kPrerolled, | |
| 358 kRunning, | |
| 359 kStopping, | |
| 360 kInEmergencyStop, | |
| 361 kError, | |
| 362 }; | |
| 363 | |
| 364 // Helper method that processes an error from MediaCodec. | |
| 365 void OnCodecError(); | |
| 366 | |
| 367 // Requests data. Ensures there is no more than one request at a time. | |
| 368 void RequestData(); | |
| 369 | |
| 370 // Prefetching callback that is posted to Media thread | |
| 371 // in the kPrefetching state. | |
| 372 void PrefetchNextChunk(); | |
| 373 | |
| 374 // The callback to do actual playback. Posted to Decoder thread | |
| 375 // in the kRunning state. | |
| 376 void ProcessNextFrame(); | |
| 377 | |
| 378 // Helper method for ProcessNextFrame. | |
| 379 // Pushes one input buffer to the MediaCodec if the codec can accept it. | |
| 380 // Returns false if there was MediaCodec error. | |
| 381 bool EnqueueInputBuffer(); | |
| 382 | |
| 383 // Helper method for EnqueueInputBuffer. | |
| 384 // Gets the next data frame from the queue, requesting more data and saving | |
| 385 // configuration changes on the way. Sets |drain_decoder| to true of any of | |
| 386 // the configuration changes requires draining the decoder. Returns the Info | |
| 387 // pointing to the current data unit ot empty Info if it got past the end of | |
| 388 // the queue. | |
| 389 AccessUnitQueue::Info AdvanceAccessUnitQueue(bool* drain_decoder); | |
| 390 | |
| 391 // Helper method for ProcessNextFrame. | |
| 392 // Pulls all currently available output frames and renders them. | |
| 393 // Returns true if we need to continue decoding process, i.e post next | |
| 394 // ProcessNextFrame method, and false if we need to stop decoding. | |
| 395 bool DepleteOutputBufferQueue(); | |
| 396 | |
| 397 DecoderState GetState() const; | |
| 398 void SetState(DecoderState state); | |
| 399 const char* AsString(DecoderState state); | |
| 400 | |
| 401 // Private Data. | |
| 402 | |
| 403 // External data request callback that is passed to decoder. | |
| 404 base::Closure external_request_data_cb_; | |
| 405 | |
| 406 // These notifications are called on corresponding conditions. | |
| 407 base::Closure prefetch_done_cb_; | |
| 408 base::Closure starvation_cb_; | |
| 409 base::Closure preroll_done_cb_; | |
| 410 base::Closure decoder_drained_cb_; | |
| 411 base::Closure stop_done_cb_; | |
| 412 base::Closure waiting_for_decryption_key_cb_; | |
| 413 base::Closure error_cb_; | |
| 414 | |
| 415 // Data request callback that is posted by decoder internally. | |
| 416 base::Closure request_data_cb_; | |
| 417 | |
| 418 // Callback used to post OnCodecError method. | |
| 419 base::Closure internal_error_cb_; | |
| 420 | |
| 421 // Callback for posting OnPrerollDone method. | |
| 422 base::Closure internal_preroll_done_cb_; | |
| 423 | |
| 424 // Internal state. | |
| 425 DecoderState state_; | |
| 426 mutable base::Lock state_lock_; | |
| 427 | |
| 428 // Preroll timestamp is set if we need preroll and cleared after we done it. | |
| 429 base::TimeDelta preroll_timestamp_; | |
| 430 | |
| 431 // Index of the dequeued and filled buffer that we keep trying to enqueue. | |
| 432 // Such buffer appears in MEDIA_CODEC_NO_KEY processing. | |
| 433 int pending_input_buf_index_; | |
| 434 | |
| 435 // Set to true when MediaCodec internal buffers are filled up. | |
| 436 bool is_prepared_; | |
| 437 | |
| 438 // Flag is set when the EOS is enqueued into MediaCodec. Reset by Flush. | |
| 439 bool eos_enqueued_; | |
| 440 | |
| 441 // Flag is set when NO_KEY error is received from QueueSecureInputBuffer. | |
| 442 // Reset after we stop. | |
| 443 bool missing_key_reported_; | |
| 444 | |
| 445 // Flag is set when the EOS is received in MediaCodec output. Reset by Flush. | |
| 446 bool completed_; | |
| 447 | |
| 448 // Flag to ensure we post last frame notification once. | |
| 449 bool last_frame_posted_; | |
| 450 | |
| 451 // Indicates whether the data request is in progress. | |
| 452 bool is_data_request_in_progress_; | |
| 453 | |
| 454 // Indicates whether the incoming data should be ignored. | |
| 455 bool is_incoming_data_invalid_; | |
| 456 | |
| 457 #ifndef NDEBUG | |
| 458 // When set, we check that the following video frame is the key frame. | |
| 459 bool verify_next_frame_is_key_; | |
| 460 #endif | |
| 461 | |
| 462 // NOTE: Weak pointers must be invalidated before all other member variables. | |
| 463 base::WeakPtrFactory<MediaCodecDecoder> weak_factory_; | |
| 464 | |
| 465 DISALLOW_COPY_AND_ASSIGN(MediaCodecDecoder); | |
| 466 }; | |
| 467 | |
| 468 } // namespace media | |
| 469 | |
| 470 #endif // MEDIA_BASE_ANDROID_MEDIA_CODEC_DECODER_H_ | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2276343005:
Delete MediaCodecPlayer, it's time! (Closed)
