| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2013 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 #ifndef CONTENT_COMMON_GPU_MEDIA_ANDROID_VIDEO_DECODE_ACCELERATOR_H_ | |
| 6 #define CONTENT_COMMON_GPU_MEDIA_ANDROID_VIDEO_DECODE_ACCELERATOR_H_ | |
| 7 | |
| 8 #include <stdint.h> | |
| 9 | |
| 10 #include <list> | |
| 11 #include <map> | |
| 12 #include <memory> | |
| 13 #include <queue> | |
| 14 #include <string> | |
| 15 #include <vector> | |
| 16 | |
| 17 #include "base/compiler_specific.h" | |
| 18 #include "base/threading/thread_checker.h" | |
| 19 #include "base/timer/timer.h" | |
| 20 #include "content/common/content_export.h" | |
| 21 #include "content/common/gpu/media/avda_state_provider.h" | |
| 22 #include "content/common/gpu/media/gpu_video_decode_accelerator_helpers.h" | |
| 23 #include "gpu/command_buffer/service/gles2_cmd_decoder.h" | |
| 24 #include "gpu/command_buffer/service/gpu_preferences.h" | |
| 25 #include "media/base/android/media_drm_bridge_cdm_context.h" | |
| 26 #include "media/base/android/sdk_media_codec_bridge.h" | |
| 27 #include "media/base/media_keys.h" | |
| 28 #include "media/video/video_decode_accelerator.h" | |
| 29 #include "ui/gl/android/scoped_java_surface.h" | |
| 30 | |
| 31 namespace gfx { | |
| 32 class SurfaceTexture; | |
| 33 } | |
| 34 | |
| 35 namespace content { | |
| 36 | |
| 37 // A VideoDecodeAccelerator implementation for Android. | |
| 38 // This class decodes the input encoded stream by using Android's MediaCodec | |
| 39 // class. http://developer.android.com/reference/android/media/MediaCodec.html | |
| 40 // It delegates attaching pictures to PictureBuffers to a BackingStrategy, but | |
| 41 // otherwise handles the work of transferring data to / from MediaCodec. | |
| 42 class CONTENT_EXPORT AndroidVideoDecodeAccelerator | |
| 43 : public media::VideoDecodeAccelerator, | |
| 44 public AVDAStateProvider { | |
| 45 public: | |
| 46 using OutputBufferMap = std::map<int32_t, media::PictureBuffer>; | |
| 47 | |
| 48 // A BackingStrategy is responsible for making a PictureBuffer's texture | |
| 49 // contain the image that a MediaCodec decoder buffer tells it to. | |
| 50 class BackingStrategy { | |
| 51 public: | |
| 52 virtual ~BackingStrategy() {} | |
| 53 | |
| 54 // Must be called before anything else. If surface_view_id is not equal to | |
| 55 // |kNoSurfaceID| it refers to a SurfaceView that the strategy must render | |
| 56 // to. | |
| 57 // Returns the Java surface to configure MediaCodec with. | |
| 58 virtual gfx::ScopedJavaSurface Initialize(int surface_view_id) = 0; | |
| 59 | |
| 60 // Called before the AVDA does any Destroy() work. This will be | |
| 61 // the last call that the BackingStrategy receives. | |
| 62 virtual void Cleanup(bool have_context, | |
| 63 const OutputBufferMap& buffer_map) = 0; | |
| 64 | |
| 65 // This returns the SurfaceTexture created by Initialize, or nullptr if | |
| 66 // the strategy was initialized with a SurfaceView. | |
| 67 virtual scoped_refptr<gfx::SurfaceTexture> GetSurfaceTexture() const = 0; | |
| 68 | |
| 69 // Return the GL texture target that the PictureBuffer textures use. | |
| 70 virtual uint32_t GetTextureTarget() const = 0; | |
| 71 | |
| 72 // Return the size to use when requesting picture buffers. | |
| 73 virtual gfx::Size GetPictureBufferSize() const = 0; | |
| 74 | |
| 75 // Make the provided PictureBuffer draw the image that is represented by | |
| 76 // the decoded output buffer at codec_buffer_index. | |
| 77 virtual void UseCodecBufferForPictureBuffer( | |
| 78 int32_t codec_buffer_index, | |
| 79 const media::PictureBuffer& picture_buffer) = 0; | |
| 80 | |
| 81 // Notify strategy that a picture buffer has been assigned. | |
| 82 virtual void AssignOnePictureBuffer( | |
| 83 const media::PictureBuffer& picture_buffer, | |
| 84 bool have_context) {} | |
| 85 | |
| 86 // Notify strategy that a picture buffer has been reused. | |
| 87 virtual void ReuseOnePictureBuffer( | |
| 88 const media::PictureBuffer& picture_buffer) {} | |
| 89 | |
| 90 // Release MediaCodec buffers. | |
| 91 virtual void ReleaseCodecBuffers( | |
| 92 const AndroidVideoDecodeAccelerator::OutputBufferMap& buffers) {} | |
| 93 | |
| 94 // Attempts to free up codec output buffers by rendering early. | |
| 95 virtual void MaybeRenderEarly() {} | |
| 96 | |
| 97 // Notify strategy that we have a new android MediaCodec instance. This | |
| 98 // happens when we're starting up or re-configuring mid-stream. Any | |
| 99 // previously provided codec should no longer be referenced. | |
| 100 virtual void CodecChanged(media::VideoCodecBridge* codec) = 0; | |
| 101 | |
| 102 // Notify the strategy that a frame is available. This callback can happen | |
| 103 // on any thread at any time. | |
| 104 virtual void OnFrameAvailable() = 0; | |
| 105 | |
| 106 // Whether the pictures produced by this backing strategy are overlayable. | |
| 107 virtual bool ArePicturesOverlayable() = 0; | |
| 108 | |
| 109 // Size may have changed due to resolution change since the last time this | |
| 110 // PictureBuffer was used. Update the size of the picture buffer to | |
| 111 // |new_size| and also update any size-dependent state (e.g. size of | |
| 112 // associated texture). Callers should set the correct GL context prior to | |
| 113 // calling. | |
| 114 virtual void UpdatePictureBufferSize(media::PictureBuffer* picture_buffer, | |
| 115 const gfx::Size& new_size) = 0; | |
| 116 }; | |
| 117 | |
| 118 AndroidVideoDecodeAccelerator( | |
| 119 const MakeGLContextCurrentCallback& make_context_current_cb, | |
| 120 const GetGLES2DecoderCallback& get_gles2_decoder_cb); | |
| 121 | |
| 122 ~AndroidVideoDecodeAccelerator() override; | |
| 123 | |
| 124 // media::VideoDecodeAccelerator implementation: | |
| 125 bool Initialize(const Config& config, Client* client) override; | |
| 126 void SetCdm(int cdm_id) override; | |
| 127 void Decode(const media::BitstreamBuffer& bitstream_buffer) override; | |
| 128 void AssignPictureBuffers( | |
| 129 const std::vector<media::PictureBuffer>& buffers) override; | |
| 130 void ReusePictureBuffer(int32_t picture_buffer_id) override; | |
| 131 void Flush() override; | |
| 132 void Reset() override; | |
| 133 void Destroy() override; | |
| 134 bool TryToSetupDecodeOnSeparateThread( | |
| 135 const base::WeakPtr<Client>& decode_client, | |
| 136 const scoped_refptr<base::SingleThreadTaskRunner>& decode_task_runner) | |
| 137 override; | |
| 138 | |
| 139 // AVDAStateProvider implementation: | |
| 140 const gfx::Size& GetSize() const override; | |
| 141 const base::ThreadChecker& ThreadChecker() const override; | |
| 142 base::WeakPtr<gpu::gles2::GLES2Decoder> GetGlDecoder() const override; | |
| 143 gpu::gles2::TextureRef* GetTextureForPicture( | |
| 144 const media::PictureBuffer& picture_buffer) override; | |
| 145 void PostError(const ::tracked_objects::Location& from_here, | |
| 146 media::VideoDecodeAccelerator::Error error) override; | |
| 147 | |
| 148 static media::VideoDecodeAccelerator::Capabilities GetCapabilities( | |
| 149 const gpu::GpuPreferences& gpu_preferences); | |
| 150 | |
| 151 // Notifies about SurfaceTexture::OnFrameAvailable. This can happen on any | |
| 152 // thread at any time! | |
| 153 void OnFrameAvailable(); | |
| 154 | |
| 155 private: | |
| 156 friend class AVDATimerManager; | |
| 157 | |
| 158 // TODO(timav): evaluate the need for more states in the AVDA state machine. | |
| 159 enum State { | |
| 160 NO_ERROR, | |
| 161 ERROR, | |
| 162 // Set when we are asynchronously constructing the codec. Will transition | |
| 163 // to NO_ERROR or ERROR depending on success. | |
| 164 WAITING_FOR_CODEC, | |
| 165 // Set when we have a codec, but it doesn't yet have a key. | |
| 166 WAITING_FOR_KEY, | |
| 167 }; | |
| 168 | |
| 169 enum DrainType { | |
| 170 DRAIN_TYPE_NONE, | |
| 171 DRAIN_FOR_FLUSH, | |
| 172 DRAIN_FOR_RESET, | |
| 173 DRAIN_FOR_DESTROY, | |
| 174 }; | |
| 175 | |
| 176 // Configuration info for MediaCodec. | |
| 177 // This is used to shuttle configuration info between threads without needing | |
| 178 // to worry about the lifetime of the AVDA instance. All of these should not | |
| 179 // be modified while |state_| is WAITING_FOR_CODEC. | |
| 180 class CodecConfig : public base::RefCountedThreadSafe<CodecConfig> { | |
| 181 public: | |
| 182 CodecConfig(); | |
| 183 | |
| 184 // Codec type. Used when we configure media codec. | |
| 185 media::VideoCodec codec_ = media::kUnknownVideoCodec; | |
| 186 | |
| 187 // Whether encryption scheme requires to use protected surface. | |
| 188 bool needs_protected_surface_ = false; | |
| 189 | |
| 190 // The surface that MediaCodec is configured to output to. It's created by | |
| 191 // the backing strategy. | |
| 192 gfx::ScopedJavaSurface surface_; | |
| 193 | |
| 194 // The MediaCrypto object is used in the MediaCodec.configure() in case of | |
| 195 // an encrypted stream. | |
| 196 media::MediaDrmBridgeCdmContext::JavaObjectPtr media_crypto_; | |
| 197 | |
| 198 // Initial coded size. The actual size might change at any time, so this | |
| 199 // is only a hint. | |
| 200 gfx::Size initial_expected_coded_size_; | |
| 201 | |
| 202 protected: | |
| 203 friend class base::RefCountedThreadSafe<CodecConfig>; | |
| 204 virtual ~CodecConfig(); | |
| 205 | |
| 206 private: | |
| 207 DISALLOW_COPY_AND_ASSIGN(CodecConfig); | |
| 208 }; | |
| 209 | |
| 210 // A part of destruction process that is sometimes postponed after the drain. | |
| 211 void ActualDestroy(); | |
| 212 | |
| 213 // Configures |media_codec_| with the given codec parameters from the client. | |
| 214 // This configuration will (probably) not be complete before this call | |
| 215 // returns. Multiple calls before completion will be ignored. |state_| | |
| 216 // must be NO_ERROR or WAITING_FOR_CODEC. Note that, once you call this, | |
| 217 // you should be careful to avoid modifying members of |codec_config_| until | |
| 218 // |state_| is no longer WAITING_FOR_CODEC. | |
| 219 void ConfigureMediaCodecAsynchronously(); | |
| 220 | |
| 221 // Like ConfigureMediaCodecAsynchronously, but synchronous. Returns true if | |
| 222 // and only if |media_codec_| is non-null. Since all configuration is done | |
| 223 // synchronously, there is no concern with modifying |codec_config_| after | |
| 224 // this returns. | |
| 225 bool ConfigureMediaCodecSynchronously(); | |
| 226 | |
| 227 // Instantiate a media codec using |codec_config|. | |
| 228 // This may be called on any thread. | |
| 229 static std::unique_ptr<media::VideoCodecBridge> | |
| 230 ConfigureMediaCodecOnAnyThread(scoped_refptr<CodecConfig> codec_config); | |
| 231 | |
| 232 // Called on the main thread to update |media_codec_| and complete codec | |
| 233 // configuration. |media_codec| will be null if configuration failed. | |
| 234 void OnCodecConfigured(std::unique_ptr<media::VideoCodecBridge> media_codec); | |
| 235 | |
| 236 // Sends the decoded frame specified by |codec_buffer_index| to the client. | |
| 237 void SendDecodedFrameToClient(int32_t codec_buffer_index, | |
| 238 int32_t bitstream_id); | |
| 239 | |
| 240 // Does pending IO tasks if any. Once this is called, it polls |media_codec_| | |
| 241 // until it finishes pending tasks. For the polling, |kDecodePollDelay| is | |
| 242 // used. | |
| 243 void DoIOTask(bool start_timer); | |
| 244 | |
| 245 // Feeds input data to |media_codec_|. This checks | |
| 246 // |pending_bitstream_buffers_| and queues a buffer to |media_codec_|. | |
| 247 // Returns true if any input was processed. | |
| 248 bool QueueInput(); | |
| 249 | |
| 250 // Dequeues output from |media_codec_| and feeds the decoded frame to the | |
| 251 // client. Returns a hint about whether calling again might produce | |
| 252 // more output. | |
| 253 bool DequeueOutput(); | |
| 254 | |
| 255 // Requests picture buffers from the client. | |
| 256 void RequestPictureBuffers(); | |
| 257 | |
| 258 // Decode the content in the |bitstream_buffer|. Note that a | |
| 259 // |bitstream_buffer| of id as -1 indicates a flush command. | |
| 260 void DecodeBuffer(const media::BitstreamBuffer& bitstream_buffer); | |
| 261 | |
| 262 // This callback is called after CDM obtained a MediaCrypto object. | |
| 263 void OnMediaCryptoReady( | |
| 264 media::MediaDrmBridgeCdmContext::JavaObjectPtr media_crypto, | |
| 265 bool needs_protected_surface); | |
| 266 | |
| 267 // This callback is called when a new key is added to CDM. | |
| 268 void OnKeyAdded(); | |
| 269 | |
| 270 // Notifies the client of the result of deferred initialization. | |
| 271 void NotifyInitializationComplete(bool success); | |
| 272 | |
| 273 // Notifies the client about the availability of a picture. | |
| 274 void NotifyPictureReady(const media::Picture& picture); | |
| 275 | |
| 276 // Notifies the client that the input buffer identifed by input_buffer_id has | |
| 277 // been processed. | |
| 278 void NotifyEndOfBitstreamBuffer(int input_buffer_id); | |
| 279 | |
| 280 // Notifies the client that the decoder was flushed. | |
| 281 void NotifyFlushDone(); | |
| 282 | |
| 283 // Notifies the client that the decoder was reset. | |
| 284 void NotifyResetDone(); | |
| 285 | |
| 286 // Notifies about decoding errors. | |
| 287 // Note: you probably don't want to call this directly. Use PostError or | |
| 288 // RETURN_ON_FAILURE, since we can defer error reporting to keep the pipeline | |
| 289 // from breaking. NotifyError will do so immediately, PostError may wait. | |
| 290 // |token| has to match |error_sequence_token_|, or else it's assumed to be | |
| 291 // from a post that's prior to a previous reset, and ignored. | |
| 292 void NotifyError(media::VideoDecodeAccelerator::Error error, int token); | |
| 293 | |
| 294 // Start or stop our work-polling timer based on whether we did any work, and | |
| 295 // how long it has been since we've done work. Calling this with true will | |
| 296 // start the timer. Calling it with false may stop the timer. | |
| 297 void ManageTimer(bool did_work); | |
| 298 | |
| 299 // Start the MediaCodec drain process by adding end_of_stream() buffer to the | |
| 300 // encoded buffers queue. When we receive EOS from the output buffer the drain | |
| 301 // process completes and we perform the action depending on the |drain_type|. | |
| 302 void StartCodecDrain(DrainType drain_type); | |
| 303 | |
| 304 // Returns true if we are currently draining the codec and doing that as part | |
| 305 // of Reset() or Destroy() VP8 workaround. (http://crbug.com/598963). We won't | |
| 306 // display any frames and disable normal errors handling. | |
| 307 bool IsDrainingForResetOrDestroy() const; | |
| 308 | |
| 309 // A helper method that performs the operation required after the drain | |
| 310 // completion (usually when we receive EOS in the output). The operation | |
| 311 // itself depends on the |drain_type_|. | |
| 312 void OnDrainCompleted(); | |
| 313 | |
| 314 // Resets MediaCodec and buffers/containers used for storing output. These | |
| 315 // components need to be reset upon EOS to decode a later stream. Input state | |
| 316 // (e.g. queued BitstreamBuffers) is not reset, as input following an EOS | |
| 317 // is still valid and should be processed. Upon competion calls |done_cb| that | |
| 318 // can be a null callback. | |
| 319 void ResetCodecState(const base::Closure& done_cb); | |
| 320 | |
| 321 // Return true if and only if we should use deferred rendering. | |
| 322 static bool UseDeferredRenderingStrategy( | |
| 323 const gpu::GpuPreferences& gpu_preferences); | |
| 324 | |
| 325 // Used to DCHECK that we are called on the correct thread. | |
| 326 base::ThreadChecker thread_checker_; | |
| 327 | |
| 328 // To expose client callbacks from VideoDecodeAccelerator. | |
| 329 Client* client_; | |
| 330 | |
| 331 // Callback to set the correct gl context. | |
| 332 MakeGLContextCurrentCallback make_context_current_cb_; | |
| 333 | |
| 334 // Callback to get the GLES2Decoder instance. | |
| 335 GetGLES2DecoderCallback get_gles2_decoder_cb_; | |
| 336 | |
| 337 // Whether the stream is encrypted. | |
| 338 bool is_encrypted_; | |
| 339 | |
| 340 // The current state of this class. For now, this is used only for setting | |
| 341 // error state. | |
| 342 State state_; | |
| 343 | |
| 344 // This map maintains the picture buffers passed to the client for decoding. | |
| 345 // The key is the picture buffer id. | |
| 346 OutputBufferMap output_picture_buffers_; | |
| 347 | |
| 348 // This keeps the free picture buffer ids which can be used for sending | |
| 349 // decoded frames to the client. | |
| 350 std::queue<int32_t> free_picture_ids_; | |
| 351 | |
| 352 // The low-level decoder which Android SDK provides. | |
| 353 std::unique_ptr<media::VideoCodecBridge> media_codec_; | |
| 354 | |
| 355 // Set to true after requesting picture buffers to the client. | |
| 356 bool picturebuffers_requested_; | |
| 357 | |
| 358 // The resolution of the stream. | |
| 359 gfx::Size size_; | |
| 360 | |
| 361 // Encoded bitstream buffers to be passed to media codec, queued until an | |
| 362 // input buffer is available. | |
| 363 std::queue<media::BitstreamBuffer> pending_bitstream_buffers_; | |
| 364 | |
| 365 // A map of presentation timestamp to bitstream buffer id for the bitstream | |
| 366 // buffers that have been submitted to the decoder but haven't yet produced an | |
| 367 // output frame with the same timestamp. Note: there will only be one entry | |
| 368 // for multiple bitstream buffers that have the same presentation timestamp. | |
| 369 std::map<base::TimeDelta, int32_t> bitstream_buffers_in_decoder_; | |
| 370 | |
| 371 // Keeps track of bitstream ids notified to the client with | |
| 372 // NotifyEndOfBitstreamBuffer() before getting output from the bitstream. | |
| 373 std::list<int32_t> bitstreams_notified_in_advance_; | |
| 374 | |
| 375 // Backing strategy that we'll use to connect PictureBuffers to frames. | |
| 376 std::unique_ptr<BackingStrategy> strategy_; | |
| 377 | |
| 378 // Helper class that manages asynchronous OnFrameAvailable callbacks. | |
| 379 class OnFrameAvailableHandler; | |
| 380 scoped_refptr<OnFrameAvailableHandler> on_frame_available_handler_; | |
| 381 | |
| 382 // Time at which we last did useful work on io_timer_. | |
| 383 base::TimeTicks most_recent_work_; | |
| 384 | |
| 385 // Type of a drain request. We need to distinguish between DRAIN_FOR_FLUSH | |
| 386 // and other types, see IsDrainingForResetOrDestroy(). | |
| 387 DrainType drain_type_; | |
| 388 | |
| 389 // CDM related stuff. | |
| 390 | |
| 391 // Holds a ref-count to the CDM to avoid using the CDM after it's destroyed. | |
| 392 scoped_refptr<media::MediaKeys> cdm_for_reference_holding_only_; | |
| 393 | |
| 394 media::MediaDrmBridgeCdmContext* media_drm_bridge_cdm_context_; | |
| 395 | |
| 396 // MediaDrmBridge requires registration/unregistration of the player, this | |
| 397 // registration id is used for this. | |
| 398 int cdm_registration_id_; | |
| 399 | |
| 400 // Configuration that we use for MediaCodec. | |
| 401 // Do not update any of its members while |state_| is WAITING_FOR_CODEC. | |
| 402 scoped_refptr<CodecConfig> codec_config_; | |
| 403 | |
| 404 // Index of the dequeued and filled buffer that we keep trying to enqueue. | |
| 405 // Such buffer appears in MEDIA_CODEC_NO_KEY processing. | |
| 406 int pending_input_buf_index_; | |
| 407 | |
| 408 // Monotonically increasing value that is used to prevent old, delayed errors | |
| 409 // from being sent after a reset. | |
| 410 int error_sequence_token_; | |
| 411 | |
| 412 // PostError will defer sending an error if and only if this is true. | |
| 413 bool defer_errors_; | |
| 414 | |
| 415 // True if and only if VDA initialization is deferred, and we have not yet | |
| 416 // called NotifyInitializationComplete. | |
| 417 bool deferred_initialization_pending_; | |
| 418 | |
| 419 // WeakPtrFactory for posting tasks back to |this|. | |
| 420 base::WeakPtrFactory<AndroidVideoDecodeAccelerator> weak_this_factory_; | |
| 421 | |
| 422 friend class AndroidVideoDecodeAcceleratorTest; | |
| 423 }; | |
| 424 | |
| 425 } // namespace content | |
| 426 | |
| 427 #endif // CONTENT_COMMON_GPU_MEDIA_ANDROID_VIDEO_DECODE_ACCELERATOR_H_ | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1882373004:
Migrate content/common/gpu/media code to media/gpu (Closed)
Base URL: https://chromium.googlesource.com/chromium/src.git@master