Add video decoding support in the CDM interface.

webkit/media/crypto/ppapi/content_decryption_module.h

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef WEBKIT_MEDIA_CRYPTO_PPAPI_CONTENT_DECRYPTION_MODULE_H_
 #define WEBKIT_MEDIA_CRYPTO_PPAPI_CONTENT_DECRYPTION_MODULE_H_
 
 #if defined(_MSC_VER)
 typedef unsigned char uint8_t;
 typedef unsigned int uint32_t;
@@ skipping 11 matching lines @@
 extern "C" {
 CDM_EXPORT cdm::ContentDecryptionModule* CreateCdmInstance();
 CDM_EXPORT void DestroyCdmInstance(cdm::ContentDecryptionModule* instance);
 CDM_EXPORT const char* GetCdmVersion();
 }
 
 namespace cdm {
 
 enum Status {
   kSuccess = 0,
-  kErrorUnknown,
-  kErrorNoKey
+  kDecryptOnly,  // The CDM does not support doing DecryptAndDecode() for the
+                 // requested decoder config but can still be used to do
+                 // Decrypt() only.
+  kNeedMoreData,  // Decoder needs more data to produce a decoded frame/sample.
+  kNoKey,  // The decryption key is not available for decryption.

[ddorwin, 2012/09/05 09:37:58]

The required decryption key is not available.

[xhwang, 2012/09/05 14:02:00]

Done.

+  kError
 };
 
 // TODO(xhwang): Use int32_t instead of uint32_t for sizes here and below and
 // update checks to include <0.
 struct KeyMessage {
   KeyMessage()
       : session_id(NULL),
         session_id_size(0),
         message(NULL),
         message_size(0),
@@ skipping 66 matching lines @@
 
   const uint8_t* checksum;
   uint32_t checksum_size;  // Size (in bytes) of the |checksum|.
 
   const struct SubsampleEntry* subsamples;
   uint32_t num_subsamples;  // Number of subsamples in |subsamples|.
 
   int64_t timestamp;  // Presentation timestamp in microseconds.
 };
 
 struct OutputBuffer {

[ddorwin, 2012/09/05 09:37:58]

// Represents an output buffer.
// Does not own the data it represents.

[xhwang, 2012/09/05 14:02:00]

Done.

   OutputBuffer()
       : data(NULL),
         data_size(0),
         timestamp(0) {}
 
   const uint8_t* data;  // Pointer to the beginning of the output data.
   uint32_t data_size;  // Size (in bytes) of |data|.
 
   int64_t timestamp;  // Presentation timestamp in microseconds.
 };
 
+// Surface formats based on FOURCC labels, see:
+// http://www.fourcc.org/yuv.php
+enum VideoFormat {
+  kUnknownVideoFormat = 0,  // Unknown format value.  Used for error reporting.
+  kEmptyVideoFrame,  // An empty frame.
+  kYv12,  // 12bpp YVU planar 1x1 Y, 2x2 VU samples.
+  kI420  // 12bpp YVU planar 1x1 Y, 2x2 UV samples.

[xhwang, 2012/09/04 15:08:18]

I am not totally clear about what formats we will use. I know VP8 decoder
supports Yv12 and I420 output so keep them here. I guess later on when people
adds decoder implementations they can always update this if needed.

+};
+
+struct VideoSize {
+  VideoSize() : width(0), height(0) {}
+  VideoSize(int32_t width, int32_t height) : width(width), height(height) {}
+
+  int32_t width;
+  int32_t height;
+};
+
+struct VideoFrame {
+  static const int32_t kMaxPlanes = 3;
+
+  VideoFrame()
+      : format(kUnknownVideoFormat),
+        timestamp(0) {
+    for (int i = 0; i < kMaxPlanes; ++i) {
+      strides[i] = 0;
+      data[i] = NULL;
+    }
+  }
+
+  VideoFormat format;

[ddorwin, 2012/09/05 09:37:58]

As discussed in 10899021, why do we need format and data_size in both the frame
and the config? How does src/media use these two copies?

[xhwang, 2012/09/05 14:02:00]

In src/media, video decoder config is only used in initialization, so each video
frame has all information needed for rendering and the renderer doesn't need to
store the config.

You are right that we don't need to duplicate them here. I actually removed
natural size etc from VideoFrame because it's only needed for rendering. It's
also safe to remove format and size here (given data_size is equivalent to
coded_size) because the caller already have all the info.

Removed format and data_size.

+
+  // Width and height of the video frame.
+  VideoSize data_size;

[xhwang, 2012/09/04 15:08:18]

scherkus@: my understanding is that this is the same as coded_size on line 205
(both are imported from media). Is that correct? If so, probably should name it
coded_size to avoid unnecessary confusion.

+
+  // Array of strides for each plane, typically greater or equal to the width
+  // of the surface divided by the horizontal sampling period.  Note that
+  // strides can be negative.
+  int32_t strides[kMaxPlanes];
+
+  // Array of data pointers to each plane.
+  uint8_t* data[kMaxPlanes];
+
+  int64_t timestamp;  // Presentation timestamp in microseconds.
+};
+
+struct VideoDecoderConfig {
+  enum VideoCodec {
+    kUnknownVideoCodec = 0,
+    kCodecVP8
+  };
+
+  enum VideoCodecProfile {
+    kUnknownVideoCodecProfile = 0,
+    kVp8ProfileMain
+  };
+
+  VideoDecoderConfig()
+      : codec(kUnknownVideoCodec),
+        profile(kUnknownVideoCodecProfile),
+        format(kUnknownVideoFormat),
+        extra_data(NULL),
+        extra_data_size() {}
+
+  VideoCodec codec;
+  VideoCodecProfile profile;
+  VideoFormat format;
+
+  // Width and height of video frame immediately post-decode. Not all pixels
+  // in this region are valid.
+  VideoSize coded_size;
+
+  // Optional byte data required to initialize video decoders, such as H.264
+  // AAVC data.
+  uint8_t* extra_data;
+  int32_t extra_data_size;
+};
+
 class ContentDecryptionModule {
  public:
   // Generates a |key_request| given the |init_data|.
   // Returns kSuccess if the key request was successfully generated,
   // in which case the callee should have allocated memory for the output
   // parameters (e.g |session_id| in |key_request|) and passed the ownership
   // to the caller.
-  // Returns kErrorUnknown otherwise, in which case the output parameters should
+  // Returns kError otherwise, in which case the output parameters should
   // not be used by the caller.
   //
   // TODO(xhwang): It's not safe to pass the ownership of the dynamically
   // allocated memory over library boundaries. Fix it after related PPAPI change
   // and sample CDM are landed.
   virtual Status GenerateKeyRequest(const uint8_t* init_data,
                                     int init_data_size,
                                     KeyMessage* key_request) = 0;
 
   // Adds the |key| to the CDM to be associated with |key_id|.
   // Returns kSuccess if the key was successfully added.
-  // Returns kErrorUnknown otherwise.
+  // Returns kError otherwise.
   virtual Status AddKey(const char* session_id,
                         int session_id_size,
                         const uint8_t* key,
                         int key_size,
                         const uint8_t* key_id,
                         int key_id_size) = 0;
 
   // Cancels any pending key request made to the CDM for |session_id|.
   // Returns kSuccess if all pending key requests for |session_id| were
   // successfully canceled or there was no key request to be canceled.
-  // Returns kErrorUnknown otherwise.
+  // Returns kError otherwise.
   virtual Status CancelKeyRequest(const char* session_id,
                                   int session_id_size) = 0;
 
   // Decrypts the |encrypted_buffer|.
   // Returns kSuccess if decryption succeeded, in which case the callee
   // should have filled the |decrypted_buffer| and passed the ownership of
   // |data| in |decrypted_buffer| to the caller.
-  // Returns kErrorNoKey if the CDM did not have the necessary decryption key
+  // Returns kNoKey if the CDM did not have the necessary decryption key
   // to decrypt.
-  // Returns kErrorUnknown if any other error happened.
-  // In these two cases, |decrypted_buffer| should not be used by the caller.
+  // Returns kError if any other error happened.
+  // If the return value is not kSuccess, |decrypted_buffer| should be discarded

[ddorwin, 2012/09/05 09:37:58]

What does "discarded" mean exactly? Ignored or destroy the data buffer?

[xhwang, 2012/09/05 14:02:00]

Done.

+  // and not be used by the caller.
   //
   // TODO(xhwang): It's not safe to pass the ownership of the dynamically
   // allocated memory over library boundaries. Fix it after related PPAPI change
   // and sample CDM are landed.
   virtual Status Decrypt(const InputBuffer& encrypted_buffer,
                          OutputBuffer* decrypted_buffer) = 0;
 
+  // Initializes the CDM video decoder with |video_decoder_config|. This

[ddorwin, 2012/09/05 09:37:58]

In 10899021, I suggested moving this up so that Decrypt*() are together.
Thoughts?

[xhwang, 2012/09/05 14:02:00]

I am grouping *Video* together. Later will also group *Audio* together.  I feel
it awkward to interleave video and audio calls. What do you think?

+  // function must be called before DecryptAndDecodeVideo() is called.
+  // Returns kSuccess if the |video_decoder_config| is supported and the CDM
+  // video decoder is successfully initialized.
+  // Returns kDecryptOnly if the |video_decoder_config| is not supported
+  // but the CDM can still be used to Decrypt() the video stream.
+  // Returns kError if |video_decoder_config| is not supported and the
+  // CDM can not be used (or is not allowed) to Decrypt() the video stream.

[ddorwin, 2012/09/05 09:37:58]

Can "allowed" really be determined during initialization? Do you expect to only
call this AFTER AddKey()? (That would seem to prevent pre-initializing
decoders.)

[xhwang, 2012/09/05 14:02:00]

Removed kDecryptOnly. If InitializeVideoDecoder() returns kError, the pipeline
will always try to use the CDM as a Decryptor. If the CDM's license doesn't
allow this, we'll get decryption error, which is okay.

+  virtual Status InitializeVideoDecoder(

[ddorwin, 2012/09/05 09:37:58]

Nothing to do now, but something to consider when we add audio:
Video/Audio labels work now because we only support 1 of each. However, at some
point in the future, we will support multiple. Thus, we may need stream IDs. I
guess we could still have separate methods for video and audio since it's not
possible to use a single value/enum for stream IDs.

[xhwang, 2012/09/05 14:02:00]

Good point. Added a todo.

+      const VideoDecoderConfig& video_decoder_config) = 0;
+
+  // Decrypts the |encrypted_buffer| and decodes the decrypted buffer into a
+  // |video_frame|. Upon end-of-stream, the caller should call this function
+  // repeatedly with empty |encrypted_buffer| (|data| being NULL) until

[ddorwin, 2012/09/05 09:37:58]

Suggestion: The following might be easier to read.
s/being/=/

[xhwang, 2012/09/05 14:02:00]

Done.

+  // only empty |video_frame| (|format| being kEmptyVideoFrame) is produced.
+  // Returns kSuccess if decryption and decoding both succeeded, in which case
+  // the callee should have filled the |video_frame| and passed the ownership of
+  // |data| in |video_frame| to the caller.
+  // Returns kNoKey if the CDM did not have the necessary decryption key
+  // to decrypt.
+  // Returns kNeedMoreData if more data was needed by the decoder to generate
+  // a decoded frame (e.g. during initialization).
+  // Returns kError if any other (decryption or decoding) error happened.
+  // If the return value is not kSuccess, |video_frame| should be discarded and

[ddorwin, 2012/09/05 09:37:58]

Same wrt "discarded".

[xhwang, 2012/09/05 14:02:00]

Done.

+  // not be used by the caller.
+  //
+  // TODO(xhwang): It's not safe to pass the ownership of the dynamically
+  // allocated memory over library boundaries. Fix it after related PPAPI change
+  // and sample CDM are landed.
+  virtual Status DecryptAndDecodeVideo(const InputBuffer& encrypted_buffer,
+                                       VideoFrame* video_frame) = 0;
+
+  // Resets the CDM video decoder to an initialized clean state.

[ddorwin, 2012/09/05 09:37:58]

... Any in-process frames are flushed.
    ^ or something to that effect.

[xhwang, 2012/09/05 14:02:00]

Done.

+  virtual void ResetVideoDecoder() = 0;

[xhwang, 2012/09/04 15:08:18]

In 10899021 it's changed to Flush*.

AudioDecoder/VideoDecoder uses Reset.
VideoRenderer and ffmpeg use Flush.
I'll keep Reset here just to be consistent with AudioDecoder/VideoDecoder, but
would be totally ok if we want to use Flush.

+
+  // Stops the CDM video decoder and set it to an uninitialized state. Note

[ddorwin, 2012/09/05 09:37:58]

s/set/sets/

[xhwang, 2012/09/05 14:02:00]

Done.

+  // that a VideoDecoder cannot be re-initialized after it has been stopped.

[xhwang, 2012/09/04 15:08:18]

This will not be true in the future if we need to Stop() then Initialize() to
support kConfigChanged.

[ddorwin, 2012/09/05 09:37:58]

We do. I think this is likely to be a V1 requirement. What additional complexity
does this involve? IOW, why add this limitation in the comment? Why would we
stop in this case rather than just reset and initialize, though?

[xhwang, 2012/09/05 14:02:00]

I chatted w/ acolwell@ offline. We are not doing stop/reinitialize right now in
FFmpegVideoDecoder when kConfigChanged. But this may change since for
GpuVideoDecoder it's more complicated. In all cases, the bottom line is that we
can always stop then initialize.

The reason that we have this comment is because we have this limit in src/media.
But you are right that we dont' need to have this restriction here. Comments
removed.

We can't do reset and initialize since resetting only flushes internal buffers
and doesn't release internal resources. Stop() will release internal resources
so we should do Stop() then Initialize().

+  virtual void StopVideoDecoder() = 0;
+
   virtual ~ContentDecryptionModule() {}
 };
 
 }  // namespace cdm
 
 #endif  // WEBKIT_MEDIA_CRYPTO_PPAPI_CONTENT_DECRYPTION_MODULE_H_