media: Let VideoFrame carry more than one native texture.

media/base/video_frame.h

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef MEDIA_BASE_VIDEO_FRAME_H_
 #define MEDIA_BASE_VIDEO_FRAME_H_
 
 #include <vector>
 
 #include "base/callback.h"
 #include "base/md5.h"
 #include "base/memory/shared_memory.h"
 #include "base/synchronization/lock.h"
+#include "gpu/command_buffer/common/mailbox_holder.h"
 #include "media/base/buffers.h"
 #include "media/base/video_frame_metadata.h"
 #include "ui/gfx/geometry/rect.h"
 #include "ui/gfx/geometry/size.h"
 
 #if defined(OS_MACOSX)
 #include <CoreVideo/CVPixelBuffer.h>
 #include "base/mac/scoped_cftyperef.h"
 #endif
 
-namespace gpu {
-struct MailboxHolder;
-}  // namespace gpu
-
 namespace media {
 
 class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> {
  public:
   enum {
     kFrameSizeAlignment = 16,
     kFrameSizePadding = 16,
     kFrameAddressAlignment = 32
   };
 
@@ skipping 24 matching lines @@
     NATIVE_TEXTURE = 6,  // Native texture.  Pixel-format agnostic.
     YV12J = 7,  // JPEG color range version of YV12
     NV12 = 8,  // 12bpp 1x1 Y plane followed by an interleaved 2x2 UV plane.
     YV24 = 9,  // 24bpp YUV planar, no subsampling.
     ARGB = 10,  // 32bpp ARGB, 1 plane.
     YV12HD = 11,  // Rec709 "HD" color space version of YV12
     // Please update UMA histogram enumeration when adding new formats here.
     FORMAT_MAX = YV12HD,  // Must always be equal to largest entry logged.
   };
 
+  // Defines the internal format and the number of the textures in the mailbox
+  // holders.
+  enum TextureFormat {

[Pawel Osciak, 2015/05/06 02:14:29]

Would it perhaps be possible to use an existing set of defines/enums for this?
We have multiple format enums already in various structures. I would be great if
we could have less of these if that would be possible.

A good set could be fourccs perhaps? They are more or less universally used and
there are defines for 2-plane and 3-plane formats as well, e.g.
https://code.google.com/p/chromium/codesearch#chromium/usr/include/drm/drm_fo....
We already use them for accelerated video in Chrome for native buffers as well.

[Daniele Castagna, 2015/05/06 18:50:24]

I'm afraid the existing defines/enums are not a good fit for what we want here.
We want to use this enum to let the compositor determine how many textures we
have and which shader needs to be used to correctly display the frame.

The DRM_FORMAT_YVU420 for example could map to at least two TextureFormat enums:
 - One texture backed by a YUV GpuMemoryBuffer.
 - Three textures backed by three RED GpuMemoryBuffers.

+    TEXTURE_RGBA,     // One RGBA texture.
+    TEXTURE_YUV_420,  // 3 RED textures one per channel. UV are 2x2 subsampled.
+  };
+
   // Returns the name of a Format as a string.
   static std::string FormatToString(Format format);
 
   // Creates a new frame in system memory with given parameters. Buffers for
   // the frame are allocated but not initialized.
   static scoped_refptr<VideoFrame> CreateFrame(
       Format format,
       const gfx::Size& coded_size,
       const gfx::Rect& visible_rect,
       const gfx::Size& natural_size,
       base::TimeDelta timestamp);
 
   // Returns true if |plane| is a valid plane number for the given format. This
   // can be used to DCHECK() plane parameters.
   static bool IsValidPlane(size_t plane, VideoFrame::Format format);
 
   // Call prior to CreateFrame to ensure validity of frame configuration. Called
   // automatically by VideoDecoderConfig::IsValidConfig().
   // TODO(scherkus): VideoDecoderConfig shouldn't call this method
-  static bool IsValidConfig(Format format, const gfx::Size& coded_size,
+  static bool IsValidConfig(Format format,
+                            const gfx::Size& coded_size,
                             const gfx::Rect& visible_rect,
                             const gfx::Size& natural_size);
 
   // CB to be called on the mailbox backing this frame when the frame is
   // destroyed.
   typedef base::Callback<void(uint32)> ReleaseMailboxCB;
 
-  // Wraps a native texture of the given parameters with a VideoFrame.  The
-  // backing of the VideoFrame is held in the mailbox held by |mailbox_holder|,
-  // and |mailbox_holder_release_cb| will be called with |mailbox_holder| as the
-  // argument when the VideoFrame is to be destroyed.
-  // |read_pixels_cb| may be used to do (slow!) readbacks from the
-  // texture to main memory.
+  // Wraps a native texture of the given parameters with a VideoFrame.
+  // The backing of the VideoFrame is held in the mailbox held by
+  // |mailbox_holder|, and |mailbox_holder_release_cb| will be called with
+  // a syncpoint as the argument when the VideoFrame is to be destroyed.
   static scoped_refptr<VideoFrame> WrapNativeTexture(
-      scoped_ptr<gpu::MailboxHolder> mailbox_holder,
+      const gpu::MailboxHolder& mailbox_holder,
       const ReleaseMailboxCB& mailbox_holder_release_cb,
       const gfx::Size& coded_size,
       const gfx::Rect& visible_rect,
       const gfx::Size& natural_size,
       base::TimeDelta timestamp,
       bool allow_overlay);
 
+  // Wraps a set of native textures representing YUV data with a VideoFrame.
+  // |mailbox_holders_release_cb| will be called with a syncpoint as the
+  // argument when the VideoFrame is to be destroyed.
+  static scoped_refptr<VideoFrame> WrapYUV420NativeTextures(
+      const gpu::MailboxHolder& y_mailbox_holder,
+      const gpu::MailboxHolder& u_mailbox_holder,
+      const gpu::MailboxHolder& v_mailbox_holder,
+      const ReleaseMailboxCB& mailbox_holders_release_cb,
+      const gfx::Size& coded_size,
+      const gfx::Rect& visible_rect,
+      const gfx::Size& natural_size,
+      base::TimeDelta timestamp,
+      bool allow_overlay);
+
   // Wraps packed image data residing in a memory buffer with a VideoFrame.
   // The image data resides in |data| and is assumed to be packed tightly in a
   // buffer of logical dimensions |coded_size| with the appropriate bit depth
   // and plane count as given by |format|.  The shared memory handle of the
   // backing allocation, if present, can be passed in with |handle|.  When the
   // frame is destroyed, |no_longer_needed_cb.Run()| will be called.
   // Returns NULL on failure.
   static scoped_refptr<VideoFrame> WrapExternalPackedMemory(
       Format format,
       const gfx::Size& coded_size,
@@ skipping 86 matching lines @@
   static scoped_refptr<VideoFrame> CreateTransparentFrame(
       const gfx::Size& size);
 
 #if defined(VIDEO_HOLE)
   // Allocates a hole frame.
   static scoped_refptr<VideoFrame> CreateHoleFrame(const gfx::Size& size);
 #endif  // defined(VIDEO_HOLE)
 
   static size_t NumPlanes(Format format);
 
+  static size_t NumTextures(TextureFormat texture_format);
+
   // Returns the required allocation size for a (tightly packed) frame of the
   // given coded size and format.
   static size_t AllocationSize(Format format, const gfx::Size& coded_size);
 
   // Returns the plane size (in bytes) for a plane of the given coded size and
   // format.
   static gfx::Size PlaneSize(Format format,
                              size_t plane,
                              const gfx::Size& coded_size);
 
@@ skipping 16 matching lines @@
   // Returns the number of rows for the given plane, format, and height.
   // The height may be aligned to format requirements.
   static size_t Rows(size_t plane, Format format, int height);
 
   // Returns the number of columns for the given plane, format, and width.
   // The width may be aligned to format requirements.
   static size_t Columns(size_t plane, Format format, int width);
 
   Format format() const { return format_; }
 
+  TextureFormat texture_format() const { return texture_format_; }
+
   const gfx::Size& coded_size() const { return coded_size_; }
   const gfx::Rect& visible_rect() const { return visible_rect_; }
   const gfx::Size& natural_size() const { return natural_size_; }
 
   int stride(size_t plane) const;
 
   // Returns the number of bytes per row and number of rows for a given plane.
   //
   // As opposed to stride(), row_bytes() refers to the bytes representing
   // frame data scanlines (coded_size.width() pixels, without stride padding).
   int row_bytes(size_t plane) const;
   int rows(size_t plane) const;
 
   // Returns pointer to the buffer for a given plane. The memory is owned by
   // VideoFrame object and must not be freed by the caller.
   const uint8* data(size_t plane) const;
   uint8* data(size_t plane);
 
   // Returns pointer to the data in the visible region of the frame. I.e. the
   // returned pointer is offsetted into the plane buffer specified by
   // visible_rect().origin(). Memory is owned by VideoFrame object and must not
   // be freed by the caller.
   const uint8* visible_data(size_t plane) const;
   uint8* visible_data(size_t plane);
 
-  // Returns the mailbox holder of the native texture wrapped by this frame.
+  // Returns a mailbox holder for a given texture.
   // Only valid to call if this is a NATIVE_TEXTURE frame. Before using the
   // mailbox, the caller must wait for the included sync point.
-  const gpu::MailboxHolder* mailbox_holder() const;
+  const gpu::MailboxHolder& mailbox_holder(size_t texture) const;
 
   // Returns the shared-memory handle, if present
   base::SharedMemoryHandle shared_memory_handle() const;
 
   // Returns the offset into the shared memory where the frame data begins.
   size_t shared_memory_offset() const;
 
   // Returns a dictionary of optional metadata.  This contains information
   // associated with the frame that downstream clients might use for frame-level
   // logging, quality/performance optimizations, signaling, etc.
@@ skipping 47 matching lines @@
   void HashFrameForTesting(base::MD5Context* context);
 
  private:
   friend class base::RefCountedThreadSafe<VideoFrame>;
 
   // Clients must use the static CreateFrame() method to create a new frame.
   VideoFrame(Format format,
              const gfx::Size& coded_size,
              const gfx::Rect& visible_rect,
              const gfx::Size& natural_size,
-             scoped_ptr<gpu::MailboxHolder> mailbox_holder,
+             const gpu::MailboxHolder(&mailbox_holders)[kMaxPlanes],
+             TextureFormat texture_format,
              base::TimeDelta timestamp,
              bool end_of_stream);
   virtual ~VideoFrame();
 
   void AllocateYUV();
 
   // Frame format.
   const Format format_;
 
+  // Format of the native textures associated with this frame.
+  const TextureFormat texture_format_;
+
   // Width and height of the video frame, in pixels. This must include pixel
   // data for the whole image; i.e. for YUV formats with subsampled chroma
   // planes, in the case that the visible portion of the image does not line up
   // on a sample boundary, |coded_size_| must be rounded up appropriately and
   // the pixel data provided for the odd pixels.
   const gfx::Size coded_size_;
 
   // Width, height, and offsets of the visible portion of the video frame. Must
   // be a subrect of |coded_size_|. Can be odd with respect to the sample
   // boundaries, e.g. for formats with subsampled chroma.
   const gfx::Rect visible_rect_;
 
   // Width and height of the visible portion of the video frame
   // (|visible_rect_.size()|) with aspect ratio taken into account.
   const gfx::Size natural_size_;
 
   // Array of strides for each plane, typically greater or equal to the width
   // of the surface divided by the horizontal sampling period.  Note that
   // strides can be negative.
   int32 strides_[kMaxPlanes];
 
   // Array of data pointers to each plane.
   uint8* data_[kMaxPlanes];
 
-  // Native texture mailbox, if this is a NATIVE_TEXTURE frame.
-  const scoped_ptr<gpu::MailboxHolder> mailbox_holder_;
-  ReleaseMailboxCB mailbox_holder_release_cb_;
+  // Native texture mailboxes, if this is a NATIVE_TEXTURE frame.
+  gpu::MailboxHolder mailbox_holders_[kMaxPlanes];

[piman, 2015/05/06 01:08:09]

drive-by: FYI, gpu::Mailbox is 64 bytes, so MailboxHolder is 72 bytes. You just
added 280 bytes to all VideoFrames. Not sure if it's a concern.

[Daniele Castagna, 2015/05/06 01:29:19]

I think it's fine. We don't have many VideoFrames allocated at the same time, in
my tests there were about 7 (let's say 14 since we've the wrapping thing going
on), and usually they have attached the frame data that is much bigger than 280
bytes.

+  ReleaseMailboxCB mailbox_holders_release_cb_;
 
   // Shared memory handle, if this frame was allocated from shared memory.
   base::SharedMemoryHandle shared_memory_handle_;
 
   // Offset in shared memory buffer.
   size_t shared_memory_offset_;
 
 #if defined(OS_POSIX)
   // Dmabufs for each plane, if this frame is wrapping memory
   // acquired via dmabuf.
@@ skipping 17 matching lines @@
   VideoFrameMetadata metadata_;
 
   bool allow_overlay_;
 
   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoFrame);
 };
 
 }  // namespace media
 
 #endif  // MEDIA_BASE_VIDEO_FRAME_H_