media: Let VideoFrame carry more than one native texture.

media/base/video_frame.h

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef MEDIA_BASE_VIDEO_FRAME_H_
 #define MEDIA_BASE_VIDEO_FRAME_H_
 
 #include <vector>
 
 #include "base/callback.h"
 #include "base/md5.h"
 #include "base/memory/shared_memory.h"
 #include "base/synchronization/lock.h"
+#include "gpu/command_buffer/common/mailbox_holder.h"
 #include "media/base/buffers.h"
 #include "media/base/video_frame_metadata.h"
 #include "ui/gfx/geometry/rect.h"
 #include "ui/gfx/geometry/size.h"
 
 #if defined(OS_MACOSX)
 #include <CoreVideo/CVPixelBuffer.h>
 #include "base/mac/scoped_cftyperef.h"
 #endif
 
-namespace gpu {
-struct MailboxHolder;
-}  // namespace gpu
-
 namespace media {
 
 class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> {
  public:
   enum {
     kFrameSizeAlignment = 16,
     kFrameSizePadding = 16,
     kFrameAddressAlignment = 32
   };
 
@@ skipping 24 matching lines @@
     NATIVE_TEXTURE = 6,  // Native texture.  Pixel-format agnostic.
     YV12J = 7,  // JPEG color range version of YV12
     NV12 = 8,  // 12bpp 1x1 Y plane followed by an interleaved 2x2 UV plane.
     YV24 = 9,  // 24bpp YUV planar, no subsampling.
     ARGB = 10,  // 32bpp ARGB, 1 plane.
     YV12HD = 11,  // Rec709 "HD" color space version of YV12
     // Please update UMA histogram enumeration when adding new formats here.
     FORMAT_MAX = YV12HD,  // Must always be equal to largest entry logged.
   };
 
+  // Defines the number and the pixel format of the textures in the mailbox
+  // holders.

[reveman, 2015/05/03 16:13:27]

hm, this defines the subsampling factor and internal format as needed to use
these textures in a shader. The actual pixel format does not need to be known.

[Daniele Castagna, 2015/05/04 15:00:15]

Changed the comment.

+  enum TextureFormat {
+    TEXTURE_NONE = 0,        // This frame doesn't contain any native texture.

[reveman, 2015/05/03 16:13:27]

I think the code would be cleaner if this enum contained only valid values and
texture_format_ field should be ignored if format is not NATIVE_TEXTURE. e.g.
the IsValidConfig change goes away.

[Daniele Castagna, 2015/05/04 15:00:15]

Done.

[DaleCurtis, 2015/05/04 16:00:14]

Hmm, I think that's fine so long as we still initialize it to RGBA or something.
Otherwise I prefer the DCHECK_NE(NONE in places to avoid waiting for valgrind to
catch an uninitialized value.

+    TEXTURE_RGBA = 1,        // One RGBA texture.

[DaleCurtis, 2015/05/02 18:21:33]

Doesn't android use ARGB or something?

[reveman, 2015/05/03 16:13:27]

I think the idea is that this just describes the internal format as far as a
shader needs to know. TEXTURE_RGBA, means it's one texture with
red/green/blue/alpha channels but the pixel layout is unknown.

[Daniele Castagna, 2015/05/04 15:00:15]

Agreed.

+    TEXTURE_YUV_R8R8R8 = 2,  // Three R8 textures one per YUV channel.

[reveman, 2015/05/03 16:13:27]

s/YUV_R8R8R8/YUV_420/ as that would make it clear the UV planes have a
subsampling factor of 2 and the actual resolution (R8R8R8) of each plane should
not be useful at this level.

[Daniele Castagna, 2015/05/04 15:00:15]

Done.

+    TEXTURE_FORMAT_MAX = TEXTURE_YUV_R8R8R8,
+  };
+
   // Returns the name of a Format as a string.
   static std::string FormatToString(Format format);
 
   // Creates a new frame in system memory with given parameters. Buffers for
   // the frame are allocated but not initialized.
   static scoped_refptr<VideoFrame> CreateFrame(
       Format format,
       const gfx::Size& coded_size,
       const gfx::Rect& visible_rect,
       const gfx::Size& natural_size,
       base::TimeDelta timestamp);
 
   // Returns true if |plane| is a valid plane number for the given format. This
   // can be used to DCHECK() plane parameters.
   static bool IsValidPlane(size_t plane, VideoFrame::Format format);
 
   // Call prior to CreateFrame to ensure validity of frame configuration. Called
   // automatically by VideoDecoderConfig::IsValidConfig().
   // TODO(scherkus): VideoDecoderConfig shouldn't call this method
-  static bool IsValidConfig(Format format, const gfx::Size& coded_size,
+  static bool IsValidConfig(Format format,
+                            TextureFormat texture_format,
+                            const gfx::Size& coded_size,
                             const gfx::Rect& visible_rect,
                             const gfx::Size& natural_size);
 
   // CB to be called on the mailbox backing this frame when the frame is
   // destroyed.
   typedef base::Callback<void(uint32)> ReleaseMailboxCB;
 
-  // Wraps a native texture of the given parameters with a VideoFrame.  The
-  // backing of the VideoFrame is held in the mailbox held by |mailbox_holder|,
-  // and |mailbox_holder_release_cb| will be called with |mailbox_holder| as the
-  // argument when the VideoFrame is to be destroyed.
-  // |read_pixels_cb| may be used to do (slow!) readbacks from the
-  // texture to main memory.
+  // Wraps a native texture of the given parameters with a VideoFrame.
+  // The backing of the VideoFrame is held in the mailbox held by
+  // |mailbox_holder|, and |mailbox_holder_release_cb| will be called with
+  // a syncpoint as the argument when the VideoFrame is to be destroyed.
   static scoped_refptr<VideoFrame> WrapNativeTexture(
-      scoped_ptr<gpu::MailboxHolder> mailbox_holder,
+      const gpu::MailboxHolder& mailbox_holder,
       const ReleaseMailboxCB& mailbox_holder_release_cb,
       const gfx::Size& coded_size,
       const gfx::Rect& visible_rect,
       const gfx::Size& natural_size,
       base::TimeDelta timestamp,
       bool allow_overlay);
 
+  // Wraps a set of native textures representing YUV data with a VideoFrame.
+  // |mailbox_holders_release_cb| will be called with a syncpoint as the
+  // argument when the VideoFrame is to be destroyed.
+  static scoped_refptr<VideoFrame> WrapYUVNativeTextures(

[reveman, 2015/05/03 16:13:27]

WrapYUV420NativeTextures? Or pass the texture format as a parameter. Or why
don't we just change the WrapNativeTexture function above into a
"WrapNativeTextures" function that take an array of mailbox holders and a
texture format parameter?

[Daniele Castagna, 2015/05/04 15:00:15]

Renamed to WrapYUV420NativeTextures.
I decided not to change WrapNativeTextures to take an array of mailboxes and a
texture format since all the other wrap functions seem to be for a specific
video format. The videoframe format is never passed from the outside but always
determined by one of this wrap function, so keeping WrapYUV420NativeTextures
seems to fit better with the rest of the class.

+      const gpu::MailboxHolder& y_mailbox_holder,
+      const gpu::MailboxHolder& u_mailbox_holder,
+      const gpu::MailboxHolder& v_mailbox_holder,
+      const ReleaseMailboxCB& mailbox_holders_release_cb,
+      const gfx::Size& coded_size,
+      const gfx::Rect& visible_rect,
+      const gfx::Size& natural_size,
+      base::TimeDelta timestamp,
+      bool allow_overlay);
+
   // Wraps packed image data residing in a memory buffer with a VideoFrame.
   // The image data resides in |data| and is assumed to be packed tightly in a
   // buffer of logical dimensions |coded_size| with the appropriate bit depth
   // and plane count as given by |format|.  The shared memory handle of the
   // backing allocation, if present, can be passed in with |handle|.  When the
   // frame is destroyed, |no_longer_needed_cb.Run()| will be called.
   // Returns NULL on failure.
   static scoped_refptr<VideoFrame> WrapExternalPackedMemory(
       Format format,
       const gfx::Size& coded_size,
@@ skipping 86 matching lines @@
   static scoped_refptr<VideoFrame> CreateTransparentFrame(
       const gfx::Size& size);
 
 #if defined(VIDEO_HOLE)
   // Allocates a hole frame.
   static scoped_refptr<VideoFrame> CreateHoleFrame(const gfx::Size& size);
 #endif  // defined(VIDEO_HOLE)
 
   static size_t NumPlanes(Format format);
 
+  static size_t NumTextures(TextureFormat texture_format);
+
   // Returns the required allocation size for a (tightly packed) frame of the
   // given coded size and format.
   static size_t AllocationSize(Format format, const gfx::Size& coded_size);
 
   // Returns the plane size (in bytes) for a plane of the given coded size and
   // format.
   static gfx::Size PlaneSize(Format format,
                              size_t plane,
                              const gfx::Size& coded_size);
 
@@ skipping 16 matching lines @@
   // Returns the number of rows for the given plane, format, and height.
   // The height may be aligned to format requirements.
   static size_t Rows(size_t plane, Format format, int height);
 
   // Returns the number of columns for the given plane, format, and width.
   // The width may be aligned to format requirements.
   static size_t Columns(size_t plane, Format format, int width);
 
   Format format() const { return format_; }
 
+  TextureFormat texture_format() const { return texture_format_; }
+
   const gfx::Size& coded_size() const { return coded_size_; }
   const gfx::Rect& visible_rect() const { return visible_rect_; }
   const gfx::Size& natural_size() const { return natural_size_; }
 
   int stride(size_t plane) const;
 
   // Returns the number of bytes per row and number of rows for a given plane.
   //
   // As opposed to stride(), row_bytes() refers to the bytes representing
   // frame data scanlines (coded_size.width() pixels, without stride padding).
   int row_bytes(size_t plane) const;
   int rows(size_t plane) const;
 
   // Returns pointer to the buffer for a given plane. The memory is owned by
   // VideoFrame object and must not be freed by the caller.
   const uint8* data(size_t plane) const;
   uint8* data(size_t plane);
 
   // Returns pointer to the data in the visible region of the frame. I.e. the
   // returned pointer is offsetted into the plane buffer specified by
   // visible_rect().origin(). Memory is owned by VideoFrame object and must not
   // be freed by the caller.
   const uint8* visible_data(size_t plane) const;
   uint8* visible_data(size_t plane);
 
-  // Returns the mailbox holder of the native texture wrapped by this frame.
+  // Returns a mailbox holder for a given texture.
   // Only valid to call if this is a NATIVE_TEXTURE frame. Before using the
   // mailbox, the caller must wait for the included sync point.
-  const gpu::MailboxHolder* mailbox_holder() const;
+  const gpu::MailboxHolder& mailbox_holder(size_t texture) const;
 
   // Returns the shared-memory handle, if present
   base::SharedMemoryHandle shared_memory_handle() const;
 
   // Returns the offset into the shared memory where the frame data begins.
   size_t shared_memory_offset() const;
 
   // Returns a dictionary of optional metadata.  This contains information
   // associated with the frame that downstream clients might use for frame-level
   // logging, quality/performance optimizations, signaling, etc.
@@ skipping 47 matching lines @@
   void HashFrameForTesting(base::MD5Context* context);
 
  private:
   friend class base::RefCountedThreadSafe<VideoFrame>;
 
   // Clients must use the static CreateFrame() method to create a new frame.
   VideoFrame(Format format,
              const gfx::Size& coded_size,
              const gfx::Rect& visible_rect,
              const gfx::Size& natural_size,
-             scoped_ptr<gpu::MailboxHolder> mailbox_holder,
+             const gpu::MailboxHolder mailbox_holders[kMaxPlanes],

[reveman, 2015/05/03 16:13:27]

Pass by reference instead of value.

[Daniele Castagna, 2015/05/04 15:00:15]

Done.

git cl format seems to be confused by the syntax though.

+             TextureFormat texture_format,
              base::TimeDelta timestamp,
              bool end_of_stream);
   virtual ~VideoFrame();
 
   void AllocateYUV();
 
   // Frame format.
   const Format format_;
 
+  // Format of the native textures associated with this frames.

[reveman, 2015/05/03 16:13:27]

nit: s/frames/frame/

[Daniele Castagna, 2015/05/04 15:00:15]

Done.

+  const TextureFormat texture_format_;
+
   // Width and height of the video frame, in pixels. This must include pixel
   // data for the whole image; i.e. for YUV formats with subsampled chroma
   // planes, in the case that the visible portion of the image does not line up
   // on a sample boundary, |coded_size_| must be rounded up appropriately and
   // the pixel data provided for the odd pixels.
   const gfx::Size coded_size_;
 
   // Width, height, and offsets of the visible portion of the video frame. Must
   // be a subrect of |coded_size_|. Can be odd with respect to the sample
   // boundaries, e.g. for formats with subsampled chroma.
   const gfx::Rect visible_rect_;
 
   // Width and height of the visible portion of the video frame
   // (|visible_rect_.size()|) with aspect ratio taken into account.
   const gfx::Size natural_size_;
 
   // Array of strides for each plane, typically greater or equal to the width
   // of the surface divided by the horizontal sampling period.  Note that
   // strides can be negative.
   int32 strides_[kMaxPlanes];
 
   // Array of data pointers to each plane.
   uint8* data_[kMaxPlanes];
 
-  // Native texture mailbox, if this is a NATIVE_TEXTURE frame.
-  const scoped_ptr<gpu::MailboxHolder> mailbox_holder_;
-  ReleaseMailboxCB mailbox_holder_release_cb_;
+  // Native texture mailboxes, if this is a NATIVE_TEXTURE frame.
+  gpu::MailboxHolder mailbox_holders_[kMaxPlanes];
+  ReleaseMailboxCB mailbox_holders_release_cb_;
 
   // Shared memory handle, if this frame was allocated from shared memory.
   base::SharedMemoryHandle shared_memory_handle_;
 
   // Offset in shared memory buffer.
   size_t shared_memory_offset_;
 
 #if defined(OS_POSIX)
   // Dmabufs for each plane, if this frame is wrapping memory
   // acquired via dmabuf.
@@ skipping 17 matching lines @@
   VideoFrameMetadata metadata_;
 
   bool allow_overlay_;
 
   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoFrame);
 };
 
 }  // namespace media
 
 #endif  // MEDIA_BASE_VIDEO_FRAME_H_