media: Let VideoFrame carry more than one native texture.

media/base/video_frame.h

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #ifndef MEDIA_BASE_VIDEO_FRAME_H_
 #define MEDIA_BASE_VIDEO_FRAME_H_
 
 #include <vector>
 
 #include "base/callback.h"
 #include "base/md5.h"
 #include "base/memory/shared_memory.h"
 #include "base/synchronization/lock.h"
+#include "gpu/command_buffer/common/mailbox_holder.h"
 #include "media/base/buffers.h"
 #include "media/base/video_frame_metadata.h"
 #include "ui/gfx/geometry/rect.h"
 #include "ui/gfx/geometry/size.h"
 
 #if defined(OS_MACOSX)
 #include <CoreVideo/CVPixelBuffer.h>
 #include "base/mac/scoped_cftyperef.h"
 #endif
 
-namespace gpu {
-struct MailboxHolder;
-}  // namespace gpu
-
 namespace media {
 
 class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> {
  public:
   enum {
     kFrameSizeAlignment = 16,
     kFrameSizePadding = 16,
     kFrameAddressAlignment = 32
   };
 
@@ skipping 24 matching lines @@
     NATIVE_TEXTURE = 6,  // Native texture.  Pixel-format agnostic.
     YV12J = 7,  // JPEG color range version of YV12
     NV12 = 8,  // 12bpp 1x1 Y plane followed by an interleaved 2x2 UV plane.
     YV24 = 9,  // 24bpp YUV planar, no subsampling.
     ARGB = 10,  // 32bpp ARGB, 1 plane.
     YV12HD = 11,  // Rec709 "HD" color space version of YV12
     // Please update UMA histogram enumeration when adding new formats here.
     FORMAT_MAX = YV12HD,  // Must always be equal to largest entry logged.
   };
 
+  // Defines the internal format and the number of the textures in the mailbox
+  // holders.
+  enum TextureFormat {
+    TEXTURE_RGBA = 1,     // One RGBA texture.

[DaleCurtis, 2015/05/04 16:12:06]

Remove =1, =2 ?

[Daniele Castagna, 2015/05/04 17:59:59]

All the other enums in the class seem to explicitly specify the values. Do you
prefer to handle this differently?

[DaleCurtis, 2015/05/04 18:12:54]

That's because they're used in UMA calculations.

[Daniele Castagna, 2015/05/04 20:51:55]

Got it.
Done.

+    TEXTURE_YUV_420 = 2,  // Three R8 textures one per YUV channel.

[hubbe, 2015/05/04 16:26:31]

FYI: In the tab capture pipeline we use RGBA instead of R8 textures to hold YUV
data. (Each RGBA pixel holds four consecutive R8 values.) This is because GLES
doesn't always support R8 textures, but RGBA textures are guaranteed to be
there.  Not sure if/how this applies to your use case, but I wanted to let you
know. Also, tab capture might want to use this facility in the future, so it
would be nice for us if it was compatible.

[Daniele Castagna, 2015/05/04 17:59:59]

Got it, we should add another texture format after this patch lands.

Just for curiousity's sake, where is the shader that draws those textures?

[hubbe, 2015/05/04 18:28:27]

If by "draws", you mean "displays on the screen", then the answer is that we
don't have any such shaders. In tab capture, we convert textures to YUV on the
gpu to read them back into cpu memory more efficiently.

The shaders that generate generate these textures are here:
https://code.google.com/p/chromium/codesearch#chromium/src/content/common/gpu...

The end goal in tab capture is to video-encode the textures. Normally we do this
in software with a vp8 encoder. On platforms where video encoding can happen on
the gpu, we'll want to not do the readback, and that is where we might want to
use this representation. (Depending on what the video codec expects as input of
course.)

+    TEXTURE_FORMAT_MAX = TEXTURE_YUV_420,
+  };

[mcasas, 2015/05/04 16:23:42]

I see here that we're creating a second layer of
types in VideoFrame, the first being for memory 
backed frames and a secondary for Texture-backed
one. This forces you to add accessors for this 
second layer. What about duplicating line
59 NATIVE_TEXTURE into e.g.
SINGLE_TEXTURE(_RGBA)
THREE_TEXTURES(_YUV)
(and we could add TWO_TEXTURES for e.g. NV21
down the road easily, if ever).

(Careful with the UMA histograms though.)

This will allow for merging WrapNativeTexture()
and WrapY420NativeTextures() into a single
method accepting as many textures as the
planarity indicates, and we'll treat the
multiple textures as planes which makes
sense semantic-wise IMHO.

(btw what's with the "NATIVE"? Are there
non-"NATIVE" textures? Consider removing it).

[Daniele Castagna, 2015/05/04 17:59:59]

We thought about the solution you're suggesting too. I don't have any strong
preference about this one or the other.
After a quick chat with dalecurtis@ he suggested to go with the approach in this
patch though.

dalecurtis@, do you mind explaining why you seemed to prefer to add another enum
for the texture_format?

[DaleCurtis, 2015/05/04 18:12:54]

I don't have a strong preference either way. NATIVE_TEXTURE explicitly says it's
pixel format agnostic though, so unless we want to rewrite what NATIVE_TEXTURE
means everywhere with say NATIVE_TEXT_ARGB, etc, this approach seems better.

+
   // Returns the name of a Format as a string.
   static std::string FormatToString(Format format);
 
   // Creates a new frame in system memory with given parameters. Buffers for
   // the frame are allocated but not initialized.
   static scoped_refptr<VideoFrame> CreateFrame(
       Format format,
       const gfx::Size& coded_size,
       const gfx::Rect& visible_rect,
       const gfx::Size& natural_size,
       base::TimeDelta timestamp);
 
   // Returns true if |plane| is a valid plane number for the given format. This
   // can be used to DCHECK() plane parameters.
   static bool IsValidPlane(size_t plane, VideoFrame::Format format);
 
   // Call prior to CreateFrame to ensure validity of frame configuration. Called
   // automatically by VideoDecoderConfig::IsValidConfig().
   // TODO(scherkus): VideoDecoderConfig shouldn't call this method
-  static bool IsValidConfig(Format format, const gfx::Size& coded_size,
+  static bool IsValidConfig(Format format,
+                            const gfx::Size& coded_size,
                             const gfx::Rect& visible_rect,
                             const gfx::Size& natural_size);
 
   // CB to be called on the mailbox backing this frame when the frame is
   // destroyed.
   typedef base::Callback<void(uint32)> ReleaseMailboxCB;
 
-  // Wraps a native texture of the given parameters with a VideoFrame.  The
-  // backing of the VideoFrame is held in the mailbox held by |mailbox_holder|,
-  // and |mailbox_holder_release_cb| will be called with |mailbox_holder| as the
-  // argument when the VideoFrame is to be destroyed.
-  // |read_pixels_cb| may be used to do (slow!) readbacks from the
-  // texture to main memory.
+  // Wraps a native texture of the given parameters with a VideoFrame.
+  // The backing of the VideoFrame is held in the mailbox held by
+  // |mailbox_holder|, and |mailbox_holder_release_cb| will be called with
+  // a syncpoint as the argument when the VideoFrame is to be destroyed.
   static scoped_refptr<VideoFrame> WrapNativeTexture(
-      scoped_ptr<gpu::MailboxHolder> mailbox_holder,
+      const gpu::MailboxHolder& mailbox_holder,
       const ReleaseMailboxCB& mailbox_holder_release_cb,
       const gfx::Size& coded_size,
       const gfx::Rect& visible_rect,
       const gfx::Size& natural_size,
       base::TimeDelta timestamp,
       bool allow_overlay);
 
+  // Wraps a set of native textures representing YUV data with a VideoFrame.
+  // |mailbox_holders_release_cb| will be called with a syncpoint as the
+  // argument when the VideoFrame is to be destroyed.
+  static scoped_refptr<VideoFrame> WrapYUV420NativeTextures(
+      const gpu::MailboxHolder& y_mailbox_holder,
+      const gpu::MailboxHolder& u_mailbox_holder,
+      const gpu::MailboxHolder& v_mailbox_holder,
+      const ReleaseMailboxCB& mailbox_holders_release_cb,
+      const gfx::Size& coded_size,
+      const gfx::Rect& visible_rect,
+      const gfx::Size& natural_size,
+      base::TimeDelta timestamp,
+      bool allow_overlay);
+
   // Wraps packed image data residing in a memory buffer with a VideoFrame.
   // The image data resides in |data| and is assumed to be packed tightly in a
   // buffer of logical dimensions |coded_size| with the appropriate bit depth
   // and plane count as given by |format|.  The shared memory handle of the
   // backing allocation, if present, can be passed in with |handle|.  When the
   // frame is destroyed, |no_longer_needed_cb.Run()| will be called.
   // Returns NULL on failure.
   static scoped_refptr<VideoFrame> WrapExternalPackedMemory(
       Format format,
       const gfx::Size& coded_size,
@@ skipping 86 matching lines @@
   static scoped_refptr<VideoFrame> CreateTransparentFrame(
       const gfx::Size& size);
 
 #if defined(VIDEO_HOLE)
   // Allocates a hole frame.
   static scoped_refptr<VideoFrame> CreateHoleFrame(const gfx::Size& size);
 #endif  // defined(VIDEO_HOLE)
 
   static size_t NumPlanes(Format format);
 
+  static size_t NumTextures(TextureFormat texture_format);

[mcasas, 2015/05/04 16:23:41]

Isn't NumTextures() going to be the same as NumPlanes()
always?

[Daniele Castagna, 2015/05/04 17:59:59]

NumPlanes is generally used to determine how data_ can be accessed, but in the
case of NATIVE_TEXTURE data_ is not set.
NumPlanes returns 0 when video_format is NATIVE_TEXTURE since there is no data.

NumTextures should be used to determine how many valid mailboxes there are. In
case of non NATIVE_TEXTURE, NumTextures should be 0 since there are no valid
mailboxes.

+
   // Returns the required allocation size for a (tightly packed) frame of the
   // given coded size and format.
   static size_t AllocationSize(Format format, const gfx::Size& coded_size);
 
   // Returns the plane size (in bytes) for a plane of the given coded size and
   // format.
   static gfx::Size PlaneSize(Format format,
                              size_t plane,
                              const gfx::Size& coded_size);
 
@@ skipping 16 matching lines @@
   // Returns the number of rows for the given plane, format, and height.
   // The height may be aligned to format requirements.
   static size_t Rows(size_t plane, Format format, int height);
 
   // Returns the number of columns for the given plane, format, and width.
   // The width may be aligned to format requirements.
   static size_t Columns(size_t plane, Format format, int width);
 
   Format format() const { return format_; }
 
+  TextureFormat texture_format() const { return texture_format_; }
+
   const gfx::Size& coded_size() const { return coded_size_; }
   const gfx::Rect& visible_rect() const { return visible_rect_; }
   const gfx::Size& natural_size() const { return natural_size_; }
 
   int stride(size_t plane) const;
 
   // Returns the number of bytes per row and number of rows for a given plane.
   //
   // As opposed to stride(), row_bytes() refers to the bytes representing
   // frame data scanlines (coded_size.width() pixels, without stride padding).
   int row_bytes(size_t plane) const;
   int rows(size_t plane) const;
 
   // Returns pointer to the buffer for a given plane. The memory is owned by
   // VideoFrame object and must not be freed by the caller.
   const uint8* data(size_t plane) const;
   uint8* data(size_t plane);
 
   // Returns pointer to the data in the visible region of the frame. I.e. the
   // returned pointer is offsetted into the plane buffer specified by
   // visible_rect().origin(). Memory is owned by VideoFrame object and must not
   // be freed by the caller.
   const uint8* visible_data(size_t plane) const;
   uint8* visible_data(size_t plane);
 
-  // Returns the mailbox holder of the native texture wrapped by this frame.
+  // Returns a mailbox holder for a given texture.
   // Only valid to call if this is a NATIVE_TEXTURE frame. Before using the
   // mailbox, the caller must wait for the included sync point.
-  const gpu::MailboxHolder* mailbox_holder() const;
+  const gpu::MailboxHolder& mailbox_holder(size_t texture) const;
 
   // Returns the shared-memory handle, if present
   base::SharedMemoryHandle shared_memory_handle() const;
 
   // Returns the offset into the shared memory where the frame data begins.
   size_t shared_memory_offset() const;
 
   // Returns a dictionary of optional metadata.  This contains information
   // associated with the frame that downstream clients might use for frame-level
   // logging, quality/performance optimizations, signaling, etc.
@@ skipping 47 matching lines @@
   void HashFrameForTesting(base::MD5Context* context);
 
  private:
   friend class base::RefCountedThreadSafe<VideoFrame>;
 
   // Clients must use the static CreateFrame() method to create a new frame.
   VideoFrame(Format format,
              const gfx::Size& coded_size,
              const gfx::Rect& visible_rect,
              const gfx::Size& natural_size,
-             scoped_ptr<gpu::MailboxHolder> mailbox_holder,
+             const gpu::MailboxHolder(&mailbox_holders)[kMaxPlanes],

[mcasas, 2015/05/04 16:23:41]

This is an unexpected change to me. Why
change the ownership semantics of the 
MailboxHolder just because we change their
multiplicity?

[Daniele Castagna, 2015/05/04 17:59:59]

MailboxHolder contains two integers and a Mailbox. Mailbox was meant to be
passed around by value and it basically contains just a string.
I didn't see any point in passing around ownership of a string and two integers
in the first place.

Getting rid of the scoped_ptr makes the interface cleaner for the new added
WrapNativeYUV420Textures and for the old WrapNativeTexture (e.g: the
mailboxholder copy in TextureWrapHelper::OnIncomingCapturedGpuMemoryBuffer).

+             TextureFormat texture_format,
              base::TimeDelta timestamp,
              bool end_of_stream);
   virtual ~VideoFrame();
 
   void AllocateYUV();
 
   // Frame format.
   const Format format_;
 
+  // Format of the native textures associated with this frame.
+  const TextureFormat texture_format_;
+
   // Width and height of the video frame, in pixels. This must include pixel
   // data for the whole image; i.e. for YUV formats with subsampled chroma
   // planes, in the case that the visible portion of the image does not line up
   // on a sample boundary, |coded_size_| must be rounded up appropriately and
   // the pixel data provided for the odd pixels.
   const gfx::Size coded_size_;
 
   // Width, height, and offsets of the visible portion of the video frame. Must
   // be a subrect of |coded_size_|. Can be odd with respect to the sample
   // boundaries, e.g. for formats with subsampled chroma.
   const gfx::Rect visible_rect_;
 
   // Width and height of the visible portion of the video frame
   // (|visible_rect_.size()|) with aspect ratio taken into account.
   const gfx::Size natural_size_;
 
   // Array of strides for each plane, typically greater or equal to the width
   // of the surface divided by the horizontal sampling period.  Note that
   // strides can be negative.
   int32 strides_[kMaxPlanes];
 
   // Array of data pointers to each plane.
   uint8* data_[kMaxPlanes];
 
-  // Native texture mailbox, if this is a NATIVE_TEXTURE frame.
-  const scoped_ptr<gpu::MailboxHolder> mailbox_holder_;
-  ReleaseMailboxCB mailbox_holder_release_cb_;
+  // Native texture mailboxes, if this is a NATIVE_TEXTURE frame.
+  gpu::MailboxHolder mailbox_holders_[kMaxPlanes];
+  ReleaseMailboxCB mailbox_holders_release_cb_;
 
   // Shared memory handle, if this frame was allocated from shared memory.
   base::SharedMemoryHandle shared_memory_handle_;
 
   // Offset in shared memory buffer.
   size_t shared_memory_offset_;
 
 #if defined(OS_POSIX)
   // Dmabufs for each plane, if this frame is wrapping memory
   // acquired via dmabuf.
@@ skipping 17 matching lines @@
   VideoFrameMetadata metadata_;
 
   bool allow_overlay_;
 
   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoFrame);
 };
 
 }  // namespace media
 
 #endif  // MEDIA_BASE_VIDEO_FRAME_H_