Pass GpuMemoryBuffer backed VideoFrame from browser to renderer processes

content/renderer/media/video_capture_impl.cc

Index: content/renderer/media/video_capture_impl.cc
diff --git a/content/renderer/media/video_capture_impl.cc b/content/renderer/media/video_capture_impl.cc
index 9dd906b39ba4563771519ea4cd13f0ee8159e73b..077b53a0227e1a3109e068312d9ac14a712451ae 100644
--- a/content/renderer/media/video_capture_impl.cc
+++ b/content/renderer/media/video_capture_impl.cc
@@ -15,6 +15,7 @@
 #include "base/stl_util.h"
 #include "base/thread_task_runner_handle.h"
 #include "content/child/child_process.h"
+#include "content/common/gpu/client/gpu_memory_buffer_impl.h"
 #include "content/common/media/video_capture_messages.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/limits.h"
@@ -38,6 +39,7 @@ void SaveReleaseSyncPoint(uint32* storage, uint32 release_sync_point) {
 
 }  // namespace
 
+// A holder of a memory-backed buffer and accessors to it.
 class VideoCaptureImpl::ClientBuffer
     : public base::RefCountedThreadSafe<ClientBuffer> {
  public:
@@ -56,6 +58,63 @@ class VideoCaptureImpl::ClientBuffer
   DISALLOW_COPY_AND_ASSIGN(ClientBuffer);
 };
 
+// A holder of a GpuMemoryBuffer-backed buffer, Map()ed on ctor and Unmap()ed on
+// dtor. Creates and owns GpuMemoryBuffer instances.
+class VideoCaptureImpl::ClientGpuMemoryBuffer
+    : public base::RefCountedThreadSafe<ClientGpuMemoryBuffer> {
+ public:
+  ClientGpuMemoryBuffer(
+      const std::vector<gfx::GpuMemoryBufferHandle>& client_gmb_handles,
+      const gfx::Size& size)
+      : gmb_handles_(client_gmb_handles),
+        size_(size),
+        format_(media::PIXEL_FORMAT_I420) {
+    Init();
+  }
+  uint8* data(int plane) { return data_[plane]; }

[mcasas, 2015/08/21 03:57:25]

const method? 
Here an l.75, 76.

[emircan, 2015/08/22 03:13:26]

Done.

+  int32 stride(int plane) { return strides_[plane]; }
+  std::vector<gfx::GpuMemoryBufferHandle> gpu_memory_buffer_handles() {
+    return gmb_handles_;
+  }
+
+ private:
+  friend class base::RefCountedThreadSafe<ClientGpuMemoryBuffer>;
+
+  virtual ~ClientGpuMemoryBuffer() {
+    for (auto& gmb : gmbs_)
+      gmb->Unmap();
+  }

[mcasas, 2015/08/21 03:57:25]

Insert blank line.

[emircan, 2015/08/22 03:13:26]

Done.

+  void Init() {
+    for (size_t i = 0; i < gmb_handles_.size(); ++i) {
+      const size_t width =
+          media::VideoFrame::Columns(i, format_, size_.width());
+      const size_t height = media::VideoFrame::Rows(i, format_, size_.height());
+      gmbs_.push_back(GpuMemoryBufferImpl::CreateFromHandle(
+          gmb_handles_[i],
+          gfx::Size(width, height),
+          gfx::BufferFormat::R_8,
+          gfx::BufferUsage::MAP,
+          base::Bind(&ClientGpuMemoryBuffer::DestroyGpuMemoryBuffer,
+                     base::Unretained(this))));
+      void* data_ptr = NULL;

[mcasas, 2015/08/21 03:57:25]

s/NULL/nullptr/

[emircan, 2015/08/22 03:13:26]

Done.

+      gmbs_[i]->Map(&data_ptr);
+      data_[i] = reinterpret_cast<uint8*>(data_ptr);
+      strides_[i] = width;
+    }
+  }
+  void DestroyGpuMemoryBuffer(uint32 sync_point) {}
+
+  const std::vector<gfx::GpuMemoryBufferHandle> gmb_handles_;
+  const gfx::Size size_;
+  const media::VideoPixelFormat format_;
+  // Owned instances of GMBs.
+  ScopedVector<gfx::GpuMemoryBuffer> gmbs_;
+  uint8* data_[media::VideoFrame::kMaxPlanes];
+  int32 strides_[media::VideoFrame::kMaxPlanes];
+
+  DISALLOW_COPY_AND_ASSIGN(ClientGpuMemoryBuffer);
+};
+
 VideoCaptureImpl::VideoCaptureImpl(
     const media::VideoCaptureSessionId session_id,
     VideoCaptureMessageFilter* filter)
@@ -146,6 +205,7 @@ void VideoCaptureImpl::StopCapture() {
   state_update_cb_.Run(VIDEO_CAPTURE_STATE_STOPPED);
   StopDevice();
   client_buffers_.clear();
+  client_gmb_buffers_.clear();
   ResetClient();
   weak_factory_.InvalidateWeakPtrs();
 }
@@ -168,9 +228,10 @@ void VideoCaptureImpl::GetDeviceFormatsInUse(
         new VideoCaptureHostMsg_GetDeviceFormatsInUse(device_id_, session_id_));
 }
 
-void VideoCaptureImpl::OnBufferCreated(base::SharedMemoryHandle handle,
-                                       int length,
-                                       int buffer_id) {
+void VideoCaptureImpl::OnBufferCreated(
+    base::SharedMemoryHandle handle,

[mcasas, 2015/08/21 03:57:25]

Revert to old indent.

[emircan, 2015/08/22 03:13:26]

Done.

+    int length,
+    int buffer_id) {
   DCHECK(io_task_runner_->BelongsToCurrentThread());
 
   // In case client calls StopCapture before the arrival of created buffer,
@@ -185,7 +246,6 @@ void VideoCaptureImpl::OnBufferCreated(base::SharedMemoryHandle handle,
     DLOG(ERROR) << "OnBufferCreated: Map failed.";
     return;
   }
-
   const bool inserted =
       client_buffers_.insert(std::make_pair(buffer_id, new ClientBuffer(
                                                            shm.Pass(), length)))
@@ -193,16 +253,39 @@ void VideoCaptureImpl::OnBufferCreated(base::SharedMemoryHandle handle,
   DCHECK(inserted);
 }
 
-void VideoCaptureImpl::OnBufferDestroyed(int buffer_id) {
+void VideoCaptureImpl::OnGpuMemoryBufferCreated(
+    const std::vector<gfx::GpuMemoryBufferHandle>& gmb_handles,
+    const gfx::Size& size,
+    int buffer_id) {
   DCHECK(io_task_runner_->BelongsToCurrentThread());
 
-  const ClientBufferMap::iterator iter = client_buffers_.find(buffer_id);
-  if (iter == client_buffers_.end())
+  // In case client calls StopCapture before the arrival of created buffer,
+  // just close this buffer and return.
+  if (state_ != VIDEO_CAPTURE_STATE_STARTED)
     return;
 
-  DCHECK(!iter->second.get() || iter->second->HasOneRef())
-      << "Instructed to delete buffer we are still using.";
-  client_buffers_.erase(iter);
+  const bool inserted =
+      client_gmb_buffers_.insert(std::make_pair(buffer_id,
+                                                new ClientGpuMemoryBuffer(
+                                                    gmb_handles, size)))
+          .second;
+  DCHECK(inserted);
+}
+
+void VideoCaptureImpl::OnBufferDestroyed(int buffer_id) {
+  DCHECK(io_task_runner_->BelongsToCurrentThread());
+
+  const auto& cb_iter = client_buffers_.find(buffer_id);
+  const auto& cgmb_iter = client_gmb_buffers_.find(buffer_id);
+  if (cb_iter != client_buffers_.end()) {
+    DCHECK(!cb_iter->second.get() || cb_iter->second->HasOneRef())
+        << "Instructed to delete buffer we are still using.";
+    client_buffers_.erase(cb_iter);
+  } else if (cgmb_iter != client_gmb_buffers_.end()) {
+    DCHECK(!cgmb_iter->second.get() || cgmb_iter->second->HasOneRef())
+        << "Instructed to delete buffer we are still using.";
+    client_gmb_buffers_.erase(cgmb_iter);
+  }
 }
 
 void VideoCaptureImpl::OnBufferReceived(
@@ -213,8 +296,9 @@ void VideoCaptureImpl::OnBufferReceived(
     media::VideoFrame::StorageType storage_type,
     const gfx::Size& coded_size,
     const gfx::Rect& visible_rect,
-    const gpu::MailboxHolder& mailbox_holder) {
+    const std::vector<gpu::MailboxHolder>& mailbox_holders) {
   DCHECK(io_task_runner_->BelongsToCurrentThread());
+  DCHECK_EQ(media::PIXEL_FORMAT_I420, pixel_format);
   if (state_ != VIDEO_CAPTURE_STATE_STARTED) {
     Send(new VideoCaptureHostMsg_BufferReady(device_id_, buffer_id, 0, -1.0));
     return;
@@ -230,46 +314,45 @@ void VideoCaptureImpl::OnBufferReceived(
 
   scoped_refptr<media::VideoFrame> frame;
   uint32* release_sync_point_storage = nullptr;
-  scoped_refptr<ClientBuffer> buffer;
-
-  if (mailbox_holder.mailbox.IsZero()) {
-    DCHECK_EQ(media::PIXEL_FORMAT_I420, pixel_format);
-    const ClientBufferMap::const_iterator iter =
-        client_buffers_.find(buffer_id);
+  if (mailbox_holders.empty()) {
+    const auto& iter = client_buffers_.find(buffer_id);
     DCHECK(iter != client_buffers_.end());
-    buffer = iter->second;
+    const scoped_refptr<ClientBuffer> buffer = iter->second;
     frame = media::VideoFrame::WrapExternalSharedMemory(
-        pixel_format,
-        coded_size,
-        visible_rect,
+        pixel_format, coded_size, visible_rect,
         gfx::Size(visible_rect.width(), visible_rect.height()),
-        reinterpret_cast<uint8*>(buffer->buffer->memory()),
-        buffer->buffer_size,
-        buffer->buffer->handle(),
-        0  /* shared_memory_offset */,
+        reinterpret_cast<uint8*>(buffer->buffer->memory()), buffer->buffer_size,
+        buffer->buffer->handle(), 0 /* shared_memory_offset */,
         timestamp - first_frame_timestamp_);
-
-  } else {
-    DCHECK_EQ(media::PIXEL_FORMAT_ARGB, pixel_format);
-    DCHECK(mailbox_holder.mailbox.Verify());  // Paranoia?
-    // To be deleted in DidFinishConsumingFrame().
+    frame->AddDestructionObserver(
+        base::Bind(&VideoCaptureImpl::DidFinishConsumingFrame,
+                   frame->metadata(), release_sync_point_storage,
+                   media::BindToCurrentLoop(base::Bind(
+                       &VideoCaptureImpl::OnClientBufferFinished,
+                       weak_factory_.GetWeakPtr(), buffer_id, buffer))));
+  } else if (storage_type == media::VideoFrame::STORAGE_OPAQUE) {
+    for (const auto& mailbox_holder : mailbox_holders)
+      DCHECK(mailbox_holder.mailbox.Verify());  // Paranoia?
+
+    const auto& gmb_iter = client_gmb_buffers_.find(buffer_id);
+    DCHECK(gmb_iter != client_gmb_buffers_.end());
+    const scoped_refptr<ClientGpuMemoryBuffer> gpu_memory_buffer =
+        gmb_iter->second;
     release_sync_point_storage = new uint32(0);
-    frame = media::VideoFrame::WrapNativeTexture(
-        pixel_format,
-        mailbox_holder,
+    frame = media::VideoFrame::WrapYUV420NativeTextures(
+        mailbox_holders[media::VideoFrame::kYPlane],
+        mailbox_holders[media::VideoFrame::kUPlane],
+        mailbox_holders[media::VideoFrame::kVPlane],
         base::Bind(&SaveReleaseSyncPoint, release_sync_point_storage),
-        coded_size,
-        gfx::Rect(coded_size),
-        coded_size,
+        coded_size, gfx::Rect(coded_size), coded_size,
         timestamp - first_frame_timestamp_);
+    frame->AddDestructionObserver(base::Bind(
+        &VideoCaptureImpl::DidFinishConsumingFrame, frame->metadata(),
+        release_sync_point_storage,
+        media::BindToCurrentLoop(base::Bind(
+            &VideoCaptureImpl::OnClientGpuMemoryBufferFinished,
+            weak_factory_.GetWeakPtr(), buffer_id, gpu_memory_buffer))));
   }
-  frame->AddDestructionObserver(
-      base::Bind(&VideoCaptureImpl::DidFinishConsumingFrame, frame->metadata(),
-                 release_sync_point_storage,
-                 media::BindToCurrentLoop(base::Bind(
-                     &VideoCaptureImpl::OnClientBufferFinished,
-                     weak_factory_.GetWeakPtr(), buffer_id, buffer))));
-
   frame->metadata()->MergeInternalValuesFrom(metadata);
   deliver_frame_cb_.Run(frame, timestamp);
 }
@@ -284,6 +367,14 @@ void VideoCaptureImpl::OnClientBufferFinished(
                                            release_sync_point,
                                            consumer_resource_utilization));
 }
+void VideoCaptureImpl::OnClientGpuMemoryBufferFinished(
+    int buffer_id,
+    const scoped_refptr<ClientGpuMemoryBuffer>& gpu_memory_buffer,
+    uint32 release_sync_point,
+    double consumer_resource_utilization) {
+  OnClientBufferFinished(buffer_id, scoped_refptr<ClientBuffer>(),
+                         release_sync_point, consumer_resource_utilization);
+}
 
 void VideoCaptureImpl::OnStateChanged(VideoCaptureState state) {
   // TODO(ajose): http://crbug.com/522155 improve this state machine.
@@ -293,6 +384,7 @@ void VideoCaptureImpl::OnStateChanged(VideoCaptureState state) {
   if (state == VIDEO_CAPTURE_STATE_STOPPED) {
     DVLOG(1) << "OnStateChanged: stopped!, device_id = " << device_id_;
     client_buffers_.clear();
+    client_gmb_buffers_.clear();
     weak_factory_.InvalidateWeakPtrs();
     return;
   }