Pass GpuMemoryBuffer backed VideoFrame from browser to renderer processes

content/renderer/media/video_capture_impl.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 //
 // Notes about usage of this object by VideoCaptureImplManager.
 //
 // VideoCaptureImplManager access this object by using a Unretained()
 // binding and tasks on the IO thread. It is then important that
 // VideoCaptureImpl never post task to itself. All operations must be
 // synchronous.
 
 #include "content/renderer/media/video_capture_impl.h"
 
 #include "base/bind.h"
 #include "base/stl_util.h"
 #include "base/thread_task_runner_handle.h"
 #include "content/child/child_process.h"
+#include "content/common/gpu/client/gpu_memory_buffer_impl.h"
 #include "content/common/media/video_capture_messages.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/limits.h"
 #include "media/base/video_frame.h"
 
 namespace content {
 
 namespace {
 
 const int kUndefinedDeviceId = 0;
 
 // This is called on an unknown thread when the VideoFrame destructor executes.
 // As of this writing, this callback mechanism is the only interface in
 // VideoFrame to provide the final value for |release_sync_point|.
 // VideoCaptureImpl::DidFinishConsumingFrame() will read the value saved here,
 // and pass it back to the IO thread to pass back to the host via the
 // BufferReady IPC.
 void SaveReleaseSyncPoint(uint32* storage, uint32 release_sync_point) {
   *storage = release_sync_point;
 }
 
 }  // namespace
 
+// A holder of a memory-backed buffer and accessors to it.
 class VideoCaptureImpl::ClientBuffer
     : public base::RefCountedThreadSafe<ClientBuffer> {
  public:
   ClientBuffer(scoped_ptr<base::SharedMemory> buffer,

[mcasas, 2015/08/21 03:57:25]

nit: Fits in one line.
   ClientBuffer(scoped_ptr<base::SharedMemory> buffer, size_t buffer_size)

[emircan, 2015/08/22 03:13:26]

Done.

                size_t buffer_size)
       : buffer(buffer.Pass()),
         buffer_size(buffer_size) {}
   const scoped_ptr<base::SharedMemory> buffer;

[mcasas, 2015/08/21 03:57:25]

Shouldn't |buffer| and |buffer_size| be private and have const getters?

[emircan, 2015/08/22 03:13:26]

Done.

   const size_t buffer_size;
 
  private:
   friend class base::RefCountedThreadSafe<ClientBuffer>;
 
   virtual ~ClientBuffer() {}
 
   DISALLOW_COPY_AND_ASSIGN(ClientBuffer);
 };
 
+// A holder of a GpuMemoryBuffer-backed buffer, Map()ed on ctor and Unmap()ed on
+// dtor. Creates and owns GpuMemoryBuffer instances.
+class VideoCaptureImpl::ClientGpuMemoryBuffer
+    : public base::RefCountedThreadSafe<ClientGpuMemoryBuffer> {
+ public:
+  ClientGpuMemoryBuffer(
+      const std::vector<gfx::GpuMemoryBufferHandle>& client_gmb_handles,
+      const gfx::Size& size)
+      : gmb_handles_(client_gmb_handles),
+        size_(size),
+        format_(media::PIXEL_FORMAT_I420) {
+    Init();
+  }
+  uint8* data(int plane) { return data_[plane]; }

[mcasas, 2015/08/21 03:57:25]

const method? 
Here an l.75, 76.

[emircan, 2015/08/22 03:13:26]

Done.

+  int32 stride(int plane) { return strides_[plane]; }
+  std::vector<gfx::GpuMemoryBufferHandle> gpu_memory_buffer_handles() {
+    return gmb_handles_;
+  }
+
+ private:
+  friend class base::RefCountedThreadSafe<ClientGpuMemoryBuffer>;
+
+  virtual ~ClientGpuMemoryBuffer() {
+    for (auto& gmb : gmbs_)
+      gmb->Unmap();
+  }

[mcasas, 2015/08/21 03:57:25]

Insert blank line.

[emircan, 2015/08/22 03:13:26]

Done.

+  void Init() {
+    for (size_t i = 0; i < gmb_handles_.size(); ++i) {
+      const size_t width =
+          media::VideoFrame::Columns(i, format_, size_.width());
+      const size_t height = media::VideoFrame::Rows(i, format_, size_.height());
+      gmbs_.push_back(GpuMemoryBufferImpl::CreateFromHandle(
+          gmb_handles_[i],
+          gfx::Size(width, height),
+          gfx::BufferFormat::R_8,
+          gfx::BufferUsage::MAP,
+          base::Bind(&ClientGpuMemoryBuffer::DestroyGpuMemoryBuffer,
+                     base::Unretained(this))));
+      void* data_ptr = NULL;

[mcasas, 2015/08/21 03:57:25]

s/NULL/nullptr/

[emircan, 2015/08/22 03:13:26]

Done.

+      gmbs_[i]->Map(&data_ptr);
+      data_[i] = reinterpret_cast<uint8*>(data_ptr);
+      strides_[i] = width;
+    }
+  }
+  void DestroyGpuMemoryBuffer(uint32 sync_point) {}
+
+  const std::vector<gfx::GpuMemoryBufferHandle> gmb_handles_;
+  const gfx::Size size_;
+  const media::VideoPixelFormat format_;
+  // Owned instances of GMBs.
+  ScopedVector<gfx::GpuMemoryBuffer> gmbs_;
+  uint8* data_[media::VideoFrame::kMaxPlanes];
+  int32 strides_[media::VideoFrame::kMaxPlanes];
+
+  DISALLOW_COPY_AND_ASSIGN(ClientGpuMemoryBuffer);
+};
+
 VideoCaptureImpl::VideoCaptureImpl(
     const media::VideoCaptureSessionId session_id,
     VideoCaptureMessageFilter* filter)
     : message_filter_(filter),
       device_id_(kUndefinedDeviceId),
       session_id_(session_id),
       state_(VIDEO_CAPTURE_STATE_STOPPED),
       weak_factory_(this) {
   DCHECK(filter);
 }
@@ skipping 70 matching lines @@
 
 void VideoCaptureImpl::StopCapture() {
   DCHECK(io_task_runner_->BelongsToCurrentThread());
   if (state_ == VIDEO_CAPTURE_STATE_STOPPED || !IsInitialized())
     return;
 
   DVLOG(1) << "StopCapture: Stopping capture.";
   state_update_cb_.Run(VIDEO_CAPTURE_STATE_STOPPED);
   StopDevice();
   client_buffers_.clear();
+  client_gmb_buffers_.clear();
   ResetClient();
   weak_factory_.InvalidateWeakPtrs();
 }
 
 void VideoCaptureImpl::GetDeviceSupportedFormats(
     const VideoCaptureDeviceFormatsCB& callback) {
   DCHECK(io_task_runner_->BelongsToCurrentThread());
   device_formats_cb_queue_.push_back(callback);
   if (device_formats_cb_queue_.size() == 1)
     Send(new VideoCaptureHostMsg_GetDeviceSupportedFormats(device_id_,
                                                            session_id_));
 }
 
 void VideoCaptureImpl::GetDeviceFormatsInUse(
     const VideoCaptureDeviceFormatsCB& callback) {
   DCHECK(io_task_runner_->BelongsToCurrentThread());
   device_formats_in_use_cb_queue_.push_back(callback);
   if (device_formats_in_use_cb_queue_.size() == 1)
     Send(
         new VideoCaptureHostMsg_GetDeviceFormatsInUse(device_id_, session_id_));
 }
 
-void VideoCaptureImpl::OnBufferCreated(base::SharedMemoryHandle handle,
-                                       int length,
-                                       int buffer_id) {
+void VideoCaptureImpl::OnBufferCreated(
+    base::SharedMemoryHandle handle,

[mcasas, 2015/08/21 03:57:25]

Revert to old indent.

[emircan, 2015/08/22 03:13:26]

Done.

+    int length,
+    int buffer_id) {
   DCHECK(io_task_runner_->BelongsToCurrentThread());
 
   // In case client calls StopCapture before the arrival of created buffer,
   // just close this buffer and return.
   if (state_ != VIDEO_CAPTURE_STATE_STARTED) {
     base::SharedMemory::CloseHandle(handle);
     return;
   }
 
   scoped_ptr<base::SharedMemory> shm(new base::SharedMemory(handle, false));
   if (!shm->Map(length)) {
     DLOG(ERROR) << "OnBufferCreated: Map failed.";
     return;
   }
-
   const bool inserted =
       client_buffers_.insert(std::make_pair(buffer_id, new ClientBuffer(
                                                            shm.Pass(), length)))
           .second;
   DCHECK(inserted);
 }
 
+void VideoCaptureImpl::OnGpuMemoryBufferCreated(
+    const std::vector<gfx::GpuMemoryBufferHandle>& gmb_handles,
+    const gfx::Size& size,
+    int buffer_id) {
+  DCHECK(io_task_runner_->BelongsToCurrentThread());
+
+  // In case client calls StopCapture before the arrival of created buffer,
+  // just close this buffer and return.
+  if (state_ != VIDEO_CAPTURE_STATE_STARTED)
+    return;
+
+  const bool inserted =
+      client_gmb_buffers_.insert(std::make_pair(buffer_id,
+                                                new ClientGpuMemoryBuffer(
+                                                    gmb_handles, size)))
+          .second;
+  DCHECK(inserted);
+}
+
 void VideoCaptureImpl::OnBufferDestroyed(int buffer_id) {
   DCHECK(io_task_runner_->BelongsToCurrentThread());
 
-  const ClientBufferMap::iterator iter = client_buffers_.find(buffer_id);
-  if (iter == client_buffers_.end())
-    return;
-
-  DCHECK(!iter->second.get() || iter->second->HasOneRef())
-      << "Instructed to delete buffer we are still using.";
-  client_buffers_.erase(iter);
+  const auto& cb_iter = client_buffers_.find(buffer_id);
+  const auto& cgmb_iter = client_gmb_buffers_.find(buffer_id);
+  if (cb_iter != client_buffers_.end()) {
+    DCHECK(!cb_iter->second.get() || cb_iter->second->HasOneRef())
+        << "Instructed to delete buffer we are still using.";
+    client_buffers_.erase(cb_iter);
+  } else if (cgmb_iter != client_gmb_buffers_.end()) {
+    DCHECK(!cgmb_iter->second.get() || cgmb_iter->second->HasOneRef())
+        << "Instructed to delete buffer we are still using.";
+    client_gmb_buffers_.erase(cgmb_iter);
+  }
 }
 
 void VideoCaptureImpl::OnBufferReceived(
     int buffer_id,
     base::TimeTicks timestamp,
     const base::DictionaryValue& metadata,
     media::VideoPixelFormat pixel_format,
     media::VideoFrame::StorageType storage_type,
     const gfx::Size& coded_size,
     const gfx::Rect& visible_rect,
-    const gpu::MailboxHolder& mailbox_holder) {
+    const std::vector<gpu::MailboxHolder>& mailbox_holders) {
   DCHECK(io_task_runner_->BelongsToCurrentThread());
+  DCHECK_EQ(media::PIXEL_FORMAT_I420, pixel_format);
   if (state_ != VIDEO_CAPTURE_STATE_STARTED) {
     Send(new VideoCaptureHostMsg_BufferReady(device_id_, buffer_id, 0, -1.0));
     return;
   }
   if (first_frame_timestamp_.is_null())
     first_frame_timestamp_ = timestamp;
 
   // Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
   TRACE_EVENT_INSTANT2("cast_perf_test", "OnBufferReceived",
                        TRACE_EVENT_SCOPE_THREAD, "timestamp",
                        timestamp.ToInternalValue(), "time_delta",
                        (timestamp - first_frame_timestamp_).ToInternalValue());
 
   scoped_refptr<media::VideoFrame> frame;
   uint32* release_sync_point_storage = nullptr;
-  scoped_refptr<ClientBuffer> buffer;
+  if (mailbox_holders.empty()) {
+    const auto& iter = client_buffers_.find(buffer_id);
+    DCHECK(iter != client_buffers_.end());
+    const scoped_refptr<ClientBuffer> buffer = iter->second;
+    frame = media::VideoFrame::WrapExternalSharedMemory(
+        pixel_format, coded_size, visible_rect,
+        gfx::Size(visible_rect.width(), visible_rect.height()),
+        reinterpret_cast<uint8*>(buffer->buffer->memory()), buffer->buffer_size,
+        buffer->buffer->handle(), 0 /* shared_memory_offset */,
+        timestamp - first_frame_timestamp_);
+    frame->AddDestructionObserver(
+        base::Bind(&VideoCaptureImpl::DidFinishConsumingFrame,
+                   frame->metadata(), release_sync_point_storage,
+                   media::BindToCurrentLoop(base::Bind(
+                       &VideoCaptureImpl::OnClientBufferFinished,
+                       weak_factory_.GetWeakPtr(), buffer_id, buffer))));
+  } else if (storage_type == media::VideoFrame::STORAGE_OPAQUE) {
+    for (const auto& mailbox_holder : mailbox_holders)
+      DCHECK(mailbox_holder.mailbox.Verify());  // Paranoia?
 
-  if (mailbox_holder.mailbox.IsZero()) {
-    DCHECK_EQ(media::PIXEL_FORMAT_I420, pixel_format);
-    const ClientBufferMap::const_iterator iter =
-        client_buffers_.find(buffer_id);
-    DCHECK(iter != client_buffers_.end());
-    buffer = iter->second;
-    frame = media::VideoFrame::WrapExternalSharedMemory(
-        pixel_format,
-        coded_size,
-        visible_rect,
-        gfx::Size(visible_rect.width(), visible_rect.height()),
-        reinterpret_cast<uint8*>(buffer->buffer->memory()),
-        buffer->buffer_size,
-        buffer->buffer->handle(),
-        0  /* shared_memory_offset */,
+    const auto& gmb_iter = client_gmb_buffers_.find(buffer_id);
+    DCHECK(gmb_iter != client_gmb_buffers_.end());
+    const scoped_refptr<ClientGpuMemoryBuffer> gpu_memory_buffer =
+        gmb_iter->second;
+    release_sync_point_storage = new uint32(0);
+    frame = media::VideoFrame::WrapYUV420NativeTextures(
+        mailbox_holders[media::VideoFrame::kYPlane],
+        mailbox_holders[media::VideoFrame::kUPlane],
+        mailbox_holders[media::VideoFrame::kVPlane],
+        base::Bind(&SaveReleaseSyncPoint, release_sync_point_storage),
+        coded_size, gfx::Rect(coded_size), coded_size,
         timestamp - first_frame_timestamp_);
-
-  } else {
-    DCHECK_EQ(media::PIXEL_FORMAT_ARGB, pixel_format);
-    DCHECK(mailbox_holder.mailbox.Verify());  // Paranoia?
-    // To be deleted in DidFinishConsumingFrame().
-    release_sync_point_storage = new uint32(0);
-    frame = media::VideoFrame::WrapNativeTexture(
-        pixel_format,
-        mailbox_holder,
-        base::Bind(&SaveReleaseSyncPoint, release_sync_point_storage),
-        coded_size,
-        gfx::Rect(coded_size),
-        coded_size,
-        timestamp - first_frame_timestamp_);
+    frame->AddDestructionObserver(base::Bind(
+        &VideoCaptureImpl::DidFinishConsumingFrame, frame->metadata(),
+        release_sync_point_storage,
+        media::BindToCurrentLoop(base::Bind(
+            &VideoCaptureImpl::OnClientGpuMemoryBufferFinished,
+            weak_factory_.GetWeakPtr(), buffer_id, gpu_memory_buffer))));
   }
-  frame->AddDestructionObserver(
-      base::Bind(&VideoCaptureImpl::DidFinishConsumingFrame, frame->metadata(),
-                 release_sync_point_storage,
-                 media::BindToCurrentLoop(base::Bind(
-                     &VideoCaptureImpl::OnClientBufferFinished,
-                     weak_factory_.GetWeakPtr(), buffer_id, buffer))));
-
   frame->metadata()->MergeInternalValuesFrom(metadata);
   deliver_frame_cb_.Run(frame, timestamp);
 }
 
 void VideoCaptureImpl::OnClientBufferFinished(
     int buffer_id,
     const scoped_refptr<ClientBuffer>& /* ignored_buffer */,
     uint32 release_sync_point,
     double consumer_resource_utilization) {
   DCHECK(io_task_runner_->BelongsToCurrentThread());
   Send(new VideoCaptureHostMsg_BufferReady(device_id_, buffer_id,
                                            release_sync_point,
                                            consumer_resource_utilization));
 }
+void VideoCaptureImpl::OnClientGpuMemoryBufferFinished(
+    int buffer_id,
+    const scoped_refptr<ClientGpuMemoryBuffer>& gpu_memory_buffer,
+    uint32 release_sync_point,
+    double consumer_resource_utilization) {
+  OnClientBufferFinished(buffer_id, scoped_refptr<ClientBuffer>(),
+                         release_sync_point, consumer_resource_utilization);
+}
 
 void VideoCaptureImpl::OnStateChanged(VideoCaptureState state) {
   // TODO(ajose): http://crbug.com/522155 improve this state machine.
   DCHECK(io_task_runner_->BelongsToCurrentThread());
   state_ = state;
 
   if (state == VIDEO_CAPTURE_STATE_STOPPED) {
     DVLOG(1) << "OnStateChanged: stopped!, device_id = " << device_id_;
     client_buffers_.clear();
+    client_gmb_buffers_.clear();
     weak_factory_.InvalidateWeakPtrs();
     return;
   }
   if (state == VIDEO_CAPTURE_STATE_ERROR) {
     DVLOG(1) << "OnStateChanged: error!, device_id = " << device_id_;
     if (!state_update_cb_.is_null())
       state_update_cb_.Run(VIDEO_CAPTURE_STATE_ERROR);
     ResetClient();
     return;
   }
@@ skipping 93 matching lines @@
 void VideoCaptureImpl::ResetClient() {
   client_params_ = media::VideoCaptureParams();
   state_update_cb_.Reset();
   deliver_frame_cb_.Reset();
   first_frame_timestamp_ = base::TimeTicks();
   device_id_ = kUndefinedDeviceId;
   state_ = VIDEO_CAPTURE_STATE_STOPPED;
 }
 
 }  // namespace content