Implement IPCs and VideoCapture::Client interfaces for texture capture

content/browser/renderer_host/media/video_capture_controller.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
+#include <map>
 #include <set>
 
 #include "base/bind.h"
 #include "base/debug/trace_event.h"
 #include "base/stl_util.h"
 #include "content/browser/renderer_host/media/media_stream_manager.h"
 #include "content/browser/renderer_host/media/video_capture_manager.h"
 #include "content/public/browser/browser_thread.h"
+#include "gpu/command_buffer/common/mailbox_holder.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_util.h"
 #include "media/base/yuv_convert.h"
 
 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
 #include "third_party/libyuv/include/libyuv.h"
 #endif
 
 using media::VideoCaptureFormat;
 
@@ skipping 42 matching lines @@
   VideoCaptureControllerEventHandler* const event_handler;
 
   // Handle to the render process that will receive the capture buffers.
   const base::ProcessHandle render_process_handle;
   const media::VideoCaptureSessionId session_id;
   const media::VideoCaptureParams parameters;
 
   // Buffers that are currently known to this client.
   std::set<int> known_buffers;
 
-  // Buffers currently held by this client.
-  std::set<int> active_buffers;
+  // Buffers currently held by this client, and syncpoint callback to call when
+  // they are returned from the client.
+  std::map<int, scoped_refptr<media::VideoFrame> > active_buffers;
 
   // State of capture session, controlled by VideoCaptureManager directly. This
   // transitions to true as soon as StopSession() occurs, at which point the
   // client is sent an OnEnded() event. However, because the client retains a
   // VideoCaptureController* pointer, its ControllerClient entry lives on until
   // it unregisters itself via RemoveClient(), which may happen asynchronously.
   //
   // TODO(nick): If we changed the semantics of VideoCaptureHost so that
   // OnEnded() events were processed synchronously (with the RemoveClient() done
   // implicitly), we could avoid tracking this state here in the Controller, and
@@ skipping 16 matching lines @@
       const base::WeakPtr<VideoCaptureController>& controller,
       const scoped_refptr<VideoCaptureBufferPool>& buffer_pool);
   virtual ~VideoCaptureDeviceClient();
 
   // VideoCaptureDevice::Client implementation.
   virtual scoped_refptr<Buffer> ReserveOutputBuffer(
       media::VideoFrame::Format format,
       const gfx::Size& size) OVERRIDE;
   virtual void OnIncomingCapturedFrame(const uint8* data,
                                        int length,
-                                       base::TimeTicks timestamp,
+                                       const VideoCaptureFormat& frame_format,
                                        int rotation,
-                                       const VideoCaptureFormat& frame_format)
-      OVERRIDE;
-  virtual void OnIncomingCapturedBuffer(const scoped_refptr<Buffer>& buffer,
-                                        media::VideoFrame::Format format,
-                                        const gfx::Size& dimensions,
-                                        base::TimeTicks timestamp,
-                                        int frame_rate) OVERRIDE;
+                                       base::TimeTicks timestamp) OVERRIDE;
+  virtual void OnIncomingCapturedVideoFrame(
+      const scoped_refptr<Buffer>& buffer,
+      const VideoCaptureFormat& buffer_format,
+      const scoped_refptr<media::VideoFrame>& frame,
+      base::TimeTicks timestamp) OVERRIDE;
   virtual void OnError() OVERRIDE;
 
  private:
   scoped_refptr<Buffer> DoReserveOutputBuffer(media::VideoFrame::Format format,
                                               const gfx::Size& dimensions);
 
   // The controller to which we post events.
   const base::WeakPtr<VideoCaptureController> controller_;
 
   // The pool of shared-memory buffers used for capturing.
@@ skipping 65 matching lines @@
     const VideoCaptureControllerID& id,
     VideoCaptureControllerEventHandler* event_handler) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   DVLOG(1) << "VideoCaptureController::RemoveClient, id " << id.device_id;
 
   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
   if (!client)
     return kInvalidMediaCaptureSessionId;
 
   // Take back all buffers held by the |client|.
-  for (std::set<int>::iterator buffer_it = client->active_buffers.begin();
+  for (std::map<int, scoped_refptr<media::VideoFrame> >::iterator buffer_it =
+           client->active_buffers.begin();
        buffer_it != client->active_buffers.end();
        ++buffer_it) {
-    int buffer_id = *buffer_it;
-    buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
+    buffer_pool_->RelinquishConsumerHold(buffer_it->first, 1);
   }
   client->active_buffers.clear();
 
   int session_id = client->session_id;
   controller_clients_.remove(client);
   delete client;
 
   return session_id;
 }
 
 void VideoCaptureController::StopSession(int session_id) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   DVLOG(1) << "VideoCaptureController::StopSession, id " << session_id;
 
   ControllerClient* client = FindClient(session_id, controller_clients_);
 
   if (client) {
     client->session_closed = true;
     client->event_handler->OnEnded(client->controller_id);
   }
 }
 
 void VideoCaptureController::ReturnBuffer(
     const VideoCaptureControllerID& id,
     VideoCaptureControllerEventHandler* event_handler,
-    int buffer_id) {
+    int buffer_id,
+    uint32 sync_point) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 
   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 
   // If this buffer is not held by this client, or this client doesn't exist
   // in controller, do nothing.
-  if (!client || !client->active_buffers.erase(buffer_id)) {
+  std::map<int, scoped_refptr<media::VideoFrame> >::iterator iter;
+  if (!client || (iter = client->active_buffers.find(buffer_id)) ==
+                     client->active_buffers.end()) {
     NOTREACHED();
     return;
   }
+  scoped_refptr<media::VideoFrame> frame = iter->second;
+  client->active_buffers.erase(iter);
+
+  if (frame->format() == media::VideoFrame::NATIVE_TEXTURE)
+    frame->mailbox_holder()->sync_point = sync_point;
 
   buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
 }
 
 const media::VideoCaptureFormat&
 VideoCaptureController::GetVideoCaptureFormat() const {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   return video_capture_format_;
 }
 
 scoped_refptr<media::VideoCaptureDevice::Client::Buffer>
 VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer(
     media::VideoFrame::Format format,
     const gfx::Size& size) {
   return DoReserveOutputBuffer(format, size);
 }
 
 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame(
     const uint8* data,
     int length,
-    base::TimeTicks timestamp,
+    const VideoCaptureFormat& frame_format,
     int rotation,
-    const VideoCaptureFormat& frame_format) {
+    base::TimeTicks timestamp) {
   TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame");
 
   if (!frame_format.IsValid())
     return;
 
   // Chopped pixels in width/height in case video capture device has odd
   // numbers for width/height.
   int chopped_width = 0;
   int chopped_height = 0;
   int new_unrotated_width = frame_format.frame_size.width();
@@ skipping 104 matching lines @@
                         new_unrotated_width,
                         new_unrotated_height,
                         rotation_mode,
                         origin_colorspace);
 #else
   // Libyuv is not linked in for Android WebView builds, but video capture is
   // not used in those builds either. Whenever libyuv is added in that build,
   // address all these #ifdef parts, see http://crbug.com/299611 .
   NOTREACHED();
 #endif  // if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
+  VideoCaptureFormat format(
+      dimensions, frame_format.frame_rate, media::PIXEL_FORMAT_I420);
+  scoped_refptr<media::VideoFrame> frame =
+      media::VideoFrame::WrapExternalPackedMemory(
+          media::VideoFrame::I420,
+          dimensions,
+          gfx::Rect(dimensions),
+          dimensions,
+          yplane,
+          media::VideoFrame::AllocationSize(media::VideoFrame::I420,
+                                            dimensions),
+          base::SharedMemory::NULLHandle(),
+          base::TimeDelta(),
+          base::Closure());
+  DCHECK(frame);
   BrowserThread::PostTask(
       BrowserThread::IO,
       FROM_HERE,
       base::Bind(
-          &VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread,
+          &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
           controller_,
           buffer,
-          dimensions,
-          frame_format.frame_rate,
+          format,
+          frame,
           timestamp));
 }
 
-void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedBuffer(
+void
+VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedVideoFrame(
     const scoped_refptr<Buffer>& buffer,
-    media::VideoFrame::Format format,
-    const gfx::Size& dimensions,
-    base::TimeTicks timestamp,
-    int frame_rate) {
-  // The capture pipeline expects I420 for now.
-  DCHECK_EQ(format, media::VideoFrame::I420)
-      << "Non-I420 output buffer returned";
-
+    const VideoCaptureFormat& buffer_format,
+    const scoped_refptr<media::VideoFrame>& frame,
+    base::TimeTicks timestamp) {
   BrowserThread::PostTask(
       BrowserThread::IO,
       FROM_HERE,
       base::Bind(
-          &VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread,
+          &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,

[mcasas, 2014/01/21 14:34:02]

I think the "unification" of VCC::VCDC::OnIncomingCapturedFrame
and VCC::VCDC::OnIncomingCapturedVideoFrame calls into a
single DoIncoming....OnIOThread is great! Perhaps ncarter@ 
would also be interested to take a look at this.

           controller_,
           buffer,
-          dimensions,
-          frame_rate,
+          buffer_format,
+          frame,
           timestamp));
 }
 
 void VideoCaptureController::VideoCaptureDeviceClient::OnError() {
   BrowserThread::PostTask(BrowserThread::IO,
       FROM_HERE,
       base::Bind(&VideoCaptureController::DoErrorOnIOThread, controller_));
 }
 
 scoped_refptr<media::VideoCaptureDevice::Client::Buffer>
 VideoCaptureController::VideoCaptureDeviceClient::DoReserveOutputBuffer(
     media::VideoFrame::Format format,
     const gfx::Size& dimensions) {
-  // The capture pipeline expects I420 for now.
-  DCHECK_EQ(format, media::VideoFrame::I420)
-      << "Non-I420 output buffer requested";
+  size_t frame_bytes = 0;
+  if (format == media::VideoFrame::NATIVE_TEXTURE) {
+    DCHECK_EQ(dimensions.width(), 0);
+    DCHECK_EQ(dimensions.height(), 0);
+  } else {
+    // The capture pipeline expects I420 for now.
+    DCHECK_EQ(format, media::VideoFrame::I420)
+        << "Non-I420 output buffer format " << format << " requested";

[mcasas, 2014/01/21 14:34:02]

Being |format| numeric, not sure this DCHECK would be very
informative as is.
Would it be possible/interesting to add some macros for capture
format -> String text?

[sheu, 2014/01/29 01:19:20]

Not very informative, but I think sufficient for a DCHECK.  For debugging
purposes you can just look up the enum value.

+    frame_bytes = media::VideoFrame::AllocationSize(format, dimensions);
+  }
 
   int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
-  const size_t frame_bytes =
-      media::VideoFrame::AllocationSize(format, dimensions);
-
   int buffer_id =
       buffer_pool_->ReserveForProducer(frame_bytes, &buffer_id_to_drop);
   if (buffer_id == VideoCaptureBufferPool::kInvalidId)
     return NULL;
   void* data;
   size_t size;
   buffer_pool_->GetBufferInfo(buffer_id, &data, &size);
 
   scoped_refptr<media::VideoCaptureDevice::Client::Buffer> output_buffer(
       new PoolBuffer(buffer_pool_, buffer_id, data, size));
 
   if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
     BrowserThread::PostTask(BrowserThread::IO,
         FROM_HERE,
         base::Bind(&VideoCaptureController::DoBufferDestroyedOnIOThread,
                    controller_, buffer_id_to_drop));
   }
 
   return output_buffer;
 }
 
 VideoCaptureController::~VideoCaptureController() {
   STLDeleteContainerPointers(controller_clients_.begin(),
                              controller_clients_.end());
 }
 
-void VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread(
-    scoped_refptr<media::VideoCaptureDevice::Client::Buffer> buffer,
-    const gfx::Size& dimensions,
-    int frame_rate,
+void VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread(
+    const scoped_refptr<media::VideoCaptureDevice::Client::Buffer>& buffer,
+    const media::VideoCaptureFormat& buffer_format,
+    const scoped_refptr<media::VideoFrame>& frame,
     base::TimeTicks timestamp) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   DCHECK_NE(buffer->id(), VideoCaptureBufferPool::kInvalidId);
 
-  VideoCaptureFormat frame_format(
-      dimensions, frame_rate, media::PIXEL_FORMAT_I420);
-
   int count = 0;
   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
     for (ControllerClients::iterator client_it = controller_clients_.begin();
          client_it != controller_clients_.end(); ++client_it) {
       ControllerClient* client = *client_it;
       if (client->session_closed)
         continue;
 
-      bool is_new_buffer = client->known_buffers.insert(buffer->id()).second;
-      if (is_new_buffer) {
-        // On the first use of a buffer on a client, share the memory handle.
-        size_t memory_size = 0;
-        base::SharedMemoryHandle remote_handle = buffer_pool_->ShareToProcess(
-            buffer->id(), client->render_process_handle, &memory_size);
-        client->event_handler->OnBufferCreated(
-            client->controller_id, remote_handle, memory_size, buffer->id());
+      if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
+        client->event_handler->OnMailboxBufferReady(client->controller_id,
+                                                    buffer->id(),
+                                                    *frame->mailbox_holder(),
+                                                    buffer_format,
+                                                    timestamp);
+      } else {
+        bool is_new_buffer = client->known_buffers.insert(buffer->id()).second;
+        if (is_new_buffer) {
+          // On the first use of a buffer on a client, share the memory handle.
+          size_t memory_size = 0;
+          base::SharedMemoryHandle remote_handle = buffer_pool_->ShareToProcess(
+              buffer->id(), client->render_process_handle, &memory_size);
+          client->event_handler->OnBufferCreated(
+              client->controller_id, remote_handle, memory_size, buffer->id());
+        }
+
+        client->event_handler->OnBufferReady(
+            client->controller_id, buffer->id(), buffer_format, timestamp);
       }
 
-      client->event_handler->OnBufferReady(
-          client->controller_id, buffer->id(), timestamp, frame_format);
-      bool inserted = client->active_buffers.insert(buffer->id()).second;
+      bool inserted =
+          client->active_buffers.insert(std::make_pair(buffer->id(), frame))
+              .second;
       DCHECK(inserted) << "Unexpected duplicate buffer: " << buffer->id();
       count++;
     }
   }
 
   buffer_pool_->HoldForConsumers(buffer->id(), count);
 }
 
 void VideoCaptureController::DoErrorOnIOThread() {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
@@ skipping 53 matching lines @@
   }
   return NULL;
 }
 
 int VideoCaptureController::GetClientCount() {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   return controller_clients_.size();
 }
 
 }  // namespace content