Implement IPCs and VideoCapture::Client interfaces for texture capture

content/browser/renderer_host/media/video_capture_controller.cc

Index: content/browser/renderer_host/media/video_capture_controller.cc
diff --git a/content/browser/renderer_host/media/video_capture_controller.cc b/content/browser/renderer_host/media/video_capture_controller.cc
index f23f50f82b2a771746ce012cbccebb0dc82cce94..1330a72a6e99ff8cb56c367783a2d89d738fbf2b 100644
--- a/content/browser/renderer_host/media/video_capture_controller.cc
+++ b/content/browser/renderer_host/media/video_capture_controller.cc
@@ -4,6 +4,7 @@
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
+#include <map>
 #include <set>
 
 #include "base/bind.h"
@@ -12,6 +13,7 @@
 #include "content/browser/renderer_host/media/media_stream_manager.h"
 #include "content/browser/renderer_host/media/video_capture_manager.h"
 #include "content/public/browser/browser_thread.h"
+#include "gpu/command_buffer/common/mailbox_holder.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_util.h"
 #include "media/base/yuv_convert.h"
@@ -74,8 +76,9 @@ struct VideoCaptureController::ControllerClient {
   // Buffers that are currently known to this client.
   std::set<int> known_buffers;
 
-  // Buffers currently held by this client.
-  std::set<int> active_buffers;
+  // Buffers currently held by this client, and syncpoint callback to call when
+  // they are returned from the client.
+  std::map<int, scoped_refptr<media::VideoFrame> > active_buffers;
 
   // State of capture session, controlled by VideoCaptureManager directly. This
   // transitions to true as soon as StopSession() occurs, at which point the
@@ -112,15 +115,14 @@ class VideoCaptureController::VideoCaptureDeviceClient
       const gfx::Size& size) OVERRIDE;
   virtual void OnIncomingCapturedFrame(const uint8* data,
                                        int length,
-                                       base::TimeTicks timestamp,
+                                       const VideoCaptureFormat& frame_format,
                                        int rotation,
-                                       const VideoCaptureFormat& frame_format)
-      OVERRIDE;
-  virtual void OnIncomingCapturedBuffer(const scoped_refptr<Buffer>& buffer,
-                                        media::VideoFrame::Format format,
-                                        const gfx::Size& dimensions,
-                                        base::TimeTicks timestamp,
-                                        int frame_rate) OVERRIDE;
+                                       base::TimeTicks timestamp) OVERRIDE;
+  virtual void OnIncomingCapturedVideoFrame(
+      const scoped_refptr<Buffer>& buffer,
+      const VideoCaptureFormat& buffer_format,
+      const scoped_refptr<media::VideoFrame>& frame,
+      base::TimeTicks timestamp) OVERRIDE;
   virtual void OnError() OVERRIDE;
 
  private:
@@ -206,11 +208,11 @@ int VideoCaptureController::RemoveClient(
     return kInvalidMediaCaptureSessionId;
 
   // Take back all buffers held by the |client|.
-  for (std::set<int>::iterator buffer_it = client->active_buffers.begin();
+  for (std::map<int, scoped_refptr<media::VideoFrame> >::iterator buffer_it =
+           client->active_buffers.begin();
        buffer_it != client->active_buffers.end();
        ++buffer_it) {
-    int buffer_id = *buffer_it;
-    buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
+    buffer_pool_->RelinquishConsumerHold(buffer_it->first, 1);
   }
   client->active_buffers.clear();
 
@@ -236,17 +238,25 @@ void VideoCaptureController::StopSession(int session_id) {
 void VideoCaptureController::ReturnBuffer(
     const VideoCaptureControllerID& id,
     VideoCaptureControllerEventHandler* event_handler,
-    int buffer_id) {
+    int buffer_id,
+    uint32 sync_point) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
 
   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 
   // If this buffer is not held by this client, or this client doesn't exist
   // in controller, do nothing.
-  if (!client || !client->active_buffers.erase(buffer_id)) {
+  std::map<int, scoped_refptr<media::VideoFrame> >::iterator iter;
+  if (!client || (iter = client->active_buffers.find(buffer_id)) ==
+                     client->active_buffers.end()) {
     NOTREACHED();
     return;
   }
+  scoped_refptr<media::VideoFrame> frame = iter->second;
+  client->active_buffers.erase(iter);
+
+  if (frame->format() == media::VideoFrame::NATIVE_TEXTURE)
+    frame->mailbox_holder()->sync_point = sync_point;
 
   buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
 }
@@ -267,9 +277,9 @@ VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer(
 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame(
     const uint8* data,
     int length,
-    base::TimeTicks timestamp,
+    const VideoCaptureFormat& frame_format,
     int rotation,
-    const VideoCaptureFormat& frame_format) {
+    base::TimeTicks timestamp) {
   TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame");
 
   if (!frame_format.IsValid())
@@ -394,37 +404,48 @@ void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame(
   // address all these #ifdef parts, see http://crbug.com/299611 .
   NOTREACHED();
 #endif  // if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
+  VideoCaptureFormat format(
+      dimensions, frame_format.frame_rate, media::PIXEL_FORMAT_I420);
+  scoped_refptr<media::VideoFrame> frame =
+      media::VideoFrame::WrapExternalPackedMemory(
+          media::VideoFrame::I420,
+          dimensions,
+          gfx::Rect(dimensions),
+          dimensions,
+          yplane,
+          media::VideoFrame::AllocationSize(media::VideoFrame::I420,
+                                            dimensions),
+          base::SharedMemory::NULLHandle(),
+          base::TimeDelta(),
+          base::Closure());
+  DCHECK(frame);
   BrowserThread::PostTask(
       BrowserThread::IO,
       FROM_HERE,
       base::Bind(
-          &VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread,
+          &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
           controller_,
           buffer,
-          dimensions,
-          frame_format.frame_rate,
+          format,
+          frame,
           timestamp));
 }
 
-void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedBuffer(
+void
+VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedVideoFrame(
     const scoped_refptr<Buffer>& buffer,
-    media::VideoFrame::Format format,
-    const gfx::Size& dimensions,
-    base::TimeTicks timestamp,
-    int frame_rate) {
-  // The capture pipeline expects I420 for now.
-  DCHECK_EQ(format, media::VideoFrame::I420)
-      << "Non-I420 output buffer returned";
-
+    const VideoCaptureFormat& buffer_format,
+    const scoped_refptr<media::VideoFrame>& frame,
+    base::TimeTicks timestamp) {
   BrowserThread::PostTask(
       BrowserThread::IO,
       FROM_HERE,
       base::Bind(
-          &VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread,
+          &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,

[mcasas, 2014/01/21 14:34:02]

I think the "unification" of VCC::VCDC::OnIncomingCapturedFrame
and VCC::VCDC::OnIncomingCapturedVideoFrame calls into a
single DoIncoming....OnIOThread is great! Perhaps ncarter@ 
would also be interested to take a look at this.

           controller_,
           buffer,
-          dimensions,
-          frame_rate,
+          buffer_format,
+          frame,
           timestamp));
 }
 
@@ -438,14 +459,18 @@ scoped_refptr<media::VideoCaptureDevice::Client::Buffer>
 VideoCaptureController::VideoCaptureDeviceClient::DoReserveOutputBuffer(
     media::VideoFrame::Format format,
     const gfx::Size& dimensions) {
-  // The capture pipeline expects I420 for now.
-  DCHECK_EQ(format, media::VideoFrame::I420)
-      << "Non-I420 output buffer requested";
+  size_t frame_bytes = 0;
+  if (format == media::VideoFrame::NATIVE_TEXTURE) {
+    DCHECK_EQ(dimensions.width(), 0);
+    DCHECK_EQ(dimensions.height(), 0);
+  } else {
+    // The capture pipeline expects I420 for now.
+    DCHECK_EQ(format, media::VideoFrame::I420)
+        << "Non-I420 output buffer format " << format << " requested";

[mcasas, 2014/01/21 14:34:02]

Being |format| numeric, not sure this DCHECK would be very
informative as is.
Would it be possible/interesting to add some macros for capture
format -> String text?

[sheu, 2014/01/29 01:19:20]

Not very informative, but I think sufficient for a DCHECK.  For debugging
purposes you can just look up the enum value.

+    frame_bytes = media::VideoFrame::AllocationSize(format, dimensions);
+  }
 
   int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
-  const size_t frame_bytes =
-      media::VideoFrame::AllocationSize(format, dimensions);
-
   int buffer_id =
       buffer_pool_->ReserveForProducer(frame_bytes, &buffer_id_to_drop);
   if (buffer_id == VideoCaptureBufferPool::kInvalidId)
@@ -472,17 +497,14 @@ VideoCaptureController::~VideoCaptureController() {
                              controller_clients_.end());
 }
 
-void VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread(
-    scoped_refptr<media::VideoCaptureDevice::Client::Buffer> buffer,
-    const gfx::Size& dimensions,
-    int frame_rate,
+void VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread(
+    const scoped_refptr<media::VideoCaptureDevice::Client::Buffer>& buffer,
+    const media::VideoCaptureFormat& buffer_format,
+    const scoped_refptr<media::VideoFrame>& frame,
     base::TimeTicks timestamp) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   DCHECK_NE(buffer->id(), VideoCaptureBufferPool::kInvalidId);
 
-  VideoCaptureFormat frame_format(
-      dimensions, frame_rate, media::PIXEL_FORMAT_I420);
-
   int count = 0;
   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
     for (ControllerClients::iterator client_it = controller_clients_.begin();
@@ -491,19 +513,30 @@ void VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread(
       if (client->session_closed)
         continue;
 
-      bool is_new_buffer = client->known_buffers.insert(buffer->id()).second;
-      if (is_new_buffer) {
-        // On the first use of a buffer on a client, share the memory handle.
-        size_t memory_size = 0;
-        base::SharedMemoryHandle remote_handle = buffer_pool_->ShareToProcess(
-            buffer->id(), client->render_process_handle, &memory_size);
-        client->event_handler->OnBufferCreated(
-            client->controller_id, remote_handle, memory_size, buffer->id());
+      if (frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
+        client->event_handler->OnMailboxBufferReady(client->controller_id,
+                                                    buffer->id(),
+                                                    *frame->mailbox_holder(),
+                                                    buffer_format,
+                                                    timestamp);
+      } else {
+        bool is_new_buffer = client->known_buffers.insert(buffer->id()).second;
+        if (is_new_buffer) {
+          // On the first use of a buffer on a client, share the memory handle.
+          size_t memory_size = 0;
+          base::SharedMemoryHandle remote_handle = buffer_pool_->ShareToProcess(
+              buffer->id(), client->render_process_handle, &memory_size);
+          client->event_handler->OnBufferCreated(
+              client->controller_id, remote_handle, memory_size, buffer->id());
+        }
+
+        client->event_handler->OnBufferReady(
+            client->controller_id, buffer->id(), buffer_format, timestamp);
       }
 
-      client->event_handler->OnBufferReady(
-          client->controller_id, buffer->id(), timestamp, frame_format);
-      bool inserted = client->active_buffers.insert(buffer->id()).second;
+      bool inserted =
+          client->active_buffers.insert(std::make_pair(buffer->id(), frame))
+              .second;
       DCHECK(inserted) << "Unexpected duplicate buffer: " << buffer->id();
       count++;
     }