Remove media::VideoFrame from media::VideoCaptureDevice::Client interface

content/browser/renderer_host/media/video_capture_controller.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
 #include <set>
 
 #include "base/bind.h"
 #include "base/debug/trace_event.h"
-#include "base/memory/scoped_ptr.h"
 #include "base/stl_util.h"
 #include "content/browser/renderer_host/media/media_stream_manager.h"
 #include "content/browser/renderer_host/media/video_capture_manager.h"
 #include "content/public/browser/browser_thread.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_util.h"
 #include "media/base/yuv_convert.h"
 
 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
 #include "third_party/libyuv/include/libyuv.h"
 #endif
 
 using media::VideoCaptureCapability;
 
 namespace content {
 
+namespace {
+
 // The number of buffers that VideoCaptureBufferPool should allocate.
-static const int kNoOfBuffers = 3;
+const int kNoOfBuffers = 3;
+
+class PoolBuffer : public media::VideoCaptureDevice::Client::Buffer {
+ public:
+  PoolBuffer(const scoped_refptr<VideoCaptureBufferPool>& pool,
+             int buffer_id,
+             void* data,
+             size_t size)
+      : Buffer(buffer_id, data, size), pool_(pool) {}

[wjia(left Chromium), 2013/11/19 01:47:35]

The |pool| shouldn't be NULL pointer here. It's used in dtor without checking.
Add a ChECK here.

[sheu, 2013/11/19 20:28:17]

Done.

+
+ private:
+  virtual ~PoolBuffer() { pool_->RelinquishProducerReservation(id()); }
+
+  const scoped_refptr<VideoCaptureBufferPool> pool_;
+};
+
+}  // anonymous namespace
 
 struct VideoCaptureController::ControllerClient {
   ControllerClient(
       const VideoCaptureControllerID& id,
       VideoCaptureControllerEventHandler* handler,
       base::ProcessHandle render_process,
       const media::VideoCaptureParams& params)
       : controller_id(id),
         event_handler(handler),
         render_process_handle(render_process),
@@ skipping 40 matching lines @@
 // v4l2_thread on Linux, and the UI thread for tab capture.
 class VideoCaptureController::VideoCaptureDeviceClient
     : public media::VideoCaptureDevice::Client {
  public:
   explicit VideoCaptureDeviceClient(
       const base::WeakPtr<VideoCaptureController>& controller,
       const scoped_refptr<VideoCaptureBufferPool>& buffer_pool);
   virtual ~VideoCaptureDeviceClient();
 
   // VideoCaptureDevice::Client implementation.
-  virtual scoped_refptr<media::VideoFrame> ReserveOutputBuffer(
+  virtual scoped_refptr<Buffer> ReserveOutputBuffer(
+      media::VideoFrame::Format format,
       const gfx::Size& size) OVERRIDE;
-  virtual void OnIncomingCapturedFrame(
-      const uint8* data,
-      int length,
-      base::Time timestamp,
-      int rotation,
-      bool flip_vert,
-      bool flip_horiz,
-      const VideoCaptureCapability& frame_info) OVERRIDE;
-  virtual void OnIncomingCapturedVideoFrame(
-      const scoped_refptr<media::VideoFrame>& frame,
-      base::Time timestamp,
-      int frame_rate) OVERRIDE;
+  virtual void OnIncomingCapturedFrame(const uint8* data,
+                                       int length,
+                                       base::Time timestamp,
+                                       int rotation,
+                                       bool flip_vert,
+                                       bool flip_horiz,
+                                       const VideoCaptureCapability& frame_info)
+      OVERRIDE;
+  virtual void OnIncomingCapturedBuffer(const scoped_refptr<Buffer>& buffer,
+                                        media::VideoFrame::Format format,
+                                        const gfx::Size& dimensions,
+                                        base::Time timestamp,
+                                        int frame_rate) OVERRIDE;
   virtual void OnError() OVERRIDE;
 
  private:
-  scoped_refptr<media::VideoFrame> DoReserveI420VideoFrame(
-      const gfx::Size& size,
-      int rotation);
+  scoped_refptr<Buffer> DoReserveOutputBuffer(media::VideoFrame::Format format,
+                                              const gfx::Size& dimensions,
+                                              int rotation);
 
   // The controller to which we post events.
   const base::WeakPtr<VideoCaptureController> controller_;
 
   // The pool of shared-memory buffers used for capturing.
   const scoped_refptr<VideoCaptureBufferPool> buffer_pool_;
+
+  // The set of buffers that have been used for rotated capturing.
+  std::set<int> rotated_buffers_;
 };
 
 VideoCaptureController::VideoCaptureController()
     : buffer_pool_(new VideoCaptureBufferPool(kNoOfBuffers)),
       state_(VIDEO_CAPTURE_STATE_STARTED),
       weak_ptr_factory_(this) {
 }
 
 VideoCaptureController::VideoCaptureDeviceClient::VideoCaptureDeviceClient(
     const base::WeakPtr<VideoCaptureController>& controller,
     const scoped_refptr<VideoCaptureBufferPool>& buffer_pool)
-    : controller_(controller),
-      buffer_pool_(buffer_pool) {}
+    : controller_(controller), buffer_pool_(buffer_pool) {}
 
 VideoCaptureController::VideoCaptureDeviceClient::~VideoCaptureDeviceClient() {}
 
 base::WeakPtr<VideoCaptureController> VideoCaptureController::GetWeakPtr() {
   return weak_ptr_factory_.GetWeakPtr();
 }
 
 scoped_ptr<media::VideoCaptureDevice::Client>
 VideoCaptureController::NewDeviceClient() {
   scoped_ptr<media::VideoCaptureDevice::Client> result(
@@ skipping 83 matching lines @@
   // If this buffer is not held by this client, or this client doesn't exist
   // in controller, do nothing.
   if (!client || !client->active_buffers.erase(buffer_id)) {
     NOTREACHED();
     return;
   }
 
   buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
 }
 
-scoped_refptr<media::VideoFrame>
+scoped_refptr<media::VideoCaptureDevice::Client::Buffer>
 VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer(
+    media::VideoFrame::Format format,
     const gfx::Size& size) {
-  return DoReserveI420VideoFrame(size, 0);
+  return DoReserveOutputBuffer(format, size, 0);
 }
 
 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame(
     const uint8* data,
     int length,
     base::Time timestamp,
     int rotation,
     bool flip_vert,
     bool flip_horiz,
     const VideoCaptureCapability& frame_info) {
@@ skipping 11 matching lines @@
 
   if (frame_info.width & 1) {
     --new_width;
     chopped_width = 1;
   }
   if (frame_info.height & 1) {
     --new_height;
     chopped_height = 1;
   }
 
-  scoped_refptr<media::VideoFrame> dst = DoReserveI420VideoFrame(
-      gfx::Size(new_width, new_height), rotation);
+  const gfx::Size dimensions(new_width, new_height);
+  scoped_refptr<Buffer> buffer =
+      DoReserveOutputBuffer(media::VideoFrame::I420, dimensions, rotation);
 
-  if (!dst.get())
+  if (!buffer)
     return;
 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
-
-  uint8* yplane = dst->data(media::VideoFrame::kYPlane);
-  uint8* uplane = dst->data(media::VideoFrame::kUPlane);
-  uint8* vplane = dst->data(media::VideoFrame::kVPlane);
+  uint8* yplane = reinterpret_cast<uint8*>(buffer->data());
+  uint8* uplane =
+      yplane +
+      media::VideoFrame::PlaneAllocationSize(
+          media::VideoFrame::I420, media::VideoFrame::kYPlane, dimensions);
+  uint8* vplane =
+      uplane +
+      media::VideoFrame::PlaneAllocationSize(
+          media::VideoFrame::I420, media::VideoFrame::kUPlane, dimensions);
   int yplane_stride = new_width;
-  int uv_plane_stride = (new_width + 1) / 2;
+  int uv_plane_stride = new_width / 2;
   int crop_x = 0;
   int crop_y = 0;
   int destination_width = new_width;
   int destination_height = new_height;
   libyuv::FourCC origin_colorspace = libyuv::FOURCC_ANY;
   // Assuming rotation happens first and flips next, we can consolidate both
   // vertical and horizontal flips together with rotation into two variables:
   // new_rotation = (rotation + 180 * vertical_flip) modulo 360
   // new_vertical_flip = horizontal_flip XOR vertical_flip
   int new_rotation_angle = (rotation + 180 * flip_vert) % 360;
@@ skipping 46 matching lines @@
   // kRGB24 on Windows start at the bottom line and has a negative stride. This
   // is not supported by libyuv, so the media API is used instead.
   if (frame_info.color == media::PIXEL_FORMAT_RGB24) {
     // Rotation and flipping is not supported in kRGB24 and OS_WIN case.
     DCHECK(!rotation && !flip_vert && !flip_horiz);
     need_convert_rgb24_on_win = true;
   }
 #endif
   if (need_convert_rgb24_on_win) {
     int rgb_stride = -3 * (new_width + chopped_width);
-    const uint8* rgb_src =
-        data + 3 * (new_width + chopped_width) *
-                   (new_height - 1 + chopped_height);
+    const uint8* rgb_src = data + 3 * (new_width + chopped_width) *
+                                      (new_height - 1 + chopped_height);
     media::ConvertRGB24ToYUV(rgb_src,
                              yplane,
                              uplane,
                              vplane,
                              new_width,
                              new_height,
                              rgb_stride,
                              yplane_stride,
                              uv_plane_stride);
   } else {
     if (new_rotation_angle==90 || new_rotation_angle==270){
       // To be compatible with non-libyuv code in RotatePlaneByPixels, when
       // rotating by 90/270, only the maximum square portion located in the
       // center of the image is rotated. F.i. 640x480 pixels, only the central
       // 480 pixels would be rotated and the leftmost and rightmost 80 columns
       // would be ignored. This process is called letterboxing.
       int letterbox_thickness = abs(new_width - new_height) / 2;
       if (destination_width > destination_height) {
         yplane += letterbox_thickness;
         uplane += letterbox_thickness / 2;
         vplane += letterbox_thickness / 2;
         destination_width = destination_height;
       } else {
         yplane += letterbox_thickness * destination_width;
         uplane += (letterbox_thickness * destination_width) / 2;
         vplane += (letterbox_thickness * destination_width) / 2;
         destination_height = destination_width;
       }
     }
-    libyuv::ConvertToI420(
-        data, length,
-        yplane, yplane_stride,
-        uplane, uv_plane_stride,
-        vplane, uv_plane_stride,
-        crop_x, crop_y,
-        new_width + chopped_width,
-        new_height * (flip_vert ^ flip_horiz ? -1 : 1),
-        destination_width,
-        destination_height,
-        rotation_mode,
-        origin_colorspace);
+    libyuv::ConvertToI420(data,
+                          length,
+                          yplane,
+                          yplane_stride,
+                          uplane,
+                          uv_plane_stride,
+                          vplane,
+                          uv_plane_stride,
+                          crop_x,
+                          crop_y,
+                          new_width + chopped_width,
+                          new_height * (flip_vert ^ flip_horiz ? -1 : 1),
+                          destination_width,
+                          destination_height,
+                          rotation_mode,
+                          origin_colorspace);
   }
 #else
   // Libyuv is not linked in for Android WebView builds, but video capture is
   // not used in those builds either. Whenever libyuv is added in that build,
   // address all these #ifdef parts, see http://crbug.com/299611 .
   NOTREACHED();
 #endif  // if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
   BrowserThread::PostTask(
       BrowserThread::IO,
       FROM_HERE,
-      base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
-                 controller_,
-                 dst,
-                 frame_info.frame_rate,
-                 timestamp));
+      base::Bind(
+          &VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread,
+          controller_,
+          buffer,
+          dimensions,
+          frame_info.frame_rate,
+          timestamp));
 }
 
-void
-VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedVideoFrame(
-    const scoped_refptr<media::VideoFrame>& frame,
+void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedBuffer(
+    const scoped_refptr<Buffer>& buffer,
+    media::VideoFrame::Format format,
+    const gfx::Size& dimensions,
     base::Time timestamp,
     int frame_rate) {
-  // If this is a frame that belongs to the buffer pool, we can forward it
-  // directly to the IO thread and be done.
-  if (buffer_pool_->RecognizeReservedBuffer(
-      frame->shared_memory_handle()) >= 0) {
-    BrowserThread::PostTask(BrowserThread::IO,
-        FROM_HERE,
-        base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
-                   controller_, frame, frame_rate, timestamp));
-    return;
-  }
+  // The capture pipeline expects I420 for now.
+  DCHECK_EQ(format, media::VideoFrame::I420)
+      << "Non-I420 output buffer returned";
 
-  NOTREACHED() << "Frames should always belong to the buffer pool.";
+  BrowserThread::PostTask(
+      BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(
+          &VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread,
+          controller_,
+          buffer,
+          dimensions,
+          frame_rate,
+          timestamp));
 }
 
 void VideoCaptureController::VideoCaptureDeviceClient::OnError() {
   BrowserThread::PostTask(BrowserThread::IO,
       FROM_HERE,
       base::Bind(&VideoCaptureController::DoErrorOnIOThread, controller_));
 }
 
-scoped_refptr<media::VideoFrame>
-VideoCaptureController::VideoCaptureDeviceClient::DoReserveI420VideoFrame(
-    const gfx::Size& size,
+scoped_refptr<media::VideoCaptureDevice::Client::Buffer>
+VideoCaptureController::VideoCaptureDeviceClient::DoReserveOutputBuffer(
+    media::VideoFrame::Format format,
+    const gfx::Size& dimensions,
     int rotation) {
+  // The capture pipeline expects I420 for now.
+  DCHECK_EQ(format, media::VideoFrame::I420)
+      << "Non-I420 output buffer requested";
+
   int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
-  scoped_refptr<media::VideoFrame> frame =
-      buffer_pool_->ReserveI420VideoFrame(size, rotation, &buffer_id_to_drop);
+  const size_t frame_bytes =
+      media::VideoFrame::AllocationSize(format, dimensions);
+
+  int buffer_id =
+      buffer_pool_->ReserveForProducer(frame_bytes, &buffer_id_to_drop);
+  if (buffer_id == VideoCaptureBufferPool::kInvalidId)
+    return NULL;
+  void* data;
+  size_t size;
+  buffer_pool_->GetBufferInfo(buffer_id, &data, &size);
+
+  scoped_refptr<media::VideoCaptureDevice::Client::Buffer> output_buffer(
+      new PoolBuffer(buffer_pool_, buffer_id, data, size));
+
   if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
     BrowserThread::PostTask(BrowserThread::IO,
         FROM_HERE,
         base::Bind(&VideoCaptureController::DoBufferDestroyedOnIOThread,
                    controller_, buffer_id_to_drop));
+    rotated_buffers_.erase(buffer_id_to_drop);
   }
-  return frame;
+
+  // If a 90/270 rotation is required, letterboxing will be required.  If the
+  // returned frame has not been rotated before, then the letterbox borders will
+  // not yet have been cleared and we should clear them now.
+  if ((rotation % 180) == 0) {
+    rotated_buffers_.erase(buffer_id);
+  } else {
+    if (rotated_buffers_.insert(buffer_id).second) {
+      memset(output_buffer->data(), 0, output_buffer->size());
+      rotated_buffers_.insert(buffer_id);

[wjia(left Chromium), 2013/11/19 01:47:35]

nit: no need to call insert again.

[sheu, 2013/11/19 20:28:17]

Done.

+    }
+  }
+
+  return output_buffer;
 }
 
 VideoCaptureController::~VideoCaptureController() {
   STLDeleteContainerPointers(controller_clients_.begin(),
                              controller_clients_.end());
 }
 
-void VideoCaptureController::DoIncomingCapturedFrameOnIOThread(
-    const scoped_refptr<media::VideoFrame>& reserved_frame,
+void VideoCaptureController::DoIncomingCapturedI420BufferOnIOThread(
+    scoped_refptr<media::VideoCaptureDevice::Client::Buffer> buffer,
+    const gfx::Size& dimensions,
     int frame_rate,
     base::Time timestamp) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
-
-  int buffer_id = buffer_pool_->RecognizeReservedBuffer(
-      reserved_frame->shared_memory_handle());
-  if (buffer_id < 0) {
-    NOTREACHED();
-    return;
-  }
+  DCHECK_NE(buffer->id(), VideoCaptureBufferPool::kInvalidId);
 
   media::VideoCaptureFormat frame_format(
-      reserved_frame->coded_size().width(),
-      reserved_frame->coded_size().height(),
+      dimensions.width(),
+      dimensions.height(),
       frame_rate,
       media::VariableResolutionVideoCaptureDevice);
 
   int count = 0;
   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
     for (ControllerClients::iterator client_it = controller_clients_.begin();
          client_it != controller_clients_.end(); ++client_it) {
       ControllerClient* client = *client_it;
       if (client->session_closed)
         continue;
 
-      bool is_new_buffer = client->known_buffers.insert(buffer_id).second;
+      bool is_new_buffer = client->known_buffers.insert(buffer->id()).second;
       if (is_new_buffer) {
         // On the first use of a buffer on a client, share the memory handle.
         size_t memory_size = 0;
         base::SharedMemoryHandle remote_handle = buffer_pool_->ShareToProcess(
-            buffer_id, client->render_process_handle, &memory_size);
-        client->event_handler->OnBufferCreated(client->controller_id,
-                                               remote_handle,
-                                               memory_size,
-                                               buffer_id);
+            buffer->id(), client->render_process_handle, &memory_size);
+        client->event_handler->OnBufferCreated(
+            client->controller_id, remote_handle, memory_size, buffer->id());
       }
 
-      client->event_handler->OnBufferReady(client->controller_id,
-                                           buffer_id, timestamp,
-                                           frame_format);
-      bool inserted = client->active_buffers.insert(buffer_id).second;
-      DCHECK(inserted) << "Unexpected duplicate buffer: " << buffer_id;
+      client->event_handler->OnBufferReady(
+          client->controller_id, buffer->id(), timestamp, frame_format);
+      bool inserted = client->active_buffers.insert(buffer->id()).second;
+      DCHECK(inserted) << "Unexpected duplicate buffer: " << buffer->id();
       count++;
     }
   }
 
-  buffer_pool_->HoldForConsumers(buffer_id, count);
+  buffer_pool_->HoldForConsumers(buffer->id(), count);
 }
 
 void VideoCaptureController::DoErrorOnIOThread() {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   state_ = VIDEO_CAPTURE_STATE_ERROR;
 
   for (ControllerClients::iterator client_it = controller_clients_.begin();
        client_it != controller_clients_.end(); ++client_it) {
     ControllerClient* client = *client_it;
     if (client->session_closed)
@@ skipping 47 matching lines @@
   }
   return NULL;
 }
 
 int VideoCaptureController::GetClientCount() {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   return controller_clients_.size();
 }
 
 }  // namespace content