Linux Video Capture: Add V4L2VideoCaptureDelegate{Single,Multi}Plane.

content/browser/renderer_host/media/video_capture_controller.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
 #include <map>
 #include <set>
 
 #include "base/bind.h"
@@ skipping 10 matching lines @@
 #include "media/base/video_frame.h"
 #include "media/base/video_util.h"
 #include "media/base/yuv_convert.h"
 #include "third_party/libyuv/include/libyuv.h"
 
 #if !defined(OS_ANDROID)
 #include "content/browser/compositor/image_transport_factory.h"
 #endif
 
 using media::VideoCaptureFormat;
+using media::VideoFrame;
 
 namespace content {
 
 namespace {
 
 static const int kInfiniteRatio = 99999;
 
 #define UMA_HISTOGRAM_ASPECT_RATIO(name, width, height) \
     UMA_HISTOGRAM_SPARSE_SLOWLY( \
         name, \
@@ skipping 19 matching lines @@
 
  private:
   ~AutoReleaseBuffer() override { pool_->RelinquishProducerReservation(id_); }
 
   const scoped_refptr<VideoCaptureBufferPool> pool_;
   const int id_;
   void* const data_;
   const size_t size_;
 };
 
-class SyncPointClientImpl : public media::VideoFrame::SyncPointClient {
+class SyncPointClientImpl : public VideoFrame::SyncPointClient {
  public:
   explicit SyncPointClientImpl(GLHelper* gl_helper) : gl_helper_(gl_helper) {}
   ~SyncPointClientImpl() override {}
   uint32 InsertSyncPoint() override { return gl_helper_->InsertSyncPoint(); }
   void WaitSyncPoint(uint32 sync_point) override {
     gl_helper_->WaitSyncPoint(sync_point);
   }
 
  private:
   GLHelper* gl_helper_;
 };
 
-void ReturnVideoFrame(const scoped_refptr<media::VideoFrame>& video_frame,
+void ReturnVideoFrame(const scoped_refptr<VideoFrame>& video_frame,
                       uint32 sync_point) {
   DCHECK_CURRENTLY_ON(BrowserThread::UI);
 #if defined(OS_ANDROID)
   NOTREACHED();
 #else
   GLHelper* gl_helper = ImageTransportFactory::GetInstance()->GetGLHelper();
   // UpdateReleaseSyncPoint() creates a new sync_point using |gl_helper|, so
   // wait the given |sync_point| using |gl_helper|.
   if (gl_helper) {
     gl_helper->WaitSyncPoint(sync_point);
@@ skipping 28 matching lines @@
   // Handle to the render process that will receive the capture buffers.
   const base::ProcessHandle render_process_handle;
   const media::VideoCaptureSessionId session_id;
   const media::VideoCaptureParams parameters;
 
   // Buffers that are currently known to this client.
   std::set<int> known_buffers;
 
   // Buffers currently held by this client, and syncpoint callback to call when
   // they are returned from the client.
-  typedef std::map<int, scoped_refptr<media::VideoFrame> > ActiveBufferMap;
+  typedef std::map<int, scoped_refptr<VideoFrame> > ActiveBufferMap;
   ActiveBufferMap active_buffers;
 
   // State of capture session, controlled by VideoCaptureManager directly. This
   // transitions to true as soon as StopSession() occurs, at which point the
   // client is sent an OnEnded() event. However, because the client retains a
   // VideoCaptureController* pointer, its ControllerClient entry lives on until
   // it unregisters itself via RemoveClient(), which may happen asynchronously.
   //
   // TODO(nick): If we changed the semantics of VideoCaptureHost so that
   // OnEnded() events were processed synchronously (with the RemoveClient() done
@@ skipping 19 matching lines @@
  public:
   explicit VideoCaptureDeviceClient(
       const base::WeakPtr<VideoCaptureController>& controller,
       const scoped_refptr<VideoCaptureBufferPool>& buffer_pool);
   ~VideoCaptureDeviceClient() override;
 
   // VideoCaptureDevice::Client implementation.
   void OnIncomingCapturedData(const uint8* data,
                               int length,
                               const VideoCaptureFormat& frame_format,
-                              int rotation,
+                              int clockwise_rotation,
                               const base::TimeTicks& timestamp) override;
-  scoped_refptr<Buffer> ReserveOutputBuffer(media::VideoFrame::Format format,
+  void OnIncomingCapturedYuvData(uint8* y_data,
+                                 uint8* u_data,
+                                 uint8* v_data,
+                                 int y_length,
+                                 int u_length,
+                                 int v_length,
+                                 const VideoCaptureFormat& frame_format,
+                                 int clockwise_rotation,
+                                 const base::TimeTicks& timestamp) override;
+  scoped_refptr<Buffer> ReserveOutputBuffer(VideoFrame::Format format,
                                             const gfx::Size& size) override;
   void OnIncomingCapturedVideoFrame(
       const scoped_refptr<Buffer>& buffer,
-      const scoped_refptr<media::VideoFrame>& frame,
+      const scoped_refptr<VideoFrame>& frame,
       const base::TimeTicks& timestamp) override;
   void OnError(const std::string& reason) override;
   void OnLog(const std::string& message) override;
 
  private:
   // The controller to which we post events.
   const base::WeakPtr<VideoCaptureController> controller_;
 
   // The pool of shared-memory buffers used for capturing.
   const scoped_refptr<VideoCaptureBufferPool> buffer_pool_;
@@ skipping 130 matching lines @@
   ControllerClient* client = FindClient(id, event_handler, controller_clients_);
 
   // If this buffer is not held by this client, or this client doesn't exist
   // in controller, do nothing.
   ControllerClient::ActiveBufferMap::iterator iter;
   if (!client || (iter = client->active_buffers.find(buffer_id)) ==
                      client->active_buffers.end()) {
     NOTREACHED();
     return;
   }
-  scoped_refptr<media::VideoFrame> frame = iter->second;
+  scoped_refptr<VideoFrame> frame = iter->second;
   client->active_buffers.erase(iter);
   buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
 
 #if defined(OS_ANDROID)
   DCHECK_EQ(0u, sync_point);
 #endif
   if (sync_point)
     BrowserThread::PostTask(BrowserThread::UI,
                             FROM_HERE,
                             base::Bind(&ReturnVideoFrame, frame, sync_point));
 }
 
 const media::VideoCaptureFormat&
 VideoCaptureController::GetVideoCaptureFormat() const {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   return video_capture_format_;
 }
 
 scoped_refptr<media::VideoCaptureDevice::Client::Buffer>
 VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer(
-    media::VideoFrame::Format format,
+    VideoFrame::Format format,
     const gfx::Size& dimensions) {
   size_t frame_bytes = 0;
-  if (format == media::VideoFrame::NATIVE_TEXTURE) {
+  if (format == VideoFrame::NATIVE_TEXTURE) {
     DCHECK_EQ(dimensions.width(), 0);
     DCHECK_EQ(dimensions.height(), 0);
   } else {
     // The capture pipeline expects I420 for now.
-    DCHECK_EQ(format, media::VideoFrame::I420)
+    DCHECK_EQ(format, VideoFrame::I420)
         << " Non-I420 output buffer format " << format << " requested";
-    frame_bytes = media::VideoFrame::AllocationSize(format, dimensions);
+    frame_bytes = VideoFrame::AllocationSize(format, dimensions);
   }
 
   int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
   int buffer_id =
       buffer_pool_->ReserveForProducer(frame_bytes, &buffer_id_to_drop);
   if (buffer_id == VideoCaptureBufferPool::kInvalidId)
     return NULL;
   void* data;
   size_t size;
   buffer_pool_->GetBufferInfo(buffer_id, &data, &size);
 
   scoped_refptr<media::VideoCaptureDevice::Client::Buffer> output_buffer(
       new AutoReleaseBuffer(buffer_pool_, buffer_id, data, size));
 
   if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
     BrowserThread::PostTask(BrowserThread::IO,
         FROM_HERE,
         base::Bind(&VideoCaptureController::DoBufferDestroyedOnIOThread,
                    controller_, buffer_id_to_drop));
   }
 
   return output_buffer;
 }
 
 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedData(
     const uint8* data,
     int length,
     const VideoCaptureFormat& frame_format,
-    int rotation,
+    int clockwise_rotation,
     const base::TimeTicks& timestamp) {
   TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedData");
 
   if (last_captured_pixel_format_ != frame_format.pixel_format) {
     OnLog("Pixel format: " + media::VideoCaptureFormat::PixelFormatToString(
                                  frame_format.pixel_format));
     last_captured_pixel_format_ = frame_format.pixel_format;
   }
 
   if (!frame_format.IsValid())
     return;
 
-  // Chopped pixels in width/height in case video capture device has odd
-  // numbers for width/height.
-  int chopped_width = 0;
-  int chopped_height = 0;
-  int new_unrotated_width = frame_format.frame_size.width();
-  int new_unrotated_height = frame_format.frame_size.height();
-
-  if (new_unrotated_width & 1) {
-    --new_unrotated_width;
-    chopped_width = 1;
-  }
-  if (new_unrotated_height & 1) {
-    --new_unrotated_height;
-    chopped_height = 1;
-  }
+  // |chopped_{width,height} and |new_unrotated_{width,height}| are the lowest
+  // bit decomposition of {width, height}, grabbing the odd and even parts.
+  const int chopped_width = frame_format.frame_size.width() & 1;
+  const int chopped_height = frame_format.frame_size.height() & 1;
+  const int new_unrotated_width = frame_format.frame_size.width() & ~1;
+  const int new_unrotated_height = frame_format.frame_size.height() & ~1;
 
   int destination_width = new_unrotated_width;
   int destination_height = new_unrotated_height;
-  if (rotation == 90 || rotation == 270) {
+  if (clockwise_rotation == 90 || clockwise_rotation == 270) {
     destination_width = new_unrotated_height;
     destination_height = new_unrotated_width;
   }
+
+  DLOG_IF(WARNING, (clockwise_rotation % 90))
+      << " Rotation must be a multiple of 90 and is " << clockwise_rotation;
+  libyuv::RotationMode rotation_mode = libyuv::kRotate0;
+  if (clockwise_rotation == 90)
+    rotation_mode = libyuv::kRotate90;
+  else if (clockwise_rotation == 180)
+    rotation_mode = libyuv::kRotate180;
+  else if (clockwise_rotation == 270)
+    rotation_mode = libyuv::kRotate270;

[emircan, 2015/03/04 02:47:47]

Consider a switch statement.

[mcasas, 2015/03/09 21:23:56]

Acknowledged.

+
   const gfx::Size dimensions(destination_width, destination_height);
-  if (!media::VideoFrame::IsValidConfig(media::VideoFrame::I420,
-                                        dimensions,
-                                        gfx::Rect(dimensions),
-                                        dimensions)) {
+  if (!VideoFrame::IsValidConfig(VideoFrame::I420, dimensions,
+                                 gfx::Rect(dimensions), dimensions)) {
     return;
   }
 
-  scoped_refptr<Buffer> buffer = ReserveOutputBuffer(media::VideoFrame::I420,
+  scoped_refptr<Buffer> buffer = ReserveOutputBuffer(VideoFrame::I420,
                                                      dimensions);
   if (!buffer.get())
     return;
-  uint8* yplane = NULL;
   bool flip = false;
-  yplane = reinterpret_cast<uint8*>(buffer->data());
-  uint8* uplane =
-      yplane +
-      media::VideoFrame::PlaneAllocationSize(
-          media::VideoFrame::I420, media::VideoFrame::kYPlane, dimensions);
-  uint8* vplane =
-      uplane +
-      media::VideoFrame::PlaneAllocationSize(
-          media::VideoFrame::I420, media::VideoFrame::kUPlane, dimensions);
-  int yplane_stride = dimensions.width();
-  int uv_plane_stride = yplane_stride / 2;
-  int crop_x = 0;
-  int crop_y = 0;
+  uint8* const yplane = reinterpret_cast<uint8*>(buffer->data());
+  uint8* const uplane =
+      yplane + VideoFrame::PlaneAllocationSize(VideoFrame::I420,
+                                               VideoFrame::kYPlane, dimensions);
+  uint8* const vplane =
+      uplane + VideoFrame::PlaneAllocationSize(VideoFrame::I420,
+                                               VideoFrame::kUPlane, dimensions);
+  const int yplane_stride = dimensions.width();
+  const int uv_plane_stride = yplane_stride / 2;
+  const int crop_x = 0;
+  const int crop_y = 0;
+
   libyuv::FourCC origin_colorspace = libyuv::FOURCC_ANY;
-
-  libyuv::RotationMode rotation_mode = libyuv::kRotate0;
-  if (rotation == 90)
-    rotation_mode = libyuv::kRotate90;
-  else if (rotation == 180)
-    rotation_mode = libyuv::kRotate180;
-  else if (rotation == 270)
-    rotation_mode = libyuv::kRotate270;
-
   switch (frame_format.pixel_format) {
     case media::PIXEL_FORMAT_UNKNOWN:  // Color format not set.
       break;
     case media::PIXEL_FORMAT_I420:
       DCHECK(!chopped_width && !chopped_height);
       origin_colorspace = libyuv::FOURCC_I420;
       break;
     case media::PIXEL_FORMAT_YV12:
       DCHECK(!chopped_width && !chopped_height);
       origin_colorspace = libyuv::FOURCC_YV12;
@@ skipping 48 matching lines @@
                             uv_plane_stride,
                             crop_x,
                             crop_y,
                             frame_format.frame_size.width(),
                             (flip ? -frame_format.frame_size.height() :
                                 frame_format.frame_size.height()),
                             new_unrotated_width,
                             new_unrotated_height,
                             rotation_mode,
                             origin_colorspace) != 0) {
-    DLOG(WARNING) << "Failed to convert buffer from"
-                  << media::VideoCaptureFormat::PixelFormatToString(
-                         frame_format.pixel_format)
-                  << "to I420.";
+    DLOG(WARNING) << "Failed to convert buffer to I420 from "
+        << VideoCaptureFormat::PixelFormatToString(frame_format.pixel_format);
     return;
   }
-  scoped_refptr<media::VideoFrame> frame =
-      media::VideoFrame::WrapExternalPackedMemory(
-          media::VideoFrame::I420,
+  scoped_refptr<VideoFrame> frame =
+      VideoFrame::WrapExternalPackedMemory(
+          VideoFrame::I420,
           dimensions,
           gfx::Rect(dimensions),
           dimensions,
           yplane,
-          media::VideoFrame::AllocationSize(media::VideoFrame::I420,
-                                            dimensions),
+          VideoFrame::AllocationSize(VideoFrame::I420, dimensions),
           base::SharedMemory::NULLHandle(),
           0,
           base::TimeDelta(),
           base::Closure());
   DCHECK(frame.get());
   frame->metadata()->SetDouble(media::VideoFrameMetadata::FRAME_RATE,
                                frame_format.frame_rate);
 
   BrowserThread::PostTask(
       BrowserThread::IO,
       FROM_HERE,
       base::Bind(
           &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
           controller_,
           buffer,
           frame,
           timestamp));
 }
 
 void
+VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedYuvData(
+    uint8* y_data,
+    uint8* u_data,
+    uint8* v_data,
+    int y_length,
+    int u_length,
+    int v_length,
+    const VideoCaptureFormat& frame_format,
+    int clockwise_rotation,
+    const base::TimeTicks& timestamp) {
+  TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedYuvData");
+  DCHECK_EQ(frame_format.pixel_format, media::PIXEL_FORMAT_I420);
+  DCHECK_EQ(clockwise_rotation, 0) << "Rotation not supported";
+  DCHECK_GE(static_cast<size_t>(y_length + u_length + v_length),
+            frame_format.ImageAllocationSize());
+
+  scoped_refptr<Buffer> buffer = ReserveOutputBuffer(VideoFrame::I420,

[emircan, 2015/03/04 02:47:47]

Consider using a scoped_ptr and passing ownership. However, it would require
function signature change on
VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread, and may be
handled in another CL.

[mcasas, 2015/03/09 21:23:56]

Good idea. Please add a bug to investigate this.

+                                                     frame_format.frame_size);
+  if (!buffer.get())
+    return;
+
+  // Blit (copy) here from y,u,v into buffer.data()). Needed so we can return
+  // the parameter buffer synchronously to the driver.
+  const size_t y_stride = y_length / frame_format.frame_size.height();
+  const size_t u_stride = u_length / frame_format.frame_size.height() / 2;
+  const size_t v_stride = v_length / frame_format.frame_size.height() / 2;
+  const size_t y_plane_size = VideoFrame::PlaneAllocationSize(VideoFrame::I420,
+      VideoFrame::kYPlane, frame_format.frame_size);
+  const size_t u_plane_size = VideoFrame::PlaneAllocationSize(VideoFrame::I420,
+    VideoFrame::kUPlane, frame_format.frame_size);
+  uint8* const dst_y = reinterpret_cast<uint8*>(buffer->data());
+  uint8* const dst_u = dst_y + y_plane_size;
+  uint8* const dst_v = dst_u + u_plane_size;
+
+  libyuv::CopyPlane(y_data, y_stride, dst_y, y_stride,
+      frame_format.frame_size.width(), frame_format.frame_size.height());
+  libyuv::CopyPlane(u_data, u_stride, dst_u, u_stride,
+      frame_format.frame_size.width() / 2,frame_format.frame_size.height() / 2);
+  libyuv::CopyPlane(v_data, v_stride, dst_v, v_stride,
+      frame_format.frame_size.width() / 2,frame_format.frame_size.height() / 2);
+
+  scoped_refptr<VideoFrame> video_frame = VideoFrame::WrapExternalYuvData(
+      VideoFrame::I420, frame_format.frame_size,
+      gfx::Rect(frame_format.frame_size), frame_format.frame_size, y_stride,
+      u_stride, v_stride, dst_y, dst_u, dst_v, base::TimeDelta(),
+      base::Closure());
+  DCHECK(video_frame.get());
+
+  BrowserThread::PostTask(
+      BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(
+          &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
+          controller_,
+          buffer,
+          //frame_format,
+          video_frame,
+          timestamp));
+};
+
+void
 VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedVideoFrame(
     const scoped_refptr<Buffer>& buffer,
-    const scoped_refptr<media::VideoFrame>& frame,
+    const scoped_refptr<VideoFrame>& frame,
     const base::TimeTicks& timestamp) {
   BrowserThread::PostTask(
       BrowserThread::IO,
       FROM_HERE,
       base::Bind(
           &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
           controller_,
           buffer,
           frame,
           timestamp));
@@ skipping 18 matching lines @@
   MediaStreamManager::SendMessageToNativeLog("Video capture: " + message);
 }
 
 VideoCaptureController::~VideoCaptureController() {
   STLDeleteContainerPointers(controller_clients_.begin(),
                              controller_clients_.end());
 }
 
 void VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread(
     const scoped_refptr<media::VideoCaptureDevice::Client::Buffer>& buffer,
-    const scoped_refptr<media::VideoFrame>& frame,
+    const scoped_refptr<VideoFrame>& frame,
     const base::TimeTicks& timestamp) {
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   DCHECK_NE(buffer->id(), VideoCaptureBufferPool::kInvalidId);
 
   int count = 0;
   if (state_ == VIDEO_CAPTURE_STATE_STARTED) {
     if (!frame->metadata()->HasKey(media::VideoFrameMetadata::FRAME_RATE)) {
       frame->metadata()->SetDouble(media::VideoFrameMetadata::FRAME_RATE,
                                    video_capture_format_.frame_rate);
     }
@@ skipping 134 matching lines @@
   DCHECK_CURRENTLY_ON(BrowserThread::IO);
   int active_client_count = 0;
   for (ControllerClient* client : controller_clients_) {
     if (!client->paused)
       ++active_client_count;
   }
   return active_client_count;
 }
 
 }  // namespace content