Video Capture: Use libyuv for color conversion in VideoCaptureController.

content/browser/renderer_host/media/video_capture_controller.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
 #include <set>
 
 #include "base/bind.h"
 #include "base/debug/trace_event.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/stl_util.h"
 #include "content/browser/renderer_host/media/media_stream_manager.h"
 #include "content/browser/renderer_host/media/video_capture_manager.h"
 #include "content/public/browser/browser_thread.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_util.h"
 #include "media/base/yuv_convert.h"
 
 #if !defined(OS_IOS) && !defined(OS_ANDROID)
 #include "third_party/libyuv/include/libyuv.h"
 #endif
 
 namespace {
 
+#if defined(OS_IOS) || defined(OS_ANDROID)
 // TODO(wjia): Support stride.
 void RotatePackedYV12Frame(
     const uint8* src,
     uint8* dest_yplane,
     uint8* dest_uplane,
     uint8* dest_vplane,
     int width,
     int height,
     int rotation,
     bool flip_vert,
     bool flip_horiz) {
   media::RotatePlaneByPixels(
       src, dest_yplane, width, height, rotation, flip_vert, flip_horiz);
   int y_size = width * height;
   src += y_size;
   media::RotatePlaneByPixels(
       src, dest_uplane, width/2, height/2, rotation, flip_vert, flip_horiz);
   src += y_size/4;
   media::RotatePlaneByPixels(
       src, dest_vplane, width/2, height/2, rotation, flip_vert, flip_horiz);
 }
+#endif  // #if defined(OS_IOS) || defined(OS_ANDROID)
 
 }  // namespace
 
 namespace content {
 
 // The number of buffers that VideoCaptureBufferPool should allocate.
 static const int kNoOfBuffers = 3;
 
 struct VideoCaptureController::ControllerClient {
   ControllerClient(
@@ skipping 194 matching lines @@
   if (!buffer_pool_.get())
     return NULL;
   return buffer_pool_->ReserveI420VideoFrame(gfx::Size(frame_info_.width,
                                                        frame_info_.height),
                                              0);
 }
 
 // Implements VideoCaptureDevice::EventHandler.
 // OnIncomingCapturedFrame is called the thread running the capture device.
 // I.e.- DirectShow thread on windows and v4l2_thread on Linux.
+#if !defined(OS_IOS) && !defined(OS_ANDROID)
 void VideoCaptureController::OnIncomingCapturedFrame(
     const uint8* data,
     int length,
+    base::Time timestamp,
+    int rotation,
+    bool flip_vert,
+    bool flip_horiz) {
+  TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame");
+
+  scoped_refptr<media::VideoFrame> dst;
+  {
+    base::AutoLock lock(buffer_pool_lock_);
+    if (!buffer_pool_.get())
+      return;
+    dst = buffer_pool_->ReserveI420VideoFrame(
+        gfx::Size(frame_info_.width, frame_info_.height), rotation);
+  }
+
+  if (!dst.get())
+    return;
+
+  uint8* yplane = dst->data(media::VideoFrame::kYPlane);
+  uint8* uplane = dst->data(media::VideoFrame::kUPlane);
+  uint8* vplane = dst->data(media::VideoFrame::kVPlane);
+  int yplane_stride = frame_info_.width;
+  int uv_plane_stride = (frame_info_.width + 1) / 2;
+  int crop_x = 0;
+  int crop_y = 0;
+  libyuv::FourCC origin_colorspace = libyuv::FOURCC_ANY;
+  // Assuming flips happen before rotations (as in libyuv), we consolidate both

[wjia(left Chromium), 2013/09/03 23:23:45]

I should have added some comments for this function. Actually, flips happen
after rotation, at least for RotatePackedYV12Frame().

[mcasas, 2013/09/04 08:47:46]

In that case the equations and the comment should be rewritten, but they happen
to be almost the same, just changing

new_rotation = (rotation + 180 * horizontal_flip) modulo 360
to
new_rotation = (rotation + 180 * vertical_flip) modulo 360

+  // vertical and horizontal flips together with rotation into two:
+  // new_rotation = (rotation + 180 * horizontal_flip) modulo 360
+  // new_vertical_flip = horizontal_flip XOR vertical_flip
+  int new_rotation_angle = (rotation + 180 * flip_horiz) % 360;
+  libyuv::RotationMode rotation_mode = libyuv::kRotate0;
+  if (new_rotation_angle == 90)
+    rotation_mode = libyuv::kRotate90;
+  else if (new_rotation_angle == 180)
+    rotation_mode = libyuv::kRotate180;
+  else if (new_rotation_angle == 270)
+    rotation_mode = libyuv::kRotate270;
+
+  switch (frame_info_.color) {
+    case media::VideoCaptureCapability::kColorUnknown:  // Color format not set.
+      break;
+    case media::VideoCaptureCapability::kI420:
+      DCHECK(!chopped_width_ && !chopped_height_);
+      origin_colorspace = libyuv::FOURCC_I420;
+      break;
+    case media::VideoCaptureCapability::kYV12:
+      DCHECK(!chopped_width_ && !chopped_height_);
+      origin_colorspace = libyuv::FOURCC_YV12;
+      break;
+    case media::VideoCaptureCapability::kNV21:
+      DCHECK(!chopped_width_ && !chopped_height_);
+      origin_colorspace = libyuv::FOURCC_NV12;
+      break;
+    case media::VideoCaptureCapability::kYUY2:
+      DCHECK(!chopped_width_ && !chopped_height_);
+      origin_colorspace = libyuv::FOURCC_YUY2;
+      break;
+    case media::VideoCaptureCapability::kRGB24:
+#if defined(OS_WIN)
+      // RGB on Windows start at the bottom line and has a negative stride. This
+      // is not supported by libyuv, so the media API is used instead.
+      {
+        int rgb_stride = -3 * (frame_info_.width + chopped_width_);
+        const uint8* rgb_src =
+            data + 3 * (frame_info_.width + chopped_width_) *
+                       (frame_info_.height - 1 + chopped_height_);
+        media::ConvertRGB24ToYUV(rgb_src,
+                                 yplane,
+                                 uplane,
+                                 vplane,
+                                 frame_info_.width,
+                                 frame_info_.height,
+                                 rgb_stride,
+                                 yplane_stride,
+                                 uv_plane_stride);
+      }
+      origin_colorspace = libyuv::FOURCC_ANY;
+#else
+      origin_colorspace = libyuv::FOURCC_RAW;
+#endif
+      break;
+    case media::VideoCaptureCapability::kARGB:
+      origin_colorspace = libyuv::FOURCC_ARGB;
+      break;
+    case media::VideoCaptureCapability::kMJPEG:
+      origin_colorspace = libyuv::FOURCC_MJPG;
+      break;
+    default:
+      NOTREACHED();
+  }
+  // Destination colorspace is always YUV I420. If destination_colorspace is
+  // libyuv::FOURCC_ANY, then ConvertToI420 just returns -1.
+  libyuv::ConvertToI420(data,

[wjia(left Chromium), 2013/09/03 23:23:45]

When origin_colorspace = libyuv::FOURCC_ANY, does this function do anything to
data (if color conversion is not done)?

Basically, I want to make sure it's correct for kRGB24 on WIN, since color
conversion is done with media::ConvertRGB24ToYUV(), but rotation should be done
by this libyuv::ConvertToI420().

[mcasas, 2013/09/04 08:47:46]

When FOURCC_ANY is used as origin colorspace, a rotation is performed if needed,
but no colorspace conversion is performed (see the lines around [1]). Rotation
is not done in place (except for YUV easy modes, see [2]), though, so a buffer
is reallocated and free'd.

[1]
https://code.google.com/p/chromium/codesearch#chromium/src/third_party/libyuv...

[2]
https://code.google.com/p/chromium/codesearch#chromium/src/third_party/libyuv...

+                        length,
+                        yplane,
+                        yplane_stride,
+                        uplane,
+                        uv_plane_stride,
+                        vplane,
+                        uv_plane_stride,
+                        crop_x,
+                        crop_y,
+                        frame_info_.width,
+                        frame_info_.height * (flip_vert ^ flip_horiz ? -1 : 1),
+                        frame_info_.width,
+                        frame_info_.height,
+                        rotation_mode,
+                        origin_colorspace);
+
+  BrowserThread::PostTask(
+      BrowserThread::IO,
+      FROM_HERE,
+      base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
+                 this,
+                 dst,
+                 timestamp));
+}
+#else
+void VideoCaptureController::OnIncomingCapturedFrame(
+    const uint8* data,
+    int length,
     base::Time timestamp,
     int rotation,
     bool flip_vert,
     bool flip_horiz) {
   DCHECK(frame_info_.color == media::VideoCaptureCapability::kI420 ||
          frame_info_.color == media::VideoCaptureCapability::kYV12 ||
          (rotation == 0 && !flip_vert && !flip_horiz));
 
   TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame");
 
@@ skipping 35 matching lines @@
       media::ConvertNV21ToYUV(data, yplane, uplane, vplane, frame_info_.width,
                               frame_info_.height);
       break;
     case media::VideoCaptureCapability::kYUY2:
       DCHECK(!chopped_width_ && !chopped_height_);
       if (frame_info_.width * frame_info_.height * 2 != length) {
         // If |length| of |data| does not match the expected width and height
         // we can't convert the frame to I420. YUY2 is 2 bytes per pixel.
         break;
       }
-
       media::ConvertYUY2ToYUV(data, yplane, uplane, vplane, frame_info_.width,
                               frame_info_.height);
       break;
     case media::VideoCaptureCapability::kRGB24: {
       int ystride = frame_info_.width;
       int uvstride = frame_info_.width / 2;
-#if defined(OS_WIN)  // RGB on Windows start at the bottom line.
-      int rgb_stride = -3 * (frame_info_.width + chopped_width_);
-      const uint8* rgb_src = data + 3 * (frame_info_.width + chopped_width_) *
-          (frame_info_.height -1 + chopped_height_);
-#else
       int rgb_stride = 3 * (frame_info_.width + chopped_width_);
       const uint8* rgb_src = data;
-#endif
       media::ConvertRGB24ToYUV(rgb_src, yplane, uplane, vplane,
                                frame_info_.width, frame_info_.height,
                                rgb_stride, ystride, uvstride);
       break;
     }
     case media::VideoCaptureCapability::kARGB:
       media::ConvertRGB32ToYUV(data, yplane, uplane, vplane, frame_info_.width,
                                frame_info_.height,
                                (frame_info_.width + chopped_width_) * 4,
                                frame_info_.width, frame_info_.width / 2);
       break;
-#if !defined(OS_IOS) && !defined(OS_ANDROID)
-    case media::VideoCaptureCapability::kMJPEG: {
-      int yplane_stride = frame_info_.width;
-      int uv_plane_stride = (frame_info_.width + 1) / 2;
-      int crop_x = 0;
-      int crop_y = 0;
-      libyuv::ConvertToI420(data, length, yplane, yplane_stride, uplane,
-                            uv_plane_stride, vplane, uv_plane_stride, crop_x,
-                            crop_y, frame_info_.width, frame_info_.height,
-                            frame_info_.width, frame_info_.height,
-                            libyuv::kRotate0, libyuv::FOURCC_MJPG);
-      break;
-    }
-#endif
     default:
       NOTREACHED();
   }
 
   BrowserThread::PostTask(BrowserThread::IO,
       FROM_HERE,
       base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
                  this, dst, timestamp));
 }
+#endif  // #if !defined(OS_IOS) && !defined(OS_ANDROID)
 
 // OnIncomingCapturedVideoFrame is called the thread running the capture device.
 void VideoCaptureController::OnIncomingCapturedVideoFrame(
     const scoped_refptr<media::VideoFrame>& frame,
     base::Time timestamp) {
 
   scoped_refptr<media::VideoFrame> target;
   {
     base::AutoLock lock(buffer_pool_lock_);
 
@@ skipping 351 matching lines @@
     controller_clients_.push_back((*client_it));
     pending_clients_.erase(client_it++);
   }
   // Request the manager to start the actual capture.
   video_capture_manager_->Start(current_params_, this);
   state_ = VIDEO_CAPTURE_STATE_STARTED;
   device_in_use_ = true;
 }
 
 }  // namespace content