Use libyuv in Android VideoCaptureController (not for Android WebView)

content/browser/renderer_host/media/video_capture_controller.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
 #include <set>
 
 #include "base/bind.h"
 #include "base/debug/trace_event.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/stl_util.h"
 #include "content/browser/renderer_host/media/media_stream_manager.h"
 #include "content/browser/renderer_host/media/video_capture_manager.h"
 #include "content/public/browser/browser_thread.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_util.h"
 #include "media/base/yuv_convert.h"
 
-#if !defined(OS_IOS) && !defined(OS_ANDROID)
 #include "third_party/libyuv/include/libyuv.h"
-#endif
-
-namespace {
-
-#if defined(OS_IOS) || defined(OS_ANDROID)
-// TODO(wjia): Support stride.
-void RotatePackedYV12Frame(
-    const uint8* src,
-    uint8* dest_yplane,
-    uint8* dest_uplane,
-    uint8* dest_vplane,
-    int width,
-    int height,
-    int rotation,
-    bool flip_vert,
-    bool flip_horiz) {
-  media::RotatePlaneByPixels(
-      src, dest_yplane, width, height, rotation, flip_vert, flip_horiz);
-  int y_size = width * height;
-  src += y_size;
-  media::RotatePlaneByPixels(
-      src, dest_uplane, width/2, height/2, rotation, flip_vert, flip_horiz);
-  src += y_size/4;
-  media::RotatePlaneByPixels(
-      src, dest_vplane, width/2, height/2, rotation, flip_vert, flip_horiz);
-}
-#endif  // #if defined(OS_IOS) || defined(OS_ANDROID)
-
-}  // namespace
 
 namespace content {
 
 // The number of buffers that VideoCaptureBufferPool should allocate.
 static const int kNoOfBuffers = 3;
 
 struct VideoCaptureController::ControllerClient {
   ControllerClient(
       const VideoCaptureControllerID& id,
       VideoCaptureControllerEventHandler* handler,
@@ skipping 71 matching lines @@
   // The pool of shared-memory buffers used for capturing.
   scoped_refptr<VideoCaptureBufferPool> buffer_pool_;
 
   // Chopped pixels in width/height in case video capture device has odd
   // numbers for width/height.
   int chopped_width_;
   int chopped_height_;
 
   // Tracks the current frame format.
   media::VideoCaptureCapability frame_info_;
-
-  // For NV21 we have to do color conversion into the intermediate buffer and
-  // from there the rotations. This variable won't be needed after
-  // http://crbug.com/292400
-#if defined(OS_IOS) || defined(OS_ANDROID)
-  scoped_ptr<uint8[]> i420_intermediate_buffer_;
-#endif  // #if defined(OS_IOS) || defined(OS_ANDROID)
+  scoped_ptr<uint8[]> rotation_temp_buffer_;
 };
 
 VideoCaptureController::VideoCaptureController()
     : state_(VIDEO_CAPTURE_STATE_STARTED),
       weak_ptr_factory_(this) {
   memset(&current_params_, 0, sizeof(current_params_));
 }
 
 VideoCaptureController::VideoCaptureDeviceClient::VideoCaptureDeviceClient(
     const base::WeakPtr<VideoCaptureController>& controller)
@@ skipping 110 matching lines @@
   buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
 }
 
 scoped_refptr<media::VideoFrame>
 VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer() {
   return buffer_pool_->ReserveI420VideoFrame(gfx::Size(frame_info_.width,
                                                        frame_info_.height),
                                              0);
 }
 
-#if !defined(OS_IOS) && !defined(OS_ANDROID)
 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame(
     const uint8* data,
     int length,
     base::Time timestamp,
     int rotation,
     bool flip_vert,
     bool flip_horiz) {
   TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame");
 
   if (!buffer_pool_.get())
     return;
   scoped_refptr<media::VideoFrame> dst = buffer_pool_->ReserveI420VideoFrame(
       gfx::Size(frame_info_.width, frame_info_.height), rotation);
 
   if (!dst.get())
     return;
 
   uint8* yplane = dst->data(media::VideoFrame::kYPlane);
   uint8* uplane = dst->data(media::VideoFrame::kUPlane);
   uint8* vplane = dst->data(media::VideoFrame::kVPlane);
   int yplane_stride = frame_info_.width;
   int uv_plane_stride = (frame_info_.width + 1) / 2;
   int crop_x = 0;
   int crop_y = 0;
+  int destination_width = frame_info_.width;
+  int destination_height = frame_info_.height;
   libyuv::FourCC origin_colorspace = libyuv::FOURCC_ANY;
   // Assuming rotation happens first and flips next, we can consolidate both
   // vertical and horizontal flips together with rotation into two variables:
   // new_rotation = (rotation + 180 * vertical_flip) modulo 360
   // new_vertical_flip = horizontal_flip XOR vertical_flip
   int new_rotation_angle = (rotation + 180 * flip_vert) % 360;
   libyuv::RotationMode rotation_mode = libyuv::kRotate0;
   if (new_rotation_angle == 90)
     rotation_mode = libyuv::kRotate90;
   else if (new_rotation_angle == 180)
@@ skipping 55 matching lines @@
     media::ConvertRGB24ToYUV(rgb_src,
                              yplane,
                              uplane,
                              vplane,
                              frame_info_.width,
                              frame_info_.height,
                              rgb_stride,
                              yplane_stride,
                              uv_plane_stride);
   } else {
+    if (new_rotation_angle==90 || new_rotation_angle==270){
+      // To be compatible with non-libyuv code in RotatePlaneByPixels, when
+      // rotating by 90/270, only the maximum square portion located in the
+      // center of the image is rotated. F.i. 640x480 pixels, only the central
+      // 480 pixels would be rotated and the leftmost and rightmost 80 columns
+      // would be ignored.
+      destination_width = destination_height;
+      int letterbox_width = abs(frame_info_.width - frame_info_.height)/2;
+      yplane += letterbox_width;
+      uplane += letterbox_width/2;
+      vplane += letterbox_width/2;

[wjia(left Chromium), 2013/09/23 18:43:49]

This handles only landscape case. What about portrait mode?

[mcasas, 2013/09/24 07:17:06]

The landscape case needs all these extra modifications because of the
circumstances described in the comment immediately above, the portrait cases
(angle==0 or angle==180) do not suffer from those complications, and is dealt
with the variables unchanged values - assigned in lines 260-268.

[wjia(left Chromium), 2013/09/25 00:38:19]

Sorry, I wasn't clear. My question is: what if the input/output frames are both
in portrait mode? That means the input frame will be rotated to become
landscape, hence cropping left/right, padding top/bottom, and leaving the center
to fit in output frame.

Take a look at media::RotatePlaneByPixels()
(https://code.google.com/p/chromium/codesearch#chromium/src/media/base/video_u...),
it handles both width>height and height>width cases when rotation is 90 or 270.

+    }
     libyuv::ConvertToI420(
-        data,
-        length,
-        yplane,
-        yplane_stride,
-        uplane,
-        uv_plane_stride,
-        vplane,
-        uv_plane_stride,
-        crop_x,
-        crop_y,
-        frame_info_.width + chopped_width_,
-        frame_info_.height * (flip_vert ^ flip_horiz ? -1 : 1),
-        frame_info_.width,
-        frame_info_.height,
-        rotation_mode,
-        origin_colorspace);
+       data, length,
+       yplane, yplane_stride,
+       uplane, uv_plane_stride,
+       vplane, uv_plane_stride,
+       crop_x, crop_y,
+       frame_info_.width + chopped_width_,
+       frame_info_.height * (flip_vert ^ flip_horiz ? -1 : 1),
+       destination_width,
+       destination_height,
+       rotation_mode,
+       origin_colorspace);
   }
   BrowserThread::PostTask(
       BrowserThread::IO,
       FROM_HERE,
       base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
                  controller_,
                  dst,
                  timestamp));
 }
-#else
-void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame(
-    const uint8* data,
-    int length,
-    base::Time timestamp,
-    int rotation,
-    bool flip_vert,
-    bool flip_horiz) {
-  DCHECK(frame_info_.color == media::PIXEL_FORMAT_I420 ||
-         frame_info_.color == media::PIXEL_FORMAT_YV12 ||
-         frame_info_.color == media::PIXEL_FORMAT_NV21 ||
-         (rotation == 0 && !flip_vert && !flip_horiz));
-
-  TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame");
-
-  if (!buffer_pool_)
-    return;
-  scoped_refptr<media::VideoFrame> dst =
-      buffer_pool_->ReserveI420VideoFrame(gfx::Size(frame_info_.width,
-                                                    frame_info_.height),
-                                          rotation);
-
-  if (!dst.get())
-    return;
-
-  uint8* yplane = dst->data(media::VideoFrame::kYPlane);
-  uint8* uplane = dst->data(media::VideoFrame::kUPlane);
-  uint8* vplane = dst->data(media::VideoFrame::kVPlane);
-
-  // Do color conversion from the camera format to I420.
-  switch (frame_info_.color) {
-    case media::PIXEL_FORMAT_UNKNOWN:  // Color format not set.
-      break;
-    case media::PIXEL_FORMAT_I420:
-      DCHECK(!chopped_width_ && !chopped_height_);
-      RotatePackedYV12Frame(
-          data, yplane, uplane, vplane, frame_info_.width, frame_info_.height,
-          rotation, flip_vert, flip_horiz);
-      break;
-    case media::PIXEL_FORMAT_YV12:
-      DCHECK(!chopped_width_ && !chopped_height_);
-      RotatePackedYV12Frame(
-          data, yplane, vplane, uplane, frame_info_.width, frame_info_.height,
-          rotation, flip_vert, flip_horiz);
-      break;
-    case media::PIXEL_FORMAT_NV21: {
-      DCHECK(!chopped_width_ && !chopped_height_);
-      int num_pixels = frame_info_.width * frame_info_.height;
-      media::ConvertNV21ToYUV(data,
-                              &i420_intermediate_buffer_[0],
-                              &i420_intermediate_buffer_[num_pixels],
-                              &i420_intermediate_buffer_[num_pixels * 5 / 4],
-                              frame_info_.width,
-                              frame_info_.height);
-      RotatePackedYV12Frame(
-          i420_intermediate_buffer_.get(), yplane, uplane, vplane,
-          frame_info_.width, frame_info_.height,
-          rotation, flip_vert, flip_horiz);
-       break;
-    }
-    case media::PIXEL_FORMAT_YUY2:
-      DCHECK(!chopped_width_ && !chopped_height_);
-      if (frame_info_.width * frame_info_.height * 2 != length) {
-        // If |length| of |data| does not match the expected width and height
-        // we can't convert the frame to I420. YUY2 is 2 bytes per pixel.
-        break;
-      }
-
-      media::ConvertYUY2ToYUV(data, yplane, uplane, vplane, frame_info_.width,
-                              frame_info_.height);
-      break;
-    case media::PIXEL_FORMAT_RGB24: {
-      int ystride = frame_info_.width;
-      int uvstride = frame_info_.width / 2;
-      int rgb_stride = 3 * (frame_info_.width + chopped_width_);
-      const uint8* rgb_src = data;
-      media::ConvertRGB24ToYUV(rgb_src, yplane, uplane, vplane,
-                               frame_info_.width, frame_info_.height,
-                               rgb_stride, ystride, uvstride);
-      break;
-    }
-    case media::PIXEL_FORMAT_ARGB:
-      media::ConvertRGB32ToYUV(data, yplane, uplane, vplane, frame_info_.width,
-                               frame_info_.height,
-                               (frame_info_.width + chopped_width_) * 4,
-                               frame_info_.width, frame_info_.width / 2);
-      break;
-    default:
-      NOTREACHED();
-  }
-
-  BrowserThread::PostTask(BrowserThread::IO,
-      FROM_HERE,
-      base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
-                 controller_, dst, timestamp));
-}
-#endif  // #if !defined(OS_IOS) && !defined(OS_ANDROID)
 
 void
 VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedVideoFrame(
     const scoped_refptr<media::VideoFrame>& frame,
     base::Time timestamp) {
   if (!buffer_pool_)
     return;
 
   // If this is a frame that belongs to the buffer pool, we can forward it
   // directly to the IO thread and be done.
@@ skipping 117 matching lines @@
     chopped_width_ = 1;
   } else {
     chopped_width_ = 0;
   }
   if (info.height & 1) {
     --frame_info_.height;
     chopped_height_ = 1;
   } else {
     chopped_height_ = 0;
   }
-#if defined(OS_IOS) || defined(OS_ANDROID)
-  if (frame_info_.color == media::PIXEL_FORMAT_NV21 &&
-      !i420_intermediate_buffer_) {
-    i420_intermediate_buffer_.reset(
-        new uint8[frame_info_.width * frame_info_.height * 12 / 8]);
-  }
-#endif  // #if defined(OS_IOS) || defined(OS_ANDROID)
+  if (!rotation_temp_buffer_.get())
+      rotation_temp_buffer_.reset(new uint8[frame_info_.width * frame_info_.height * 12 / 8]);

[wjia(left Chromium), 2013/09/23 18:43:49]

80 chars per line.
2 space indent for a new line.
This rotation_temp_buffer_ is not used anywhere in this file.

[mcasas, 2013/09/24 07:17:06]

Absolutely, I removed it locally but forgot to cl upload it.
Done.

 
   DCHECK(!buffer_pool_.get());
 
   // TODO(nick): Give BufferPool the same lifetime as the controller, have it
   // support frame size changes, and stop checking it for NULL everywhere.
   // http://crbug.com/266082
   buffer_pool_ = new VideoCaptureBufferPool(
       media::VideoFrame::AllocationSize(
             media::VideoFrame::I420,
             gfx::Size(frame_info_.width, frame_info_.height)),
@@ skipping 155 matching lines @@
   }
   return NULL;
 }
 
 int VideoCaptureController::GetClientCount() {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   return controller_clients_.size();
 }
 
 }  // namespace content