Use libyuv in Android VideoCaptureController (not for Android WebView)

content/browser/renderer_host/media/video_capture_controller.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_controller.h"
 
 #include <set>
 
 #include "base/bind.h"
 #include "base/debug/trace_event.h"
 #include "base/memory/scoped_ptr.h"
 #include "base/stl_util.h"
 #include "content/browser/renderer_host/media/media_stream_manager.h"
 #include "content/browser/renderer_host/media/video_capture_manager.h"
 #include "content/public/browser/browser_thread.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_util.h"
 #include "media/base/yuv_convert.h"
 
-#if !defined(OS_IOS) && !defined(OS_ANDROID)
+#if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
 #include "third_party/libyuv/include/libyuv.h"
 #endif
 
-namespace {
-
-#if defined(OS_IOS) || defined(OS_ANDROID)
-// TODO(wjia): Support stride.
-void RotatePackedYV12Frame(
-    const uint8* src,
-    uint8* dest_yplane,
-    uint8* dest_uplane,
-    uint8* dest_vplane,
-    int width,
-    int height,
-    int rotation,
-    bool flip_vert,
-    bool flip_horiz) {
-  media::RotatePlaneByPixels(
-      src, dest_yplane, width, height, rotation, flip_vert, flip_horiz);
-  int y_size = width * height;
-  src += y_size;
-  media::RotatePlaneByPixels(
-      src, dest_uplane, width/2, height/2, rotation, flip_vert, flip_horiz);
-  src += y_size/4;
-  media::RotatePlaneByPixels(
-      src, dest_vplane, width/2, height/2, rotation, flip_vert, flip_horiz);
-}
-#endif  // #if defined(OS_IOS) || defined(OS_ANDROID)
-
-}  // namespace
-
 namespace content {
 
 // The number of buffers that VideoCaptureBufferPool should allocate.
 static const int kNoOfBuffers = 3;
 
 struct VideoCaptureController::ControllerClient {
   ControllerClient(
       const VideoCaptureControllerID& id,
       VideoCaptureControllerEventHandler* handler,
       base::ProcessHandle render_process,
@@ skipping 70 matching lines @@
   // The pool of shared-memory buffers used for capturing.
   scoped_refptr<VideoCaptureBufferPool> buffer_pool_;
 
   // Chopped pixels in width/height in case video capture device has odd
   // numbers for width/height.
   int chopped_width_;
   int chopped_height_;
 
   // Tracks the current frame format.
   media::VideoCaptureCapability frame_info_;
-
-  // For NV21 we have to do color conversion into the intermediate buffer and
-  // from there the rotations. This variable won't be needed after
-  // http://crbug.com/292400
-#if defined(OS_IOS) || defined(OS_ANDROID)
-  scoped_ptr<uint8[]> i420_intermediate_buffer_;
-#endif  // #if defined(OS_IOS) || defined(OS_ANDROID)
 };
 
 VideoCaptureController::VideoCaptureController()
     : state_(VIDEO_CAPTURE_STATE_STARTED),
       weak_ptr_factory_(this) {
   memset(&current_params_, 0, sizeof(current_params_));
 }
 
 VideoCaptureController::VideoCaptureDeviceClient::VideoCaptureDeviceClient(
     const base::WeakPtr<VideoCaptureController>& controller)
@@ skipping 110 matching lines @@
   buffer_pool_->RelinquishConsumerHold(buffer_id, 1);
 }
 
 scoped_refptr<media::VideoFrame>
 VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer() {
   return buffer_pool_->ReserveI420VideoFrame(gfx::Size(frame_info_.width,
                                                        frame_info_.height),
                                              0);
 }
 
-#if !defined(OS_IOS) && !defined(OS_ANDROID)
 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame(
     const uint8* data,
     int length,
     base::Time timestamp,
     int rotation,
     bool flip_vert,
     bool flip_horiz) {
   TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame");
 
   if (!buffer_pool_.get())
     return;
   scoped_refptr<media::VideoFrame> dst = buffer_pool_->ReserveI420VideoFrame(
       gfx::Size(frame_info_.width, frame_info_.height), rotation);
 
   if (!dst.get())
     return;
+#if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)

[wjia(left Chromium), 2013/09/25 18:50:25]

The exclusion here will create video frames with incorrect content, since
rotation is not done. There should be an "else" branch for that case.

[mcasas, 2013/09/25 19:14:38]

After much debate with the WebView team, we agreed that, as a temporary stage
before integrating WebRTC in Android WebView build and, given the fact that in
Android WebView there is no Video Capture using this path, it does not matter to
pass around what is effectively black screens. Hence the lack of "#else" path. 

Please note that Android WebView build (trybot android_aosp) is different from
normal chromium WebView build.

 
   uint8* yplane = dst->data(media::VideoFrame::kYPlane);
   uint8* uplane = dst->data(media::VideoFrame::kUPlane);
   uint8* vplane = dst->data(media::VideoFrame::kVPlane);
   int yplane_stride = frame_info_.width;
   int uv_plane_stride = (frame_info_.width + 1) / 2;
   int crop_x = 0;
   int crop_y = 0;
+  int destination_width = frame_info_.width;
+  int destination_height = frame_info_.height;
   libyuv::FourCC origin_colorspace = libyuv::FOURCC_ANY;
   // Assuming rotation happens first and flips next, we can consolidate both
   // vertical and horizontal flips together with rotation into two variables:
   // new_rotation = (rotation + 180 * vertical_flip) modulo 360
   // new_vertical_flip = horizontal_flip XOR vertical_flip
   int new_rotation_angle = (rotation + 180 * flip_vert) % 360;
   libyuv::RotationMode rotation_mode = libyuv::kRotate0;
   if (new_rotation_angle == 90)
     rotation_mode = libyuv::kRotate90;
   else if (new_rotation_angle == 180)
@@ skipping 55 matching lines @@
     media::ConvertRGB24ToYUV(rgb_src,
                              yplane,
                              uplane,
                              vplane,
                              frame_info_.width,
                              frame_info_.height,
                              rgb_stride,
                              yplane_stride,
                              uv_plane_stride);
   } else {
+    if (new_rotation_angle==90 || new_rotation_angle==270){
+      // To be compatible with non-libyuv code in RotatePlaneByPixels, when
+      // rotating by 90/270, only the maximum square portion located in the
+      // center of the image is rotated. F.i. 640x480 pixels, only the central
+      // 480 pixels would be rotated and the leftmost and rightmost 80 columns
+      // would be ignored. This process is called letterboxing.
+      int letterbox_thickness = abs(frame_info_.width - frame_info_.height)/2;
+      if (destination_width > destination_height){
+        yplane += letterbox_thickness;
+        uplane += letterbox_thickness/2;

[wjia(left Chromium), 2013/09/25 18:50:25]

nit: one space before and after operator "/". This one and below.

[mcasas, 2013/09/26 11:28:56]

Done.

+        vplane += letterbox_thickness/2;
+        destination_width = destination_height;
+      } else {
+        yplane += letterbox_thickness * destination_width;
+        uplane += (letterbox_thickness * destination_width)/2;
+        vplane += (letterbox_thickness * destination_width)/2;
+        destination_height = destination_width;
+      }

[wjia(left Chromium), 2013/09/25 18:50:25]

Please make sure both cases are tested and work correctly.

[mcasas, 2013/09/25 19:14:38]

They do, but if I understand it correctly, in normal operating conditions case
height>width cannot be exercised since all the modes supported by getUserMedia
are landscape, right?

+    }
     libyuv::ConvertToI420(
-        data,
-        length,
-        yplane,
-        yplane_stride,
-        uplane,
-        uv_plane_stride,
-        vplane,
-        uv_plane_stride,
-        crop_x,
-        crop_y,
+        data, length,
+        yplane, yplane_stride,
+        uplane, uv_plane_stride,
+        vplane, uv_plane_stride,
+        crop_x, crop_y,
         frame_info_.width + chopped_width_,
         frame_info_.height * (flip_vert ^ flip_horiz ? -1 : 1),
-        frame_info_.width,
-        frame_info_.height,
+        destination_width,
+        destination_height,
         rotation_mode,
         origin_colorspace);
   }
+#endif  // if defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW)
   BrowserThread::PostTask(
       BrowserThread::IO,
       FROM_HERE,
       base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
                  controller_,
                  dst,
                  timestamp));
 }
-#else
-void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame(
-    const uint8* data,
-    int length,
-    base::Time timestamp,
-    int rotation,
-    bool flip_vert,
-    bool flip_horiz) {
-  DCHECK(frame_info_.color == media::PIXEL_FORMAT_I420 ||
-         frame_info_.color == media::PIXEL_FORMAT_YV12 ||
-         frame_info_.color == media::PIXEL_FORMAT_NV21 ||
-         (rotation == 0 && !flip_vert && !flip_horiz));
-
-  TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame");
-
-  if (!buffer_pool_)
-    return;
-  scoped_refptr<media::VideoFrame> dst =
-      buffer_pool_->ReserveI420VideoFrame(gfx::Size(frame_info_.width,
-                                                    frame_info_.height),
-                                          rotation);
-
-  if (!dst.get())
-    return;
-
-  uint8* yplane = dst->data(media::VideoFrame::kYPlane);
-  uint8* uplane = dst->data(media::VideoFrame::kUPlane);
-  uint8* vplane = dst->data(media::VideoFrame::kVPlane);
-
-  // Do color conversion from the camera format to I420.
-  switch (frame_info_.color) {
-    case media::PIXEL_FORMAT_UNKNOWN:  // Color format not set.
-      break;
-    case media::PIXEL_FORMAT_I420:
-      DCHECK(!chopped_width_ && !chopped_height_);
-      RotatePackedYV12Frame(
-          data, yplane, uplane, vplane, frame_info_.width, frame_info_.height,
-          rotation, flip_vert, flip_horiz);
-      break;
-    case media::PIXEL_FORMAT_YV12:
-      DCHECK(!chopped_width_ && !chopped_height_);
-      RotatePackedYV12Frame(
-          data, yplane, vplane, uplane, frame_info_.width, frame_info_.height,
-          rotation, flip_vert, flip_horiz);
-      break;
-    case media::PIXEL_FORMAT_NV21: {
-      DCHECK(!chopped_width_ && !chopped_height_);
-      int num_pixels = frame_info_.width * frame_info_.height;
-      media::ConvertNV21ToYUV(data,
-                              &i420_intermediate_buffer_[0],
-                              &i420_intermediate_buffer_[num_pixels],
-                              &i420_intermediate_buffer_[num_pixels * 5 / 4],
-                              frame_info_.width,
-                              frame_info_.height);
-      RotatePackedYV12Frame(
-          i420_intermediate_buffer_.get(), yplane, uplane, vplane,
-          frame_info_.width, frame_info_.height,
-          rotation, flip_vert, flip_horiz);
-       break;
-    }
-    case media::PIXEL_FORMAT_YUY2:
-      DCHECK(!chopped_width_ && !chopped_height_);
-      if (frame_info_.width * frame_info_.height * 2 != length) {
-        // If |length| of |data| does not match the expected width and height
-        // we can't convert the frame to I420. YUY2 is 2 bytes per pixel.
-        break;
-      }
-
-      media::ConvertYUY2ToYUV(data, yplane, uplane, vplane, frame_info_.width,
-                              frame_info_.height);
-      break;
-    case media::PIXEL_FORMAT_RGB24: {
-      int ystride = frame_info_.width;
-      int uvstride = frame_info_.width / 2;
-      int rgb_stride = 3 * (frame_info_.width + chopped_width_);
-      const uint8* rgb_src = data;
-      media::ConvertRGB24ToYUV(rgb_src, yplane, uplane, vplane,
-                               frame_info_.width, frame_info_.height,
-                               rgb_stride, ystride, uvstride);
-      break;
-    }
-    case media::PIXEL_FORMAT_ARGB:
-      media::ConvertRGB32ToYUV(data, yplane, uplane, vplane, frame_info_.width,
-                               frame_info_.height,
-                               (frame_info_.width + chopped_width_) * 4,
-                               frame_info_.width, frame_info_.width / 2);
-      break;
-    default:
-      NOTREACHED();
-  }
-
-  BrowserThread::PostTask(BrowserThread::IO,
-      FROM_HERE,
-      base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread,
-                 controller_, dst, timestamp));
-}
-#endif  // #if !defined(OS_IOS) && !defined(OS_ANDROID)
 
 void
 VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedVideoFrame(
     const scoped_refptr<media::VideoFrame>& frame,
     base::Time timestamp) {
   if (!buffer_pool_)
     return;
 
   // If this is a frame that belongs to the buffer pool, we can forward it
   // directly to the IO thread and be done.
@@ skipping 117 matching lines @@
     chopped_width_ = 1;
   } else {
     chopped_width_ = 0;
   }
   if (info.height & 1) {
     --frame_info_.height;
     chopped_height_ = 1;
   } else {
     chopped_height_ = 0;
   }
-#if defined(OS_IOS) || defined(OS_ANDROID)
-  if (frame_info_.color == media::PIXEL_FORMAT_NV21 &&
-      !i420_intermediate_buffer_) {
-    i420_intermediate_buffer_.reset(
-        new uint8[frame_info_.width * frame_info_.height * 12 / 8]);
-  }
-#endif  // #if defined(OS_IOS) || defined(OS_ANDROID)
 
   DCHECK(!buffer_pool_.get());
 
   // TODO(nick): Give BufferPool the same lifetime as the controller, have it
   // support frame size changes, and stop checking it for NULL everywhere.
   // http://crbug.com/266082
   buffer_pool_ = new VideoCaptureBufferPool(
       media::VideoFrame::AllocationSize(
             media::VideoFrame::I420,
             gfx::Size(frame_info_.width, frame_info_.height)),
@@ skipping 155 matching lines @@
   }
   return NULL;
 }
 
 int VideoCaptureController::GetClientCount() {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   return controller_clients_.size();
 }
 
 }  // namespace content