16 bpp video stream capture, render and createImageBitmap(video) using (CPU) shared memory buffers

media/capture/video/video_capture_device_client.cc

Index: media/capture/video/video_capture_device_client.cc
diff --git a/media/capture/video/video_capture_device_client.cc b/media/capture/video/video_capture_device_client.cc
index d9874a78814cbb6d8c36d5f2a82822436908e9f4..a34735aee9cfbd2bbbfc85b56c0aa7811d852e4f 100644
--- a/media/capture/video/video_capture_device_client.cc
+++ b/media/capture/video/video_capture_device_client.cc
@@ -27,6 +27,14 @@ using media::VideoCaptureFormat;
 using media::VideoFrame;
 using media::VideoFrameMetadata;
 
+namespace {
+
+bool IsFormatSupported(media::VideoPixelFormat pixel_format) {
+  return (pixel_format == media::PIXEL_FORMAT_I420 ||
+          pixel_format == media::PIXEL_FORMAT_Y16);
+}
+}
+
 namespace media {
 
 // Class combining a Client::Buffer interface implementation and a pool buffer
@@ -106,12 +114,23 @@ void VideoCaptureDeviceClient::OnIncomingCapturedData(
   if (!frame_format.IsValid())
     return;
 
+  // The input |length| can be greater than the required buffer size because of
+  // paddings and/or alignments, but it cannot be smaller.
+  DCHECK_GE(static_cast<size_t>(length), frame_format.ImageAllocationSize());

[mcasas, 2016/10/27 00:44:42]

This line is duplicated in l.244.

[aleksandar.stojiljkovic, 2016/10/27 21:27:22]

Done.

+
+  const bool convert_to_I420 =
+      frame_format.pixel_format != media::PIXEL_FORMAT_Y16;

[mcasas, 2016/10/27 00:44:42]

There's too many special-cases for Y16 and
anyway this format doesn't need conversion
rotation, nor cropping, which are the points of this 
method, so I suggest: write up a private method
OnIncomingCaptureY16Data() and bundle there the
specifics, leaving OnIncomingCapturedData()
mostly untouched. Jump to it if needed in
or around l.117. WDYT?

[aleksandar.stojiljkovic, 2016/10/27 21:27:22]

Done.

+
   // |chopped_{width,height} and |new_unrotated_{width,height}| are the lowest
   // bit decomposition of {width, height}, grabbing the odd and even parts.
   const int chopped_width = frame_format.frame_size.width() & 1;
   const int chopped_height = frame_format.frame_size.height() & 1;
-  const int new_unrotated_width = frame_format.frame_size.width() & ~1;
-  const int new_unrotated_height = frame_format.frame_size.height() & ~1;
+  const int new_unrotated_width = convert_to_I420
+                                      ? (frame_format.frame_size.width() & ~1)
+                                      : frame_format.frame_size.width();
+  const int new_unrotated_height = convert_to_I420
+                                       ? (frame_format.frame_size.height() & ~1)
+                                       : frame_format.frame_size.height();
 
   int destination_width = new_unrotated_width;
   int destination_height = new_unrotated_height;
@@ -129,16 +148,22 @@ void VideoCaptureDeviceClient::OnIncomingCapturedData(
     rotation_mode = libyuv::kRotate270;
 
   const gfx::Size dimensions(destination_width, destination_height);
-  uint8_t *y_plane_data, *u_plane_data, *v_plane_data;
+  uint8_t* y_plane_data = nullptr;
+  uint8_t* u_plane_data = nullptr;
+  uint8_t* v_plane_data = nullptr;
+
   std::unique_ptr<Buffer> buffer(
-      ReserveI420OutputBuffer(dimensions, media::PIXEL_STORAGE_CPU,
-                              &y_plane_data, &u_plane_data, &v_plane_data));
+      convert_to_I420
+          ? ReserveI420OutputBuffer(dimensions, media::PIXEL_STORAGE_CPU,
+                                    &y_plane_data, &u_plane_data, &v_plane_data)
+          : ReserveOutputBuffer(dimensions, frame_format.pixel_format,
+                                media::PIXEL_STORAGE_CPU));
 #if DCHECK_IS_ON()
   dropped_frame_counter_ = buffer.get() ? 0 : dropped_frame_counter_ + 1;
   if (dropped_frame_counter_ >= kMaxDroppedFrames)
     OnError(FROM_HERE, "Too many frames dropped");
 #endif
-  // Failed to reserve I420 output buffer, so drop the frame.
+  // Failed to reserve output buffer, so drop the frame.
   if (!buffer.get())
     return;
 
@@ -208,6 +233,8 @@ void VideoCaptureDeviceClient::OnIncomingCapturedData(
     case media::PIXEL_FORMAT_MJPEG:
       origin_colorspace = libyuv::FOURCC_MJPG;
       break;
+    case media::PIXEL_FORMAT_Y16:
+      break;
     default:
       NOTREACHED();
   }
@@ -231,21 +258,28 @@ void VideoCaptureDeviceClient::OnIncomingCapturedData(
     }
   }
 
-  if (libyuv::ConvertToI420(data, length, y_plane_data, yplane_stride,
-                            u_plane_data, uv_plane_stride, v_plane_data,
-                            uv_plane_stride, crop_x, crop_y,
-                            frame_format.frame_size.width(),
-                            (flip ? -1 : 1) * frame_format.frame_size.height(),
-                            new_unrotated_width, new_unrotated_height,
-                            rotation_mode, origin_colorspace) != 0) {
-    DLOG(WARNING) << "Failed to convert buffer's pixel format to I420 from "
-                  << media::VideoPixelFormatToString(frame_format.pixel_format);
-    return;
+  VideoPixelFormat pixel_format = frame_format.pixel_format;
+  if (convert_to_I420) {
+    pixel_format = media::PIXEL_FORMAT_I420;
+    if (libyuv::ConvertToI420(
+            data, length, y_plane_data, yplane_stride, u_plane_data,
+            uv_plane_stride, v_plane_data, uv_plane_stride, crop_x, crop_y,
+            frame_format.frame_size.width(),
+            (flip ? -1 : 1) * frame_format.frame_size.height(),
+            new_unrotated_width, new_unrotated_height, rotation_mode,
+            origin_colorspace) != 0) {
+      DLOG(WARNING) << "Failed to convert buffer's pixel format to I420 from "
+                    << media::VideoPixelFormatToString(
+                           frame_format.pixel_format);
+      return;
+    }
+  } else {
+    memcpy(buffer->data(), data, length);
   }
 
   const VideoCaptureFormat output_format =
-      VideoCaptureFormat(dimensions, frame_format.frame_rate,
-                         media::PIXEL_FORMAT_I420, media::PIXEL_STORAGE_CPU);
+      VideoCaptureFormat(dimensions, frame_format.frame_rate, pixel_format,
+                         media::PIXEL_STORAGE_CPU);
   OnIncomingCapturedBuffer(std::move(buffer), output_format, reference_time,
                            timestamp);
 }
@@ -257,8 +291,7 @@ VideoCaptureDeviceClient::ReserveOutputBuffer(
     media::VideoPixelStorage pixel_storage) {
   DCHECK_GT(frame_size.width(), 0);
   DCHECK_GT(frame_size.height(), 0);
-  // Currently, only I420 pixel format is supported.
-  DCHECK_EQ(media::PIXEL_FORMAT_I420, pixel_format);
+  DCHECK(IsFormatSupported(pixel_format));
 
   // TODO(mcasas): For PIXEL_STORAGE_GPUMEMORYBUFFER, find a way to indicate if
   // it's a ShMem GMB or a DmaBuf GMB.
@@ -278,8 +311,7 @@ void VideoCaptureDeviceClient::OnIncomingCapturedBuffer(
     const VideoCaptureFormat& frame_format,
     base::TimeTicks reference_time,
     base::TimeDelta timestamp) {
-  // Currently, only I420 pixel format is supported.
-  DCHECK_EQ(media::PIXEL_FORMAT_I420, frame_format.pixel_format);
+  DCHECK(IsFormatSupported(frame_format.pixel_format));
   DCHECK_EQ(media::PIXEL_STORAGE_CPU, frame_format.pixel_storage);
 
   scoped_refptr<VideoFrame> frame;
@@ -288,10 +320,10 @@ void VideoCaptureDeviceClient::OnIncomingCapturedBuffer(
     frame->set_timestamp(timestamp);
   } else {
     frame = VideoFrame::WrapExternalSharedMemory(
-        media::PIXEL_FORMAT_I420, frame_format.frame_size,
+        frame_format.pixel_format, frame_format.frame_size,
         gfx::Rect(frame_format.frame_size), frame_format.frame_size,
         reinterpret_cast<uint8_t*>(buffer->data()),
-        VideoFrame::AllocationSize(media::PIXEL_FORMAT_I420,
+        VideoFrame::AllocationSize(frame_format.pixel_format,
                                    frame_format.frame_size),
         base::SharedMemory::NULLHandle(), 0u, timestamp);
   }