16 bpp video stream capture, render and WebGL usage - Realsense R200 & SR300 support.

content/browser/renderer_host/media/video_capture_device_client.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/media/video_capture_device_client.h"
 
 #include <algorithm>
 #include <utility>
 
 #include "base/bind.h"
@@ skipping 10 matching lines @@
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/media_switches.h"
 #include "media/base/video_capture_types.h"
 #include "media/base/video_frame.h"
 #include "third_party/libyuv/include/libyuv.h"
 
 using media::VideoCaptureFormat;
 using media::VideoFrame;
 using media::VideoFrameMetadata;
 
+namespace {
+
+bool isFormatSupported(media::VideoPixelFormat pixel_format) {
+  // Currently, only I420, Y8 and Y16 pixel formats are supported.
+  return (pixel_format == media::PIXEL_FORMAT_I420 ||
+          pixel_format == media::PIXEL_FORMAT_Y8 ||
+          pixel_format == media::PIXEL_FORMAT_Y16);
+}
+
+}
+
 namespace content {
 
 // Class combining a Client::Buffer interface implementation and a pool buffer
 // implementation to guarantee proper cleanup on destruction on our side.
 class AutoReleaseBuffer : public media::VideoCaptureDevice::Client::Buffer {
  public:
   AutoReleaseBuffer(const scoped_refptr<VideoCaptureBufferPool>& pool,
                     int buffer_id)
       : id_(buffer_id),
         pool_(pool),
@@ skipping 60 matching lines @@
       external_jpeg_decoder_.reset(new VideoCaptureGpuJpegDecoder(base::Bind(
           &VideoCaptureController::DoIncomingCapturedVideoFrameOnIOThread,
           controller_)));
       external_jpeg_decoder_->Initialize();
     }
   }
 
   if (!frame_format.IsValid())
     return;
 
+  // The input |length| can be greater than the required buffer size because of
+  // paddings and/or alignments, but it cannot be smaller.
+  DCHECK_GE(static_cast<size_t>(length), frame_format.ImageAllocationSize());
+
+  const bool useFullSize =
+      frame_format.pixel_format == media::PIXEL_FORMAT_Y8 ||
+      frame_format.pixel_format == media::PIXEL_FORMAT_Y16;
+
   // |chopped_{width,height} and |new_unrotated_{width,height}| are the lowest
   // bit decomposition of {width, height}, grabbing the odd and even parts.
   const int chopped_width = frame_format.frame_size.width() & 1;
   const int chopped_height = frame_format.frame_size.height() & 1;
-  const int new_unrotated_width = frame_format.frame_size.width() & ~1;
-  const int new_unrotated_height = frame_format.frame_size.height() & ~1;
+  const int new_unrotated_width = (useFullSize)
+                                  ? frame_format.frame_size.width()
+                                  : (frame_format.frame_size.width() & ~1);
+  const int new_unrotated_height = (useFullSize)
+                                   ? frame_format.frame_size.height()
+                                   : (frame_format.frame_size.height() & ~1);
 
   int destination_width = new_unrotated_width;
   int destination_height = new_unrotated_height;
   if (rotation == 90 || rotation == 270)
     std::swap(destination_width, destination_height);
 
   DCHECK_EQ(0, rotation % 90)
       << " Rotation must be a multiple of 90, now: " << rotation;
   libyuv::RotationMode rotation_mode = libyuv::kRotate0;
   if (rotation == 90)
     rotation_mode = libyuv::kRotate90;
   else if (rotation == 180)
     rotation_mode = libyuv::kRotate180;
   else if (rotation == 270)
     rotation_mode = libyuv::kRotate270;
 
   const gfx::Size dimensions(destination_width, destination_height);
   const media::VideoPixelStorage output_pixel_storage =
       use_gpu_memory_buffers_ ? media::PIXEL_STORAGE_GPUMEMORYBUFFER
                               : media::PIXEL_STORAGE_CPU;
   uint8_t *y_plane_data, *u_plane_data, *v_plane_data;
-  std::unique_ptr<Buffer> buffer(
-      ReserveI420OutputBuffer(dimensions, output_pixel_storage, &y_plane_data,
-                              &u_plane_data, &v_plane_data));
+
+  if (frame_format.pixel_format == media::PIXEL_FORMAT_Y8 ||
+      frame_format.pixel_format == media::PIXEL_FORMAT_Y16) {
+    std::unique_ptr<Buffer> buffer(ReserveOutputBuffer(dimensions,
+        frame_format.pixel_format, output_pixel_storage));
+    if (!buffer.get()) {
+      DLOG(WARNING) 
+          << "Failed to reserve output buffer:"
+          << media::VideoPixelFormatToString(frame_format.pixel_format);
+      return;
+    }
+    memcpy(buffer->data(), data, length);

[aleksandar.stojiljkovic, 2016/07/11 22:15:14]

normalized unsigned short -| half_float conversion could happen here.

+    const VideoCaptureFormat output_format = VideoCaptureFormat(
+        dimensions, frame_format.frame_rate,
+        frame_format.pixel_format, output_pixel_storage);
+    OnIncomingCapturedBuffer(std::move(buffer), output_format, reference_time,
+                             timestamp);
+    return;
+  }
+
+  std::unique_ptr<Buffer> buffer(ReserveI420OutputBuffer(dimensions,
+      output_pixel_storage, &y_plane_data, &u_plane_data, &v_plane_data));
   if (!buffer.get()) {
     DLOG(WARNING) << "Failed to reserve I420 output buffer.";
     return;
   }
 
   const int yplane_stride = dimensions.width();
   const int uv_plane_stride = yplane_stride / 2;
   int crop_x = 0;
   int crop_y = 0;
   libyuv::FourCC origin_colorspace = libyuv::FOURCC_ANY;
@@ skipping 55 matching lines @@
     case media::PIXEL_FORMAT_ARGB:
       origin_colorspace = libyuv::FOURCC_ARGB;
       break;
     case media::PIXEL_FORMAT_MJPEG:
       origin_colorspace = libyuv::FOURCC_MJPG;
       break;
     default:
       NOTREACHED();
   }
 
-  // The input |length| can be greater than the required buffer size because of
-  // paddings and/or alignments, but it cannot be smaller.
-  DCHECK_GE(static_cast<size_t>(length), frame_format.ImageAllocationSize());
-
   if (external_jpeg_decoder_) {
     const VideoCaptureGpuJpegDecoder::STATUS status =
         external_jpeg_decoder_->GetStatus();
     if (status == VideoCaptureGpuJpegDecoder::FAILED) {
       external_jpeg_decoder_.reset();
     } else if (status == VideoCaptureGpuJpegDecoder::INIT_PASSED &&
         frame_format.pixel_format == media::PIXEL_FORMAT_MJPEG &&
         rotation == 0 && !flip) {
       external_jpeg_decoder_->DecodeCapturedData(data, length, frame_format,
                                                  reference_time, timestamp,
@@ skipping 30 matching lines @@
                            timestamp);
 }
 
 std::unique_ptr<media::VideoCaptureDevice::Client::Buffer>
 VideoCaptureDeviceClient::ReserveOutputBuffer(
     const gfx::Size& frame_size,
     media::VideoPixelFormat pixel_format,
     media::VideoPixelStorage pixel_storage) {
   DCHECK_GT(frame_size.width(), 0);
   DCHECK_GT(frame_size.height(), 0);
-  // Currently, only I420 pixel format is supported.
-  DCHECK_EQ(media::PIXEL_FORMAT_I420, pixel_format);
+  DCHECK(isFormatSupported(pixel_format));
 
   // TODO(mcasas): For PIXEL_STORAGE_GPUMEMORYBUFFER, find a way to indicate if
   // it's a ShMem GMB or a DmaBuf GMB.
   int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId;
   const int buffer_id = buffer_pool_->ReserveForProducer(
       frame_size, pixel_format, pixel_storage, &buffer_id_to_drop);
   if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) {
     BrowserThread::PostTask(BrowserThread::IO,
         FROM_HERE,
         base::Bind(&VideoCaptureController::DoBufferDestroyedOnIOThread,
                    controller_, buffer_id_to_drop));
   }
   if (buffer_id == VideoCaptureBufferPool::kInvalidId)
     return nullptr;
   return base::WrapUnique<Buffer>(
       new AutoReleaseBuffer(buffer_pool_, buffer_id));
 }
 
 void VideoCaptureDeviceClient::OnIncomingCapturedBuffer(
     std::unique_ptr<Buffer> buffer,
     const VideoCaptureFormat& frame_format,
     base::TimeTicks reference_time,
     base::TimeDelta timestamp) {
-  // Currently, only I420 pixel format is supported.
-  DCHECK_EQ(media::PIXEL_FORMAT_I420, frame_format.pixel_format);
+  DCHECK(isFormatSupported(frame_format.pixel_format));
 
   scoped_refptr<VideoFrame> frame;
   switch (frame_format.pixel_storage) {
     case media::PIXEL_STORAGE_GPUMEMORYBUFFER: {
+      // TODO(astojilj) Check Y8 and Y16 support.
+      DCHECK_EQ(media::PIXEL_FORMAT_I420, frame_format.pixel_format);
       // Create a VideoFrame to set the correct storage_type and pixel_format.
       gfx::GpuMemoryBufferHandle handle;
       frame = VideoFrame::WrapExternalYuvGpuMemoryBuffers(
           media::PIXEL_FORMAT_I420, frame_format.frame_size,
           gfx::Rect(frame_format.frame_size), frame_format.frame_size, 0, 0, 0,
           reinterpret_cast<uint8_t*>(buffer->data(media::VideoFrame::kYPlane)),
           reinterpret_cast<uint8_t*>(buffer->data(media::VideoFrame::kUPlane)),
           reinterpret_cast<uint8_t*>(buffer->data(media::VideoFrame::kVPlane)),
           handle, handle, handle, timestamp);
       break;
     }
     case media::PIXEL_STORAGE_CPU:
       frame = VideoFrame::WrapExternalSharedMemory(
-          media::PIXEL_FORMAT_I420, frame_format.frame_size,
+          frame_format.pixel_format, frame_format.frame_size,
           gfx::Rect(frame_format.frame_size), frame_format.frame_size,
           reinterpret_cast<uint8_t*>(buffer->data()),
-          VideoFrame::AllocationSize(media::PIXEL_FORMAT_I420,
+          VideoFrame::AllocationSize(frame_format.pixel_format,
                                      frame_format.frame_size),
           base::SharedMemory::NULLHandle(), 0u, timestamp);
       break;
   }
   if (!frame)
     return;
   frame->metadata()->SetDouble(media::VideoFrameMetadata::FRAME_RATE,
                                frame_format.frame_rate);
   frame->metadata()->SetTimeTicks(media::VideoFrameMetadata::REFERENCE_TIME,
                                   reference_time);
@@ skipping 89 matching lines @@
           reinterpret_cast<uint8_t*>(buffer->data(VideoFrame::kUPlane));
       *v_plane_data =
           reinterpret_cast<uint8_t*>(buffer->data(VideoFrame::kVPlane));
       return buffer;
   }
   NOTREACHED();
   return std::unique_ptr<Buffer>();
 }
 
 }  // namespace content