16 bpp video stream capture, render and WebGL usage - Realsense R200 & SR300 support.

cc/resources/video_resource_updater.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "cc/resources/video_resource_updater.h"
 
 #include <stddef.h>
 #include <stdint.h>
 
 #include <algorithm>
@@ skipping 51 matching lines @@
       switch (video_frame->mailbox_holder(0).texture_target) {
         case GL_TEXTURE_EXTERNAL_OES:
           return VideoFrameExternalResources::YUV_RESOURCE;
         case GL_TEXTURE_RECTANGLE_ARB:
           return VideoFrameExternalResources::RGB_RESOURCE;
         default:
           NOTREACHED();
           break;
       }
       break;
+    case media::PIXEL_FORMAT_Y16:
+      return VideoFrameExternalResources::Y_RESOURCE;
+      break;
     case media::PIXEL_FORMAT_YV12:
     case media::PIXEL_FORMAT_YV16:
     case media::PIXEL_FORMAT_YV24:
     case media::PIXEL_FORMAT_YV12A:
     case media::PIXEL_FORMAT_NV21:
     case media::PIXEL_FORMAT_YUY2:
     case media::PIXEL_FORMAT_RGB24:
     case media::PIXEL_FORMAT_RGB32:
     case media::PIXEL_FORMAT_MJPEG:
     case media::PIXEL_FORMAT_MT21:
     case media::PIXEL_FORMAT_YUV420P9:
     case media::PIXEL_FORMAT_YUV422P9:
     case media::PIXEL_FORMAT_YUV444P9:
     case media::PIXEL_FORMAT_YUV420P10:
     case media::PIXEL_FORMAT_YUV422P10:
     case media::PIXEL_FORMAT_YUV444P10:
     case media::PIXEL_FORMAT_YUV420P12:
     case media::PIXEL_FORMAT_YUV422P12:
     case media::PIXEL_FORMAT_YUV444P12:
     case media::PIXEL_FORMAT_Y8:
-    case media::PIXEL_FORMAT_Y16:
     case media::PIXEL_FORMAT_UNKNOWN:
       break;
   }
   return VideoFrameExternalResources::NONE;
 }
 
 class SyncTokenClientImpl : public media::VideoFrame::SyncTokenClient {
  public:
   SyncTokenClientImpl(gpu::gles2::GLES2Interface* gl,
                       const gpu::SyncToken& sync_token)
@@ skipping 202 matching lines @@
   // Maximum value used in |src|.
   int max_value = (1 << bits_per_channel) - 1;
   int rows = 1;
   libyuv::HalfFloatPlane(src, stride, dst, stride, 1.0f / max_value, num, rows);
 }
 
 VideoFrameExternalResources VideoResourceUpdater::CreateForSoftwarePlanes(
     scoped_refptr<media::VideoFrame> video_frame) {
   TRACE_EVENT0("cc", "VideoResourceUpdater::CreateForSoftwarePlanes");
   const media::VideoPixelFormat input_frame_format = video_frame->format();
-
   // TODO(hubbe): Make this a video frame method.
   int bits_per_channel = 0;
   switch (input_frame_format) {
     case media::PIXEL_FORMAT_UNKNOWN:
       NOTREACHED();
     // Fall through!
     case media::PIXEL_FORMAT_I420:
     case media::PIXEL_FORMAT_YV12:
     case media::PIXEL_FORMAT_YV16:
     case media::PIXEL_FORMAT_YV12A:
@@ skipping 24 matching lines @@
     case media::PIXEL_FORMAT_YUV420P12:
     case media::PIXEL_FORMAT_YUV422P12:
     case media::PIXEL_FORMAT_YUV444P12:
       bits_per_channel = 12;
       break;
     case media::PIXEL_FORMAT_Y16:
       bits_per_channel = 16;
       break;
   }
 
-  // TODO(dshwang): support PIXEL_FORMAT_Y16. crbug.com/624436
-  DCHECK_NE(bits_per_channel, 16);
-
-  // Only YUV software video frames are supported.
-  if (!media::IsYuvPlanar(input_frame_format)) {
+  // Only YUV and Y16 software video frames are supported.
+  const bool isYuvPlanar = media::IsYuvPlanar(input_frame_format);
+  if (!(isYuvPlanar || input_frame_format == media::PIXEL_FORMAT_Y16)) {
     NOTREACHED() << media::VideoPixelFormatToString(input_frame_format);
     return VideoFrameExternalResources();
   }
 
   const bool software_compositor = context_provider_ == NULL;
 
   ResourceFormat output_resource_format =
-      resource_provider_->YuvResourceFormat(bits_per_channel);
+      (input_frame_format == media::PIXEL_FORMAT_Y16)
+          ? resource_provider_->Y16ResourceFormat()
+          : resource_provider_->YuvResourceFormat(bits_per_channel);
 
   // If GPU compositing is enabled, but the output resource format
   // returned by the resource provider is RGBA_8888, then a GPU driver
   // bug workaround requires that YUV frames must be converted to RGB
   // before texture upload.
   bool texture_needs_rgb_conversion =
       !software_compositor &&
-      output_resource_format == ResourceFormat::RGBA_8888;
+      output_resource_format == ResourceFormat::RGBA_8888 &&
+      input_frame_format != media::PIXEL_FORMAT_Y16;
   size_t output_plane_count = media::VideoFrame::NumPlanes(input_frame_format);
 
   // TODO(skaslev): If we're in software compositing mode, we do the YUV -> RGB
   // conversion here. That involves an extra copy of each frame to a bitmap.
   // Obviously, this is suboptimal and should be addressed once ubercompositor
   // starts shaping up.
   if (software_compositor || texture_needs_rgb_conversion) {
     output_resource_format = kRGBResourceFormat;
     output_plane_count = 1;
     bits_per_channel = 8;
@@ skipping 88 matching lines @@
       external_resources.release_callbacks.push_back(base::Bind(
           &RecycleResource, AsWeakPtr(), plane_resource.resource_id()));
       external_resources.type = VideoFrameExternalResources::RGBA_RESOURCE;
     }
     return external_resources;
   }
 
   for (size_t i = 0; i < plane_resources.size(); ++i) {
     PlaneResource& plane_resource = *plane_resources[i];
     // Update each plane's resource id with its content.
-    DCHECK_EQ(plane_resource.resource_format(),
-              resource_provider_->YuvResourceFormat(bits_per_channel));
+    DCHECK_EQ(plane_resource.resource_format(), output_resource_format);
 
     if (!plane_resource.Matches(video_frame->unique_id(), i)) {
       // TODO(hubbe): Move all conversion (and upload?) code to media/.
       // We need to transfer data from |video_frame| to the plane resource.
       // TODO(reveman): Can use GpuMemoryBuffers here to improve performance.
 
       // The |resource_size_pixels| is the size of the resource we want to
       // upload to.
       gfx::Size resource_size_pixels = plane_resource.resource_size();
       // The |video_stride_bytes| is the width of the video frame we are
@@ skipping 34 matching lines @@
           // https://en.wikipedia.org/wiki/Half-precision_floating-point_format
           //
           // PLEASE NOTE:
           // All planes are assumed to use the same multiplier/offset.
           external_resources.offset = 0.5f;
           // Max value from input data.
           int max_input_value = (1 << bits_per_channel) - 1;
           // 2 << 11 = 2048 would be 1.0 with our exponent.
           external_resources.multiplier = 2048.0 / max_input_value;
         }
+      } else if (input_frame_format == media::PIXEL_FORMAT_Y16) {
+        if (plane_resource.resource_format() == RGBA_8888)
+          needs_conversion = true;
       } else if (bits_per_channel > 8) {
         // If bits_per_channel > 8 and we can't use LUMINANCE_F16, we need to
         // shift the data down and create an 8-bit texture.
         needs_conversion = true;
         shift = bits_per_channel - 8;
       }
       const uint8_t* pixels;
       if (static_cast<int>(upload_image_stride) == video_stride_bytes &&
           !needs_conversion) {
         pixels = video_frame->data(i);
@@ skipping 22 matching lines @@
               MakeHalfFloats(src, bits_per_channel, bytes_per_row / 2, dst);
             }
           } else if (shift != 0) {
             // We have more-than-8-bit input which we need to shift
             // down to fit it into an 8-bit texture.
             uint8_t* dst = &upload_pixels_[upload_image_stride * row];
             const uint16_t* src = reinterpret_cast<uint16_t*>(
                 video_frame->data(i) + (video_stride_bytes * row));
             for (size_t i = 0; i < bytes_per_row; i++)
               dst[i] = src[i] >> shift;
+          } else if (input_frame_format == media::PIXEL_FORMAT_Y16 &&
+                     plane_resource.resource_format() == RGBA_8888) {
+            uint32_t* dst = reinterpret_cast<uint32_t*>(
+                &upload_pixels_[upload_image_stride * row]);
+            const uint16_t* src = reinterpret_cast<uint16_t*>(
+                video_frame->data(i) + (video_stride_bytes * row));
+            for (size_t i = 0; i < bytes_per_row / 4; ++i)
+              *dst++ = *src++;
           } else {
             // Input and output are the same size and format, but
             // differ in stride, copy one row at a time.
             uint8_t* dst = &upload_pixels_[upload_image_stride * row];
             const uint8_t* src =
                 video_frame->data(i) + (video_stride_bytes * row);
             memcpy(dst, src, bytes_per_row);
           }
         }
         pixels = &upload_pixels_[0];
       }
 
       resource_provider_->CopyToResource(plane_resource.resource_id(), pixels,
                                          resource_size_pixels);
       plane_resource.SetUniqueId(video_frame->unique_id(), i);
     }
 
 
     // VideoResourceUpdater shares a context with the compositor so a
     // sync token is not required.
     TextureMailbox mailbox(plane_resource.mailbox(), gpu::SyncToken(),
                            resource_provider_->GetResourceTextureTarget(
                                plane_resource.resource_id()));
     mailbox.set_color_space(video_frame->ColorSpace());
     external_resources.mailboxes.push_back(mailbox);
     external_resources.release_callbacks.push_back(base::Bind(
         &RecycleResource, AsWeakPtr(), plane_resource.resource_id()));
   }
 
-  external_resources.type = VideoFrameExternalResources::YUV_RESOURCE;
+  external_resources.type = (input_frame_format == media::PIXEL_FORMAT_Y16)
+                                ? VideoFrameExternalResources::Y_RESOURCE
+                                : VideoFrameExternalResources::YUV_RESOURCE;
   return external_resources;
 }
 
 // static
 void VideoResourceUpdater::ReturnTexture(
     base::WeakPtr<VideoResourceUpdater> updater,
     const scoped_refptr<media::VideoFrame>& video_frame,
     const gpu::SyncToken& sync_token,
     bool lost_resource,
     BlockingTaskRunner* main_thread_task_runner) {
@@ skipping 96 matching lines @@
                              video_frame->coded_size(),
                              video_frame->metadata()->IsTrue(
                                  media::VideoFrameMetadata::ALLOW_OVERLAY),
                              false);
       mailbox.set_color_space(video_frame->ColorSpace());
       external_resources.mailboxes.push_back(mailbox);
       external_resources.release_callbacks.push_back(
           base::Bind(&ReturnTexture, AsWeakPtr(), video_frame));
     }
   }
+
+  external_resources.bits_per_channel =
+      (video_frame->format() == media::PIXEL_FORMAT_Y16) ? 16 : 8;
   return external_resources;
 }
 
 // static
 void VideoResourceUpdater::RecycleResource(
     base::WeakPtr<VideoResourceUpdater> updater,
     ResourceId resource_id,
     const gpu::SyncToken& sync_token,
     bool lost_resource,
     BlockingTaskRunner* main_thread_task_runner) {
@@ skipping 18 matching lines @@
   if (lost_resource) {
     resource_it->clear_refs();
     updater->DeleteResource(resource_it);
     return;
   }
 
   resource_it->remove_ref();
 }
 
 }  // namespace cc