16 bpp video stream capture, render and createImageBitmap(video) using (CPU) shared memory buffers

media/renderers/skcanvas_video_renderer.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/renderers/skcanvas_video_renderer.h"
 
 #include <limits>
 
 #include "base/macros.h"
 #include "gpu/GLES2/gl2extchromium.h"
@@ skipping 329 matching lines @@
     return;
   }
 
   SkRect dest;
   dest.set(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom());
 
   // Paint black rectangle if there isn't a frame available or the
   // frame has an unexpected format.
   if (!video_frame.get() || video_frame->natural_size().IsEmpty() ||
       !(media::IsYuvPlanar(video_frame->format()) ||
+        video_frame->format() == media::PIXEL_FORMAT_Y16 ||
         video_frame->HasTextures())) {
     SkPaint blackWithAlphaPaint;
     blackWithAlphaPaint.setAlpha(paint.getAlpha());
     canvas->drawRect(dest, blackWithAlphaPaint);
     canvas->flush();
     return;
   }
 
   gpu::gles2::GLES2Interface* gl = context_3d.gl;
   if (!UpdateLastImage(video_frame, context_3d))
@@ skipping 153 matching lines @@
     for (int row = 0; row < video_frame->rows(plane); row++) {
       for (int x = 0; x < width; x++) {
         dst[x] = src[x] >> shift;
       }
       src += video_frame->stride(plane) / 2;
       dst += ret->stride(plane);
     }
   }
   return ret;
 }
+
+// We take the upper 8 bits of 16-bit data and convert it as luminance to ARGB.
+// We loose the precision here, but it is important not to render Y16 as RG_88.
+// To get the full precision use float textures with WebGL1 and e.g. R16UI or
+// R32F textures with WebGL2.
+void ConvertY16ToARGB(const VideoFrame* video_frame,
+                      void* argb_pixels,
+                      size_t argb_row_bytes) {
+  const uint8_t* row_head = video_frame->visible_data(0);
+  uint8_t* out = reinterpret_cast<uint8_t*>(argb_pixels);

[xhwang, 2016/10/28 16:54:17]

static_cast?

[aleksandar.stojiljkovic, 2016/10/28 18:58:55]

Done.

+  const size_t stride = video_frame->stride(0);
+  for (int i = 0; i < video_frame->visible_rect().height(); ++i) {
+    uint32_t* rgba = reinterpret_cast<uint32_t*>(out);
+    const uint8_t* row_end = row_head + video_frame->visible_rect().width() * 2;
+    for (const uint8_t* row = row_head; row < row_end; ++row) {
+      uint32_t gray_value = *++row;
+      *rgba++ = SkColorSetARGB(0xFF, gray_value, gray_value, gray_value);

[xhwang, 2016/10/28 16:54:17]

nit: There's a SkColorSetRGB() we can use.

[aleksandar.stojiljkovic, 2016/10/28 18:58:55]

Done.

+    }
+    out += argb_row_bytes;

[xhwang, 2016/10/28 16:54:17]

shall we dcheck that |argb_row_bytes| is big enough? also, should it be a
multiple of 4?

[aleksandar.stojiljkovic, 2016/10/28 18:58:55]

I wouldn't check it in our generator (other SkImageGenerator implementations
don't do it)- argb_row_bytes is SkBitmap row stride (SkBitmap::rowBytes()) in
this case destination buffer stride.
[1]
https://cs.chromium.org/chromium/src/third_party/skia/src/core/SkImageGenerat...

+    row_head += stride;
+  }
 }
 
+}  // anonymous namespace
+
 // static
 void SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
     const VideoFrame* video_frame,
     void* rgb_pixels,
     size_t row_bytes) {
   if (!video_frame->IsMappable()) {
     NOTREACHED() << "Cannot extract pixels from non-CPU frame formats.";
     return;
   }
-  if (!media::IsYuvPlanar(video_frame->format())) {
-    NOTREACHED() << "Non YUV formats are not supported";
-    return;
-  }
 
   switch (video_frame->format()) {
     case PIXEL_FORMAT_YV12:
     case PIXEL_FORMAT_I420:
       if (CheckColorSpace(video_frame, COLOR_SPACE_JPEG)) {
         LIBYUV_J420_TO_ARGB(video_frame->visible_data(VideoFrame::kYPlane),
                             video_frame->stride(VideoFrame::kYPlane),
                             video_frame->visible_data(VideoFrame::kUPlane),
                             video_frame->stride(VideoFrame::kUPlane),
                             video_frame->visible_data(VideoFrame::kVPlane),
@@ skipping 72 matching lines @@
     case PIXEL_FORMAT_YUV420P12:
     case PIXEL_FORMAT_YUV422P12:
     case PIXEL_FORMAT_YUV444P12: {
       scoped_refptr<VideoFrame> temporary_frame =
           DownShiftHighbitVideoFrame(video_frame);
       ConvertVideoFrameToRGBPixels(temporary_frame.get(), rgb_pixels,
                                    row_bytes);
       break;
     }
 
+    case PIXEL_FORMAT_Y16:
+      ConvertY16ToARGB(video_frame, rgb_pixels, row_bytes);
+      break;
+
     case PIXEL_FORMAT_NV12:
     case PIXEL_FORMAT_NV21:
     case PIXEL_FORMAT_UYVY:
     case PIXEL_FORMAT_YUY2:
     case PIXEL_FORMAT_ARGB:
     case PIXEL_FORMAT_XRGB:
     case PIXEL_FORMAT_RGB24:
     case PIXEL_FORMAT_RGB32:
     case PIXEL_FORMAT_MJPEG:
     case PIXEL_FORMAT_MT21:
-    // TODO(dshwang): Use either I400ToARGB or J400ToARGB depending if we want
-    // BT.601 constrained range of 16 to 240, or JPEG full range BT.601
-    // coefficients. Implement it when Y8/16 foramt is supported.
-    // crbug.com/624436
     case PIXEL_FORMAT_Y8:
-    case PIXEL_FORMAT_Y16:
     case PIXEL_FORMAT_UNKNOWN:
-      NOTREACHED();
+      NOTREACHED() << "Only YUV formats and Y16 are supported.";
   }
 }
 
 // static
 void SkCanvasVideoRenderer::CopyVideoFrameSingleTextureToGLTexture(
     gpu::gles2::GLES2Interface* gl,
     VideoFrame* video_frame,
     unsigned int texture,
     unsigned int internal_format,
     unsigned int type,
@@ skipping 145 matching lines @@
       last_image_->bounds().contains(visible_rect)) {
     last_image_ = last_image_->makeSubset(visible_rect);
   }
 }
 
 SkISize SkCanvasVideoRenderer::LastImageDimensionsForTesting() {
   return last_image_dimensions_for_testing_;
 }
 
 }  // namespace media