16 bpp video stream capture, render and createImageBitmap(video) using (CPU) shared memory buffers

media/renderers/skcanvas_video_renderer.cc

Index: media/renderers/skcanvas_video_renderer.cc
diff --git a/media/renderers/skcanvas_video_renderer.cc b/media/renderers/skcanvas_video_renderer.cc
index c3387ae15db6db9f9918d5192865ca36e5827313..becf201efc3f2db987618b7a8bd2f2e3ef196dae 100644
--- a/media/renderers/skcanvas_video_renderer.cc
+++ b/media/renderers/skcanvas_video_renderer.cc
@@ -4,12 +4,14 @@
 
 #include "media/renderers/skcanvas_video_renderer.h"
 
+#include <GLES3/gl3.h>
 #include <limits>
 
 #include "base/macros.h"
 #include "gpu/GLES2/gl2extchromium.h"
 #include "gpu/command_buffer/client/gles2_interface.h"
 #include "gpu/command_buffer/common/mailbox_holder.h"
+#include "media/base/data_buffer.h"
 #include "media/base/video_frame.h"
 #include "media/base/yuv_convert.h"
 #include "skia/ext/texture_handle.h"
@@ -347,6 +349,7 @@ void SkCanvasVideoRenderer::Paint(const scoped_refptr<VideoFrame>& video_frame,
   // frame has an unexpected format.
   if (!video_frame.get() || video_frame->natural_size().IsEmpty() ||
       !(media::IsYuvPlanar(video_frame->format()) ||
+        video_frame->format() == media::PIXEL_FORMAT_Y16 ||

[mcasas, 2016/10/21 00:10:51]

PIXEL_FORMAT_Y16 is not IsYuvPlanar() ?

[aleksandar.stojiljkovic, 2016/10/21 22:11:11]

No. It is a 16-bit single plane, e.g. D16 / z-buffer.

         video_frame->HasTextures())) {
     SkPaint blackWithAlphaPaint;
     blackWithAlphaPaint.setAlpha(paint.getAlpha());
@@ -520,6 +523,77 @@ scoped_refptr<VideoFrame> DownShiftHighbitVideoFrame(
   }
   return ret;
 }
+
+void ConvertY16ToARGB(const VideoFrame* video_frame,
+                      void* argb_pixels,
+                      size_t argb_row_bytes) {
+  const uint8_t* source =
+      reinterpret_cast<const uint8_t*>(video_frame->visible_data(0));
+  uint8_t* out = reinterpret_cast<uint8_t*>(argb_pixels);
+  const size_t stride = video_frame->stride(0);
+  for (int i = 0; i < video_frame->visible_rect().height(); ++i) {
+    const uint8_t* row = source;
+    uint32_t* rgba = reinterpret_cast<uint32_t*>(out);
+    for (const uint8_t* row_end = row + video_frame->row_bytes(0);
+         row < row_end; ++row) {
+      // We loose the precision here and take only upper 8 bits of 16 bit data.
+      // It is important not to render Y16 as RG_88. To get the full precision
+      // use float textures with WebGL1 and e.g. R16UI or R32F textures with
+      // WebGL2.
+      uint32_t green = *++row;
+      *rgba++ = SkColorSetARGB(0xFF, green, green, green);
+    }
+    out += argb_row_bytes;
+    source += stride;
+  }
+}
+
+void FlipAndConvertY16(const uint8_t* input,
+                       uint8_t* output,
+                       unsigned format,
+                       unsigned type,
+                       bool flip_y,
+                       size_t row_bytes,
+                       size_t stride,
+                       size_t output_row_bytes,
+                       size_t height) {
+  DCHECK(input != output);
+  for (size_t i = 0; i < height; ++i) {
+    const uint16_t* in = reinterpret_cast<const uint16_t*>(input + i * stride);
+    uint8_t* out = flip_y ? output + output_row_bytes * (height - i - 1)
+                          : output + output_row_bytes * i;
+    if (type == GL_FLOAT) {
+      float* out_row = reinterpret_cast<float*>(out);
+      const uint16_t* in_end = in + row_bytes / 2;
+      if (format == GL_RGBA) {
+        while (in < in_end) {
+          float red = *in++ / 65536.f;
+          *out_row++ = red;
+          *out_row++ = red;
+          *out_row++ = red;
+          *out_row++ = 1.0f;
+        }
+      } else if (format == GL_RGB) {
+        while (in < in_end) {
+          float red = *in++ / 65536.f;
+          *out_row++ = red;
+          *out_row++ = red;
+          *out_row++ = red;
+        }
+      } else if (type == GL_RED) {
+        while (in < in_end)
+          *out_row++ = *in++ / 65536.f;
+      } else {
+        NOTREACHED();
+      }
+    } else if ((format == GL_RG && type == GL_UNSIGNED_BYTE) ||
+               (format == GL_RED_INTEGER && type == GL_UNSIGNED_SHORT)) {
+      memcpy(out, input + i * stride, row_bytes);
+    } else {
+      NOTREACHED();
+    }
+  }
+}
 }
 
 // static
@@ -531,10 +605,6 @@ void SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
     NOTREACHED() << "Cannot extract pixels from non-CPU frame formats.";
     return;
   }
-  if (!media::IsYuvPlanar(video_frame->format())) {
-    NOTREACHED() << "Non YUV formats are not supported";
-    return;
-  }
 
   switch (video_frame->format()) {
     case PIXEL_FORMAT_YV12:
@@ -627,6 +697,10 @@ void SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
       break;
     }
 
+    case PIXEL_FORMAT_Y16:
+      ConvertY16ToARGB(video_frame, rgb_pixels, row_bytes);
+      break;
+
     case PIXEL_FORMAT_NV12:
     case PIXEL_FORMAT_NV21:
     case PIXEL_FORMAT_UYVY:
@@ -637,14 +711,9 @@ void SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
     case PIXEL_FORMAT_RGB32:
     case PIXEL_FORMAT_MJPEG:
     case PIXEL_FORMAT_MT21:
-    // TODO(dshwang): Use either I400ToARGB or J400ToARGB depending if we want
-    // BT.601 constrained range of 16 to 240, or JPEG full range BT.601
-    // coefficients. Implement it when Y8/16 foramt is supported.
-    // crbug.com/624436
     case PIXEL_FORMAT_Y8:
-    case PIXEL_FORMAT_Y16:
     case PIXEL_FORMAT_UNKNOWN:
-      NOTREACHED();
+      NOTREACHED() << "Only YUV formats and Y16 are supported.";
   }
 }
 
@@ -749,7 +818,87 @@ bool SkCanvasVideoRenderer::CopyVideoFrameTexturesToGLTexture(
                                            texture, internal_format, type,
                                            premultiply_alpha, flip_y);
   }
+  return true;
+}
 
+bool SkCanvasVideoRenderer::TexImageImpl(const char* functionID,
+                                         unsigned target,
+                                         gpu::gles2::GLES2Interface* gl,
+                                         VideoFrame* frame,
+                                         int level,
+                                         int internalformat,
+                                         unsigned format,
+                                         unsigned type,
+                                         int xoffset,
+                                         int yoffset,
+                                         int zoffset,
+                                         bool flip_y,
+                                         bool premultiplyAlpha) {
+  DCHECK(frame);
+  DCHECK(!frame->HasTextures());
+
+  bool has_alpha = false;
+  unsigned output_bytes_per_pixel;
+  switch (frame->format()) {
+    case PIXEL_FORMAT_Y16:
+      // Allow reinterpreting RG8 buffer here as R component in FLOAT.
+      switch (format) {
+        case GL_RGBA:
+          if (type == GL_FLOAT) {
+            output_bytes_per_pixel = 4 * sizeof(GLfloat);
+            break;
+          }
+        // Pass through.
+        case GL_RGB:
+          if (type == GL_FLOAT) {
+            output_bytes_per_pixel = 3 * sizeof(GLfloat);
+            break;
+          }
+        // Pass through.
+        default:
+          return false;
+      }
+      break;
+    default:
+      return false;
+  }
+  unsigned source_bytes_per_pixel =
+      VideoFrame::PlaneBitsPerPixel(frame->format(), 0) / 8;
+  DCHECK_EQ(VideoFrame::PlaneBitsPerPixel(frame->format(), 0) % 8, 0);
+
+  if (has_alpha && premultiplyAlpha) {
+    NOTREACHED() << "Premultiply alpha is not supported.";
+    return false;
+  }
+  if (xoffset || yoffset || zoffset) {
+    NOTREACHED() << "Offsets are not supported.";
+    return false;
+  }
+
+  uint8_t* data;
+  scoped_refptr<DataBuffer> temp_buffer;
+  size_t width = frame->visible_rect().width();
+  size_t height = frame->visible_rect().height();
+  size_t output_row_bytes =
+      frame->row_bytes(0) * output_bytes_per_pixel / source_bytes_per_pixel;
+  temp_buffer = new DataBuffer(output_row_bytes * height);
+  data = temp_buffer->writable_data();
+  DCHECK_EQ(frame->format(), PIXEL_FORMAT_Y16);
+  FlipAndConvertY16(frame->visible_data(0), data, format, type, flip_y,
+                    frame->row_bytes(0), frame->stride(0), output_row_bytes,
+                    height);
+
+  if (!strcmp(functionID, "texImage2D")) {
+    gl->TexImage2D(target, level, internalformat, width, height, 0, format,
+                   type, data);
+  } else if (!strcmp(functionID, "texSubImage2D")) {
+    gl->TexSubImage2D(target, level, xoffset, yoffset, width, height, format,
+                      type, data);
+  } else {
+    DCHECK(!strcmp(functionID, "texSubImage3D"));
+    gl->TexSubImage3D(target, level, xoffset, yoffset, zoffset, width, height,
+                      1, format, type, data);
+  }
   return true;
 }