16 bpp video stream capture, render and WebGL usage - Realsense R200 & SR300 support.

media/renderers/skcanvas_video_renderer.cc

Index: media/renderers/skcanvas_video_renderer.cc
diff --git a/media/renderers/skcanvas_video_renderer.cc b/media/renderers/skcanvas_video_renderer.cc
index 74e72bc5a4c5338b654097de6de29fa321a9b611..7bfdc393b5dbc267ef5f6c31553c774a57df02cb 100644
--- a/media/renderers/skcanvas_video_renderer.cc
+++ b/media/renderers/skcanvas_video_renderer.cc
@@ -4,12 +4,14 @@
 
 #include "media/renderers/skcanvas_video_renderer.h"
 
+#include <GLES3/gl3.h>
 #include <limits>
 
 #include "base/macros.h"
 #include "gpu/GLES2/gl2extchromium.h"
 #include "gpu/command_buffer/client/gles2_interface.h"
 #include "gpu/command_buffer/common/mailbox_holder.h"
+#include "media/base/data_buffer.h"
 #include "media/base/video_frame.h"
 #include "media/base/yuv_convert.h"
 #include "skia/ext/texture_handle.h"
@@ -209,10 +211,11 @@ sk_sp<SkImage> NewSkImageFromVideoFrameNative(VideoFrame* video_frame,
 // Generates an RGB image from a VideoFrame. Convert YUV to RGB plain on GPU.
 class VideoImageGenerator : public SkImageGenerator {
  public:
-  VideoImageGenerator(const scoped_refptr<VideoFrame>& frame)
-      : SkImageGenerator(
-            SkImageInfo::MakeN32Premul(frame->visible_rect().width(),
-                                       frame->visible_rect().height())),
+  VideoImageGenerator(const scoped_refptr<VideoFrame>& frame, SkColorType type)
+      : SkImageGenerator(SkImageInfo::Make(frame->visible_rect().width(),
+                                           frame->visible_rect().height(),
+                                           type,
+                                           kPremul_SkAlphaType)),
         frame_(frame) {
     DCHECK(!frame_->HasTextures());
   }
@@ -350,7 +353,8 @@ void SkCanvasVideoRenderer::Paint(const scoped_refptr<VideoFrame>& video_frame,
   // Paint black rectangle if there isn't a frame available or the
   // frame has an unexpected format.
   if (!video_frame.get() || video_frame->natural_size().IsEmpty() ||
-      !(media::IsYuvPlanar(video_frame->format()) ||
+      !((media::IsYuvPlanar(video_frame->format()) ||
+         video_frame->format() == media::PIXEL_FORMAT_Y16) ||
         video_frame->HasTextures())) {
     canvas->drawRect(dest, paint);
     canvas->flush();
@@ -377,7 +381,10 @@ void SkCanvasVideoRenderer::Paint(const scoped_refptr<VideoFrame>& video_frame,
             NewSkImageFromVideoFrameNative(video_frame.get(), context_3d);
       }
     } else {
-      auto* video_generator = new VideoImageGenerator(video_frame);
+      auto* video_generator = new VideoImageGenerator(
+          video_frame, (video_frame->format() == media::PIXEL_FORMAT_Y16)
+                           ? kRGBA_8888_SkColorType
+                           : kN32_SkColorType);
       last_image_ = SkImage::MakeFromGenerator(video_generator);
     }
     if (!last_image_)  // Couldn't create the SkImage.
@@ -522,6 +529,41 @@ scoped_refptr<VideoFrame> DownShiftHighbitVideoFrame(
   }
   return ret;
 }
+
+void ConvertY16ToABGR(const VideoFrame* video_frame,
+                      void* argb_pixels,
+                      size_t argb_row_bytes) {
+  const uint8_t* source =
+      reinterpret_cast<const uint8_t*>(video_frame->visible_data(0));
+  uint8_t* out = reinterpret_cast<uint8_t*>(argb_pixels);
+  const size_t stride = video_frame->stride(0);
+  for (int i = 0; i < video_frame->visible_rect().height(); ++i) {
+    const uint16_t* row = reinterpret_cast<const uint16_t*>(source);
+    uint32_t* argb = reinterpret_cast<uint32_t*>(out);
+    for (const uint16_t* row_end = row + video_frame->visible_rect().width();
+         row < row_end;)
+      *argb++ = ((*row++) >> 8) | 0xFF000000;
+    out += argb_row_bytes;
+    source += stride;
+  }
+}
+
+void FlipAndMultiply(const uint8_t* input,
+                     uint8_t* output,
+                     unsigned format,
+                     unsigned type,
+                     bool flipY,
+                     bool premultiplyAlpha,
+                     size_t row_bytes,
+                     size_t stride,
+                     size_t height) {
+  // Premultiply alpha is not set for formats that this method supports.
+  DCHECK(!premultiplyAlpha);
+  DCHECK(flipY || premultiplyAlpha);
+  DCHECK(input != output);
+  for (int i = 0; i < height; ++i)
+    memcpy(output + stride * (height - i - 1), input + i * stride, row_bytes);
+}
 }
 
 // static
@@ -533,10 +575,6 @@ void SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
     NOTREACHED() << "Cannot extract pixels from non-CPU frame formats.";
     return;
   }
-  if (!media::IsYuvPlanar(video_frame->format())) {
-    NOTREACHED() << "Non YUV formats are not supported";
-    return;
-  }
 
   switch (video_frame->format()) {
     case PIXEL_FORMAT_YV12:
@@ -626,6 +664,10 @@ void SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
       break;
     }
 
+    case PIXEL_FORMAT_Y16:
+      ConvertY16ToABGR(video_frame, rgb_pixels, row_bytes);
+      break;
+
     case PIXEL_FORMAT_NV12:
     case PIXEL_FORMAT_NV21:
     case PIXEL_FORMAT_UYVY:
@@ -636,8 +678,9 @@ void SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
     case PIXEL_FORMAT_RGB32:
     case PIXEL_FORMAT_MJPEG:
     case PIXEL_FORMAT_MT21:
+    case PIXEL_FORMAT_Y8:
     case PIXEL_FORMAT_UNKNOWN:
-      NOTREACHED();
+      NOTREACHED() << "Only YUV formats and Y16 are supported.";
   }
 }
 
@@ -678,6 +721,74 @@ void SkCanvasVideoRenderer::CopyVideoFrameSingleTextureToGLTexture(
   video_frame->UpdateReleaseSyncToken(&client);
 }
 
+bool SkCanvasVideoRenderer::TexImageImpl(const char* functionID,
+                                         unsigned target,
+                                         gpu::gles2::GLES2Interface* gl,
+                                         VideoFrame* frame,
+                                         int level,
+                                         int internalformat,
+                                         unsigned format,
+                                         unsigned type,
+                                         int xoffset,
+                                         int yoffset,
+                                         int zoffset,
+                                         bool flipY,
+                                         bool premultiplyAlpha) {
+  DCHECK(frame);
+
+  bool hasAlpha = false;
+  switch (frame->format()) {
+    case PIXEL_FORMAT_Y16:
+      // Allow reinterpreting R16 buffer here as RG8.
+      if ((format != GL_RED_INTEGER || type != GL_UNSIGNED_SHORT) &&
+          (format != GL_RG || type != GL_UNSIGNED_BYTE))
+        return false;
+      break;
+    case PIXEL_FORMAT_Y8:
+      if (format != GL_RED || type != GL_UNSIGNED_BYTE)
+        return false;
+      break;
+    // TODO(astojilj): ARGB, XRGB....v
+    default:
+      return false;
+  }
+
+  size_t width = frame->visible_rect().width();
+  size_t height = frame->visible_rect().height();
+  if (hasAlpha && premultiplyAlpha) {
+    NOTREACHED() << "Premultiply alpha is not supported.";
+    return false;
+  }
+  if (xoffset || yoffset || zoffset) {
+    NOTREACHED() << "Offsets are not supported.";
+    return false;
+  }
+
+  uint8_t* data;
+  scoped_refptr<DataBuffer> temp_buffer;
+  if (flipY) {
+    temp_buffer = new DataBuffer(frame->stride(0) * height);
+    data = temp_buffer->writable_data();
+    FlipAndMultiply(frame->visible_data(0), data, format, type, flipY, false,
+                    frame->row_bytes(0), frame->stride(0), height);
+  } else {
+    data = frame->visible_data(0);

[Ken Russell (switch to Gerrit), 2016/08/16 00:22:27]

Is there ever a situation where the video frame's data will already be stored in
a texture on the GPU, and the call to writeable_data() or visible_data() will
cause a GPU-to-CPU readback? Obviously, for performance reasons, it's crucial to
avoid such readbacks.

[aleksandar.stojiljkovic, 2016/08/16 12:02:31]

Of course. In this case VideoFrame::storage_type() is STORAGE_SHMEM.
If video_frame->HasTextures(), it would be handled using
copyVideoTextureToPlatformTexture, but there is a need to check that here.

[Ken Russell (switch to Gerrit), 2016/08/16 18:27:37]

OK. Do you plan to add this code path?

If this code path is added, is it practical to support a few types of packing
when copying from the source to the destination texture? For example, RG8 ->
R16UI? Or has that been the plan all along?

[aleksandar.stojiljkovic, 2016/09/20 12:22:55]

Done. Reinterpreting RG8 to R32F when storage is GPU native memory buffer is
done now on GPU in this file. 

+  }
+
+  if (!strcmp(functionID, "texImage2D")) {
+    gl->TexImage2D(target, level, internalformat, width, height, 0, format,
+                   type, data);
+  } else if (!strcmp(functionID, "texSubImage2D")) {
+    gl->TexSubImage2D(target, level, xoffset, yoffset, width, height, format,
+                      type, data);
+  } else {
+    DCHECK(!strcmp(functionID, "texSubImage3D"));
+    gl->TexSubImage3D(target, level, xoffset, yoffset, zoffset, width, height,
+                      1, format, type, data);
+  }
+  return true;
+}
+
 void SkCanvasVideoRenderer::ResetCache() {
   DCHECK(thread_checker_.CalledOnValidThread());
   // Clear cached values.