16 bpp video stream capture, render and WebGL usage - Realsense R200 & SR300 support.

media/renderers/skcanvas_video_renderer.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/renderers/skcanvas_video_renderer.h"
 
+#include <GLES3/gl3.h>
 #include <limits>
 
 #include "base/macros.h"
 #include "gpu/GLES2/gl2extchromium.h"
 #include "gpu/command_buffer/client/gles2_interface.h"
 #include "gpu/command_buffer/common/mailbox_holder.h"
+#include "media/base/data_buffer.h"
 #include "media/base/video_frame.h"
 #include "media/base/yuv_convert.h"
 #include "skia/ext/texture_handle.h"
 #include "third_party/libyuv/include/libyuv.h"
 #include "third_party/skia/include/core/SkCanvas.h"
 #include "third_party/skia/include/core/SkImage.h"
 #include "third_party/skia/include/core/SkImageGenerator.h"
 #include "third_party/skia/include/gpu/GrContext.h"
 #include "third_party/skia/include/gpu/GrPaint.h"
 #include "third_party/skia/include/gpu/GrTexture.h"
@@ skipping 179 matching lines @@
   desc.fTextureHandle =
       skia::GrGLTextureInfoToGrBackendObject(source_texture_info);
   return SkImage::MakeFromAdoptedTexture(context_3d.gr_context, desc);
 }
 
 }  // anonymous namespace
 
 // Generates an RGB image from a VideoFrame. Convert YUV to RGB plain on GPU.
 class VideoImageGenerator : public SkImageGenerator {
  public:
-  VideoImageGenerator(const scoped_refptr<VideoFrame>& frame)
-      : SkImageGenerator(
-            SkImageInfo::MakeN32Premul(frame->visible_rect().width(),
-                                       frame->visible_rect().height())),
+  VideoImageGenerator(const scoped_refptr<VideoFrame>& frame, SkColorType type)
+      : SkImageGenerator(SkImageInfo::Make(frame->visible_rect().width(),
+                                           frame->visible_rect().height(),
+                                           type,
+                                           kPremul_SkAlphaType)),
         frame_(frame) {
     DCHECK(!frame_->HasTextures());
   }
   ~VideoImageGenerator() override {}
 
  protected:
   bool onGetPixels(const SkImageInfo& info,
                    void* pixels,
                    size_t row_bytes,
                    SkPMColor ctable[],
@@ skipping 117 matching lines @@
 
   SkRect dest;
   dest.set(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom());
 
   SkPaint paint;
   paint.setAlpha(alpha);
 
   // Paint black rectangle if there isn't a frame available or the
   // frame has an unexpected format.
   if (!video_frame.get() || video_frame->natural_size().IsEmpty() ||
-      !(media::IsYuvPlanar(video_frame->format()) ||
+      !((media::IsYuvPlanar(video_frame->format()) ||
+         video_frame->format() == media::PIXEL_FORMAT_Y16) ||
         video_frame->HasTextures())) {
     canvas->drawRect(dest, paint);
     canvas->flush();
     return;
   }
 
   gpu::gles2::GLES2Interface* gl = context_3d.gl;
 
   if (!last_image_ || video_frame->timestamp() != last_timestamp_) {
     ResetCache();
     // Generate a new image.
     // Note: Skia will hold onto |video_frame| via |video_generator| only when
     // |video_frame| is software.
     // Holding |video_frame| longer than this call when using GPUVideoDecoder
     // could cause problems since the pool of VideoFrames has a fixed size.
     if (video_frame->HasTextures()) {
       DCHECK(context_3d.gr_context);
       DCHECK(gl);
       if (media::VideoFrame::NumPlanes(video_frame->format()) > 1) {
         last_image_ =
             NewSkImageFromVideoFrameYUVTextures(video_frame.get(), context_3d);
       } else {
         last_image_ =
             NewSkImageFromVideoFrameNative(video_frame.get(), context_3d);
       }
     } else {
-      auto* video_generator = new VideoImageGenerator(video_frame);
+      auto* video_generator = new VideoImageGenerator(
+          video_frame, (video_frame->format() == media::PIXEL_FORMAT_Y16)
+                           ? kRGBA_8888_SkColorType
+                           : kN32_SkColorType);
       last_image_ = SkImage::MakeFromGenerator(video_generator);
     }
     if (!last_image_)  // Couldn't create the SkImage.
       return;
     last_timestamp_ = video_frame->timestamp();
   }
   last_image_deleting_timer_.Reset();
 
   paint.setXfermodeMode(mode);
   paint.setFilterQuality(kLow_SkFilterQuality);
@@ skipping 124 matching lines @@
     for (int row = 0; row < video_frame->rows(plane); row++) {
       for (int x = 0; x < width; x++) {
         dst[x] = src[x] >> shift;
       }
       src += video_frame->stride(plane) / 2;
       dst += ret->stride(plane);
     }
   }
   return ret;
 }
+
+void ConvertY16ToABGR(const VideoFrame* video_frame,
+                      void* argb_pixels,
+                      size_t argb_row_bytes) {
+  const uint8_t* source =
+      reinterpret_cast<const uint8_t*>(video_frame->visible_data(0));
+  uint8_t* out = reinterpret_cast<uint8_t*>(argb_pixels);
+  const size_t stride = video_frame->stride(0);
+  for (int i = 0; i < video_frame->visible_rect().height(); ++i) {
+    const uint16_t* row = reinterpret_cast<const uint16_t*>(source);
+    uint32_t* argb = reinterpret_cast<uint32_t*>(out);
+    for (const uint16_t* row_end = row + video_frame->visible_rect().width();
+         row < row_end;)
+      *argb++ = ((*row++) >> 8) | 0xFF000000;
+    out += argb_row_bytes;
+    source += stride;
+  }
+}
+
+void FlipAndMultiply(const uint8_t* input,
+                     uint8_t* output,
+                     unsigned format,
+                     unsigned type,
+                     bool flipY,
+                     bool premultiplyAlpha,
+                     size_t row_bytes,
+                     size_t stride,
+                     size_t height) {
+  // Premultiply alpha is not set for formats that this method supports.
+  DCHECK(!premultiplyAlpha);
+  DCHECK(flipY || premultiplyAlpha);
+  DCHECK(input != output);
+  for (int i = 0; i < height; ++i)
+    memcpy(output + stride * (height - i - 1), input + i * stride, row_bytes);
+}
 }
 
 // static
 void SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
     const VideoFrame* video_frame,
     void* rgb_pixels,
     size_t row_bytes) {
   if (!video_frame->IsMappable()) {
     NOTREACHED() << "Cannot extract pixels from non-CPU frame formats.";
     return;
   }
-  if (!media::IsYuvPlanar(video_frame->format())) {
-    NOTREACHED() << "Non YUV formats are not supported";
-    return;
-  }
 
   switch (video_frame->format()) {
     case PIXEL_FORMAT_YV12:
     case PIXEL_FORMAT_I420:
       if (CheckColorSpace(video_frame, COLOR_SPACE_JPEG)) {
         LIBYUV_J420_TO_ARGB(video_frame->visible_data(VideoFrame::kYPlane),
                             video_frame->stride(VideoFrame::kYPlane),
                             video_frame->visible_data(VideoFrame::kUPlane),
                             video_frame->stride(VideoFrame::kUPlane),
                             video_frame->visible_data(VideoFrame::kVPlane),
@@ skipping 69 matching lines @@
     case PIXEL_FORMAT_YUV420P10:
     case PIXEL_FORMAT_YUV422P10:
     case PIXEL_FORMAT_YUV444P10: {
       scoped_refptr<VideoFrame> temporary_frame =
           DownShiftHighbitVideoFrame(video_frame);
       ConvertVideoFrameToRGBPixels(temporary_frame.get(), rgb_pixels,
                                    row_bytes);
       break;
     }
 
+    case PIXEL_FORMAT_Y16:
+      ConvertY16ToABGR(video_frame, rgb_pixels, row_bytes);
+      break;
+
     case PIXEL_FORMAT_NV12:
     case PIXEL_FORMAT_NV21:
     case PIXEL_FORMAT_UYVY:
     case PIXEL_FORMAT_YUY2:
     case PIXEL_FORMAT_ARGB:
     case PIXEL_FORMAT_XRGB:
     case PIXEL_FORMAT_RGB24:
     case PIXEL_FORMAT_RGB32:
     case PIXEL_FORMAT_MJPEG:
     case PIXEL_FORMAT_MT21:
+    case PIXEL_FORMAT_Y8:
     case PIXEL_FORMAT_UNKNOWN:
-      NOTREACHED();
+      NOTREACHED() << "Only YUV formats and Y16 are supported.";
   }
 }
 
 // static
 void SkCanvasVideoRenderer::CopyVideoFrameSingleTextureToGLTexture(
     gpu::gles2::GLES2Interface* gl,
     VideoFrame* video_frame,
     unsigned int texture,
     unsigned int internal_format,
     unsigned int type,
@@ skipping 20 matching lines @@
   // "flip_y == false" means to keep the intrinsic orientation.
   gl->CopyTextureCHROMIUM(source_texture, texture, internal_format, type,
                           flip_y, premultiply_alpha, false);
   gl->DeleteTextures(1, &source_texture);
   gl->Flush();
 
   SyncTokenClientImpl client(gl);
   video_frame->UpdateReleaseSyncToken(&client);
 }
 
+bool SkCanvasVideoRenderer::TexImageImpl(const char* functionID,
+                                         unsigned target,
+                                         gpu::gles2::GLES2Interface* gl,
+                                         VideoFrame* frame,
+                                         int level,
+                                         int internalformat,
+                                         unsigned format,
+                                         unsigned type,
+                                         int xoffset,
+                                         int yoffset,
+                                         int zoffset,
+                                         bool flipY,
+                                         bool premultiplyAlpha) {
+  DCHECK(frame);
+
+  bool hasAlpha = false;
+  switch (frame->format()) {
+    case PIXEL_FORMAT_Y16:
+      // Allow reinterpreting R16 buffer here as RG8.
+      if ((format != GL_RED_INTEGER || type != GL_UNSIGNED_SHORT) &&
+          (format != GL_RG || type != GL_UNSIGNED_BYTE))
+        return false;
+      break;
+    case PIXEL_FORMAT_Y8:
+      if (format != GL_RED || type != GL_UNSIGNED_BYTE)
+        return false;
+      break;
+    // TODO(astojilj): ARGB, XRGB....v
+    default:
+      return false;
+  }
+
+  size_t width = frame->visible_rect().width();
+  size_t height = frame->visible_rect().height();
+  if (hasAlpha && premultiplyAlpha) {
+    NOTREACHED() << "Premultiply alpha is not supported.";
+    return false;
+  }
+  if (xoffset || yoffset || zoffset) {
+    NOTREACHED() << "Offsets are not supported.";
+    return false;
+  }
+
+  uint8_t* data;
+  scoped_refptr<DataBuffer> temp_buffer;
+  if (flipY) {
+    temp_buffer = new DataBuffer(frame->stride(0) * height);
+    data = temp_buffer->writable_data();
+    FlipAndMultiply(frame->visible_data(0), data, format, type, flipY, false,
+                    frame->row_bytes(0), frame->stride(0), height);
+  } else {
+    data = frame->visible_data(0);

[Ken Russell (switch to Gerrit), 2016/08/16 00:22:27]

Is there ever a situation where the video frame's data will already be stored in
a texture on the GPU, and the call to writeable_data() or visible_data() will
cause a GPU-to-CPU readback? Obviously, for performance reasons, it's crucial to
avoid such readbacks.

[aleksandar.stojiljkovic, 2016/08/16 12:02:31]

Of course. In this case VideoFrame::storage_type() is STORAGE_SHMEM.
If video_frame->HasTextures(), it would be handled using
copyVideoTextureToPlatformTexture, but there is a need to check that here.

[Ken Russell (switch to Gerrit), 2016/08/16 18:27:37]

OK. Do you plan to add this code path?

If this code path is added, is it practical to support a few types of packing
when copying from the source to the destination texture? For example, RG8 ->
R16UI? Or has that been the plan all along?

[aleksandar.stojiljkovic, 2016/09/20 12:22:55]

Done. Reinterpreting RG8 to R32F when storage is GPU native memory buffer is
done now on GPU in this file. 

+  }
+
+  if (!strcmp(functionID, "texImage2D")) {
+    gl->TexImage2D(target, level, internalformat, width, height, 0, format,
+                   type, data);
+  } else if (!strcmp(functionID, "texSubImage2D")) {
+    gl->TexSubImage2D(target, level, xoffset, yoffset, width, height, format,
+                      type, data);
+  } else {
+    DCHECK(!strcmp(functionID, "texSubImage3D"));
+    gl->TexSubImage3D(target, level, xoffset, yoffset, zoffset, width, height,
+                      1, format, type, data);
+  }
+  return true;
+}
+
 void SkCanvasVideoRenderer::ResetCache() {
   DCHECK(thread_checker_.CalledOnValidThread());
   // Clear cached values.
   last_image_ = nullptr;
   last_timestamp_ = kNoTimestamp();
 }
 
 }  // namespace media