16 bpp video stream capture, render and WebGL usage - Realsense R200 & SR300 support.

media/renderers/skcanvas_video_renderer.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/renderers/skcanvas_video_renderer.h"
 
+#include <GLES3/gl3.h>
 #include <limits>
 
 #include "base/macros.h"
 #include "gpu/GLES2/gl2extchromium.h"
 #include "gpu/command_buffer/client/gles2_interface.h"
 #include "gpu/command_buffer/common/mailbox_holder.h"
+#include "media/base/data_buffer.h"
 #include "media/base/video_frame.h"
 #include "media/base/yuv_convert.h"
 #include "skia/ext/texture_handle.h"
 #include "third_party/libyuv/include/libyuv.h"
 #include "third_party/skia/include/core/SkCanvas.h"
 #include "third_party/skia/include/core/SkImage.h"
 #include "third_party/skia/include/core/SkImageGenerator.h"
 #include "third_party/skia/include/gpu/GrContext.h"
 #include "third_party/skia/include/gpu/GrPaint.h"
 #include "third_party/skia/include/gpu/GrTexture.h"
@@ skipping 135 matching lines @@
   return img;
 }
 
 // Creates a SkImage from a |video_frame| backed by native resources.
 // The SkImage will take ownership of the underlying resource.
 sk_sp<SkImage> NewSkImageFromVideoFrameNative(VideoFrame* video_frame,
                                               const Context3D& context_3d) {
   DCHECK(PIXEL_FORMAT_ARGB == video_frame->format() ||
          PIXEL_FORMAT_XRGB == video_frame->format() ||
          PIXEL_FORMAT_NV12 == video_frame->format() ||
-         PIXEL_FORMAT_UYVY == video_frame->format());
+         PIXEL_FORMAT_UYVY == video_frame->format() ||
+         PIXEL_FORMAT_Y8 == video_frame->format() ||
+         PIXEL_FORMAT_Y16 == video_frame->format());
 
   const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(0);
   DCHECK(mailbox_holder.texture_target == GL_TEXTURE_2D ||
          mailbox_holder.texture_target == GL_TEXTURE_RECTANGLE_ARB ||
          mailbox_holder.texture_target == GL_TEXTURE_EXTERNAL_OES)
       << mailbox_holder.texture_target;
 
   gpu::gles2::GLES2Interface* gl = context_3d.gl;
   unsigned source_texture = 0;
   if (mailbox_holder.texture_target != GL_TEXTURE_2D) {
@@ skipping 137 matching lines @@
   scoped_refptr<VideoFrame> frame_;
 
   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoImageGenerator);
 };
 
 SkCanvasVideoRenderer::SkCanvasVideoRenderer()
     : last_image_deleting_timer_(
           FROM_HERE,
           base::TimeDelta::FromSeconds(kTemporaryResourceDeletionDelay),
           this,
-          &SkCanvasVideoRenderer::ResetCache) {}
+          &SkCanvasVideoRenderer::ResetCache),
+      rg8_to_red_program_(0) {}
 
 SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {
   ResetCache();
 }
 
 void SkCanvasVideoRenderer::Paint(const scoped_refptr<VideoFrame>& video_frame,
                                   SkCanvas* canvas,
                                   const gfx::RectF& dest_rect,
                                   uint8_t alpha,
                                   SkXfermode::Mode mode,
@@ skipping 146 matching lines @@
     for (int row = 0; row < video_frame->rows(plane); row++) {
       for (int x = 0; x < width; x++) {
         dst[x] = src[x] >> shift;
       }
       src += video_frame->stride(plane) / 2;
       dst += ret->stride(plane);
     }
   }
   return ret;
 }
+
+void FlipAndConvertY16(const uint8_t* input,
+                       uint8_t* output,
+                       unsigned format,
+                       unsigned type,
+                       bool flip_y,
+                       size_t row_bytes,
+                       size_t stride,
+                       size_t output_row_bytes,
+                       size_t height) {
+  DCHECK(input != output);
+  for (size_t i = 0; i < height; ++i) {
+    const uint16_t* in = reinterpret_cast<const uint16_t*>(input + i * stride);
+    uint8_t* out = flip_y ? output + output_row_bytes * (height - i - 1)
+                          : output + output_row_bytes * height;
+    if ((format == GL_RG && type == GL_UNSIGNED_BYTE) ||
+        (format == GL_RED_INTEGER && type == GL_UNSIGNED_SHORT)) {
+      memcpy(out, input + i * stride, row_bytes);
+    } else if (format == GL_RED && type == GL_FLOAT) {
+      float* out_row = reinterpret_cast<float*>(out);
+      const uint16_t* in_end = in + row_bytes / 2;
+      while (in < in_end)
+        *out_row++ = *in++ / 65536.f;
+    } else {
+      NOTREACHED();
+    }
+  }
+}
+
+bool isSingleRedComponentFormat(unsigned internal_format) {
+  switch (internal_format) {
+    case GL_R8:
+    case GL_R16F:
+    case GL_R32F:
+    case GL_R8UI:
+    case GL_R8I:
+    case GL_R16UI:
+    case GL_R16I:
+    case GL_R32UI:
+    case GL_R32I:
+      return true;
+    default:
+      return false;
+  }
+}
 }
 
 // static
 void SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
     const VideoFrame* video_frame,
     void* rgb_pixels,
     size_t row_bytes) {
   if (!video_frame->IsMappable()) {
     NOTREACHED() << "Cannot extract pixels from non-CPU frame formats.";
     return;
   }
-  if (!media::IsYuvPlanar(video_frame->format())) {
-    NOTREACHED() << "Non YUV formats are not supported";
-    return;
-  }
 
   switch (video_frame->format()) {
     case PIXEL_FORMAT_YV12:
     case PIXEL_FORMAT_I420:
       if (CheckColorSpace(video_frame, COLOR_SPACE_JPEG)) {
         LIBYUV_J420_TO_ARGB(video_frame->visible_data(VideoFrame::kYPlane),
                             video_frame->stride(VideoFrame::kYPlane),
                             video_frame->visible_data(VideoFrame::kUPlane),
                             video_frame->stride(VideoFrame::kUPlane),
                             video_frame->visible_data(VideoFrame::kVPlane),
@@ skipping 79 matching lines @@
     case PIXEL_FORMAT_NV12:
     case PIXEL_FORMAT_NV21:
     case PIXEL_FORMAT_UYVY:
     case PIXEL_FORMAT_YUY2:
     case PIXEL_FORMAT_ARGB:
     case PIXEL_FORMAT_XRGB:
     case PIXEL_FORMAT_RGB24:
     case PIXEL_FORMAT_RGB32:
     case PIXEL_FORMAT_MJPEG:
     case PIXEL_FORMAT_MT21:
+    case PIXEL_FORMAT_Y8:
+    case PIXEL_FORMAT_Y16:
     case PIXEL_FORMAT_UNKNOWN:
-      NOTREACHED();
+      NOTREACHED() << "Only YUV formats and Y16 are supported.";
   }
 }
 
 // static
 void SkCanvasVideoRenderer::CopyVideoFrameSingleTextureToGLTexture(
     gpu::gles2::GLES2Interface* gl,
     VideoFrame* video_frame,
     unsigned int texture,
     unsigned int internal_format,
     unsigned int type,
@@ skipping 79 matching lines @@
     // canvas context.
     const GLuint64 dest_fence_sync = destination_gl->InsertFenceSyncCHROMIUM();
     destination_gl->ShallowFlushCHROMIUM();
     gpu::SyncToken dest_sync_token;
     destination_gl->GenSyncTokenCHROMIUM(dest_fence_sync,
                                          dest_sync_token.GetData());
     canvas_gl->WaitSyncTokenCHROMIUM(dest_sync_token.GetConstData());
 
     SyncTokenClientImpl client(canvas_gl);
     video_frame->UpdateReleaseSyncToken(&client);
+  } else if (video_frame->format() == PIXEL_FORMAT_Y16 &&
+             isSingleRedComponentFormat(internal_format)) {
+    CopyRG8ToRedTextureData(context_3d, destination_gl, video_frame.get(),
+                            texture, internal_format, type, flip_y);
   } else {
     CopyVideoFrameSingleTextureToGLTexture(destination_gl, video_frame.get(),
                                            texture, internal_format, type,
                                            premultiply_alpha, flip_y);
   }
-
   return true;
 }
 
+bool SkCanvasVideoRenderer::TexImageImpl(const char* functionID,
+                                         unsigned target,
+                                         gpu::gles2::GLES2Interface* gl,
+                                         VideoFrame* frame,
+                                         int level,
+                                         int internalformat,
+                                         unsigned format,
+                                         unsigned type,
+                                         int xoffset,
+                                         int yoffset,
+                                         int zoffset,
+                                         bool flip_y,
+                                         bool premultiplyAlpha) {
+  DCHECK(frame);
+  DCHECK(!frame->HasTextures());
+
+  bool has_alpha = false;
+  bool need_conversion = false;
+  unsigned output_bits_per_pixel;
+  switch (frame->format()) {
+    case PIXEL_FORMAT_Y16:
+      // Allow reinterpreting RG8 buffer here as R16 or FLOAT.
+      if ((format != GL_RED_INTEGER || type != GL_UNSIGNED_SHORT) &&
+          (format != GL_RG || type != GL_UNSIGNED_BYTE) &&
+          (format != GL_RED || type != GL_FLOAT))
+        return false;
+      need_conversion =
+          !(format == GL_RED_INTEGER && type == GL_UNSIGNED_SHORT) &&
+          !(format == GL_RG && type == GL_UNSIGNED_BYTE);
+      output_bits_per_pixel = (type == GL_FLOAT) ? 32 : 16;
+      break;
+    case PIXEL_FORMAT_Y8:
+      if (format != GL_RED || type != GL_UNSIGNED_BYTE)
+        return false;
+      output_bits_per_pixel = 8;
+      break;
+    default:
+      return false;
+  }
+  need_conversion =
+      need_conversion || (frame->stride(0) != frame->row_bytes(0));
+  unsigned source_bits_per_pixel =
+      VideoFrame::PlaneBitsPerPixel(frame->format(), 0);
+  DCHECK_EQ(source_bits_per_pixel % 8, 0U);
+
+  if (has_alpha && premultiplyAlpha) {
+    NOTREACHED() << "Premultiply alpha is not supported.";
+    return false;
+  }
+  if (xoffset || yoffset || zoffset) {
+    NOTREACHED() << "Offsets are not supported.";
+    return false;
+  }
+
+  // Handle other than Y16 formats here only if there is no repacking required.
+  // The reason for this is that this method handles Y16 specifics and provides
+  // optimization (compared to default implementation in
+  // WebGLRenderingContextBase::texImageImpl) for situations where there is no
+  // need to do repacking pixel data.
+  if (frame->format() != PIXEL_FORMAT_Y16 && (flip_y || need_conversion))
+    return false;
+
+  uint8_t* data;
+  scoped_refptr<DataBuffer> temp_buffer;
+  size_t width = frame->visible_rect().width();
+  size_t height = frame->visible_rect().height();
+  if (flip_y || need_conversion) {
+    size_t output_row_bytes =
+        frame->row_bytes(0) * output_bits_per_pixel / source_bits_per_pixel;
+    temp_buffer = new DataBuffer(output_row_bytes * height);
+    data = temp_buffer->writable_data();
+    DCHECK_EQ(frame->format(), PIXEL_FORMAT_Y16);
+    FlipAndConvertY16(frame->visible_data(0), data, format, type, flip_y,
+                      frame->row_bytes(0), frame->stride(0), output_row_bytes,
+                      height);
+  } else {
+    data = frame->visible_data(0);
+  }
+
+  if (!strcmp(functionID, "texImage2D")) {
+    gl->TexImage2D(target, level, internalformat, width, height, 0, format,
+                   type, data);
+  } else if (!strcmp(functionID, "texSubImage2D")) {
+    gl->TexSubImage2D(target, level, xoffset, yoffset, width, height, format,
+                      type, data);
+  } else {
+    DCHECK(!strcmp(functionID, "texSubImage3D"));
+    gl->TexSubImage3D(target, level, xoffset, yoffset, zoffset, width, height,
+                      1, format, type, data);
+  }
+  return true;
+}
+
+void SkCanvasVideoRenderer::CopyRG8ToRedTextureData(
+    const Context3D& context_3d,
+    gpu::gles2::GLES2Interface* target_gl,
+    VideoFrame* frame,
+    unsigned texture,
+    unsigned internal_format,
+    unsigned type,
+    bool flip_y) {
+  // TODO(astojilj): After GLES 3.2, use glCopyImageSubData for RG8 to R16UI
+  // conversion. glCopyImageSubData is already available as extension on some of
+  // the Android devices (GL_EXT_COPY_IMAGE).
+  gpu::gles2::GLES2Interface* local_gl = context_3d.gl;
+
+  // Get source texture on local_gl.
+  unsigned source_texture;
+  {
+    const gpu::MailboxHolder& mailbox_holder = frame->mailbox_holder(0);
+    DCHECK(mailbox_holder.texture_target == GL_TEXTURE_2D)
+        << mailbox_holder.texture_target;
+
+    local_gl->WaitSyncTokenCHROMIUM(mailbox_holder.sync_token.GetConstData());
+    source_texture = local_gl->CreateAndConsumeTextureCHROMIUM(
+        mailbox_holder.texture_target, mailbox_holder.mailbox.name);
+  }
+
+  // Reinterpret RG8 to R32F on local_gl. Later, do the copy on target_gl.
+  unsigned intermediate_texture;
+  local_gl->GenTextures(1, &intermediate_texture);
+  DoCopyRG8ToRedTextureData(local_gl, source_texture, intermediate_texture,

[aleksandar.stojiljkovic, 2016/09/20 12:22:55]

https://codereview.chromium.org/2121043002/diff/60001/media/renderers/skcanva...
https://codereview.chromium.org/2121043002/diff/60001/media/renderers/skcanva...
It is implemented here - 16 bit value laid out in two RG bytes is normalized and
converted (rendered) to R32F texture.
The approach follows YUV planes conversion approach - local canvas's GL is used
for conversion, output then promoted to destination GL and  CopyTextureCHROMIUM
used to copy to target texture.
+sandersd, +dshwang, +jbauman, 
could you please review this file changes? Thanks.

+                            GL_R32F, GL_FLOAT, frame->visible_rect().width(),
+                            frame->visible_rect().height(), flip_y);
+  local_gl->DeleteTextures(1, &source_texture);
+
+  // Get intermediate_texture to target_gl.
+  {
+    gpu::MailboxHolder mailbox_holder;
+    mailbox_holder.texture_target = GL_TEXTURE_2D;
+    local_gl->GenMailboxCHROMIUM(mailbox_holder.mailbox.name);
+    local_gl->ProduceTextureDirectCHROMIUM(intermediate_texture,
+                                           mailbox_holder.texture_target,
+                                           mailbox_holder.mailbox.name);
+    // Wait for mailbox creation on local context before consuming it and
+    // copying from it on the consumer context.
+    const GLuint64 fence_sync = local_gl->InsertFenceSyncCHROMIUM();
+    local_gl->ShallowFlushCHROMIUM();
+    local_gl->GenSyncTokenCHROMIUM(fence_sync,
+                                   mailbox_holder.sync_token.GetData());
+
+    target_gl->WaitSyncTokenCHROMIUM(mailbox_holder.sync_token.GetConstData());
+    uint32_t target_intermediate_texture =
+        target_gl->CreateAndConsumeTextureCHROMIUM(
+            mailbox_holder.texture_target, mailbox_holder.mailbox.name);
+
+    local_gl->DeleteTextures(1, &intermediate_texture);
+    intermediate_texture = target_intermediate_texture;
+  }
+
+  // Copy intermediate_texture to texture.
+  target_gl->CopyTextureCHROMIUM(intermediate_texture, texture, internal_format,
+                                 type, false, false, false);
+  target_gl->DeleteTextures(1, &intermediate_texture);
+
+  // Undo gr_context state changes introduced in this function.
+  context_3d.gr_context->resetContext(
+      kMisc_GrGLBackendState | kBlend_GrGLBackendState |
+      kView_GrGLBackendState | kStencil_GrGLBackendState |
+      kVertex_GrGLBackendState | kProgram_GrGLBackendState |
+      kTextureBinding_GrGLBackendState);
+  SyncTokenClientImpl client(target_gl);
+  frame->UpdateReleaseSyncToken(&client);
+}
+
+void SkCanvasVideoRenderer::DoCopyRG8ToRedTextureData(
+    gpu::gles2::GLES2Interface* gl,
+    unsigned source_texture,
+    unsigned texture,
+    unsigned internal_format,
+    unsigned type,
+    unsigned width,
+    unsigned height,
+    bool flip_y) {
+  GLuint buffer;
+  gl->GenBuffers(1, &buffer);
+  gl->BindBuffer(GL_ARRAY_BUFFER, buffer);
+  const GLfloat vertices[] = {-1.f, -1.f, 1.f, -1.f, -1.0f, 1.f, 1.f, 1.f};
+  gl->BufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);
+  gl->EnableVertexAttribArray(0);
+  gl->VertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 0, 0);
+
+  if (!rg8_to_red_program_) {
+    rg8_to_red_program_ = gl->CreateProgram();

[jbauman, 2016/09/20 20:51:22]

I don't think the lifetime of this is correct if the main thread context is lost
(GPU process crashes, etc.). I'm not sure where to put it otherwise, though.
Previously I've put it into skia, where the GrContext has the correct lifetime.
Another option is to somehow put it into CopyTextureCHROMIUM.

[aleksandar.stojiljkovic, 2016/09/30 14:26:32]

Done.
Using GrContext::uniqueID to control the lifecycle of cached resources.
Added unit test to cover this:
SkCanvasVideoRendererTest, Y16CachedGLResourcesContextRestore
https://codereview.chromium.org/2121043002/diff/120001/media/renderers/skcanv...

+    GLuint vertex_shader = gl->CreateShader(GL_VERTEX_SHADER);
+    std::string source = std::string(
+        "\
+          #ifdef GL_ES\n\
+          precision mediump float;\n\
+          #define TexCoordPrecision mediump\n\
+          #else\n\
+          #define TexCoordPrecision\n\
+          #endif\n\
+          uniform vec2 coord_transform;\n\
+          attribute vec2 a_pos;\n\
+          varying TexCoordPrecision vec2 v_uv;\n\
+          void main(void) {\n\
+            gl_Position = vec4(a_pos, 0, 1.0);\n\
+            v_uv = (a_pos * coord_transform) + 0.5;\n\
+          }\n");
+    const char* vertex_source = source.c_str();
+    gl->ShaderSource(vertex_shader, 1, &vertex_source, 0);
+    gl->CompileShader(vertex_shader);
+    gl->AttachShader(rg8_to_red_program_, vertex_shader);
+
+    GLuint fragment_shader = gl->CreateShader(GL_FRAGMENT_SHADER);
+    source = std::string(
+        "\
+          #ifdef GL_ES\n\
+          precision mediump float;\n\
+          #define TexCoordPrecision mediump\n\
+          #else\n\
+          #define TexCoordPrecision\n\
+          #endif\n\
+          uniform sampler2D u_sampler;\n\
+          varying TexCoordPrecision vec2 v_uv;\n\
+          void main(void) {\n\
+            vec4 color = texture2D(u_sampler, v_uv);\n\
+            gl_FragColor.r = (color.r * 0.00390625) + color.g;\n\
+          }\n");
+    const char* fragment_source = source.c_str();
+    gl->ShaderSource(fragment_shader, 1, &fragment_source, 0);
+    gl->CompileShader(fragment_shader);
+
+    gl->AttachShader(rg8_to_red_program_, fragment_shader);
+
+    gl->BindAttribLocation(rg8_to_red_program_, 0, "a_pos");
+    gl->LinkProgram(rg8_to_red_program_);
+  }
+
+  gl->UseProgram(rg8_to_red_program_);
+  gl->Uniform1i(gl->GetUniformLocation(rg8_to_red_program_, "u_sampler"), 0);
+  gl->Uniform2f(gl->GetUniformLocation(rg8_to_red_program_, "coord_transform"),
+                0.5, flip_y ? -0.5 : 0.5);
+
+  GLuint fbo;
+  gl->GenFramebuffers(1, &fbo);
+  gl->ActiveTexture(GL_TEXTURE0);
+  unsigned target = GL_TEXTURE_2D;
+  gl->BindTexture(target, texture);
+  gl->TexImage2D(GL_TEXTURE_2D, 0, internal_format, width, height, 0, GL_RED,
+                 type, 0);
+
+  gl->TexParameterf(target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+  gl->TexParameterf(target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+  gl->TexParameteri(target, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+  gl->TexParameteri(target, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+  gl->BindFramebuffer(GL_FRAMEBUFFER, fbo);
+  gl->FramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, target,
+                           texture, 0);
+
+  gl->BindTexture(target, source_texture);
+  gl->TexParameterf(target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+  gl->TexParameterf(target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+  gl->TexParameteri(target, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+  gl->TexParameteri(target, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+
+  gl->Disable(GL_DEPTH_TEST);
+  gl->Disable(GL_STENCIL_TEST);
+  gl->Disable(GL_CULL_FACE);
+  gl->ColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+  gl->DepthMask(GL_FALSE);
+  gl->Disable(GL_BLEND);
+  gl->Disable(GL_SCISSOR_TEST);
+  gl->Viewport(0, 0, width, height);
+  gl->DrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+  gl->Flush();
+  gl->BindFramebuffer(GL_FRAMEBUFFER, 0);
+  gl->DeleteBuffers(1, &buffer);
+  gl->DeleteFramebuffers(1, &fbo);
+}
+
 void SkCanvasVideoRenderer::ResetCache() {
   DCHECK(thread_checker_.CalledOnValidThread());
   // Clear cached values.
   last_image_ = nullptr;
   last_timestamp_ = kNoTimestamp;
 }
 
 bool SkCanvasVideoRenderer::UpdateLastImage(
     const scoped_refptr<VideoFrame>& video_frame,
     const Context3D& context_3d) {
@@ skipping 21 matching lines @@
     if (!last_image_)  // Couldn't create the SkImage.
       return false;
     last_timestamp_ = video_frame->timestamp();
   }
   last_image_deleting_timer_.Reset();
   DCHECK(!!last_image_);
   return true;
 }
 
 }  // namespace media