16 bpp video stream capture, render and WebGL usage - Realsense R200 & SR300 support.

media/renderers/skcanvas_video_renderer.cc

Index: media/renderers/skcanvas_video_renderer.cc
diff --git a/media/renderers/skcanvas_video_renderer.cc b/media/renderers/skcanvas_video_renderer.cc
index 8c71b3edfbdd00fd7da334e1bfbe1f838f8edcc1..2a0d3f5b0e9dd04a925d1086bb5eadce73fb8b0c 100644
--- a/media/renderers/skcanvas_video_renderer.cc
+++ b/media/renderers/skcanvas_video_renderer.cc
@@ -4,12 +4,14 @@
 
 #include "media/renderers/skcanvas_video_renderer.h"
 
+#include <GLES3/gl3.h>
 #include <limits>
 
 #include "base/macros.h"
 #include "gpu/GLES2/gl2extchromium.h"
 #include "gpu/command_buffer/client/gles2_interface.h"
 #include "gpu/command_buffer/common/mailbox_holder.h"
+#include "media/base/data_buffer.h"
 #include "media/base/video_frame.h"
 #include "media/base/yuv_convert.h"
 #include "skia/ext/texture_handle.h"
@@ -165,7 +167,9 @@ sk_sp<SkImage> NewSkImageFromVideoFrameNative(VideoFrame* video_frame,
   DCHECK(PIXEL_FORMAT_ARGB == video_frame->format() ||
          PIXEL_FORMAT_XRGB == video_frame->format() ||
          PIXEL_FORMAT_NV12 == video_frame->format() ||
-         PIXEL_FORMAT_UYVY == video_frame->format());
+         PIXEL_FORMAT_UYVY == video_frame->format() ||
+         PIXEL_FORMAT_Y8 == video_frame->format() ||
+         PIXEL_FORMAT_Y16 == video_frame->format());
 
   const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(0);
   DCHECK(mailbox_holder.texture_target == GL_TEXTURE_2D ||
@@ -323,7 +327,8 @@ SkCanvasVideoRenderer::SkCanvasVideoRenderer()
           FROM_HERE,
           base::TimeDelta::FromSeconds(kTemporaryResourceDeletionDelay),
           this,
-          &SkCanvasVideoRenderer::ResetCache) {}
+          &SkCanvasVideoRenderer::ResetCache),
+      rg8_to_red_program_(0) {}
 
 SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {
   ResetCache();
@@ -490,6 +495,51 @@ scoped_refptr<VideoFrame> DownShiftHighbitVideoFrame(
   }
   return ret;
 }
+
+void FlipAndConvertY16(const uint8_t* input,
+                       uint8_t* output,
+                       unsigned format,
+                       unsigned type,
+                       bool flip_y,
+                       size_t row_bytes,
+                       size_t stride,
+                       size_t output_row_bytes,
+                       size_t height) {
+  DCHECK(input != output);
+  for (size_t i = 0; i < height; ++i) {
+    const uint16_t* in = reinterpret_cast<const uint16_t*>(input + i * stride);
+    uint8_t* out = flip_y ? output + output_row_bytes * (height - i - 1)
+                          : output + output_row_bytes * height;
+    if ((format == GL_RG && type == GL_UNSIGNED_BYTE) ||
+        (format == GL_RED_INTEGER && type == GL_UNSIGNED_SHORT)) {
+      memcpy(out, input + i * stride, row_bytes);
+    } else if (format == GL_RED && type == GL_FLOAT) {
+      float* out_row = reinterpret_cast<float*>(out);
+      const uint16_t* in_end = in + row_bytes / 2;
+      while (in < in_end)
+        *out_row++ = *in++ / 65536.f;
+    } else {
+      NOTREACHED();
+    }
+  }
+}
+
+bool isSingleRedComponentFormat(unsigned internal_format) {
+  switch (internal_format) {
+    case GL_R8:
+    case GL_R16F:
+    case GL_R32F:
+    case GL_R8UI:
+    case GL_R8I:
+    case GL_R16UI:
+    case GL_R16I:
+    case GL_R32UI:
+    case GL_R32I:
+      return true;
+    default:
+      return false;
+  }
+}
 }
 
 // static
@@ -501,10 +551,6 @@ void SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
     NOTREACHED() << "Cannot extract pixels from non-CPU frame formats.";
     return;
   }
-  if (!media::IsYuvPlanar(video_frame->format())) {
-    NOTREACHED() << "Non YUV formats are not supported";
-    return;
-  }
 
   switch (video_frame->format()) {
     case PIXEL_FORMAT_YV12:
@@ -604,8 +650,10 @@ void SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
     case PIXEL_FORMAT_RGB32:
     case PIXEL_FORMAT_MJPEG:
     case PIXEL_FORMAT_MT21:
+    case PIXEL_FORMAT_Y8:
+    case PIXEL_FORMAT_Y16:
     case PIXEL_FORMAT_UNKNOWN:
-      NOTREACHED();
+      NOTREACHED() << "Only YUV formats and Y16 are supported.";
   }
 }
 
@@ -705,15 +753,290 @@ bool SkCanvasVideoRenderer::CopyVideoFrameTexturesToGLTexture(
 
     SyncTokenClientImpl client(canvas_gl);
     video_frame->UpdateReleaseSyncToken(&client);
+  } else if (video_frame->format() == PIXEL_FORMAT_Y16 &&
+             isSingleRedComponentFormat(internal_format)) {
+    CopyRG8ToRedTextureData(context_3d, destination_gl, video_frame.get(),
+                            texture, internal_format, type, flip_y);
   } else {
     CopyVideoFrameSingleTextureToGLTexture(destination_gl, video_frame.get(),
                                            texture, internal_format, type,
                                            premultiply_alpha, flip_y);
   }
+  return true;
+}
+
+bool SkCanvasVideoRenderer::TexImageImpl(const char* functionID,
+                                         unsigned target,
+                                         gpu::gles2::GLES2Interface* gl,
+                                         VideoFrame* frame,
+                                         int level,
+                                         int internalformat,
+                                         unsigned format,
+                                         unsigned type,
+                                         int xoffset,
+                                         int yoffset,
+                                         int zoffset,
+                                         bool flip_y,
+                                         bool premultiplyAlpha) {
+  DCHECK(frame);
+  DCHECK(!frame->HasTextures());
+
+  bool has_alpha = false;
+  bool need_conversion = false;
+  unsigned output_bits_per_pixel;
+  switch (frame->format()) {
+    case PIXEL_FORMAT_Y16:
+      // Allow reinterpreting RG8 buffer here as R16 or FLOAT.
+      if ((format != GL_RED_INTEGER || type != GL_UNSIGNED_SHORT) &&
+          (format != GL_RG || type != GL_UNSIGNED_BYTE) &&
+          (format != GL_RED || type != GL_FLOAT))
+        return false;
+      need_conversion =
+          !(format == GL_RED_INTEGER && type == GL_UNSIGNED_SHORT) &&
+          !(format == GL_RG && type == GL_UNSIGNED_BYTE);
+      output_bits_per_pixel = (type == GL_FLOAT) ? 32 : 16;
+      break;
+    case PIXEL_FORMAT_Y8:
+      if (format != GL_RED || type != GL_UNSIGNED_BYTE)
+        return false;
+      output_bits_per_pixel = 8;
+      break;
+    default:
+      return false;
+  }
+  need_conversion =
+      need_conversion || (frame->stride(0) != frame->row_bytes(0));
+  unsigned source_bits_per_pixel =
+      VideoFrame::PlaneBitsPerPixel(frame->format(), 0);
+  DCHECK_EQ(source_bits_per_pixel % 8, 0U);
+
+  if (has_alpha && premultiplyAlpha) {
+    NOTREACHED() << "Premultiply alpha is not supported.";
+    return false;
+  }
+  if (xoffset || yoffset || zoffset) {
+    NOTREACHED() << "Offsets are not supported.";
+    return false;
+  }
 
+  // Handle other than Y16 formats here only if there is no repacking required.
+  // The reason for this is that this method handles Y16 specifics and provides
+  // optimization (compared to default implementation in
+  // WebGLRenderingContextBase::texImageImpl) for situations where there is no
+  // need to do repacking pixel data.
+  if (frame->format() != PIXEL_FORMAT_Y16 && (flip_y || need_conversion))
+    return false;
+
+  uint8_t* data;
+  scoped_refptr<DataBuffer> temp_buffer;
+  size_t width = frame->visible_rect().width();
+  size_t height = frame->visible_rect().height();
+  if (flip_y || need_conversion) {
+    size_t output_row_bytes =
+        frame->row_bytes(0) * output_bits_per_pixel / source_bits_per_pixel;
+    temp_buffer = new DataBuffer(output_row_bytes * height);
+    data = temp_buffer->writable_data();
+    DCHECK_EQ(frame->format(), PIXEL_FORMAT_Y16);
+    FlipAndConvertY16(frame->visible_data(0), data, format, type, flip_y,
+                      frame->row_bytes(0), frame->stride(0), output_row_bytes,
+                      height);
+  } else {
+    data = frame->visible_data(0);
+  }
+
+  if (!strcmp(functionID, "texImage2D")) {
+    gl->TexImage2D(target, level, internalformat, width, height, 0, format,
+                   type, data);
+  } else if (!strcmp(functionID, "texSubImage2D")) {
+    gl->TexSubImage2D(target, level, xoffset, yoffset, width, height, format,
+                      type, data);
+  } else {
+    DCHECK(!strcmp(functionID, "texSubImage3D"));
+    gl->TexSubImage3D(target, level, xoffset, yoffset, zoffset, width, height,
+                      1, format, type, data);
+  }
   return true;
 }
 
+void SkCanvasVideoRenderer::CopyRG8ToRedTextureData(
+    const Context3D& context_3d,
+    gpu::gles2::GLES2Interface* target_gl,
+    VideoFrame* frame,
+    unsigned texture,
+    unsigned internal_format,
+    unsigned type,
+    bool flip_y) {
+  // TODO(astojilj): After GLES 3.2, use glCopyImageSubData for RG8 to R16UI
+  // conversion. glCopyImageSubData is already available as extension on some of
+  // the Android devices (GL_EXT_COPY_IMAGE).
+  gpu::gles2::GLES2Interface* local_gl = context_3d.gl;
+
+  // Get source texture on local_gl.
+  unsigned source_texture;
+  {
+    const gpu::MailboxHolder& mailbox_holder = frame->mailbox_holder(0);
+    DCHECK(mailbox_holder.texture_target == GL_TEXTURE_2D)
+        << mailbox_holder.texture_target;
+
+    local_gl->WaitSyncTokenCHROMIUM(mailbox_holder.sync_token.GetConstData());
+    source_texture = local_gl->CreateAndConsumeTextureCHROMIUM(
+        mailbox_holder.texture_target, mailbox_holder.mailbox.name);
+  }
+
+  // Reinterpret RG8 to R32F on local_gl. Later, do the copy on target_gl.
+  unsigned intermediate_texture;
+  local_gl->GenTextures(1, &intermediate_texture);
+  DoCopyRG8ToRedTextureData(local_gl, source_texture, intermediate_texture,

[aleksandar.stojiljkovic, 2016/09/20 12:22:55]

https://codereview.chromium.org/2121043002/diff/60001/media/renderers/skcanva...
https://codereview.chromium.org/2121043002/diff/60001/media/renderers/skcanva...
It is implemented here - 16 bit value laid out in two RG bytes is normalized and
converted (rendered) to R32F texture.
The approach follows YUV planes conversion approach - local canvas's GL is used
for conversion, output then promoted to destination GL and  CopyTextureCHROMIUM
used to copy to target texture.
+sandersd, +dshwang, +jbauman, 
could you please review this file changes? Thanks.

+                            GL_R32F, GL_FLOAT, frame->visible_rect().width(),
+                            frame->visible_rect().height(), flip_y);
+  local_gl->DeleteTextures(1, &source_texture);
+
+  // Get intermediate_texture to target_gl.
+  {
+    gpu::MailboxHolder mailbox_holder;
+    mailbox_holder.texture_target = GL_TEXTURE_2D;
+    local_gl->GenMailboxCHROMIUM(mailbox_holder.mailbox.name);
+    local_gl->ProduceTextureDirectCHROMIUM(intermediate_texture,
+                                           mailbox_holder.texture_target,
+                                           mailbox_holder.mailbox.name);
+    // Wait for mailbox creation on local context before consuming it and
+    // copying from it on the consumer context.
+    const GLuint64 fence_sync = local_gl->InsertFenceSyncCHROMIUM();
+    local_gl->ShallowFlushCHROMIUM();
+    local_gl->GenSyncTokenCHROMIUM(fence_sync,
+                                   mailbox_holder.sync_token.GetData());
+
+    target_gl->WaitSyncTokenCHROMIUM(mailbox_holder.sync_token.GetConstData());
+    uint32_t target_intermediate_texture =
+        target_gl->CreateAndConsumeTextureCHROMIUM(
+            mailbox_holder.texture_target, mailbox_holder.mailbox.name);
+
+    local_gl->DeleteTextures(1, &intermediate_texture);
+    intermediate_texture = target_intermediate_texture;
+  }
+
+  // Copy intermediate_texture to texture.
+  target_gl->CopyTextureCHROMIUM(intermediate_texture, texture, internal_format,
+                                 type, false, false, false);
+  target_gl->DeleteTextures(1, &intermediate_texture);
+
+  // Undo gr_context state changes introduced in this function.
+  context_3d.gr_context->resetContext(
+      kMisc_GrGLBackendState | kBlend_GrGLBackendState |
+      kView_GrGLBackendState | kStencil_GrGLBackendState |
+      kVertex_GrGLBackendState | kProgram_GrGLBackendState |
+      kTextureBinding_GrGLBackendState);
+  SyncTokenClientImpl client(target_gl);
+  frame->UpdateReleaseSyncToken(&client);
+}
+
+void SkCanvasVideoRenderer::DoCopyRG8ToRedTextureData(
+    gpu::gles2::GLES2Interface* gl,
+    unsigned source_texture,
+    unsigned texture,
+    unsigned internal_format,
+    unsigned type,
+    unsigned width,
+    unsigned height,
+    bool flip_y) {
+  GLuint buffer;
+  gl->GenBuffers(1, &buffer);
+  gl->BindBuffer(GL_ARRAY_BUFFER, buffer);
+  const GLfloat vertices[] = {-1.f, -1.f, 1.f, -1.f, -1.0f, 1.f, 1.f, 1.f};
+  gl->BufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);
+  gl->EnableVertexAttribArray(0);
+  gl->VertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 0, 0);
+
+  if (!rg8_to_red_program_) {
+    rg8_to_red_program_ = gl->CreateProgram();

[jbauman, 2016/09/20 20:51:22]

I don't think the lifetime of this is correct if the main thread context is lost
(GPU process crashes, etc.). I'm not sure where to put it otherwise, though.
Previously I've put it into skia, where the GrContext has the correct lifetime.
Another option is to somehow put it into CopyTextureCHROMIUM.

[aleksandar.stojiljkovic, 2016/09/30 14:26:32]

Done.
Using GrContext::uniqueID to control the lifecycle of cached resources.
Added unit test to cover this:
SkCanvasVideoRendererTest, Y16CachedGLResourcesContextRestore
https://codereview.chromium.org/2121043002/diff/120001/media/renderers/skcanv...

+    GLuint vertex_shader = gl->CreateShader(GL_VERTEX_SHADER);
+    std::string source = std::string(
+        "\
+          #ifdef GL_ES\n\
+          precision mediump float;\n\
+          #define TexCoordPrecision mediump\n\
+          #else\n\
+          #define TexCoordPrecision\n\
+          #endif\n\
+          uniform vec2 coord_transform;\n\
+          attribute vec2 a_pos;\n\
+          varying TexCoordPrecision vec2 v_uv;\n\
+          void main(void) {\n\
+            gl_Position = vec4(a_pos, 0, 1.0);\n\
+            v_uv = (a_pos * coord_transform) + 0.5;\n\
+          }\n");
+    const char* vertex_source = source.c_str();
+    gl->ShaderSource(vertex_shader, 1, &vertex_source, 0);
+    gl->CompileShader(vertex_shader);
+    gl->AttachShader(rg8_to_red_program_, vertex_shader);
+
+    GLuint fragment_shader = gl->CreateShader(GL_FRAGMENT_SHADER);
+    source = std::string(
+        "\
+          #ifdef GL_ES\n\
+          precision mediump float;\n\
+          #define TexCoordPrecision mediump\n\
+          #else\n\
+          #define TexCoordPrecision\n\
+          #endif\n\
+          uniform sampler2D u_sampler;\n\
+          varying TexCoordPrecision vec2 v_uv;\n\
+          void main(void) {\n\
+            vec4 color = texture2D(u_sampler, v_uv);\n\
+            gl_FragColor.r = (color.r * 0.00390625) + color.g;\n\
+          }\n");
+    const char* fragment_source = source.c_str();
+    gl->ShaderSource(fragment_shader, 1, &fragment_source, 0);
+    gl->CompileShader(fragment_shader);
+
+    gl->AttachShader(rg8_to_red_program_, fragment_shader);
+
+    gl->BindAttribLocation(rg8_to_red_program_, 0, "a_pos");
+    gl->LinkProgram(rg8_to_red_program_);
+  }
+
+  gl->UseProgram(rg8_to_red_program_);
+  gl->Uniform1i(gl->GetUniformLocation(rg8_to_red_program_, "u_sampler"), 0);
+  gl->Uniform2f(gl->GetUniformLocation(rg8_to_red_program_, "coord_transform"),
+                0.5, flip_y ? -0.5 : 0.5);
+
+  GLuint fbo;
+  gl->GenFramebuffers(1, &fbo);
+  gl->ActiveTexture(GL_TEXTURE0);
+  unsigned target = GL_TEXTURE_2D;
+  gl->BindTexture(target, texture);
+  gl->TexImage2D(GL_TEXTURE_2D, 0, internal_format, width, height, 0, GL_RED,
+                 type, 0);
+
+  gl->TexParameterf(target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+  gl->TexParameterf(target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+  gl->TexParameteri(target, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+  gl->TexParameteri(target, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+  gl->BindFramebuffer(GL_FRAMEBUFFER, fbo);
+  gl->FramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, target,
+                           texture, 0);
+
+  gl->BindTexture(target, source_texture);
+  gl->TexParameterf(target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+  gl->TexParameterf(target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+  gl->TexParameteri(target, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+  gl->TexParameteri(target, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+
+  gl->Disable(GL_DEPTH_TEST);
+  gl->Disable(GL_STENCIL_TEST);
+  gl->Disable(GL_CULL_FACE);
+  gl->ColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+  gl->DepthMask(GL_FALSE);
+  gl->Disable(GL_BLEND);
+  gl->Disable(GL_SCISSOR_TEST);
+  gl->Viewport(0, 0, width, height);
+  gl->DrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+  gl->Flush();
+  gl->BindFramebuffer(GL_FRAMEBUFFER, 0);
+  gl->DeleteBuffers(1, &buffer);
+  gl->DeleteFramebuffers(1, &fbo);
+}
+
 void SkCanvasVideoRenderer::ResetCache() {
   DCHECK(thread_checker_.CalledOnValidThread());
   // Clear cached values.