gpu: Add YCbCr 420v extension.

ui/gl/gl_image_io_surface.mm

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "ui/gl/gl_image_io_surface.h"
 
 #include <map>
 
+#include "base/callback_helpers.h"
 #include "base/lazy_instance.h"
 #include "base/mac/foundation_util.h"
+#include "base/strings/stringize_macros.h"
 #include "base/trace_event/memory_allocator_dump.h"
 #include "base/trace_event/memory_dump_manager.h"
 #include "base/trace_event/process_memory_dump.h"
 #include "ui/gl/gl_bindings.h"
 #include "ui/gl/gl_context.h"
+#include "ui/gl/gl_helper.h"
+#include "ui/gl/scoped_binders.h"
 
 // Note that this must be included after gl_bindings.h to avoid conflicts.
 #include <OpenGL/CGLIOSurface.h>
 #include <Quartz/Quartz.h>
 
 using gfx::BufferFormat;
 
 namespace gl {
 namespace {
 
 using WidgetToLayerMap = std::map<gfx::AcceleratedWidget, CALayer*>;
 base::LazyInstance<WidgetToLayerMap> g_widget_to_layer_map;
 
 bool ValidInternalFormat(unsigned internalformat) {
   switch (internalformat) {
     case GL_R8:
     case GL_BGRA_EXT:
     case GL_RGB:
+    case GL_RGB_YCBCR_420V_CHROMIUM:
     case GL_RGB_YCBCR_422_CHROMIUM:
     case GL_RGBA:
       return true;
     default:
       return false;
   }
 }
 
 bool ValidFormat(BufferFormat format) {
   switch (format) {
@@ skipping 15 matching lines @@
       return false;
   }
 
   NOTREACHED();
   return false;
 }
 
 GLenum TextureFormat(BufferFormat format) {
   switch (format) {
     case BufferFormat::R_8:
-    case BufferFormat::YUV_420_BIPLANAR:
       return GL_RED;
     case BufferFormat::BGRA_8888:
     case BufferFormat::RGBA_8888:
       return GL_RGBA;
     case BufferFormat::UYVY_422:
-      return GL_RGB;
     case BufferFormat::ATC:
     case BufferFormat::ATCIA:
     case BufferFormat::DXT1:
     case BufferFormat::DXT5:
     case BufferFormat::ETC1:
     case BufferFormat::RGBA_4444:
     case BufferFormat::RGBX_8888:
     case BufferFormat::BGRX_8888:
     case BufferFormat::YUV_420:
+    case BufferFormat::YUV_420_BIPLANAR:
       NOTREACHED();
       return 0;
   }
 
   NOTREACHED();
   return 0;
 }
 
 GLenum DataFormat(BufferFormat format) {
   switch (format) {
     case BufferFormat::R_8:
-    case BufferFormat::YUV_420_BIPLANAR:
       return GL_RED;
     case BufferFormat::BGRA_8888:
     case BufferFormat::RGBA_8888:
       return GL_BGRA;
     case BufferFormat::UYVY_422:
       return GL_YCBCR_422_APPLE;
-      break;
     case BufferFormat::ATC:
     case BufferFormat::ATCIA:
     case BufferFormat::DXT1:
     case BufferFormat::DXT5:
     case BufferFormat::ETC1:
     case BufferFormat::RGBA_4444:
     case BufferFormat::RGBX_8888:
     case BufferFormat::BGRX_8888:
     case BufferFormat::YUV_420:
+    case BufferFormat::YUV_420_BIPLANAR:
       NOTREACHED();
       return 0;
   }
 
   NOTREACHED();
   return 0;
 }
 
 GLenum DataType(BufferFormat format) {
   switch (format) {
     case BufferFormat::R_8:
-    case BufferFormat::YUV_420_BIPLANAR:
       return GL_UNSIGNED_BYTE;
     case BufferFormat::BGRA_8888:
     case BufferFormat::RGBA_8888:
       return GL_UNSIGNED_INT_8_8_8_8_REV;
     case BufferFormat::UYVY_422:
       return GL_UNSIGNED_SHORT_8_8_APPLE;
       break;
     case BufferFormat::ATC:
     case BufferFormat::ATCIA:
     case BufferFormat::DXT1:
     case BufferFormat::DXT5:
     case BufferFormat::ETC1:
     case BufferFormat::RGBA_4444:
     case BufferFormat::RGBX_8888:
     case BufferFormat::BGRX_8888:
     case BufferFormat::YUV_420:
+    case BufferFormat::YUV_420_BIPLANAR:
       NOTREACHED();
       return 0;
   }
 
   NOTREACHED();
   return 0;
 }
 
+GLuint SetupVertexBuffer() {
+  GLuint vertex_buffer = 0;
+  glGenBuffersARB(1, &vertex_buffer);
+  gfx::ScopedBufferBinder sbb(GL_ARRAY_BUFFER, vertex_buffer);

[reveman, 2015/10/29 22:08:04]

nit: s/sbb/scoped_buffer_binder/ or buffer_binder, or whatever we typically use
for the texture binder equivalent in glimage code. 

[Daniele Castagna, 2015/10/30 23:51:57]

Renamed just buffer_binder, also changed the name of the other Scoped* things.

+
+  GLfloat data[] = {-1.f, -1.f, 1.f, -1.f, -1.f, 1.f, 1.f, 1.f};
+  glBufferData(GL_ARRAY_BUFFER, sizeof(data), data, GL_STATIC_DRAW);
+  glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, sizeof(GLfloat) * 2, 0);
+
+  return vertex_buffer;
+}
+
 }  // namespace
 
+// clang-format off
+const char GLImageIOSurface::kVertexShader[] =
+STRINGIZE(
+  attribute vec2 a_position;
+  uniform vec2 a_size;
+  varying vec2 v_texCoord;
+  void main() {
+    gl_Position = vec4(a_position.x, a_position.y, 0.0, 1.0);
+    v_texCoord = (a_position + vec2(1,1)) * 0.5 * a_size;
+  }
+);
+
+const char GLImageIOSurface::kFragmentShader[] =
+STRINGIZE(
+  uniform sampler2DRect a_y_texture;
+  uniform sampler2DRect a_uv_texture;
+  varying vec2 v_texCoord;
+  void main() {
+    vec3 yuv_adj = vec3(-0.0625, -0.5, -0.5);
+    mat3 yuv_matrix = mat3(vec3(1.164, 1.164, 1.164),
+                           vec3(0.0, -.391, 2.018),
+                           vec3(1.596, -.813, 0.0));
+    vec3 yuv = vec3(texture2DRect(a_y_texture, v_texCoord).r,
+        texture2DRect(a_uv_texture, v_texCoord * 0.5).rg);
+    gl_FragColor = vec4( yuv_matrix * (yuv + yuv_adj), 1.0);
+  }
+);
+// clang-format on
+
 GLImageIOSurface::GLImageIOSurface(const gfx::Size& size,
                                    unsigned internalformat)
     : size_(size),
       internalformat_(internalformat),
       format_(BufferFormat::RGBA_8888) {}
 
 GLImageIOSurface::~GLImageIOSurface() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(!io_surface_);
 }
@@ skipping 15 matching lines @@
   }
 
   format_ = format;
   io_surface_.reset(io_surface, base::scoped_policy::RETAIN);
   io_surface_id_ = io_surface_id;
   return true;
 }
 
 void GLImageIOSurface::Destroy(bool have_context) {
   DCHECK(thread_checker_.CalledOnValidThread());
+  if (have_context) {
+    glDeleteProgram(program_);

[reveman, 2015/10/29 22:08:04]

what if you haven't created a program?

[Daniele Castagna, 2015/10/30 23:51:57]

In that case program_ would be 0.
The documentation says: "A value of 0 for program will be silently ignored."
Same for all the other calls.

+    glDeleteShader(vertex_shader_);
+    glDeleteShader(fragment_shader_);
+    glDeleteBuffersARB(1, &vertex_buffer_);
+    glDeleteFramebuffersEXT(1, &framebuffer_);
+  }
   io_surface_.reset();
 }
 
 gfx::Size GLImageIOSurface::GetSize() {
   return size_;
 }
 
 unsigned GLImageIOSurface::GetInternalFormat() { return internalformat_; }
 
 bool GLImageIOSurface::BindTexImage(unsigned target) {
   DCHECK(thread_checker_.CalledOnValidThread());
+
+  if (format_ == BufferFormat::YUV_420_BIPLANAR)

[reveman, 2015/10/29 22:08:04]

Can you add a comment here to explain that this format needs to be converted to
RGB and as a result, BindTexImage is not supported but CopyTexImage is?

[Daniele Castagna, 2015/10/30 23:51:57]

Done.

+    return false;
+
   if (target != GL_TEXTURE_RECTANGLE_ARB) {
     // This might be supported in the future. For now, perform strict
     // validation so we know what's going on.
     LOG(ERROR) << "IOSurface requires TEXTURE_RECTANGLE_ARB target";
     return false;
   }
 
   CGLContextObj cgl_context =
       static_cast<CGLContextObj>(gfx::GLContext::GetCurrent()->GetHandle());
 
   DCHECK(io_surface_);
   CGLError cgl_error =
       CGLTexImageIOSurface2D(cgl_context, target, TextureFormat(format_),
                              size_.width(), size_.height(), DataFormat(format_),
                              DataType(format_), io_surface_.get(), 0);
   if (cgl_error != kCGLNoError) {
     LOG(ERROR) << "Error in CGLTexImageIOSurface2D";
     return false;
   }
 
   return true;
 }
 
 bool GLImageIOSurface::CopyTexImage(unsigned target) {
-  return false;
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(io_surface_);
+  if (format_ != BufferFormat::YUV_420_BIPLANAR)
+    return false;
+
+  if (target != GL_TEXTURE_2D) {
+    LOG(ERROR) << "YUV_420_BIPLANAR requires TEXTURE_2D target";
+    return false;
+  }
+
+  if (!framebuffer_) {
+    glGenFramebuffersEXT(1, &framebuffer_);
+    vertex_buffer_ = SetupVertexBuffer();
+    vertex_shader_ = gfx::GLHelper::LoadShader(GL_VERTEX_SHADER, kVertexShader);
+    fragment_shader_ =
+        gfx::GLHelper::LoadShader(GL_FRAGMENT_SHADER, kFragmentShader);
+    program_ = gfx::GLHelper::SetupProgram(vertex_shader_, fragment_shader_);
+    gfx::ScopedUseProgram pr(program_);
+
+    size_location_ = glGetUniformLocation(program_, "a_size");
+    DCHECK_NE(-1, size_location_);
+    int y_sampler_location = glGetUniformLocation(program_, "a_y_texture");
+    DCHECK_NE(-1, y_sampler_location);
+    int uv_sampler_location = glGetUniformLocation(program_, "a_uv_texture");
+    DCHECK_NE(-1, uv_sampler_location);
+
+    glUniform1i(y_sampler_location, 0);
+    glUniform1i(uv_sampler_location, 1);
+  }
+
+  CGLContextObj cgl_context =
+      static_cast<CGLContextObj>(gfx::GLContext::GetCurrent()->GetHandle());
+
+  GLuint yuv_textures[2];
+  glGenTextures(2, yuv_textures);
+  DCHECK(yuv_textures[0] && yuv_textures[1]);
+
+  GLint service_texture = 0;

[reveman, 2015/10/29 22:08:04]

nit: service_texture -> target_texture as there's technically no such thing as a
"service" from the pov of this code

[Daniele Castagna, 2015/10/30 23:51:57]

Done.

+  glGetIntegerv(GL_TEXTURE_BINDING_2D, &service_texture);
+  DCHECK(service_texture);
+
+  GLint previous_active_texture = 0;
+  glGetIntegerv(GL_ACTIVE_TEXTURE, &previous_active_texture);
+
+  CGLError cgl_error = kCGLNoError;
+  {
+    glActiveTexture(GL_TEXTURE0);
+    gfx::ScopedTextureBinder b(GL_TEXTURE_RECTANGLE_ARB, yuv_textures[0]);
+    cgl_error = CGLTexImageIOSurface2D(
+        cgl_context, GL_TEXTURE_RECTANGLE_ARB, GL_RED, size_.width(),
+        size_.height(), GL_RED, GL_UNSIGNED_BYTE, io_surface_.get(), 0);
+    if (cgl_error != kCGLNoError) {
+      LOG(ERROR) << "Error in CGLTexImageIOSurface2D for the Y plane. "
+                 << cgl_error;
+      glDeleteTextures(2, yuv_textures);

[reveman, 2015/10/29 22:08:04]

Do we need to restore the active texture here? Maybe add a ScopedActiveTexture?

[Daniele Castagna, 2015/10/30 23:51:57]

Done.

+      return false;
+    }
+    {
+      glActiveTexture(GL_TEXTURE1);
+      gfx::ScopedTextureBinder b(GL_TEXTURE_RECTANGLE_ARB, yuv_textures[1]);
+      cgl_error = CGLTexImageIOSurface2D(
+          cgl_context, GL_TEXTURE_RECTANGLE_ARB, GL_RG, size_.width() / 2,
+          size_.height() / 2, GL_RG, GL_UNSIGNED_BYTE, io_surface_.get(), 1);
+      if (cgl_error != kCGLNoError) {
+        LOG(ERROR) << "Error in CGLTexImageIOSurface2D for the UV plane. "
+                   << cgl_error;
+        glDeleteTextures(2, yuv_textures);
+        return false;
+      }
+
+      gfx::ScopedFrameBufferBinder fb(framebuffer_);

[reveman, 2015/10/29 22:08:04]

nit: s/fb/scoped_framebuffer/

[Daniele Castagna, 2015/10/30 23:51:57]

Changed all of them!

+      glViewport(0, 0, size_.width(), size_.height());

[reveman, 2015/10/29 22:08:04]

do we need to restore the viewport before we exit?

[Daniele Castagna, 2015/10/30 23:51:57]

Added another Scoped* class...

+      glFramebufferTexture2DEXT(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
+                                GL_TEXTURE_2D, service_texture, 0);
+      DCHECK_EQ(static_cast<GLenum>(GL_FRAMEBUFFER_COMPLETE),
+                glCheckFramebufferStatusEXT(GL_FRAMEBUFFER));
+
+      gfx::ScopedUseProgram pr(program_);

[reveman, 2015/10/29 22:08:04]

nit: please remove "pr" abbreviation here and similar below

[Daniele Castagna, 2015/10/30 23:51:57]

Done.

+      glUniform2f(size_location_, size_.width(), size_.height());
+
+      gfx::ScopedEnableVertexAttribArray vaa(0);
+      gfx::ScopedBufferBinder bb(GL_ARRAY_BUFFER, vertex_buffer_);
+      glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+
+      // Detach the service texture from the fbo.

[reveman, 2015/10/29 22:08:04]

target texture

[Daniele Castagna, 2015/10/30 23:51:57]

Done.

+      glFramebufferTexture2DEXT(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
+                                GL_TEXTURE_2D, 0, 0);
+
+    }  // GL_TEXTURE1
+    glActiveTexture(GL_TEXTURE0);

[reveman, 2015/10/29 22:08:04]

previous_active_texture?

[Daniele Castagna, 2015/10/30 23:51:57]

ack.

+  }  // GL_TEXTURE0
+  return true;
 }
 
 bool GLImageIOSurface::CopyTexSubImage(unsigned target,
                                        const gfx::Point& offset,
                                        const gfx::Rect& rect) {
   return false;
 }
 
 bool GLImageIOSurface::ScheduleOverlayPlane(gfx::AcceleratedWidget widget,
                                             int z_order,
@@ skipping 30 matching lines @@
 // static
 void GLImageIOSurface::SetLayerForWidget(gfx::AcceleratedWidget widget,
                                          CALayer* layer) {
   if (layer)
     g_widget_to_layer_map.Pointer()->insert(std::make_pair(widget, layer));
   else
     g_widget_to_layer_map.Pointer()->erase(widget);
 }
 
 }  // namespace gfx