Improving GestureNav screenshotting performance - Part 1 - gl_helper

content/common/gpu/client/gl_helper.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/common/gpu/client/gl_helper.h"
 
 #include <queue>
 #include <string>
 
 #include "base/bind.h"
@@ skipping 124 matching lines @@
                                  uint32 sync_point) {
     return helper_->ConsumeMailboxToTexture(mailbox, sync_point);
   }
 
   void CropScaleReadbackAndCleanTexture(
       GLuint src_texture,
       const gfx::Size& src_size,
       const gfx::Rect& src_subrect,
       const gfx::Size& dst_size,
       unsigned char* out,
-      const SkColorType color_type,
+      const SkColorType out_color_type,
       const base::Callback<void(bool)>& callback,
       GLHelper::ScalerQuality quality);
 
   void ReadbackTextureSync(GLuint texture,
                            const gfx::Rect& src_rect,
                            unsigned char* out,
                            SkColorType format);
 
   void ReadbackTextureAsync(GLuint texture,
                             const gfx::Size& dst_size,
@@ skipping 162 matching lines @@
   // |src_size| is the size of |src_texture|.
   GLuint ScaleTexture(GLuint src_texture,
                       const gfx::Size& src_size,
                       const gfx::Rect& src_subrect,
                       const gfx::Size& dst_size,
                       bool vertically_flip_texture,
                       bool swizzle,
                       SkColorType color_type,
                       GLHelper::ScalerQuality quality);
 
+  // Converts each four consecutive pixels of the source texture into one pixel
+  // in the result texture with each pixel channel representing the grayscale
+  // color of one of the four original pixels:
+  // R1G1B1A1 R2G2B2A2 R3G3B3A3 R4G4B4A4 -> X1X2X3X4
+  // The resulting texture is still an RGBA texture (which is ~4 times narrower
+  // than the original). If rendered directly, it wouldn't show anything useful,
+  // but the data in it can be used to construct a grayscale image.
+  // |encoded_texture_size| is the exact size of the resulting RGBA texture. It
+  // is equal to src_size.width()/4 rounded upwards. Some channels in the last
+  // pixel ((-src_size.width()) % 4) to be exact) are padding and don't contain
+  // useful data.
+  // If swizzle is set to true, the transformed pixels are reordered:
+  // R1G1B1A1 R2G2B2A2 R3G3B3A3 R4G4B4A4 -> X3X2X1X4.
+  GLuint EncodeTextureAsGrayscale(GLuint src_texture,
+                                  const gfx::Size& src_size,
+                                  gfx::Size* const encoded_texture_size,
+                                  bool vertically_flip_texture,
+                                  bool swizzle);
+
   static void nullcallback(bool success) {}
   void ReadbackDone(Request *request, int bytes_per_pixel);
   void FinishRequest(Request* request, bool result);
   void CancelRequests();
 
   static const float kRGBtoYColorWeights[];
   static const float kRGBtoUColorWeights[];
   static const float kRGBtoVColorWeights[];
+  static const float kRGBtoGrayscaleColorWeights[];
 
   GLES2Interface* gl_;
   gpu::ContextSupport* context_support_;
   GLHelper* helper_;
 
   // A scoped flush that will ensure all resource deletions are flushed when
   // this object is destroyed. Must be declared before other Scoped* fields.
   ScopedFlush flush_;
 
   std::queue<Request*> request_queue_;
@@ skipping 51 matching lines @@
       helper_->CreateScaler(quality,
                             src_size,
                             src_subrect,
                             dst_size,
                             vertically_flip_texture,
                             swizzle));
   scaler->Scale(src_texture, dst_texture);
   return dst_texture;
 }
 
+GLuint GLHelper::CopyTextureToImpl::EncodeTextureAsGrayscale(
+    GLuint src_texture,
+    const gfx::Size& src_size,
+    gfx::Size* const encoded_texture_size,
+    bool vertically_flip_texture,
+    bool swizzle) {
+  GLuint dst_texture = 0u;
+  gl_->GenTextures(1, &dst_texture);
+  // The size of the encoded texture.
+  *encoded_texture_size =
+      gfx::Size((src_size.width() + 3) / 4, src_size.height());
+  {
+    ScopedTextureBinder<GL_TEXTURE_2D> texture_binder(gl_, dst_texture);
+    gl_->TexImage2D(GL_TEXTURE_2D,
+                    0,
+                    GL_RGBA,
+                    encoded_texture_size->width(),
+                    encoded_texture_size->height(),
+                    0,
+                    GL_RGBA,
+                    GL_UNSIGNED_BYTE,
+                    NULL);
+  }
+
+  helper_->InitScalerImpl();
+  scoped_ptr<ScalerInterface> grayscale_scaler(
+      helper_->scaler_impl_.get()->CreatePlanarScaler(
+          src_size,
+          gfx::Rect(0, 0, (src_size.width() + 3) & ~3, src_size.height()),
+          *encoded_texture_size,
+          vertically_flip_texture,
+          swizzle,
+          kRGBtoGrayscaleColorWeights));
+  grayscale_scaler->Scale(src_texture, dst_texture);
+  return dst_texture;
+}
+
 void GLHelper::CopyTextureToImpl::ReadbackAsync(
     const gfx::Size& dst_size,
     int32 bytes_per_row,
     int32 row_stride_bytes,
     unsigned char* out,
     GLenum format,
     GLenum type,
     size_t bytes_per_pixel,
     const base::Callback<void(bool)>& callback) {
   TRACE_EVENT0("mirror", "GLHelper::CopyTextureToImpl::ReadbackAsync");
@@ skipping 19 matching lines @@
                   format,
                   type,
                   NULL);
   gl_->EndQueryEXT(GL_ASYNC_PIXEL_PACK_COMPLETED_CHROMIUM);
   gl_->BindBuffer(GL_PIXEL_PACK_TRANSFER_BUFFER_CHROMIUM, 0);
   context_support_->SignalQuery(
       request->query,
       base::Bind(&CopyTextureToImpl::ReadbackDone, AsWeakPtr(),
                  request, bytes_per_pixel));
 }
+
 void GLHelper::CopyTextureToImpl::CropScaleReadbackAndCleanTexture(
     GLuint src_texture,
     const gfx::Size& src_size,
     const gfx::Rect& src_subrect,
     const gfx::Size& dst_size,
     unsigned char* out,
-    const SkColorType color_type,
+    const SkColorType out_color_type,
     const base::Callback<void(bool)>& callback,
     GLHelper::ScalerQuality quality) {
   GLenum format, type;
   size_t bytes_per_pixel;
-  FormatSupport supported =
-      GetReadbackConfig(color_type, true, &format, &type, &bytes_per_pixel);
+  SkColorType readback_color_type = out_color_type;
+  // Single-component textures are not supported by all GPUs, so  we implement
+  // kAlpha_8_SkColorType support here via a special encoding (see below) using
+  // a 32-bit texture to represent an 8-bit image.
+  // Thus we use generic 32-bit readback in this case.
+  if (out_color_type == kAlpha_8_SkColorType) {
+    readback_color_type = kN32_SkColorType;

[no sievers, 2014/08/20 20:29:51]

I don't think it makes sense to check for readback support for the
platform-specific preferred native format here (kN32_SkColorType = RGBA on
Android, BGRA otherwise I believe), since we are not using that format. You
hardcode the format for the alpha texture to RGBA, so we should check that here.
Esp. since the additional format we query (and might prefer) depends on the
format of the bound texture when we query it. And the way this check works is
that the texture is created with the format matching the SkColorType we pass
into GetReadbackConfig() (so if you pass BGRA here it tells you whether you can
read back a native BGRA framebuffer which is not what we want to know)...

[mfomitchev, 2014/08/22 18:01:33]

GetReadbackConfig() returns the readback format.  hubbe@ was making a point in
crbug.com/379983 that we should be doing the readback using BGRA on CrOS. So
this is what I wanted to ensure. My naive assumption was that if I pass the
native format to GetReadbackConfig(), it would also return the readback format
which is more efficient. I don't really care which format it tells me to use, I
just want the processing + readback to be as fast as possible. Although I do
need to swizzle if the readback is using BGRA since EncodeAsGrayscale writes the
pixels in RGBA order.
Ok, this is the part I am pretty confused about and asked for clarification from
hubbe@ above. It sounds like you are saying that the format we are using with
TexImage2D() does matter and we should match it to the format we pass to
GetReadbackConfig()? I tried changing the format of the alpha texture to BGRA,
and it didn't seem to make any difference at all. Do you mind explaining this? I
ended up making it RGBA only because we use RGBA or RGB everywhere else in gl
helper. We don't seem to use BGRA anywhere here with TexImage2D() (even though
we do use BGRA for readback). Why is that?

[no sievers, 2014/08/22 18:44:13]

I kow this is really confusing :/

GetReadbackFormat() returns what it thinks is the best readback format for
getting back readback_color_type assuming that the internal format of the
texture also matches readback_color_type.

For the 'additional format' it queries see how it creates a dummy texture that
matches the skia color format you pass in. Note that the GL query we do there is
dependent on the format of the bound framebuffer.

I'm still a bit confused about this, also see comments in
https://codereview.chromium.org/412453002/.

I'm not sure why reading back a texture, whose internal format is RGBA, as BGRA
would ever be faster.

However, I think you need to
- either set |readback_color_type| to RGBA above since that is the format of
your texture being read back for the query to do the right thing
- or allocate the texture as BGRA or RGBA respectively so that it matches the
kN32_SkColorType layout

[hubbe, 2014/08/22 19:19:20]

The reason reading back as BGRA is faster than RGBA is that some intel/mesa
drivers always stores the textures as BGRA internally, so weather you ask for
RGBA or BGRA makes no difference. Since it makes no difference, we should use
RGBA since that is what GLES is guaranteed to support.

(Unless I'm wrong, but this is what I've understood from other people.)

On 2014/08/22 18:44:13, sievers wrote:

[no sievers, 2014/08/22 20:54:48]

Thanks hubbe+piman, that makes sense.

In that case we should still use kRGBA_8888_SkColorType in line 521 to get the
correct answer for which readback format to use for the texture that we created
with RGBA. The driver overriding the format internally is handled by this code
in GetReadbackConfig():

   case kRGBA_8888_SkColorType:
      *format = GL_RGBA;
      if (can_swizzle) {
        // If GL_BGRA_EXT is advertised as the readback format through
        // GL_IMPLEMENTATION_COLOR_READ_FORMAT then assume it is preferred by
        // the implementation for performance.
        GetAdditionalFormat(*format, *type, &new_format, &new_type);

+  }
+
+  FormatSupport supported = GetReadbackConfig(readback_color_type,
+                                              true,
+                                              &format,
+                                              &type,
+                                              &bytes_per_pixel);
+
   if (supported == GLHelperReadbackSupport::NOT_SUPPORTED) {
     callback.Run(false);
     return;
   }
 
-  GLuint texture = ScaleTexture(src_texture,
-                                src_size,
-                                src_subrect,
-                                dst_size,
-                                true,
-                                (supported == GLHelperReadbackSupport::SWIZZLE),
-                                color_type,
-                                quality);
-  DCHECK(texture);
+  GLuint texture = src_texture;
+
+  // Scale texture if needed
+  // Optimization: SCALER_QUALITY_FAST is just a single bilinear pass, which we
+  // can do just as well in EncodeTextureAsGrayscale, which we will do if
+  // out_color_type is kAlpha_8_SkColorType, so let's skip the scaling step
+  // in that case.
+  bool scale_texture = out_color_type != kAlpha_8_SkColorType ||
+                       quality != GLHelper::SCALER_QUALITY_FAST;
+  if (scale_texture) {
+    // Don't swizzle during the scale step for kAlpha_8_SkColorType.
+    // We will swizzle in the encode step below if needed.
+    bool scale_swizzle =
+        out_color_type == kAlpha_8_SkColorType ?
+            false : supported == GLHelperReadbackSupport::SWIZZLE;
+    texture =
+        ScaleTexture(src_texture,
+                     src_size,
+                     src_subrect,
+                     dst_size,
+                     true,
+                     scale_swizzle,
+                     out_color_type == kAlpha_8_SkColorType ? kN32_SkColorType
+                                                            : out_color_type,
+                     quality);
+    DCHECK(texture);
+  }
+
+  gfx::Size readback_texture_size = dst_size;
+  // Encode texture to grayscale if needed.
+  if (out_color_type == kAlpha_8_SkColorType) {
+    // Do the vertical flip here if we haven't already done it when we scaled
+    // the texture.
+    bool encode_as_grayscale_vertical_flip = !scale_texture;
+    // EncodeTextureAsGrayscale by default creates a texture which should be
+    // read back as RGBA, so need to swizzle if the readback format is BGRA.

[no sievers, 2014/08/20 20:29:51]

...and this check is still correct then, I think. If format == BGRA here then
GetReadbackConfig() believes it's better to read back a RGBA framebuffer as BGRA
(according to the comment there). For that to yield the correct results, we have
to swizzle the data first to neutralize the swizzle that will then happen during
readback.

+    bool encode_as_grayscale_swizzle = format == GL_BGRA_EXT;
+    texture = EncodeTextureAsGrayscale(

[no sievers, 2014/08/20 20:29:51]

Do we leak |texture| here if we scaled above, as in the scaled RGBA version?

[mfomitchev, 2014/08/22 18:01:33]

Done.

+        texture,
+        dst_size,
+        &readback_texture_size,
+        encode_as_grayscale_vertical_flip,
+        encode_as_grayscale_swizzle);
+    DCHECK(texture);
+  }
+
+  // Readback the pixels of the resulting texture
   ScopedFramebuffer dst_framebuffer(gl_);
   ScopedFramebufferBinder<GL_FRAMEBUFFER> framebuffer_binder(gl_,
                                                              dst_framebuffer);
   ScopedTextureBinder<GL_TEXTURE_2D> texture_binder(gl_, texture);
   gl_->FramebufferTexture2D(GL_FRAMEBUFFER,
                             GL_COLOR_ATTACHMENT0,
                             GL_TEXTURE_2D,
                             texture,
                             0);
-  ReadbackAsync(dst_size,
-                dst_size.width() * bytes_per_pixel,
-                dst_size.width() * bytes_per_pixel,
+
+  int32 bytes_per_row =
+      out_color_type == kAlpha_8_SkColorType ?
+          dst_size.width() : dst_size.width() * bytes_per_pixel;
+
+  ReadbackAsync(readback_texture_size,
+                bytes_per_row,
+                bytes_per_row,
                 out,
                 format,
                 type,
                 bytes_per_pixel,
                 callback);
   gl_->DeleteTextures(1, &texture);
 }
 
 void GLHelper::CopyTextureToImpl::ReadbackTextureSync(
     GLuint texture,
@@ skipping 154 matching lines @@
       readback_support_(new GLHelperReadbackSupport(gl)) {}
 
 GLHelper::~GLHelper() {}
 
 void GLHelper::CropScaleReadbackAndCleanTexture(
     GLuint src_texture,
     const gfx::Size& src_size,
     const gfx::Rect& src_subrect,
     const gfx::Size& dst_size,
     unsigned char* out,
-    const SkColorType color_type,
+    const SkColorType out_color_type,
     const base::Callback<void(bool)>& callback,
     GLHelper::ScalerQuality quality) {
   InitCopyTextToImpl();
-  copy_texture_to_impl_->CropScaleReadbackAndCleanTexture(
-      src_texture,
-      src_size,
-      src_subrect,
-      dst_size,
-      out,
-      color_type,
-      callback,
-      quality);
+  copy_texture_to_impl_->CropScaleReadbackAndCleanTexture(src_texture,
+                                                          src_size,
+                                                          src_subrect,
+                                                          dst_size,
+                                                          out,
+                                                          out_color_type,
+                                                          callback,
+                                                          quality);
 }
 
 void GLHelper::CropScaleReadbackAndCleanMailbox(
     const gpu::Mailbox& src_mailbox,
     uint32 sync_point,
     const gfx::Size& src_size,
     const gfx::Rect& src_subrect,
     const gfx::Size& dst_size,
     unsigned char* out,
-    const SkColorType color_type,
+    const SkColorType out_color_type,
     const base::Callback<void(bool)>& callback,
     GLHelper::ScalerQuality quality) {
   GLuint mailbox_texture = ConsumeMailboxToTexture(src_mailbox, sync_point);
-  CropScaleReadbackAndCleanTexture(
-      mailbox_texture, src_size, src_subrect, dst_size, out,
-      color_type,
-      callback,
-      quality);
+  CropScaleReadbackAndCleanTexture(mailbox_texture,
+                                   src_size,
+                                   src_subrect,
+                                   dst_size,
+                                   out,
+                                   out_color_type,
+                                   callback,
+                                   quality);
   gl_->DeleteTextures(1, &mailbox_texture);
 }
 
 void GLHelper::ReadbackTextureSync(GLuint texture,
                                    const gfx::Rect& src_rect,
                                    unsigned char* out,
                                    SkColorType format) {
   InitCopyTextToImpl();
   copy_texture_to_impl_->ReadbackTextureSync(texture, src_rect, out, format);
 }
@@ skipping 194 matching lines @@
                 4,
                 callback);
 }
 
 const float GLHelper::CopyTextureToImpl::kRGBtoYColorWeights[] = {
     0.257f, 0.504f, 0.098f, 0.0625f};
 const float GLHelper::CopyTextureToImpl::kRGBtoUColorWeights[] = {
     -0.148f, -0.291f, 0.439f, 0.5f};
 const float GLHelper::CopyTextureToImpl::kRGBtoVColorWeights[] = {
     0.439f, -0.368f, -0.071f, 0.5f};
+const float GLHelper::CopyTextureToImpl::kRGBtoGrayscaleColorWeights[] = {
+    0.213f, 0.715f, 0.072f, 0.0f};
 
 // YUV readback constructors. Initiates the main scaler pipeline and
 // one planar scaler for each of the Y, U and V planes.
 GLHelper::CopyTextureToImpl::ReadbackYUVImpl::ReadbackYUVImpl(
     GLES2Interface* gl,
     CopyTextureToImpl* copy_impl,
     GLHelperScaling* scaler_impl,
     GLHelper::ScalerQuality quality,
     const gfx::Size& src_size,
     const gfx::Rect& src_subrect,
@@ skipping 321 matching lines @@
   return copy_texture_to_impl_->CreateReadbackPipelineYUV(quality,
                                                           src_size,
                                                           src_subrect,
                                                           dst_size,
                                                           dst_subrect,
                                                           flip_vertically,
                                                           use_mrt);
 }
 
 }  // namespace content