Apply the Surface Texture transformation matrix during texture copy.

content/common/gpu/stream_texture_android.cc

 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/common/gpu/stream_texture_android.h"
 
 #include "base/bind.h"
+#include "base/strings/stringize_macros.h"
 #include "content/common/android/surface_texture_peer.h"
 #include "content/common/gpu/gpu_channel.h"
 #include "content/common/gpu/gpu_messages.h"
 #include "gpu/command_buffer/service/context_group.h"
 #include "gpu/command_buffer/service/context_state.h"
 #include "gpu/command_buffer/service/gles2_cmd_decoder.h"
 #include "gpu/command_buffer/service/texture_manager.h"
 #include "ui/gfx/geometry/size.h"
 #include "ui/gl/gl_context.h"
+#include "ui/gl/gl_helper.h"
+#include "ui/gl/scoped_binders.h"
 #include "ui/gl/scoped_make_current.h"
 
 namespace content {
 
 using gpu::gles2::ContextGroup;
 using gpu::gles2::GLES2Decoder;
 using gpu::gles2::TextureManager;
 using gpu::gles2::TextureRef;
 
 // static
@@ skipping 30 matching lines @@
 StreamTexture::StreamTexture(GpuCommandBufferStub* owner_stub,
                              int32 route_id,
                              uint32 texture_id)
     : surface_texture_(gfx::SurfaceTexture::Create(texture_id)),
       size_(0, 0),
       has_valid_frame_(false),
       has_pending_frame_(false),
       owner_stub_(owner_stub),
       route_id_(route_id),
       has_listener_(false),
+      matrix_modified_(false),
       texture_id_(texture_id),
+      framebuffer_(0),
+      vertex_shader_(0),
+      fragment_shader_(0),
+      program_(0),
+      vertex_buffer_(0),
+      u_xform_loc_(-1),
       weak_factory_(this) {
   owner_stub->AddDestructionObserver(this);
   memset(current_matrix_, 0, sizeof(current_matrix_));
   owner_stub->channel()->AddRoute(route_id, this);
   surface_texture_->SetFrameAvailableCallback(base::Bind(
       &StreamTexture::OnFrameAvailable, weak_factory_.GetWeakPtr()));
 }
 
 StreamTexture::~StreamTexture() {
   if (owner_stub_) {
     owner_stub_->RemoveDestructionObserver(this);
     owner_stub_->channel()->RemoveRoute(route_id_);
   }
 }
 
 void StreamTexture::OnWillDestroyStub() {
   owner_stub_->RemoveDestructionObserver(this);
   owner_stub_->channel()->RemoveRoute(route_id_);
+
+  if (framebuffer_) {
+    scoped_ptr<ui::ScopedMakeCurrent> scoped_make_current(MakeStubCurrent());
+
+    glDeleteProgram(program_);
+    glDeleteShader(vertex_shader_);
+    glDeleteShader(fragment_shader_);
+    glDeleteBuffersARB(1, &vertex_buffer_);
+    glDeleteFramebuffersEXT(1, &framebuffer_);
+    program_ = 0;
+    vertex_shader_ = 0;
+    fragment_shader_ = 0;
+    vertex_buffer_ = 0;
+    framebuffer_ = 0;
+    u_xform_loc_ = -1;
+  }
+
   owner_stub_ = NULL;
 
   // If the owner goes away, there is no need to keep the SurfaceTexture around.
   // The GL texture will keep working regardless with the currently bound frame.
   surface_texture_ = NULL;
 }
 
 void StreamTexture::Destroy(bool have_context) {
   NOTREACHED();
 }
 
+scoped_ptr<ui::ScopedMakeCurrent> StreamTexture::MakeStubCurrent() {
+  scoped_ptr<ui::ScopedMakeCurrent> scoped_make_current;
+  bool needs_make_current =
+      !owner_stub_->decoder()->GetGLContext()->IsCurrent(NULL);
+  // On Android we should not have to perform a real context switch here when
+  // using virtual contexts.
+  DCHECK(!needs_make_current ||
+         !owner_stub_->decoder()
+              ->GetContextGroup()
+              ->feature_info()
+              ->workarounds()
+              .use_virtualized_gl_contexts);
+  if (needs_make_current) {
+    scoped_make_current.reset(new ui::ScopedMakeCurrent(
+        owner_stub_->decoder()->GetGLContext(), owner_stub_->surface()));
+  }
+  return scoped_make_current;
+}
+
+void StreamTexture::DoUpdateTexImage() {
+  DCHECK(surface_texture_.get());
+
+  scoped_ptr<ui::ScopedMakeCurrent> scoped_make_current(MakeStubCurrent());
+
+  surface_texture_->UpdateTexImage();
+  has_valid_frame_ = true;
+  has_pending_frame_ = false;
+
+  if (scoped_make_current.get()) {
+    // UpdateTexImage() implies glBindTexture().
+    // The cmd decoder takes care of restoring the binding for this GLImage as
+    // far as the current context is concerned, but if we temporarily change
+    // it, we have to keep the state intact in *that* context also.
+    const gpu::gles2::ContextState* state =
+        owner_stub_->decoder()->GetContextState();
+    const gpu::gles2::TextureUnit& active_unit =
+        state->texture_units[state->active_texture_unit];
+    glBindTexture(GL_TEXTURE_EXTERNAL_OES,
+                  active_unit.bound_texture_external_oes.get()
+                      ? active_unit.bound_texture_external_oes->service_id()
+                      : 0);
+  }
+}
+
+bool StreamTexture::DoCopyTexture(const gfx::Point& offset,
+                                  const gfx::Rect& rect) {
+  if (!owner_stub_ || !surface_texture_.get())
+    return true;
+
+  if (!offset.IsOrigin()) {
+    LOG(ERROR) << "Non-origin offset is not supported";
+    return false;
+  }
+
+  if (rect != gfx::Rect(size_)) {
+    LOG(ERROR) << "Sub-rectangle is not supported";
+    return false;
+  }
+  GLint target_texture = 0;
+  glGetIntegerv(GL_TEXTURE_BINDING_2D, &target_texture);
+  DCHECK(target_texture);
+
+  DoUpdateTexImage();
+  UpdateTransformMatrix();
+
+  if (!framebuffer_) {
+    glGenFramebuffersEXT(1, &framebuffer_);
+
+    // clang-format off
+    const char kVertexShader[] = STRINGIZE(
+      attribute vec2 a_position;
+      varying vec2 v_texCoord;
+      uniform mat4 u_xform;
+      void main() {
+        gl_Position = vec4(a_position.x, a_position.y, 0.0, 1.0);
+        vec2 uv_untransformed = (a_position + vec2(1.0, 1.0)) * 0.5;
+        v_texCoord = (u_xform * vec4(uv_untransformed, 0.0, 1.0)).xy;
+      }
+    );
+    const char kFragmentShader[] =
+      "#extension GL_OES_EGL_image_external : require\n" STRINGIZE(
+      precision mediump float;
+      uniform samplerExternalOES a_texture;
+      varying vec2 v_texCoord;
+      void main() {
+        gl_FragColor = texture2D(a_texture, v_texCoord);
+      }
+    );
+    // clang-format on
+
+    vertex_buffer_ = gfx::GLHelper::SetupQuadVertexBuffer();
+    vertex_shader_ = gfx::GLHelper::LoadShader(GL_VERTEX_SHADER, kVertexShader);
+    fragment_shader_ =
+        gfx::GLHelper::LoadShader(GL_FRAGMENT_SHADER, kFragmentShader);
+    program_ = gfx::GLHelper::SetupProgram(vertex_shader_, fragment_shader_);
+    gfx::ScopedUseProgram use_program(program_);
+    int sampler_location = glGetUniformLocation(program_, "a_texture");
+    DCHECK_NE(-1, sampler_location);
+    glUniform1i(sampler_location, 0);
+    u_xform_loc_ = glGetUniformLocation(program_, "u_xform");
+    DCHECK_NE(-1, u_xform_loc_);
+  }
+
+  gfx::ScopedActiveTexture active_texture(GL_TEXTURE0);
+  // UpdateTexImage() call below will bind the surface texture to
+  // TEXTURE_EXTERNAL_OES. This scoped texture binder will restore the current
+  // binding before this function returns.
+  gfx::ScopedTextureBinder texture_binder(GL_TEXTURE_EXTERNAL_OES, texture_id_);
+
+  {
+    gfx::ScopedFrameBufferBinder framebuffer_binder(framebuffer_);
+    gfx::ScopedViewport viewport(0, 0, size_.width(), size_.height());
+    glFramebufferTexture2DEXT(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
+                              GL_TEXTURE_2D, target_texture, 0);
+    DCHECK_EQ(static_cast<GLenum>(GL_FRAMEBUFFER_COMPLETE),
+              glCheckFramebufferStatusEXT(GL_FRAMEBUFFER));
+    gfx::ScopedUseProgram use_program(program_);
+
+    glUniformMatrix4fv(u_xform_loc_, 1, false, current_matrix_);
+    gfx::GLHelper::DrawQuad(vertex_buffer_);
+
+    // Detach the output texture from the fbo.
+    glFramebufferTexture2DEXT(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
+                              GL_TEXTURE_2D, 0, 0);
+  }
+
+  return true;
+}
+
 bool StreamTexture::CopyTexImage(unsigned target) {

[reveman, 2015/12/23 23:11:49]

CopyTexImage unlike CopySubTexImage is supposed to define the storage for
texture. See CopyTexImage version in https://codereview.chromium.org/1419623008
where I call TexImage2D. I think we need to do the same here.

Btw, can we do the same I'm doing in that patch and call CopyTexSubImage here
instead of adding a DoCopyTexture function?

[Tobias Sargeant, 2015/12/24 01:28:34]

Done.

+  if (target == GL_TEXTURE_2D) {
+    return DoCopyTexture(gfx::Point(), gfx::Rect(size_));
+  }
+
   if (target != GL_TEXTURE_EXTERNAL_OES)
     return false;
 
   if (!owner_stub_ || !surface_texture_.get())
     return true;
 
   GLint texture_id;
   glGetIntegerv(GL_TEXTURE_BINDING_EXTERNAL_OES, &texture_id);
   DCHECK(texture_id);
 
   // The following code only works if we're being asked to copy into
   // |texture_id_|. Copying into a different texture is not supported.
   if (static_cast<unsigned>(texture_id) != texture_id_)
     return false;
 
-  if (has_pending_frame_) {
-    scoped_ptr<ui::ScopedMakeCurrent> scoped_make_current;
-    bool needs_make_current =
-        !owner_stub_->decoder()->GetGLContext()->IsCurrent(NULL);
-    // On Android we should not have to perform a real context switch here when
-    // using virtual contexts.
-    DCHECK(!needs_make_current || !owner_stub_->decoder()
-                                       ->GetContextGroup()
-                                       ->feature_info()
-                                       ->workarounds()
-                                       .use_virtualized_gl_contexts);
-    if (needs_make_current) {
-      scoped_make_current.reset(new ui::ScopedMakeCurrent(
-          owner_stub_->decoder()->GetGLContext(), owner_stub_->surface()));
-    }
-    surface_texture_->UpdateTexImage();
-    has_valid_frame_ = true;
-    has_pending_frame_ = false;
-    if (scoped_make_current.get()) {
-      // UpdateTexImage() implies glBindTexture().
-      // The cmd decoder takes care of restoring the binding for this GLImage as
-      // far as the current context is concerned, but if we temporarily change
-      // it, we have to keep the state intact in *that* context also.
-      const gpu::gles2::ContextState* state =
-          owner_stub_->decoder()->GetContextState();
-      const gpu::gles2::TextureUnit& active_unit =
-          state->texture_units[state->active_texture_unit];
-      glBindTexture(GL_TEXTURE_EXTERNAL_OES,
-                    active_unit.bound_texture_external_oes.get()
-                        ? active_unit.bound_texture_external_oes->service_id()
-                        : 0);
-    }
-  }
+  DoUpdateTexImage();
 
   TextureManager* texture_manager =
       owner_stub_->decoder()->GetContextGroup()->texture_manager();
   gpu::gles2::Texture* texture =
       texture_manager->GetTextureForServiceId(texture_id_);
   if (texture) {
     // By setting image state to UNBOUND instead of COPIED we ensure that
     // CopyTexImage() is called each time the surface texture is used for
     // drawing.
     texture->SetLevelImage(GL_TEXTURE_EXTERNAL_OES, 0, this,
                            gpu::gles2::Texture::UNBOUND);
   }
 
-  if (has_listener_ && has_valid_frame_) {
+  UpdateTransformMatrix();
+  return true;
+}
+
+void StreamTexture::UpdateTransformMatrix() {

[reveman, 2015/12/23 23:11:49]

can we move the contents of this function into DoUpdateTexImage instead?

maybe use surface_texture_->GetTransformMatrix directly in the copy code and
limit current_matrix_ to track what has been sent to the client as before? that
way we don't have to add matrix_modified_..

[Tobias Sargeant, 2015/12/24 01:28:34]

Done. matrix_modfied_ didn't need to be added in any case. I'm still a bit
worried that if the listener changes (or is set after the matrix is first
updated) then it will never be told what the current matrix is. But given that
the existing code doesn't exhibit that problem, I think it's not an issue in
practice.

[reveman, 2015/12/24 02:14:43]

Right, seems like we should call UpdateTexImage when adding the stub and
listener but that can be done in a follow up as not a new problem.

+  if (has_valid_frame_) {
     float mtx[16];
     surface_texture_->GetTransformMatrix(mtx);
 
     // Only query the matrix once we have bound a valid frame.
     if (memcmp(current_matrix_, mtx, sizeof(mtx)) != 0) {
+      matrix_modified_ = true;
       memcpy(current_matrix_, mtx, sizeof(mtx));
+    }
 
+    if (has_listener_ && owner_stub_) {
       GpuStreamTextureMsg_MatrixChanged_Params params;
       memcpy(&params.m00, mtx, sizeof(mtx));
       owner_stub_->channel()->Send(
           new GpuStreamTextureMsg_MatrixChanged(route_id_, params));
+      matrix_modified_ = false;
     }
   }
-
-  return true;
 }
 
 void StreamTexture::OnFrameAvailable() {
   has_pending_frame_ = true;
   if (has_listener_ && owner_stub_) {
     owner_stub_->channel()->Send(
         new GpuStreamTextureMsg_FrameAvailable(route_id_));
   }
 }
 
@@ skipping 14 matching lines @@
     IPC_MESSAGE_UNHANDLED(handled = false)
   IPC_END_MESSAGE_MAP()
 
   DCHECK(handled);
   return handled;
 }
 
 void StreamTexture::OnStartListening() {
   DCHECK(!has_listener_);
   has_listener_ = true;
+  matrix_modified_ = true;
 }
 
 void StreamTexture::OnEstablishPeer(int32 primary_id, int32 secondary_id) {
   if (!owner_stub_)
     return;
 
   base::ProcessHandle process = owner_stub_->channel()->GetClientPID();
 
   SurfaceTexturePeer::GetInstance()->EstablishSurfaceTexturePeer(
       process, surface_texture_, primary_id, secondary_id);
 }
 
 bool StreamTexture::BindTexImage(unsigned target) {
   NOTREACHED();
   return false;
 }
 
 void StreamTexture::ReleaseTexImage(unsigned target) {
   NOTREACHED();
 }
 
 bool StreamTexture::CopyTexSubImage(unsigned target,
                                     const gfx::Point& offset,
                                     const gfx::Rect& rect) {
+  if (target == GL_TEXTURE_2D) {
+    return DoCopyTexture(offset, rect);
+  }
+
   return false;
 }
 
 bool StreamTexture::ScheduleOverlayPlane(gfx::AcceleratedWidget widget,
                                          int z_order,
                                          gfx::OverlayTransform transform,
                                          const gfx::Rect& bounds_rect,
                                          const gfx::RectF& crop_rect) {
   NOTREACHED();
   return false;
 }
 
 void StreamTexture::OnMemoryDump(base::trace_event::ProcessMemoryDump* pmd,
                                  uint64_t process_tracing_id,
                                  const std::string& dump_name) {
   // TODO(ericrk): Add OnMemoryDump for GLImages. crbug.com/514914
 }
 
 }  // namespace content