WebGL & 16-bit video stream: upload to FLOAT texture.

third_party/WebKit/Source/modules/webgl/WebGLRenderingContextBase.cpp

 /*
  * Copyright (C) 2009 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
@@ skipping 483 matching lines @@
  public:
   explicit ScopedFramebufferRestorer(WebGLRenderingContextBase* context)
       : m_context(context) {}
 
   ~ScopedFramebufferRestorer() { m_context->restoreCurrentFramebuffer(); }
 
  private:
   Member<WebGLRenderingContextBase> m_context;
 };
 
+class ScopedUnpackParametersResetRestore {
+  STACK_ALLOCATED();
+
+ public:
+  explicit ScopedUnpackParametersResetRestore(
+      WebGLRenderingContextBase* context,
+      bool enabled = true)

[pdr., 2016/11/22 19:16:01]

This restore class is a nice way of cleaning up the reset/restore calls but the
bool for enabling it is not obvious at the callsites. WDYT of using
base::Optional for this?

The pattern would look something like when there is a conditional:
Optional<ScopedUnpackParametersResetRestore> resetRestoreScope;
if (should reset restore)
  resetRestoreScope.emplace(this);

The pattern would look something like this when there's no conditional:
ScopedUnpackParametersResetRestore resetRestoreScope(this);

[Ken Russell (switch to Gerrit), 2016/11/22 22:47:44]

+1, this is nice.
I prefer the way Alex has done this. The Optional class seems more complex and
harder to understand to me.

+      : m_context(context), m_enabled(enabled) {
+    if (enabled)
+      m_context->resetUnpackParameters();
+  }
+
+  ~ScopedUnpackParametersResetRestore() {
+    if (m_enabled)
+      m_context->restoreUnpackParameters();
+  }
+
+ private:
+  Member<WebGLRenderingContextBase> m_context;
+  bool m_enabled;
+};
+
 static void formatWebGLStatusString(const StringView& glInfo,
                                     const StringView& infoString,
                                     StringBuilder& builder) {
   if (infoString.isEmpty())
     return;
   builder.append(", ");
   builder.append(glInfo);
   builder.append(" = ");
   builder.append(infoString);
 }
@@ skipping 3896 matching lines @@
             image, imagePixelData, format, type, flipY, alphaOp,
             sourceDataFormat, imageExtractor.imageWidth(),
             imageExtractor.imageHeight(), adjustedSourceImageRect, depth,
             imageExtractor.imageSourceUnpackAlignment(), unpackImageHeight,
             data)) {
       synthesizeGLError(GL_INVALID_VALUE, funcName, "packImage error");
       return;
     }
   }
 
-  resetUnpackParameters();
+  ScopedUnpackParametersResetRestore temporaryResetUnpack(this);
   if (functionID == TexImage2D) {
     texImage2DBase(target, level, internalformat,
                    adjustedSourceImageRect.width(),
                    adjustedSourceImageRect.height(), 0, format, type,
                    needConversion ? data.data() : imagePixelData);
   } else if (functionID == TexSubImage2D) {
     contextGL()->TexSubImage2D(target, level, xoffset, yoffset,
                                adjustedSourceImageRect.width(),
                                adjustedSourceImageRect.height(), format, type,
                                needConversion ? data.data() : imagePixelData);
   } else {
     // 3D functions.
     if (functionID == TexImage3D) {
       contextGL()->TexImage3D(
           target, level, internalformat, adjustedSourceImageRect.width(),
           adjustedSourceImageRect.height(), depth, 0, format, type,
           needConversion ? data.data() : imagePixelData);
     } else {
       DCHECK_EQ(functionID, TexSubImage3D);
       contextGL()->TexSubImage3D(
           target, level, xoffset, yoffset, zoffset,
           adjustedSourceImageRect.width(), adjustedSourceImageRect.height(),
           depth, format, type, needConversion ? data.data() : imagePixelData);
     }
   }
-  restoreUnpackParameters();
 }
 
 bool WebGLRenderingContextBase::validateTexFunc(
     const char* functionName,
     TexImageFunctionType functionType,
     TexFuncValidationSourceType sourceType,
     GLenum target,
     GLint level,
     GLenum internalformat,
     GLsizei width,
@@ skipping 186 matching lines @@
                             convertTexInternalFormat(internalformat, type),
                             width, height, depth, border, format, type, data);
     return;
   }
   if (functionID == TexSubImage3D) {
     contextGL()->TexSubImage3D(target, level, xoffset, yoffset, zoffset, width,
                                height, depth, format, type, data);
     return;
   }
 
-  if (changeUnpackAlignment)
-    resetUnpackParameters();
+  ScopedUnpackParametersResetRestore temporaryResetUnpack(
+      this, changeUnpackAlignment);
   if (functionID == TexImage2D)
     texImage2DBase(target, level, internalformat, width, height, border, format,
                    type, data);
   else if (functionID == TexSubImage2D)
     contextGL()->TexSubImage2D(target, level, xoffset, yoffset, width, height,
                                format, type, data);
-  if (changeUnpackAlignment)
-    restoreUnpackParameters();
 }
 
 void WebGLRenderingContextBase::texImage2D(GLenum target,
                                            GLint level,
                                            GLint internalformat,
                                            GLsizei width,
                                            GLsizei height,
                                            GLint border,
                                            GLenum format,
                                            GLenum type,
@@ skipping 67 matching lines @@
     }
     if (!WebGLImageConversion::extractImageData(
             pixels->data()->data(),
             WebGLImageConversion::DataFormat::DataFormatRGBA8, pixels->size(),
             adjustedSourceImageRect, depth, unpackImageHeight, format, type,
             m_unpackFlipY, m_unpackPremultiplyAlpha, data)) {
       synthesizeGLError(GL_INVALID_VALUE, funcName, "bad image data");
       return;
     }
   }
-  resetUnpackParameters();
+  ScopedUnpackParametersResetRestore temporaryResetUnpack(this);
   const uint8_t* bytes = needConversion ? data.data() : pixels->data()->data();
   if (functionID == TexImage2D) {
     DCHECK_EQ(unpackImageHeight, 0);
     texImage2DBase(
         target, level, internalformat, adjustedSourceImageRect.width(),
         adjustedSourceImageRect.height(), border, format, type, bytes);
   } else if (functionID == TexSubImage2D) {
     DCHECK_EQ(unpackImageHeight, 0);
     contextGL()->TexSubImage2D(
         target, level, xoffset, yoffset, adjustedSourceImageRect.width(),
         adjustedSourceImageRect.height(), format, type, bytes);
   } else {
     GLint uploadHeight = adjustedSourceImageRect.height();
     if (unpackImageHeight) {
       // GL_UNPACK_IMAGE_HEIGHT overrides the passed-in height.
       uploadHeight = unpackImageHeight;
     }
     if (functionID == TexImage3D) {
       contextGL()->TexImage3D(target, level, internalformat,
                               adjustedSourceImageRect.width(), uploadHeight,
                               depth, border, format, type, bytes);
     } else {
       DCHECK_EQ(functionID, TexSubImage3D);
       contextGL()->TexSubImage3D(target, level, xoffset, yoffset, zoffset,
                                  adjustedSourceImageRect.width(), uploadHeight,
                                  depth, format, type, bytes);
     }
   }
-  restoreUnpackParameters();
 }
 
 void WebGLRenderingContextBase::texImage2D(GLenum target,
                                            GLint level,
                                            GLint internalformat,
                                            GLenum format,
                                            GLenum type,
                                            ImageData* pixels) {
   texImageHelperImageData(TexImage2D, target, level, internalformat, 0, format,
                           type, 1, 0, 0, 0, pixels, getImageDataSize(pixels),
@@ skipping 349 matching lines @@
   if (!validateTexFunc(funcName, functionType, SourceHTMLVideoElement, target,
                        level, internalformat, video->videoWidth(),
                        video->videoHeight(), 1, 0, format, type, xoffset,
                        yoffset, zoffset))
     return;
 
   bool sourceImageRectIsDefault =
       sourceImageRect == sentinelEmptyRect() ||
       sourceImageRect ==
           IntRect(0, 0, video->videoWidth(), video->videoHeight());
-  if (functionID == TexImage2D && sourceImageRectIsDefault && depth == 1) {
+  if (functionID == TexImage2D && sourceImageRectIsDefault && depth == 1 &&
+      GL_TEXTURE_2D == target && Extensions3DUtil::canUseCopyTextureCHROMIUM(
+                                     target, internalformat, type, level)) {

[Ken Russell (switch to Gerrit), 2016/11/22 22:47:44]

This change to the if-test will make the semi-accelerated code path below (using
an accelerated image buffer) be taken less often. What is the rationale for
making this change? Can you figure out a way to preserve the current properties
of the code? If you want to inject the attempted call to
HTMLVideoElement::texImageImpl, can you figure out a slightly different code
structure to achieve this?

[aleksandar.stojiljkovic, 2016/11/22 23:21:06]

The change here is orthogonal in respect to the rest of the code. It is
resolving performance issue of using float textures regardless what the video
format is.
The only difference is that semi-accelerated code is under
Extensions3DUtil::canUseCopyTextureCHROMIUM.
And it should be, as ImageBuffer::copyToPlatformtexture l.5207 returns early
false (for e.g. float formats after canUseCopyTextureCHROMIUM) in [1]. 
So, there is no need to do new AcceleratedImageBufferSurface,
video->paintCurrentFrame and texImage2DBase in 5204 as the code would just early
leave call to l.5207.

[1]
https://cs.chromium.org/chromium/src/third_party/WebKit/Source/platform/graph...

 

[aleksandar.stojiljkovic, 2016/11/29 11:59:53]

Done. This landed in crrev.com/2527343002.

     DCHECK_EQ(xoffset, 0);
     DCHECK_EQ(yoffset, 0);
     DCHECK_EQ(zoffset, 0);
     // Go through the fast path doing a GPU-GPU textures copy without a readback
     // to system memory if possible.  Otherwise, it will fall back to the normal
     // SW path.
-    if (GL_TEXTURE_2D == target) {
-      if (Extensions3DUtil::canUseCopyTextureCHROMIUM(target, internalformat,
-                                                      type, level) &&
-          video->copyVideoTextureToPlatformTexture(
-              contextGL(), texture->object(), internalformat, type,
-              m_unpackPremultiplyAlpha, m_unpackFlipY)) {
-        return;
-      }
+    if (video->copyVideoTextureToPlatformTexture(
+            contextGL(), texture->object(), internalformat, type,
+            m_unpackPremultiplyAlpha, m_unpackFlipY)) {
+      return;
+    }
 
-      // Try using an accelerated image buffer, this allows YUV conversion to be
-      // done on the GPU.
-      std::unique_ptr<ImageBufferSurface> surface =
-          wrapUnique(new AcceleratedImageBufferSurface(
-              IntSize(video->videoWidth(), video->videoHeight())));
-      if (surface->isValid()) {
-        std::unique_ptr<ImageBuffer> imageBuffer(
-            ImageBuffer::create(std::move(surface)));
-        if (imageBuffer) {
-          // The video element paints an RGBA frame into our surface here. By
-          // using an AcceleratedImageBufferSurface, we enable the
-          // WebMediaPlayer implementation to do any necessary color space
-          // conversion on the GPU (though it
-          // may still do a CPU conversion and upload the results).
-          video->paintCurrentFrame(
-              imageBuffer->canvas(),
-              IntRect(0, 0, video->videoWidth(), video->videoHeight()),
-              nullptr);
+    // Try using an accelerated image buffer, this allows YUV conversion to be
+    // done on the GPU.
+    std::unique_ptr<ImageBufferSurface> surface =
+        wrapUnique(new AcceleratedImageBufferSurface(
+            IntSize(video->videoWidth(), video->videoHeight())));
+    if (surface->isValid()) {
+      std::unique_ptr<ImageBuffer> imageBuffer(
+          ImageBuffer::create(std::move(surface)));
+      if (imageBuffer) {
+        // The video element paints an RGBA frame into our surface here. By
+        // using an AcceleratedImageBufferSurface, we enable the WebMediaPlayer
+        // implementation to do any necessary color space conversion on the GPU
+        // (though it may still do a CPU conversion and upload the results).
+        video->paintCurrentFrame(
+            imageBuffer->canvas(),
+            IntRect(0, 0, video->videoWidth(), video->videoHeight()), nullptr);
 
-          // This is a straight GPU-GPU copy, any necessary color space
-          // conversion was handled in the paintCurrentFrameInContext() call.
+        // This is a straight GPU-GPU copy, any necessary color space conversion
+        // was handled in the paintCurrentFrameInContext() call.
 
-          // Note that copyToPlatformTexture no longer allocates the
-          // destination texture.
-          texImage2DBase(target, level, internalformat, video->videoWidth(),
-                         video->videoHeight(), 0, format, type, nullptr);
+        // Note that copyToPlatformTexture no longer allocates the destination
+        // texture.
+        texImage2DBase(target, level, internalformat, video->videoWidth(),
+                       video->videoHeight(), 0, format, type, nullptr);
 
-          if (imageBuffer->copyToPlatformTexture(
-                  contextGL(), texture->object(), internalformat, type, level,
-                  m_unpackPremultiplyAlpha, m_unpackFlipY, IntPoint(0, 0),
-                  IntRect(0, 0, video->videoWidth(), video->videoHeight()))) {
-            return;
-          }
+        if (imageBuffer->copyToPlatformTexture(
+                contextGL(), texture->object(), internalformat, type, level,
+                m_unpackPremultiplyAlpha, m_unpackFlipY, IntPoint(0, 0),
+                IntRect(0, 0, video->videoWidth(), video->videoHeight()))) {
+          return;
         }
       }
     }
   }
 
+  {
+    // Try using optimized CPU-GPU path for some formats: e.g. Y16 and Y8. It
+    // leaves early for other formats or if frame is stored on GPU.

[Ken Russell (switch to Gerrit), 2016/11/22 22:47:44]

This now needs to take into consideration the fact that a source sub-rectangle
might be specified for WebGL 2.0. The easiest way for the moment would be to
skip attempting this block if (!sourceImageRectIsDefault). It can be expanded
later to handle sub-rectangle uploads.

[aleksandar.stojiljkovic, 2016/11/22 23:21:06]

Thanks. I'll do so.

[aleksandar.stojiljkovic, 2016/11/27 20:44:00]

Done.

+    ScopedUnpackParametersResetRestore(
+        this, m_unpackFlipY || m_unpackPremultiplyAlpha);
+    if (video->texImageImpl(
+            static_cast<WebMediaPlayer::TexImageFunctionID>(functionID), target,
+            contextGL(), level, convertTexInternalFormat(internalformat, type),
+            format, type, xoffset, yoffset, zoffset, m_unpackFlipY,
+            m_unpackPremultiplyAlpha &&
+                m_unpackColorspaceConversion == GL_NONE))
+      return;
+  }
+
   RefPtr<Image> image = videoFrameToImage(video);
   if (!image)
     return;
   texImageImpl(functionID, target, level, internalformat, xoffset, yoffset,
                zoffset, format, type, image.get(),
                WebGLImageConversion::HtmlDomVideo, m_unpackFlipY,
                m_unpackPremultiplyAlpha, sourceImageRect, depth,
                unpackImageHeight);
 }
 
@@ skipping 121 matching lines @@
              type, false, false, data)) ||
         (isPixelDataRGBA &&
          !WebGLImageConversion::extractImageData(
              pixelDataPtr, WebGLImageConversion::DataFormat::DataFormatRGBA8,
              bitmap->size(), sourceSubRect, depth, unpackImageHeight, format,
              type, false, false, data))) {
       synthesizeGLError(GL_INVALID_VALUE, funcName, "bad image data");
       return;
     }
   }
-  resetUnpackParameters();
+  ScopedUnpackParametersResetRestore temporaryResetUnpack(this);
   if (functionID == TexImage2D) {
     texImage2DBase(target, level, internalformat, width, height, 0, format,
                    type, needConversion ? data.data() : pixelDataPtr);
   } else if (functionID == TexSubImage2D) {
     contextGL()->TexSubImage2D(target, level, xoffset, yoffset, width, height,
                                format, type,
                                needConversion ? data.data() : pixelDataPtr);
   } else if (functionID == TexImage3D) {
     contextGL()->TexImage3D(target, level, internalformat, width, height, depth,
                             0, format, type,
                             needConversion ? data.data() : pixelDataPtr);
   } else {
     DCHECK_EQ(functionID, TexSubImage3D);
     contextGL()->TexSubImage3D(target, level, xoffset, yoffset, zoffset, width,
                                height, depth, format, type,
                                needConversion ? data.data() : pixelDataPtr);
   }
-  restoreUnpackParameters();
 }
 
 void WebGLRenderingContextBase::texImage2D(GLenum target,
                                            GLint level,
                                            GLint internalformat,
                                            GLenum format,
                                            GLenum type,
                                            ImageBitmap* bitmap,
                                            ExceptionState& exceptionState) {
   texImageHelperImageBitmap(TexImage2D, target, level, internalformat, format,
@@ skipping 2398 matching lines @@
 
 void WebGLRenderingContextBase::getHTMLOrOffscreenCanvas(
     HTMLCanvasElementOrOffscreenCanvas& result) const {
   if (canvas())
     result.setHTMLCanvasElement(canvas());
   else
     result.setOffscreenCanvas(getOffscreenCanvas());
 }
 
 }  // namespace blink