Synchronize OffscreenCanvas content with the placeholder canvas

third_party/WebKit/Source/platform/graphics/OffscreenCanvasFrameDispatcherImpl.cpp

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "platform/graphics/OffscreenCanvasFrameDispatcherImpl.h"
 
 #include "cc/output/compositor_frame.h"
 #include "cc/quads/texture_draw_quad.h"
 #include "gpu/command_buffer/client/gles2_interface.h"
+#include "platform/CrossThreadFunctional.h"
 #include "platform/Histogram.h"
+#include "platform/WebTaskRunner.h"
+#include "platform/graphics/OffscreenCanvasPlaceholder.h"
 #include "platform/graphics/gpu/SharedGpuContext.h"
 #include "public/platform/InterfaceProvider.h"
 #include "public/platform/Platform.h"
 #include "public/platform/WebGraphicsContext3DProvider.h"
 #include "public/platform/modules/offscreencanvas/offscreen_canvas_surface.mojom-blink.h"
 #include "third_party/khronos/GLES2/gl2.h"
 #include "third_party/khronos/GLES2/gl2ext.h"
 #include "third_party/skia/include/core/SkColor.h"
 #include "third_party/skia/include/core/SkImage.h"
 #include "ui/gfx/geometry/rect.h"
 #include "ui/gfx/transform.h"
 #include "wtf/typed_arrays/ArrayBuffer.h"
 #include "wtf/typed_arrays/Uint8Array.h"
 
 namespace blink {
 
 OffscreenCanvasFrameDispatcherImpl::OffscreenCanvasFrameDispatcherImpl(
     uint32_t clientId,
     uint32_t sinkId,
     uint32_t localId,
     uint64_t nonce,
+    int canvasId,
     int width,
     int height)
     : m_surfaceId(cc::FrameSinkId(clientId, sinkId),
                   cc::LocalFrameId(localId, nonce)),
       m_width(width),
       m_height(height),
       m_nextResourceId(1u),
-      m_binding(this) {
+      m_binding(this),
+      m_placeholderCanvasId(canvasId) {
   DCHECK(!m_sink.is_bound());
   mojom::blink::OffscreenCanvasCompositorFrameSinkProviderPtr provider;
   Platform::current()->interfaceProvider()->getInterface(
       mojo::GetProxy(&provider));
   provider->CreateCompositorFrameSink(m_surfaceId,
                                       m_binding.CreateInterfacePtrAndBind(),
                                       mojo::GetProxy(&m_sink));
 }
 
 void OffscreenCanvasFrameDispatcherImpl::setTransferableResourceToSharedBitmap(
@@ skipping 89 matching lines @@
   resource.mailbox_holder =
       gpu::MailboxHolder(image->mailbox(), image->syncToken(), GL_TEXTURE_2D);
   resource.read_lock_fences_enabled = false;
   resource.is_software = false;
 
   // Hold ref to |image|, to keep it alive until the browser ReclaimResources.
   // It guarantees that the resource is not re-used or deleted.
   m_cachedImages.add(m_nextResourceId, std::move(image));
 }
 
+namespace {
+
+void updatePlaceholderImage(WeakPtr<OffscreenCanvasFrameDispatcher> dispatcher,
+                            std::unique_ptr<WebTaskRunner> taskRunner,
+                            int placeholderCanvasId,
+                            RefPtr<blink::Image> image,
+                            unsigned resourceId) {
+  DCHECK(isMainThread());
+  OffscreenCanvasPlaceholder* placeholderCanvas =
+      OffscreenCanvasPlaceholder::getPlaceholderById(placeholderCanvasId);
+  if (placeholderCanvas) {
+    placeholderCanvas->setPlaceholderFrame(std::move(image),
+                                           std::move(dispatcher),
+                                           std::move(taskRunner), resourceId);
+  }
+}
+
+}  // namespace
+
 void OffscreenCanvasFrameDispatcherImpl::dispatchFrame(
     RefPtr<StaticBitmapImage> image,
     double commitStartTime,
     bool isWebGLSoftwareRendering /* This flag is true when WebGL's commit is
     called on SwiftShader. */) {
   if (!image)
     return;
   if (!verifyImageSize(image->size()))
     return;
   cc::CompositorFrame frame;
@@ skipping 45 matching lines @@
         !isWebGLSoftwareRendering) {
       // Case 3: canvas is not gpu-accelerated, but compositor is
       commitType = CommitSoftwareCanvasGPUCompositing;
       setTransferableResourceToSharedGPUContext(resource, image);
     } else {
       // Case 4: both canvas and compositor are not gpu accelerated.
       commitType = CommitSoftwareCanvasSoftwareCompositing;
       setTransferableResourceToSharedBitmap(resource, image);
     }
   }
+
+  // After this point, |image| can only be used on the main thread, until
+  // it is returned.
+  image->transfer();

[xlai (Olivia), 2016/11/11 16:28:33]

In the case of gpu-accelerated offscreencanvas, this image is kept as a resource
in OffscreenCanvasFrameDispatcher in worker. I'm not sure if you transferring it
to main thread here will cause any trouble.

[Justin Novosad, 2016/11/11 22:09:00]

The idea is that We don't need to use this snapshot on the worker except for
when comes the time to delete the resource. So this should be fine.
Except that I was missing the reciprocal transfer in
OffscreenCanvasPlaceholder::releasePlaceholderFrame.  That is fixed now.

+  std::unique_ptr<WebTaskRunner> dispatcherTaskRunner =
+      Platform::current()->currentThread()->getWebTaskRunner()->clone();
+
+  Platform::current()->mainThread()->getWebTaskRunner()->postTask(

[xlai (Olivia), 2016/11/11 16:28:33]

This dispatchFrame() is invoked in every commit(). Whilst I understand that you
need to refresh the new frame and resource id for every commit(), you may end up
posting certain duplicate information of (dispatcherTaskRunner,
this->createWeakPtr()) to the main thread repeatedly. This is an overhead that
may become more and more painful when there are lots of commit().

A better place to send the dispatcherTaskRunner and this->createWeakPtr() should
be in OffscreenCanvas::getContext(), which is guaranteed to be invoked once in a
lifetime of an OffscreenCanvas.

[Justin Novosad, 2016/11/11 22:09:00]

I tried that and it is actually more complicated with no real reduction in
overhead.  The problem is that if OffscreenCanvasPlaceholder retains a
long-lived WeakPtr, then it can't std::move the weakptr when posting the call to
releaseFrameToDispatcher.  To solve that, you'd have to pass a weakPtr to the
OffscreenCanvasPlaceholder so that we can access m_frameDispatcherTaskRunner. 
And we're right back where we were, creating a weakPtr at every frame, except
that now, we've also added a weakptr factory on OffscreenCanvasPlaceholder.

+      BLINK_FROM_HERE,
+      crossThreadBind(updatePlaceholderImage, this->createWeakPtr(),
+                      passed(std::move(dispatcherTaskRunner)),
+                      m_placeholderCanvasId, std::move(image), resource.id));
+  m_spareResourceLocks.add(m_nextResourceId);
+
   commitTypeHistogram.count(commitType);
 
   m_nextResourceId++;
   frame.resource_list.push_back(std::move(resource));
 
   cc::TextureDrawQuad* quad =
       pass->CreateAndAppendDrawQuad<cc::TextureDrawQuad>();
   gfx::Size rectSize(m_width, m_height);
 
   const bool needsBlending = true;
   // TOOD(crbug.com/645993): this should be inherited from WebGL context's
   // creation settings.
   const bool premultipliedAlpha = true;
   const gfx::PointF uvTopLeft(0.f, 0.f);
   const gfx::PointF uvBottomRight(1.f, 1.f);
   float vertexOpacity[4] = {1.f, 1.f, 1.f, 1.f};
   // TODO(crbug.com/645994): this should be true when using style
   // "image-rendering: pixelated".
   const bool nearestNeighbor = false;
   quad->SetAll(sqs, bounds, bounds, bounds, needsBlending, resource.id,
                gfx::Size(), premultipliedAlpha, uvTopLeft, uvBottomRight,
                SK_ColorTRANSPARENT, vertexOpacity, yflipped, nearestNeighbor,
                false);
 
   frame.render_pass_list.push_back(std::move(pass));
 
   double elapsedTime = WTF::monotonicallyIncreasingTime() - commitStartTime;
+
+  // TODO(crbug.com/663916): The off-main-thread metrics are commented-out
+  // because they cause thread check errors (static variable accessed in many
+  // threads)
   switch (commitType) {
     case CommitGPUCanvasGPUCompositing:
       if (isMainThread()) {
         DEFINE_STATIC_LOCAL(
             CustomCountHistogram, commitGPUCanvasGPUCompositingMainTimer,
             ("Blink.Canvas.OffscreenCommit.GPUCanvasGPUCompositingMain", 0,
              10000000, 50));
         commitGPUCanvasGPUCompositingMainTimer.count(elapsedTime * 1000000.0);
-      } else {
+      } /* else {
         DEFINE_STATIC_LOCAL(
             CustomCountHistogram, commitGPUCanvasGPUCompositingWorkerTimer,
             ("Blink.Canvas.OffscreenCommit.GPUCanvasGPUCompositingWorker", 0,
              10000000, 50));
         commitGPUCanvasGPUCompositingWorkerTimer.count(elapsedTime * 1000000.0);
-      }
+      } */
       break;
     case CommitGPUCanvasSoftwareCompositing:
       if (isMainThread()) {
         DEFINE_STATIC_LOCAL(
             CustomCountHistogram, commitGPUCanvasSoftwareCompositingMainTimer,
             ("Blink.Canvas.OffscreenCommit.GPUCanvasSoftwareCompositingMain", 0,
              10000000, 50));
         commitGPUCanvasSoftwareCompositingMainTimer.count(elapsedTime *
                                                           1000000.0);
-      } else {
+      } /* else {
         DEFINE_STATIC_LOCAL(
             CustomCountHistogram, commitGPUCanvasSoftwareCompositingWorkerTimer,
             ("Blink.Canvas.OffscreenCommit.GPUCanvasSoftwareCompositingWorker",
              0, 10000000, 50));
         commitGPUCanvasSoftwareCompositingWorkerTimer.count(elapsedTime *
                                                             1000000.0);
-      }
+      } */
       break;
     case CommitSoftwareCanvasGPUCompositing:
       if (isMainThread()) {
         DEFINE_STATIC_LOCAL(
             CustomCountHistogram, commitSoftwareCanvasGPUCompositingMainTimer,
             ("Blink.Canvas.OffscreenCommit.SoftwareCanvasGPUCompositingMain", 0,
              10000000, 50));
         commitSoftwareCanvasGPUCompositingMainTimer.count(elapsedTime *
                                                           1000000.0);
-      } else {
+      } /* else {
         DEFINE_STATIC_LOCAL(
             CustomCountHistogram, commitSoftwareCanvasGPUCompositingWorkerTimer,
             ("Blink.Canvas.OffscreenCommit.SoftwareCanvasGPUCompositingWorker",
              0, 10000000, 50));
         commitSoftwareCanvasGPUCompositingWorkerTimer.count(elapsedTime *
                                                             1000000.0);
-      }
+      } */
       break;
     case CommitSoftwareCanvasSoftwareCompositing:
       if (isMainThread()) {
         DEFINE_STATIC_LOCAL(CustomCountHistogram,
                             commitSoftwareCanvasSoftwareCompositingMainTimer,
                             ("Blink.Canvas.OffscreenCommit."
                              "SoftwareCanvasSoftwareCompositingMain",
                              0, 10000000, 50));
         commitSoftwareCanvasSoftwareCompositingMainTimer.count(elapsedTime *
                                                                1000000.0);
-      } else {
+      } /* else {
         DEFINE_STATIC_LOCAL(CustomCountHistogram,
                             commitSoftwareCanvasSoftwareCompositingWorkerTimer,
                             ("Blink.Canvas.OffscreenCommit."
                              "SoftwareCanvasSoftwareCompositingWorker",
                              0, 10000000, 50));
         commitSoftwareCanvasSoftwareCompositingWorkerTimer.count(elapsedTime *
                                                                  1000000.0);
-      }
+      } */
       break;
     case OffscreenCanvasCommitTypeCount:
       NOTREACHED();
   }
 
   m_sink->SubmitCompositorFrame(std::move(frame));
 }
 
 void OffscreenCanvasFrameDispatcherImpl::DidReceiveCompositorFrameAck() {
   // TODO(fsamuel): Implement this.
 }
 
 void OffscreenCanvasFrameDispatcherImpl::OnBeginFrame(
     const cc::BeginFrameArgs& beginFrameArgs) {}
 
 void OffscreenCanvasFrameDispatcherImpl::ReclaimResources(
     const cc::ReturnedResourceArray& resources) {
   for (const auto& resource : resources) {
     RefPtr<StaticBitmapImage> image = m_cachedImages.get(resource.id);
     if (image)
       image->updateSyncToken(resource.sync_token);
-    m_cachedImages.remove(resource.id);
-    m_sharedBitmaps.remove(resource.id);
-    m_cachedTextureIds.remove(resource.id);
+    reclaimResource(resource.id);
   }
 }
 
+void OffscreenCanvasFrameDispatcherImpl::reclaimResource(unsigned resourceId) {
+  // An image resource needs to be returned by both the
+  // CompositorFrameSink and the HTMLCanvasElement. These
+  // events can happen in any order.  The first of the two
+  // to return a given resource will result in the spare
+  // resource lock being lifted, and the second will delete
+  // the resource for real.
+  if (m_spareResourceLocks.contains(resourceId)) {
+    m_spareResourceLocks.remove(resourceId);
+    return;
+  }
+  m_cachedImages.remove(resourceId);
+  m_sharedBitmaps.remove(resourceId);
+  m_cachedTextureIds.remove(resourceId);
+}
+
 bool OffscreenCanvasFrameDispatcherImpl::verifyImageSize(
     const IntSize imageSize) {
   if (imageSize.width() == m_width && imageSize.height() == m_height)
     return true;
   return false;
 }
 
 }  // namespace blink