media: Optimize HW Video to 2D Canvas copy.

content/renderer/media/android/webmediaplayer_android.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/android/webmediaplayer_android.h"
 
 #include <limits>
 
 #include "base/android/build_info.h"
 #include "base/bind.h"
@@ skipping 24 matching lines @@
 #include "media/base/android/media_common_android.h"
 #include "media/base/android/media_player_android.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/media_keys.h"
 #include "media/base/media_log.h"
 #include "media/base/media_switches.h"
 #include "media/base/video_frame.h"
 #include "media/blink/webmediaplayer_delegate.h"
 #include "media/blink/webmediaplayer_util.h"
 #include "net/base/mime_util.h"
-#include "third_party/WebKit/public/platform/Platform.h"
 #include "third_party/WebKit/public/platform/WebContentDecryptionModuleResult.h"
-#include "third_party/WebKit/public/platform/WebGraphicsContext3DProvider.h"
+#include "third_party/WebKit/public/platform/WebGraphicsContext3D.h"
 #include "third_party/WebKit/public/platform/WebMediaPlayerClient.h"
 #include "third_party/WebKit/public/platform/WebString.h"
 #include "third_party/WebKit/public/platform/WebURL.h"
 #include "third_party/WebKit/public/web/WebDocument.h"
 #include "third_party/WebKit/public/web/WebFrame.h"
 #include "third_party/WebKit/public/web/WebRuntimeFeatures.h"
 #include "third_party/WebKit/public/web/WebSecurityOrigin.h"
 #include "third_party/WebKit/public/web/WebView.h"
 #include "third_party/skia/include/core/SkCanvas.h"
-#include "third_party/skia/include/core/SkPaint.h"
 #include "third_party/skia/include/core/SkTypeface.h"
 #include "ui/gfx/image/image.h"
+#include "webkit/common/gpu/context_provider_web_context.h"
+#include "webkit/common/gpu/webgraphicscontext3d_impl.h"
 
 static const uint32 kGLTextureExternalOES = 0x8D65;
 static const int kSDKVersionToSupportSecurityOriginCheck = 20;
 
 using blink::WebMediaPlayer;
 using blink::WebSize;
 using blink::WebString;
 using blink::WebURL;
 using gpu::gles2::GLES2Interface;
 using media::MediaPlayerAndroid;
 using media::VideoFrame;
 
 namespace {
 // Prefix for histograms related to Encrypted Media Extensions.
 const char* kMediaEme = "Media.EME.";
 
 // File-static function is to allow it to run even after WMPA is deleted.
 void OnReleaseTexture(
     const scoped_refptr<content::StreamTextureFactory>& factories,
     uint32 texture_id,
     uint32 release_sync_point) {
   GLES2Interface* gl = factories->ContextGL();
   gl->WaitSyncPointCHROMIUM(release_sync_point);
   gl->DeleteTextures(1, &texture_id);
 }
 
-class SyncPointClientImpl : public media::VideoFrame::SyncPointClient {
- public:
-  explicit SyncPointClientImpl(
-      blink::WebGraphicsContext3D* web_graphics_context)
-      : web_graphics_context_(web_graphics_context) {}
-  virtual ~SyncPointClientImpl() {}
-  virtual uint32 InsertSyncPoint() override {
-    return web_graphics_context_->insertSyncPoint();
-  }
-  virtual void WaitSyncPoint(uint32 sync_point) override {
-    web_graphics_context_->waitSyncPoint(sync_point);
-  }
-
- private:
-  blink::WebGraphicsContext3D* web_graphics_context_;
-};
-
 // Used for calls to decryptor_ready_cb_ where the result can be ignored.
 void DoNothing(bool) {
 }
 
 }  // namespace
 
 namespace content {
 
 WebMediaPlayerAndroid::WebMediaPlayerAndroid(
     blink::WebFrame* frame,
@@ skipping 420 matching lines @@
   const blink::WebTimeRange seekable_range(0.0, duration());
   return blink::WebTimeRanges(&seekable_range, 1);
 }
 
 bool WebMediaPlayerAndroid::didLoadingProgress() {
   bool ret = did_loading_progress_;
   did_loading_progress_ = false;
   return ret;
 }
 
-bool WebMediaPlayerAndroid::EnsureTextureBackedSkBitmap(GrContext* gr,
-                                                        SkBitmap& bitmap,
-                                                        const WebSize& size,
-                                                        GrSurfaceOrigin origin,
-                                                        GrPixelConfig config) {
-  DCHECK(main_thread_checker_.CalledOnValidThread());
-  if (!bitmap.getTexture() || bitmap.width() != size.width
-      || bitmap.height() != size.height) {
-    if (!gr)
-      return false;
-    GrTextureDesc desc;
-    desc.fConfig = config;
-    desc.fFlags = kRenderTarget_GrTextureFlagBit | kNoStencil_GrTextureFlagBit;
-    desc.fSampleCnt = 0;
-    desc.fOrigin = origin;
-    desc.fWidth = size.width;
-    desc.fHeight = size.height;
-    skia::RefPtr<GrTexture> texture;
-    texture = skia::AdoptRef(gr->createUncachedTexture(desc, 0, 0));
-    if (!texture.get())
-      return false;
-
-    SkImageInfo info = SkImageInfo::MakeN32Premul(desc.fWidth, desc.fHeight);
-    SkGrPixelRef* pixelRef = SkNEW_ARGS(SkGrPixelRef, (info, texture.get()));
-    if (!pixelRef)
-      return false;
-    bitmap.setInfo(info);
-    bitmap.setPixelRef(pixelRef)->unref();
-  }
-
-  return true;
-}
-
 void WebMediaPlayerAndroid::paint(blink::WebCanvas* canvas,
                                   const blink::WebRect& rect,
                                   unsigned char alpha,
                                   SkXfermode::Mode mode) {
   DCHECK(main_thread_checker_.CalledOnValidThread());
-  scoped_ptr<blink::WebGraphicsContext3DProvider> provider =
-    scoped_ptr<blink::WebGraphicsContext3DProvider>(blink::Platform::current(
-      )->createSharedOffscreenGraphicsContext3DProvider());
-  if (!provider)
-    return;
-  blink::WebGraphicsContext3D* context3D = provider->context3d();
-  if (!context3D)
+  // Don't allow clients to copy an encrypted video frame.
+  if (needs_external_surface_)
     return;
 
-  // Copy video texture into a RGBA texture based bitmap first as video texture
-  // on Android is GL_TEXTURE_EXTERNAL_OES which is not supported by Skia yet.
-  // The bitmap's size needs to be the same as the video and use naturalSize()
-  // here. Check if we could reuse existing texture based bitmap.
-  // Otherwise, release existing texture based bitmap and allocate
-  // a new one based on video size.
-  if (!EnsureTextureBackedSkBitmap(provider->grContext(), bitmap_,
-      naturalSize(), kTopLeft_GrSurfaceOrigin, kSkia8888_GrPixelConfig)) {
-    return;
+  scoped_refptr<VideoFrame> video_frame;
+  {
+    base::AutoLock auto_lock(current_frame_lock_);
+    video_frame = current_frame_;
   }
 
-  unsigned textureId = static_cast<unsigned>(
-    (bitmap_.getTexture())->getTextureHandle());
-  if (!copyVideoTextureToPlatformTexture(context3D, textureId, 0,
-      GL_RGBA, GL_UNSIGNED_BYTE, true, false)) {
-    return;
+  gfx::Rect gfx_rect(rect);
+  media::Context3DProvider provider{nullptr, nullptr};
+  if (canvas->getGrContext() ||
+      (video_frame.get() &&

[scherkus (not reviewing), 2014/10/30 20:40:11]

nit: is the .get() needed?

[danakj, 2014/10/30 21:02:52]

It is for scoped_refptr for now, which this is. But not for scoped_ptr, which
makes this seem inconsistent. scoped_refptr doesn't have a bool-like operator
yet. (Part of transition to get rid of implicit T* conversions.)

[dshwang, 2014/10/31 09:29:16]

danakj, thank you for answering.
scoped_refptr::operator T*() is defined in the strange guard and in linux it's
disabled.
https://code.google.com/p/chromium/codesearch#chromium/src/base/memory/ref_co...

+       video_frame->format() == media::VideoFrame::NATIVE_TEXTURE)) {
+    scoped_refptr<cc::ContextProvider> context_provider =
+        RenderThreadImpl::current()->SharedMainThreadContextProvider();
+    // GPU Process crashed.
+    if (!context_provider.get())
+      return;
+    provider = {context_provider->ContextGL(), context_provider->GrContext()};
   }
-
-  // Draw the texture based bitmap onto the Canvas. If the canvas is
-  // hardware based, this will do a GPU-GPU texture copy.
-  // If the canvas is software based, the texture based bitmap will be
-  // readbacked to system memory then draw onto the canvas.
-  SkRect dest;
-  dest.set(rect.x, rect.y, rect.x + rect.width, rect.y + rect.height);
-  SkPaint paint;
-  paint.setAlpha(alpha);
-  paint.setXfermodeMode(mode);
-  // It is not necessary to pass the dest into the drawBitmap call since all
-  // the context have been set up before calling paintCurrentFrameInContext.
-  canvas->drawBitmapRect(bitmap_, 0, dest, &paint);
+  skcanvas_video_renderer_.Paint(video_frame.get(),
+                                 canvas,
+                                 gfx_rect,
+                                 alpha,
+                                 mode,
+                                 media::VIDEO_ROTATION_0,
+                                 provider);
 }
 
 bool WebMediaPlayerAndroid::copyVideoTextureToPlatformTexture(
     blink::WebGraphicsContext3D* web_graphics_context,
     unsigned int texture,
     unsigned int level,
     unsigned int internal_format,
     unsigned int type,
     bool premultiply_alpha,
     bool flip_y) {
   DCHECK(main_thread_checker_.CalledOnValidThread());
   // Don't allow clients to copy an encrypted video frame.
   if (needs_external_surface_)
     return false;
 
   scoped_refptr<VideoFrame> video_frame;
   {
     base::AutoLock auto_lock(current_frame_lock_);
     video_frame = current_frame_;
   }
 
   if (!video_frame.get() ||
       video_frame->format() != media::VideoFrame::NATIVE_TEXTURE)
     return false;
+
+#if DCHECK_IS_ON
   const gpu::MailboxHolder* mailbox_holder = video_frame->mailbox_holder();
   DCHECK((!is_remote_ &&
           mailbox_holder->texture_target == GL_TEXTURE_EXTERNAL_OES) ||
          (is_remote_ && mailbox_holder->texture_target == GL_TEXTURE_2D));
+#endif
 
-  web_graphics_context->waitSyncPoint(mailbox_holder->sync_point);
-
-  // Ensure the target of texture is set before copyTextureCHROMIUM, otherwise
-  // an invalid texture target may be used for copy texture.
-  uint32 src_texture = web_graphics_context->createAndConsumeTextureCHROMIUM(
-      mailbox_holder->texture_target, mailbox_holder->mailbox.name);
-
-  // The video is stored in an unmultiplied format, so premultiply if
-  // necessary.
-  web_graphics_context->pixelStorei(GL_UNPACK_PREMULTIPLY_ALPHA_CHROMIUM,
-                                    premultiply_alpha);
-
-  // Application itself needs to take care of setting the right flip_y
-  // value down to get the expected result.
-  // flip_y==true means to reverse the video orientation while
-  // flip_y==false means to keep the intrinsic orientation.
-  web_graphics_context->pixelStorei(GL_UNPACK_FLIP_Y_CHROMIUM, flip_y);
-  web_graphics_context->copyTextureCHROMIUM(GL_TEXTURE_2D, src_texture,
-                                            texture, level, internal_format,
-                                            type);
-  web_graphics_context->pixelStorei(GL_UNPACK_FLIP_Y_CHROMIUM, false);
-  web_graphics_context->pixelStorei(GL_UNPACK_PREMULTIPLY_ALPHA_CHROMIUM,
-                                    false);
-
-  web_graphics_context->deleteTexture(src_texture);
-  web_graphics_context->flush();
-
-  SyncPointClientImpl client(web_graphics_context);
-  video_frame->UpdateReleaseSyncPoint(&client);
+  // TODO(dshwang): need more elegant way to convert WebGraphicsContext3D to
+  // GLES2Interface.
+  gpu::gles2::GLES2Interface* gl =
+      static_cast<webkit::gpu::WebGraphicsContext3DImpl*>(web_graphics_context)
+          ->GetGLInterface();
+  media::SkCanvasVideoRenderer::CopyVideoFrameTextureToGLTexture(
+      gl,
+      video_frame.get(),

[scherkus (not reviewing), 2014/10/30 20:40:11]

nit: is .get() needed?

[dshwang, 2014/10/31 09:29:16]

not needed. done.

+      texture,
+      level,
+      internal_format,
+      type,
+      premultiply_alpha,
+      flip_y);
   return true;
 }
 
 bool WebMediaPlayerAndroid::hasSingleSecurityOrigin() const {
   DCHECK(main_thread_checker_.CalledOnValidThread());
   if (player_type_ != MEDIA_PLAYER_TYPE_URL)
     return true;
 
   if (!info_loader_ || !info_loader_->HasSingleOrigin())
     return false;
@@ skipping 1137 matching lines @@
 
 bool WebMediaPlayerAndroid::IsHLSStream() const {
   std::string mime;
   GURL url = redirected_url_.is_empty() ? url_ : redirected_url_;
   if (!net::GetMimeTypeFromFile(base::FilePath(url.path()), &mime))
     return false;
   return !mime.compare("application/x-mpegurl");
 }
 
 }  // namespace content