media: Optimize HW Video to 2D Canvas copy.

media/filters/skcanvas_video_renderer.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/filters/skcanvas_video_renderer.h"
 
 #include "base/logging.h"
+#include "gpu/GLES2/gl2extchromium.h"
+#include "gpu/command_buffer/client/gles2_interface.h"
+#include "gpu/command_buffer/common/mailbox_holder.h"
 #include "media/base/video_frame.h"
 #include "media/base/yuv_convert.h"
+#include "skia/ext/refptr.h"
 #include "third_party/libyuv/include/libyuv.h"
-#include "third_party/skia/include/core/SkCanvas.h"
 #include "third_party/skia/include/core/SkImageGenerator.h"
 #include "third_party/skia/include/gpu/GrContext.h"
+#include "third_party/skia/include/gpu/SkGrPixelRef.h"
 #include "ui/gfx/skbitmap_operations.h"
 
 // Skia internal format depends on a platform. On Android it is ABGR, on others
 // it is ARGB.
 #if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \
     SK_A32_SHIFT == 24
 #define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB
 #define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB
 #elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \
     SK_A32_SHIFT == 24
@@ skipping 183 matching lines @@
           row_bytes);
       video_frame->ReadPixelsFromNativeTexture(tmp);
       break;
     }
     default:
       NOTREACHED();
       break;
   }
 }
 
+bool IsSkBitmapProperlySizedTexture(const SkBitmap* bitmap,
+                                    const gfx::Size& size) {
+  return bitmap->getTexture() && bitmap->width() == size.width() &&
+         bitmap->height() == size.height();
+}
+
+bool AllocateSkBitmapTexture(GrContext* gr,
+                             SkBitmap* bitmap,
+                             const gfx::Size& size) {
+  DCHECK(gr);
+  GrTextureDesc desc;
+  // Use kRGBA_8888_GrPixelConfig, not kSkia8888_GrPixelConfig, because
+  // glCopyTextureChromium doesn't support GL_BGRA_EXT as internal format.
+  desc.fConfig = kRGBA_8888_GrPixelConfig;
+  desc.fFlags = kRenderTarget_GrTextureFlagBit | kNoStencil_GrTextureFlagBit;
+  desc.fSampleCnt = 0;
+  desc.fOrigin = kTopLeft_GrSurfaceOrigin;
+  desc.fWidth = size.width();
+  desc.fHeight = size.height();
+  skia::RefPtr<GrTexture> texture = skia::AdoptRef(
+      gr->refScratchTexture(desc, GrContext::kExact_ScratchTexMatch));
+  if (!texture.get())
+    return false;
+
+  SkImageInfo info = SkImageInfo::MakeN32Premul(desc.fWidth, desc.fHeight);
+  SkGrPixelRef* pixelRef = SkNEW_ARGS(SkGrPixelRef, (info, texture.get()));

[scherkus (not reviewing), 2014/10/30 20:40:12]

pixelRef -> pixel_ref

[dshwang, 2014/10/31 09:29:17]

oops, Thank you.

+  if (!pixelRef)
+    return false;
+  bitmap->setInfo(info);
+  bitmap->setPixelRef(pixelRef)->unref();
+  return true;
+}
+
+bool CopyVideoFrameTextureToSkBitmapTexture(
+    VideoFrame* video_frame,

[scherkus (not reviewing), 2014/10/30 20:40:13]

const scoped_refptr<VideoFrame>&

[dshwang, 2014/10/31 09:29:17]

not needed because it doesn't take a ref

+    SkBitmap* bitmap,
+    const Context3DProvider& context_provider) {
+  // Check if we could reuse existing texture based bitmap.
+  // Otherwise, release existing texture based bitmap and allocate
+  // a new one based on video size.
+  if (!IsSkBitmapProperlySizedTexture(bitmap,
+                                      video_frame->visible_rect().size())) {
+    if (!AllocateSkBitmapTexture(context_provider.gr_context,
+                                 bitmap,
+                                 video_frame->visible_rect().size())) {
+      return false;
+    }
+  }
+
+  unsigned texture_id =
+      static_cast<unsigned>((bitmap->getTexture())->getTextureHandle());
+  // If CopyVideoFrameToTexture() changes the state of the |texture_id|, it's

[scherkus (not reviewing), 2014/10/30 20:40:12]

"CopyVideoFrameToTexture" doesn't refer to a function that exists

I believe you mean CopyVideoFrameTextureToGLTexture()

[dshwang, 2014/10/31 09:29:17]

Thank you!

+  // needed to invalidate the state cached in skia, but currently the state
+  // isn't changed.
+  SkCanvasVideoRenderer::CopyVideoFrameTextureToGLTexture(context_provider.gl,
+                                                          video_frame,
+                                                          texture_id,
+                                                          0,
+                                                          GL_RGBA,
+                                                          GL_UNSIGNED_BYTE,
+                                                          true,
+                                                          false);
+  return true;
+}
+
+class SyncPointClientImpl : public VideoFrame::SyncPointClient {
+ public:
+  explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl) : gl_(gl) {}
+  virtual ~SyncPointClientImpl() {}

[scherkus (not reviewing), 2014/10/30 20:40:12]

s/virtual/override/ everywhere

[dshwang, 2014/10/31 09:29:17]

interesting. Done.

+  virtual uint32 InsertSyncPoint() override {
+    return gl_->InsertSyncPointCHROMIUM();
+  }
+  virtual void WaitSyncPoint(uint32 sync_point) override {
+    gl_->WaitSyncPointCHROMIUM(sync_point);
+  }
+
+ private:
+  gpu::gles2::GLES2Interface* gl_;
+};

[scherkus (not reviewing), 2014/10/30 20:40:12]

DISALLOW_COPY_AND_ASSIGN()

[dshwang, 2014/10/31 09:29:17]

Done.

+
 }  // anonymous namespace
 
 // Generates an RGB image from a VideoFrame. Convert YUV to RGB plain on GPU.
 class VideoImageGenerator : public SkImageGenerator {
  public:
   VideoImageGenerator(const scoped_refptr<VideoFrame>& frame) : frame_(frame) {
     DCHECK(frame_.get());
   }
   ~VideoImageGenerator() override {}
 
@@ skipping 72 matching lines @@
   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoImageGenerator);
 };
 
 SkCanvasVideoRenderer::SkCanvasVideoRenderer()
     : last_frame_timestamp_(media::kNoTimestamp()),
       frame_deleting_timer_(
           FROM_HERE,
           base::TimeDelta::FromSeconds(kTemporaryResourceDeletionDelay),
           this,
           &SkCanvasVideoRenderer::ResetLastFrame),
-      accelerated_generator_(NULL),
+      accelerated_generator_(nullptr),
       accelerated_last_frame_timestamp_(media::kNoTimestamp()),
       accelerated_frame_deleting_timer_(
           FROM_HERE,
           base::TimeDelta::FromSeconds(kTemporaryResourceDeletionDelay),
           this,
           &SkCanvasVideoRenderer::ResetAcceleratedLastFrame) {
   last_frame_.setIsVolatile(true);
 }
 
 SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {}
 
 void SkCanvasVideoRenderer::Paint(const scoped_refptr<VideoFrame>& video_frame,
                                   SkCanvas* canvas,
                                   const gfx::RectF& dest_rect,
                                   uint8 alpha,
                                   SkXfermode::Mode mode,
-                                  VideoRotation video_rotation) {
+                                  VideoRotation video_rotation,
+                                  const Context3DProvider& context_provider) {
   if (alpha == 0) {
     return;
   }
 
   SkRect dest;
   dest.set(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom());
 
   SkPaint paint;
   paint.setAlpha(alpha);
 
   // Paint black rectangle if there isn't a frame available or the
   // frame has an unexpected format.
   if (!video_frame.get() || video_frame->natural_size().IsEmpty() ||
       !IsYUVOrNative(video_frame->format())) {
     canvas->drawRect(dest, paint);
     canvas->flush();
     return;
   }
 
-  SkBitmap* target_frame = NULL;
+  SkBitmap* target_frame = nullptr;
   if (canvas->getGrContext()) {
+    // TODO(dshwang): Android video decoder doesn't update the timestamp on a
+    // VideoFrame. To reduce redundant copy, Android should update the
+    // timestamp.
     if (accelerated_last_frame_.isNull() ||
+#if defined(OS_ANDROID)
+        video_frame->timestamp() == base::TimeDelta() ||
+#endif
         video_frame->timestamp() != accelerated_last_frame_timestamp_) {
-      accelerated_generator_ = new VideoImageGenerator(video_frame);
+      if (video_frame->format() == media::VideoFrame::NATIVE_TEXTURE) {
+        // Draw HW Video on HW Canvas.
+        DCHECK(context_provider.gl && context_provider.gr_context);

[scherkus (not reviewing), 2014/10/30 20:40:13]

nit: break out into two DCHECKs so we know which one was NULL if they fail

[dshwang, 2014/10/31 09:29:17]

Done.

+        DCHECK(!accelerated_generator_);
+        if (!CopyVideoFrameTextureToSkBitmapTexture(video_frame.get(),

[scherkus (not reviewing), 2014/10/30 20:40:13]

you can drop the .get() if you switch to using const scoped_refptr<VideoFrame>&
as suggested above

[dshwang, 2014/10/31 09:29:17]

CopyVideoFrameTextureToSkBitmapTexture doesn't take a ref, so I preserve it.

+                                                    &accelerated_last_frame_,
+                                                    context_provider)) {
+          NOTREACHED();
+          return;
+        }
+      } else {
+        // Draw SW Video on HW Canvas.
+        accelerated_generator_ = new VideoImageGenerator(video_frame);
 
-      // Note: This takes ownership of |accelerated_generator_|.
-      if (!SkInstallDiscardablePixelRef(accelerated_generator_,
-                                        &accelerated_last_frame_)) {
-        NOTREACHED();
+        // Note: This takes ownership of |accelerated_generator_|.
+        if (!SkInstallDiscardablePixelRef(accelerated_generator_,
+                                          &accelerated_last_frame_)) {
+          NOTREACHED();
+          return;
+        }
       }
       DCHECK(video_frame->visible_rect().width() ==
                  accelerated_last_frame_.width() &&
              video_frame->visible_rect().height() ==
                  accelerated_last_frame_.height());
 
       accelerated_last_frame_timestamp_ = video_frame->timestamp();
-    } else {
+    } else if (accelerated_generator_) {
       accelerated_generator_->set_frame(video_frame);
     }
     target_frame = &accelerated_last_frame_;
     accelerated_frame_deleting_timer_.Reset();
   } else {
-    // Check if we should convert and update |last_frame_|.
+    // Draw both SW and HW Video on SW Canvas.
     if (last_frame_.isNull() ||
         video_frame->timestamp() != last_frame_timestamp_) {
       // Check if |bitmap| needs to be (re)allocated.
       if (last_frame_.isNull() ||
           last_frame_.width() != video_frame->visible_rect().width() ||
           last_frame_.height() != video_frame->visible_rect().height()) {
         last_frame_.allocN32Pixels(video_frame->visible_rect().width(),
                                    video_frame->visible_rect().height());
         last_frame_.setIsVolatile(true);
       }
@@ skipping 47 matching lines @@
         SkFloatToScalar(rotated_dest_size.height() / target_frame->height()));
     canvas->translate(-SkFloatToScalar(target_frame->width() * 0.5f),
                       -SkFloatToScalar(target_frame->height() * 0.5f));
   }
   canvas->drawBitmap(*target_frame, 0, 0, &paint);
   if (need_transform)
     canvas->restore();
   canvas->flush();
   // SkCanvas::flush() causes the generator to generate SkImage, so delete
   // |video_frame| not to be outlived.
-  if (canvas->getGrContext())
-    accelerated_generator_->set_frame(NULL);
+  if (canvas->getGrContext() && accelerated_generator_)
+    accelerated_generator_->set_frame(nullptr);
 }
 
 void SkCanvasVideoRenderer::Copy(const scoped_refptr<VideoFrame>& video_frame,
                                  SkCanvas* canvas) {
   Paint(video_frame,
         canvas,
         video_frame->visible_rect(),
         0xff,
         SkXfermode::kSrc_Mode,
-        media::VIDEO_ROTATION_0);
+        media::VIDEO_ROTATION_0,
+        media::Context3DProvider());
+}
+
+// static
+void SkCanvasVideoRenderer::CopyVideoFrameTextureToGLTexture(
+    gpu::gles2::GLES2Interface* gl,
+    VideoFrame* video_frame,
+    unsigned int texture,
+    unsigned int level,
+    unsigned int internal_format,
+    unsigned int type,
+    bool premultiply_alpha,
+    bool flip_y) {
+  DCHECK(video_frame && video_frame->format() == VideoFrame::NATIVE_TEXTURE);
+  const gpu::MailboxHolder* mailbox_holder = video_frame->mailbox_holder();
+  DCHECK(mailbox_holder->texture_target == GL_TEXTURE_2D ||
+         mailbox_holder->texture_target == GL_TEXTURE_EXTERNAL_OES);
+
+  gl->WaitSyncPointCHROMIUM(mailbox_holder->sync_point);
+  uint32 source_texture = gl->CreateAndConsumeTextureCHROMIUM(
+      mailbox_holder->texture_target, mailbox_holder->mailbox.name);
+
+  // The video is stored in a unmultiplied format, so premultiply
+  // if necessary.
+  gl->PixelStorei(GL_UNPACK_PREMULTIPLY_ALPHA_CHROMIUM, premultiply_alpha);
+  // Application itself needs to take care of setting the right |flip_y|
+  // value down to get the expected result.
+  // "flip_y == true" means to reverse the video orientation while
+  // "flip_y == false" means to keep the intrinsic orientation.
+  gl->PixelStorei(GL_UNPACK_FLIP_Y_CHROMIUM, flip_y);
+  gl->CopyTextureCHROMIUM(
+      GL_TEXTURE_2D, source_texture, texture, level, internal_format, type);
+  gl->PixelStorei(GL_UNPACK_FLIP_Y_CHROMIUM, false);
+  gl->PixelStorei(GL_UNPACK_PREMULTIPLY_ALPHA_CHROMIUM, false);
+
+  gl->DeleteTextures(1, &source_texture);
+  gl->Flush();
+
+  SyncPointClientImpl client(gl);
+  video_frame->UpdateReleaseSyncPoint(&client);
 }
 
 void SkCanvasVideoRenderer::ResetLastFrame() {
   last_frame_.reset();
   last_frame_timestamp_ = media::kNoTimestamp();
 }
 
 void SkCanvasVideoRenderer::ResetAcceleratedLastFrame() {
   accelerated_last_frame_.reset();
   accelerated_generator_ = nullptr;
   accelerated_last_frame_timestamp_ = media::kNoTimestamp();
 }
 
 }  // namespace media