media: Refactor SkCanvasVideoRenderer to use SkImages and not SkBitmaps.

media/blink/skcanvas_video_renderer.cc

Index: media/blink/skcanvas_video_renderer.cc
diff --git a/media/blink/skcanvas_video_renderer.cc b/media/blink/skcanvas_video_renderer.cc
index 2d3a416b3fd80fa4dc4d69e73ad2fd833e0cb9d2..969f2bd7000b1556fb4c26da5ad03e557a6b3f87 100644
--- a/media/blink/skcanvas_video_renderer.cc
+++ b/media/blink/skcanvas_video_renderer.cc
@@ -12,11 +12,10 @@
 #include "skia/ext/refptr.h"
 #include "third_party/libyuv/include/libyuv.h"
 #include "third_party/skia/include/core/SkCanvas.h"
+#include "third_party/skia/include/core/SkImage.h"
 #include "third_party/skia/include/core/SkImageGenerator.h"
 #include "third_party/skia/include/gpu/GrContext.h"
 #include "third_party/skia/include/gpu/GrTextureProvider.h"
-#include "third_party/skia/include/gpu/SkGrPixelRef.h"
-#include "ui/gfx/skbitmap_operations.h"
 
 // Skia internal format depends on a platform. On Android it is ABGR, on others
 // it is ARGB.
@@ -36,12 +35,6 @@ namespace media {
 
 namespace {
 
-// This class keeps two temporary resources; software bitmap, hardware bitmap.
-// If both bitmap are created and then only software bitmap is updated every
-// frame, hardware bitmap outlives until the media player dies. So we delete
-// a temporary resource if it is not used for 3 sec.
-const int kTemporaryResourceDeletionDelay = 3;  // Seconds;
-
 bool CheckColorSpace(const scoped_refptr<VideoFrame>& video_frame,
                      VideoFrame::ColorSpace color_space) {
   int result;
@@ -50,66 +43,6 @@ bool CheckColorSpace(const scoped_refptr<VideoFrame>& video_frame,
          result == color_space;
 }
 
-bool IsSkBitmapProperlySizedTexture(const SkBitmap* bitmap,
-                                    const gfx::Size& size) {
-  return bitmap->getTexture() && bitmap->width() == size.width() &&
-         bitmap->height() == size.height();
-}
-
-bool AllocateSkBitmapTexture(GrContext* gr,
-                             SkBitmap* bitmap,
-                             const gfx::Size& size) {
-  DCHECK(gr);
-  GrTextureDesc desc;
-  // Use kRGBA_8888_GrPixelConfig, not kSkia8888_GrPixelConfig, to avoid
-  // RGBA to BGRA conversion.
-  desc.fConfig = kRGBA_8888_GrPixelConfig;
-  desc.fFlags = kRenderTarget_GrSurfaceFlag;
-  desc.fSampleCnt = 0;
-  desc.fOrigin = kTopLeft_GrSurfaceOrigin;
-  desc.fWidth = size.width();
-  desc.fHeight = size.height();
-  skia::RefPtr<GrTexture> texture = skia::AdoptRef(
-      gr->textureProvider()->refScratchTexture(
-          desc, GrTextureProvider::kExact_ScratchTexMatch));
-  if (!texture.get())
-    return false;
-
-  SkImageInfo info = SkImageInfo::MakeN32Premul(desc.fWidth, desc.fHeight);
-  SkGrPixelRef* pixel_ref = SkNEW_ARGS(SkGrPixelRef, (info, texture.get()));
-  if (!pixel_ref)
-    return false;
-  bitmap->setInfo(info);
-  bitmap->setPixelRef(pixel_ref)->unref();
-  return true;
-}
-
-bool CopyVideoFrameTextureToSkBitmapTexture(VideoFrame* video_frame,
-                                            SkBitmap* bitmap,
-                                            const Context3D& context_3d) {
-  // Check if we could reuse existing texture based bitmap.
-  // Otherwise, release existing texture based bitmap and allocate
-  // a new one based on video size.
-  if (!IsSkBitmapProperlySizedTexture(bitmap,
-                                      video_frame->visible_rect().size())) {
-    if (!AllocateSkBitmapTexture(context_3d.gr_context, bitmap,
-                                 video_frame->visible_rect().size())) {
-      return false;
-    }
-  }
-
-  unsigned texture_id =
-      static_cast<unsigned>((bitmap->getTexture())->getTextureHandle());
-  // If CopyVideoFrameTextureToGLTexture() changes the state of the
-  // |texture_id|, it's needed to invalidate the state cached in skia,
-  // but currently the state isn't changed.
-  SkCanvasVideoRenderer::CopyVideoFrameTextureToGLTexture(
-      context_3d.gl, video_frame, texture_id, GL_RGBA, GL_UNSIGNED_BYTE, true,
-      false);
-  bitmap->notifyPixelsChanged();
-  return true;
-}
-
 class SyncPointClientImpl : public VideoFrame::SyncPointClient {
  public:
   explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl) : gl_(gl) {}
@@ -125,21 +58,62 @@ class SyncPointClientImpl : public VideoFrame::SyncPointClient {
   DISALLOW_IMPLICIT_CONSTRUCTORS(SyncPointClientImpl);
 };
 
+// Creates a SkImage from a |video_frame| backed by native resources.
+// The SkImage will not take ownership of the underlying resources and the
+// caller will have to delete them after the SkImage is used.
+// |source_textures| is a pointer to an array that will be filled with
+// texture ids backing the returned image.
+skia::RefPtr<SkImage> NewSkImageFromVideoFrameNative(
+    const scoped_refptr<VideoFrame>& video_frame,
+    const Context3D& context_3d,
+    unsigned* source_textures) {
+  // TODO(dcastagna): Add support for YUV420.
+  DCHECK_EQ(VideoFrame::ARGB, video_frame->format());
+
+  const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(0);
+  DCHECK(mailbox_holder.texture_target == GL_TEXTURE_2D ||
+         mailbox_holder.texture_target == GL_TEXTURE_RECTANGLE_ARB ||
+         mailbox_holder.texture_target == GL_TEXTURE_EXTERNAL_OES)
+      << mailbox_holder.texture_target;
+
+  gpu::gles2::GLES2Interface* gl = context_3d.gl;
+  if (mailbox_holder.texture_target != GL_TEXTURE_2D) {
+    // TODO(dcastagna): At the moment Skia doesn't support targets different
+    // than GL_TEXTURE_2D.  Avoid this copy once
+    // https://code.google.com/p/skia/issues/detail?id=3868 is addressed.
+    gl->GenTextures(1, source_textures);
+    DCHECK(*source_textures);
+    gl->BindTexture(GL_TEXTURE_2D, *source_textures);
+    SkCanvasVideoRenderer::CopyVideoFrameTextureToGLTexture(
+        gl, video_frame.get(), *source_textures, GL_RGBA, GL_UNSIGNED_BYTE,
+        true, false);
+  } else {
+    gl->WaitSyncPointCHROMIUM(mailbox_holder.sync_point);
+    *source_textures = gl->CreateAndConsumeTextureCHROMIUM(
+        mailbox_holder.texture_target, mailbox_holder.mailbox.name);

[dshwang, 2015/06/05 18:58:16]

Following two lines is needed to insert sync point here.

SyncPointClientImpl client(gl);
video_frame->UpdateReleaseSyncPoint(&client);

[dshwang, 2015/06/05 19:08:41]

ah, I found you insert sync point at the end of drawing. sorry for noise.

[dshwang, 2015/06/08 12:21:43]

In this case, this class should not cache the texture. Gpu decoder can reuse the
texture after VideoFrame destruction. Keeping this texture might cause data
race.
In addition, cache is meaningless in this case, because there is no copy.

On the other hand, it's fine to cache the copied texture.

[Daniele Castagna, 2015/06/09 23:21:11]

The lifetime of the texture id is currently not well defined since SkImage does
not take the ownership of it.
Let's discuss this once we fix that.

[dshwang, 2015/06/10 07:17:39]

I guess final solution would be to not cache. why don't you not cache in this
case here? I would like to not cache it in TEXTURE_2D case as final solution.

+  }
+  GrBackendTextureDesc desc;
+  desc.fOrigin = kTopLeft_GrSurfaceOrigin;
+  desc.fWidth = video_frame->coded_size().width();
+  desc.fHeight = video_frame->coded_size().height();
+  desc.fConfig = kRGBA_8888_GrPixelConfig;
+  desc.fTextureHandle = *source_textures;
+  return skia::AdoptRef(SkImage::NewFromTexture(context_3d.gr_context, desc));
+}
+
 }  // anonymous namespace
 
 // Generates an RGB image from a VideoFrame. Convert YUV to RGB plain on GPU.
 class VideoImageGenerator : public SkImageGenerator {
  public:
-  VideoImageGenerator(const scoped_refptr<VideoFrame>& frame)
+  VideoImageGenerator(VideoFrame* frame)
       : SkImageGenerator(
             SkImageInfo::MakeN32Premul(frame->visible_rect().width(),
-                                       frame->visible_rect().height()))
-      , frame_(frame) {
-    DCHECK(frame_.get());
-  }
+                                       frame->visible_rect().height())),
+        frame_(frame) {}
   ~VideoImageGenerator() override {}
 
-  void set_frame(const scoped_refptr<VideoFrame>& frame) { frame_ = frame; }
+  void set_frame(VideoFrame* frame) { frame_ = frame; }
 
  protected:
   Result onGetPixels(const SkImageInfo& info,
@@ -147,7 +121,7 @@ class VideoImageGenerator : public SkImageGenerator {
                      size_t row_bytes,
                      SkPMColor ctable[],
                      int* ctable_count) override {
-    if (!frame_.get())
+    if (!frame_)
       return kInvalidInput;
     // If skia couldn't do the YUV conversion on GPU, we will on CPU.
     SkCanvasVideoRenderer::ConvertVideoFrameToRGBPixels(
@@ -159,7 +133,7 @@ class VideoImageGenerator : public SkImageGenerator {
                        void* planes[3],
                        size_t row_bytes[3],
                        SkYUVColorSpace* color_space) override {
-    if (!frame_.get() || !media::VideoFrame::IsYuvPlanar(frame_->format()) ||
+    if (!frame_ || !media::VideoFrame::IsYuvPlanar(frame_->format()) ||
         // TODO(rileya): Skia currently doesn't support Rec709 YUV conversion,
         // or YUVA conversion. Remove this case once it does. As-is we will
         // fall back on the pure-software path in this case.
@@ -225,29 +199,17 @@ class VideoImageGenerator : public SkImageGenerator {
   }
 
  private:
-  scoped_refptr<VideoFrame> frame_;
+  VideoFrame* frame_;
 
   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoImageGenerator);
 };
 
-SkCanvasVideoRenderer::SkCanvasVideoRenderer()
-    : last_frame_timestamp_(media::kNoTimestamp()),
-      frame_deleting_timer_(

[dshwang, 2015/06/05 19:08:41]

I think it's needed.
e.g. if some canvas draws video and then doesn't draw anymore, we want to delete
cache earlier than video life cycle.
other example, if user switches tab, we want to delete cache soon.
This caches can consumes few MB of GPU memory.

[Daniele Castagna, 2015/06/05 22:05:32]

SkImage caches the result internally and I'd assume it purges the cache when
it's out of budget.

|last_texture_id_| unfortunately takes GPU memory keeping the texture object
alive. That will go away as soon as we have Skia support for different texture
targets. At that point we won't need to copy the texture and we'll be able to
get rid of |last_texture_id_|.

Would you still suggest to leave that 3 second timer to reset the cache? Why was
that added in the first place?

[dshwang, 2015/06/08 12:21:43]

3 second timer was introduced to eagerly remove cache to not wait when
it's out of budget.
I think it's needed, but I'm not strongly claiming.  DaleCurtis, mcasas, what do
you think?

[Daniele Castagna, 2015/06/09 23:21:11]

My rationale for removing the delaytimer is that now we have a plan to get rid
of the cache completely.

[dshwang, 2015/06/10 07:17:39]

How will you remove cache for software path?

-          FROM_HERE,
-          base::TimeDelta::FromSeconds(kTemporaryResourceDeletionDelay),
-          this,
-          &SkCanvasVideoRenderer::ResetLastFrame),
-      accelerated_generator_(nullptr),
-      accelerated_last_frame_timestamp_(media::kNoTimestamp()),
-      accelerated_frame_deleting_timer_(
-          FROM_HERE,
-          base::TimeDelta::FromSeconds(kTemporaryResourceDeletionDelay),
-          this,
-          &SkCanvasVideoRenderer::ResetAcceleratedLastFrame) {
-  last_frame_.setIsVolatile(true);
+SkCanvasVideoRenderer::SkCanvasVideoRenderer() {
 }
 
-SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {}
+SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {
+  ResetCache();
+}
 
 void SkCanvasVideoRenderer::Paint(const scoped_refptr<VideoFrame>& video_frame,
                                   SkCanvas* canvas,
@@ -276,83 +238,30 @@ void SkCanvasVideoRenderer::Paint(const scoped_refptr<VideoFrame>& video_frame,
     return;
   }
 
-  SkBitmap* target_frame = nullptr;
+  gpu::gles2::GLES2Interface* gl = context_3d.gl;
 
-  if (video_frame->storage_type() == VideoFrame::STORAGE_TEXTURE) {
-    // Draw HW Video on both SW and HW Canvas.
-    // In SW Canvas case, rely on skia drawing Ganesh SkBitmap on SW SkCanvas.
-    if (accelerated_last_frame_.isNull() ||
-        video_frame->timestamp() != accelerated_last_frame_timestamp_) {
-      DCHECK(context_3d.gl);
-      DCHECK(context_3d.gr_context);
-      if (accelerated_generator_) {
-        // Reset SkBitmap used in SWVideo-to-HWCanvas path.
-        accelerated_last_frame_.reset();
-        accelerated_generator_ = nullptr;
-      }
-      if (!CopyVideoFrameTextureToSkBitmapTexture(
-              video_frame.get(), &accelerated_last_frame_, context_3d)) {
-        NOTREACHED();
-        return;
-      }
-      DCHECK(video_frame->visible_rect().width() ==
-                 accelerated_last_frame_.width() &&
-             video_frame->visible_rect().height() ==
-                 accelerated_last_frame_.height());
-
-      accelerated_last_frame_timestamp_ = video_frame->timestamp();
-    }
-    target_frame = &accelerated_last_frame_;
-    accelerated_frame_deleting_timer_.Reset();
-  } else if (canvas->getGrContext()) {
-    if (accelerated_last_frame_.isNull() ||
-        video_frame->timestamp() != accelerated_last_frame_timestamp_) {
-      // Draw SW Video on HW Canvas.
-      if (!accelerated_generator_ && !accelerated_last_frame_.isNull()) {
-        // Reset SkBitmap used in HWVideo-to-HWCanvas path.
-        accelerated_last_frame_.reset();
-      }
-      accelerated_generator_ = new VideoImageGenerator(video_frame);
-
-      // Note: This takes ownership of |accelerated_generator_|.
-      if (!SkInstallDiscardablePixelRef(accelerated_generator_,
-                                        &accelerated_last_frame_)) {
-        NOTREACHED();
-        return;
-      }
-      DCHECK(video_frame->visible_rect().width() ==
-                 accelerated_last_frame_.width() &&
-             video_frame->visible_rect().height() ==
-                 accelerated_last_frame_.height());
-
-      accelerated_last_frame_timestamp_ = video_frame->timestamp();
-    } else if (accelerated_generator_) {
-      accelerated_generator_->set_frame(video_frame);
-    }
-    target_frame = &accelerated_last_frame_;
-    accelerated_frame_deleting_timer_.Reset();
+  // This might be wrong, two different videos could be drawn to the same
+  // canvas and the two different videoframes could have the same timestamp.
+  if (last_image_ && video_frame->timestamp() == last_timestamp_) {
+    // |last_image_| can be reused.
+    // |video_generator_| will be set only if the last cache videoframe was
+    // a software videoframe.
+    if (video_generator_)
+      video_generator_->set_frame(video_frame.get());
   } else {
-    // Draw SW Video on SW Canvas.
-    DCHECK(VideoFrame::IsMappable(video_frame->storage_type()));
-    if (last_frame_.isNull() ||
-        video_frame->timestamp() != last_frame_timestamp_) {
-      // Check if |bitmap| needs to be (re)allocated.
-      if (last_frame_.isNull() ||
-          last_frame_.width() != video_frame->visible_rect().width() ||
-          last_frame_.height() != video_frame->visible_rect().height()) {
-        last_frame_.allocN32Pixels(video_frame->visible_rect().width(),
-                                   video_frame->visible_rect().height());
-        last_frame_.setIsVolatile(true);
-      }
-      last_frame_.lockPixels();
-      ConvertVideoFrameToRGBPixels(
-          video_frame, last_frame_.getPixels(), last_frame_.rowBytes());

[dshwang, 2015/06/08 12:21:43]

does VideoGenerator way have same performance to create SkBitmap explicitly?
+bsalomon?

-      last_frame_.notifyPixelsChanged();
-      last_frame_.unlockPixels();
-      last_frame_timestamp_ = video_frame->timestamp();
+    ResetCache();
+    // Generate a new image.
+    if (video_frame->storage_type() == VideoFrame::STORAGE_TEXTURE) {
+      DCHECK(context_3d.gr_context);
+      DCHECK(gl);
+      last_image_ = NewSkImageFromVideoFrameNative(video_frame, context_3d,
+                                                   &last_texture_id_);
+      last_gl_ = gl;
+    } else {
+      video_generator_ = new VideoImageGenerator(video_frame.get());
+      last_image_ = skia::AdoptRef(SkImage::NewFromGenerator(video_generator_));
     }
-    target_frame = &last_frame_;
-    frame_deleting_timer_.Reset();
+    last_timestamp_ = video_frame->timestamp();
   }
 
   paint.setXfermodeMode(mode);
@@ -390,19 +299,31 @@ void SkCanvasVideoRenderer::Paint(const scoped_refptr<VideoFrame>& video_frame,
           gfx::SizeF(rotated_dest_size.height(), rotated_dest_size.width());
     }
     canvas->scale(
-        SkFloatToScalar(rotated_dest_size.width() / target_frame->width()),
-        SkFloatToScalar(rotated_dest_size.height() / target_frame->height()));
-    canvas->translate(-SkFloatToScalar(target_frame->width() * 0.5f),
-                      -SkFloatToScalar(target_frame->height() * 0.5f));
+        SkFloatToScalar(rotated_dest_size.width() / last_image_->width()),
+        SkFloatToScalar(rotated_dest_size.height() / last_image_->height()));
+    canvas->translate(-SkFloatToScalar(last_image_->width() * 0.5f),
+                      -SkFloatToScalar(last_image_->height() * 0.5f));
   }
-  canvas->drawBitmap(*target_frame, 0, 0, &paint);
+
+  canvas->drawImage(last_image_.get(), 0, 0, &paint);
   if (need_transform)
     canvas->restore();
+  // Make sure to flush so we can remove the videoframe from the generator.
   canvas->flush();
-  // SkCanvas::flush() causes the generator to generate SkImage, so delete
-  // |video_frame| not to be outlived.
-  if (canvas->getGrContext() && accelerated_generator_)
-    accelerated_generator_->set_frame(nullptr);
+
+  if (last_texture_id_) {
+    SyncPointClientImpl client(gl);
+    // Make sure the video_frame can go back to the pool after the native
+    // texture has been used for drawing.
+    video_frame->UpdateReleaseSyncPoint(&client);

[dshwang, 2015/06/08 12:21:43]

It's needed only for GL_TEXTURE_2D case.

[Daniele Castagna, 2015/06/09 23:21:11]

That's true, but considering that eventually it will be needed for all the
texture target, I'd prefer not to add specific cases to improve performance for
a code path we're planning to remove.

[dshwang, 2015/06/10 07:17:39]

Acknowledged.

+  }
+
+  // TODO(dcastagna): here we're assuming |video_generator_| will still be valid
+  // after SkImage::NewFromGenerator took the ownership.
+  // Fix this once https://code.google.com/p/skia/issues/detail?id=3870 is
+  // addressed.
+  if (video_generator_)
+    video_generator_->set_frame(nullptr);
 }
 
 void SkCanvasVideoRenderer::Copy(const scoped_refptr<VideoFrame>& video_frame,
@@ -563,7 +484,8 @@ void SkCanvasVideoRenderer::CopyVideoFrameTextureToGLTexture(
   const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(0);
   DCHECK(mailbox_holder.texture_target == GL_TEXTURE_2D ||
          mailbox_holder.texture_target == GL_TEXTURE_RECTANGLE_ARB ||
-         mailbox_holder.texture_target == GL_TEXTURE_EXTERNAL_OES);
+         mailbox_holder.texture_target == GL_TEXTURE_EXTERNAL_OES)
+      << mailbox_holder.texture_target;
 
   gl->WaitSyncPointCHROMIUM(mailbox_holder.sync_point);
   uint32 source_texture = gl->CreateAndConsumeTextureCHROMIUM(
@@ -589,15 +511,16 @@ void SkCanvasVideoRenderer::CopyVideoFrameTextureToGLTexture(
   video_frame->UpdateReleaseSyncPoint(&client);
 }
 
-void SkCanvasVideoRenderer::ResetLastFrame() {
-  last_frame_.reset();
-  last_frame_timestamp_ = media::kNoTimestamp();
-}
-
-void SkCanvasVideoRenderer::ResetAcceleratedLastFrame() {
-  accelerated_last_frame_.reset();
-  accelerated_generator_ = nullptr;
-  accelerated_last_frame_timestamp_ = media::kNoTimestamp();
+void SkCanvasVideoRenderer::ResetCache() {
+  // Clear cached values.
+  video_generator_ = nullptr;
+  last_image_ = nullptr;
+  last_timestamp_ = kNoTimestamp();
+  if (last_texture_id_) {
+    DCHECK(last_gl_);
+    last_gl_->DeleteTextures(1, &last_texture_id_);
+    last_texture_id_ = 0;
+  }
 }
 
 }  // namespace media