media: Refactor SkCanvasVideoRenderer to use SkImages and not SkBitmaps.

media/blink/skcanvas_video_renderer.cc

   // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/blink/skcanvas_video_renderer.h"
 
 #include "gpu/GLES2/gl2extchromium.h"
 #include "gpu/command_buffer/client/gles2_interface.h"
 #include "gpu/command_buffer/common/mailbox_holder.h"
 #include "media/base/video_frame.h"
 #include "media/base/yuv_convert.h"
 #include "skia/ext/refptr.h"
 #include "third_party/libyuv/include/libyuv.h"
 #include "third_party/skia/include/core/SkCanvas.h"
+#include "third_party/skia/include/core/SkImage.h"
 #include "third_party/skia/include/core/SkImageGenerator.h"
 #include "third_party/skia/include/gpu/GrContext.h"
 #include "third_party/skia/include/gpu/GrTextureProvider.h"
 #include "third_party/skia/include/gpu/SkGrPixelRef.h"
-#include "ui/gfx/skbitmap_operations.h"
 
 // Skia internal format depends on a platform. On Android it is ABGR, on others
 // it is ARGB.
 #if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \
     SK_A32_SHIFT == 24
 #define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB
 #define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB
 #elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \
     SK_A32_SHIFT == 24
 #define LIBYUV_I420_TO_ARGB libyuv::I420ToABGR
 #define LIBYUV_I422_TO_ARGB libyuv::I422ToABGR
 #else
 #error Unexpected Skia ARGB_8888 layout!
 #endif
 
 namespace media {
 
 namespace {
 
-// This class keeps two temporary resources; software bitmap, hardware bitmap.
-// If both bitmap are created and then only software bitmap is updated every
-// frame, hardware bitmap outlives until the media player dies. So we delete
-// a temporary resource if it is not used for 3 sec.
-const int kTemporaryResourceDeletionDelay = 3;  // Seconds;
+bool IsCanvasDrawingOnPicture(SkCanvas* canvas) {
+  SkImageInfo info;
+  size_t rowBytes;
+  return !canvas->getGrContext() && !canvas->peekPixels(&info, &rowBytes);
+}
 
 bool IsYUV(media::VideoFrame::Format format) {
   switch (format) {
     case VideoFrame::YV12:
     case VideoFrame::YV16:
     case VideoFrame::I420:
     case VideoFrame::YV12A:
     case VideoFrame::YV24:
     case VideoFrame::NV12:
       return true;
@@ skipping 14 matching lines @@
   int result;
   return video_frame->metadata()->GetInteger(
              media::VideoFrameMetadata::COLOR_SPACE, &result) &&
          result == color_space;
 }
 
 bool IsYUVOrNative(media::VideoFrame::Format format) {
   return IsYUV(format) || format == media::VideoFrame::NATIVE_TEXTURE;
 }
 
-bool IsSkBitmapProperlySizedTexture(const SkBitmap* bitmap,
-                                    const gfx::Size& size) {
-  return bitmap->getTexture() && bitmap->width() == size.width() &&
-         bitmap->height() == size.height();
-}
-
-bool AllocateSkBitmapTexture(GrContext* gr,
-                             SkBitmap* bitmap,
-                             const gfx::Size& size) {
-  DCHECK(gr);
-  GrTextureDesc desc;
-  // Use kRGBA_8888_GrPixelConfig, not kSkia8888_GrPixelConfig, to avoid
-  // RGBA to BGRA conversion.
-  desc.fConfig = kRGBA_8888_GrPixelConfig;
-  desc.fFlags = kRenderTarget_GrSurfaceFlag;
-  desc.fSampleCnt = 0;
-  desc.fOrigin = kTopLeft_GrSurfaceOrigin;
-  desc.fWidth = size.width();
-  desc.fHeight = size.height();
-  skia::RefPtr<GrTexture> texture = skia::AdoptRef(
-      gr->textureProvider()->refScratchTexture(
-          desc, GrTextureProvider::kExact_ScratchTexMatch));
-  if (!texture.get())
-    return false;
-
-  SkImageInfo info = SkImageInfo::MakeN32Premul(desc.fWidth, desc.fHeight);
-  SkGrPixelRef* pixel_ref = SkNEW_ARGS(SkGrPixelRef, (info, texture.get()));
-  if (!pixel_ref)
-    return false;
-  bitmap->setInfo(info);
-  bitmap->setPixelRef(pixel_ref)->unref();
-  return true;
-}
-
-bool CopyVideoFrameTextureToSkBitmapTexture(VideoFrame* video_frame,
-                                            SkBitmap* bitmap,
-                                            const Context3D& context_3d) {
-  // Check if we could reuse existing texture based bitmap.
-  // Otherwise, release existing texture based bitmap and allocate
-  // a new one based on video size.
-  if (!IsSkBitmapProperlySizedTexture(bitmap,
-                                      video_frame->visible_rect().size())) {
-    if (!AllocateSkBitmapTexture(context_3d.gr_context, bitmap,
-                                 video_frame->visible_rect().size())) {
-      return false;
-    }
-  }
-
-  unsigned texture_id =
-      static_cast<unsigned>((bitmap->getTexture())->getTextureHandle());
-  // If CopyVideoFrameTextureToGLTexture() changes the state of the
-  // |texture_id|, it's needed to invalidate the state cached in skia,
-  // but currently the state isn't changed.
-  SkCanvasVideoRenderer::CopyVideoFrameTextureToGLTexture(
-      context_3d.gl, video_frame, texture_id, GL_RGBA, GL_UNSIGNED_BYTE, true,
-      false);
-  bitmap->notifyPixelsChanged();
-  return true;
-}
-
 class SyncPointClientImpl : public VideoFrame::SyncPointClient {
  public:
   explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl) : gl_(gl) {}
   ~SyncPointClientImpl() override {}
   uint32 InsertSyncPoint() override { return gl_->InsertSyncPointCHROMIUM(); }
   void WaitSyncPoint(uint32 sync_point) override {
     gl_->WaitSyncPointCHROMIUM(sync_point);
   }
 
  private:
   gpu::gles2::GLES2Interface* gl_;
 
   DISALLOW_IMPLICIT_CONSTRUCTORS(SyncPointClientImpl);
 };
 
+// Creates a SkImage from a |video_frame| backed by native resources.
+// The SkImage will not take ownership of the underlying resources and the
+// caller will have to delete them after the SkImage is used.
+// |source_textures| is a pointer to an array that will be filled with
+// texture ids backing the returned image.
+SkImage* NewSkImageFromVideoFrameNative(
+    const scoped_refptr<VideoFrame>& video_frame,
+    const Context3D& context_3d,
+    unsigned* source_textures) {
+  // TODO(dcastagna): Add support for YUV420.
+  DCHECK_EQ(VideoFrame::TEXTURE_RGBA, video_frame->texture_format());
+
+  // Get the texture from the mailbox and wrap it in a GrTexture.
+  const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(0);
+  DCHECK(mailbox_holder.texture_target == GL_TEXTURE_2D ||
+         mailbox_holder.texture_target == GL_TEXTURE_RECTANGLE_ARB ||
+         mailbox_holder.texture_target == GL_TEXTURE_EXTERNAL_OES);

[DaleCurtis, 2015/05/27 22:37:16]

Add << texture_target?

[Daniele Castagna, 2015/05/28 22:48:12]

Done.

+  gpu::gles2::GLES2Interface* gl = context_3d.gl;
+  gl->WaitSyncPointCHROMIUM(mailbox_holder.sync_point);
+  *source_textures = gl->CreateAndConsumeTextureCHROMIUM(
+      mailbox_holder.texture_target, mailbox_holder.mailbox.name);

[dshwang, 2015/05/28 15:54:17]

Need to insert new sync point so that GpuVideoDecoder makes sure all client
completes to use this mailbox before deleting this VideoFrame.
It's why SkCanvasVideoRenderer::CopyVideoFrameTextureToGLTexture uses
VideoFrame::UpdateReleaseSyncPoint api.
IMO this fucntion should reuse CopyVideoFrameTextureToGLTexture().

[Daniele Castagna, 2015/05/28 22:48:12]

We keep a scoped_refptr to the VideoFrame, the GpuVideoDecoder will not get back
this VideoFrame until we've completely used this mailbox. A syncpoint is not
needed in this case.

Btw, a syncpoint wouldn't be needed even if we were not keeping a ref to the
videoframe. A syncpoint here would be needed only if there were the possibility
that the storage for this texture could be reused somewhere else.
Why would you want to copy the texture another time, when you can simply wrap it
and draw it to a canvas without an additional copy?

[dshwang, 2015/05/29 08:49:46]

That's true in Renderer, but it's not true in GPU process. How can GPU process
know the mailbox is completely used?
No, this code should insert new sync point.
Let's assume following case
1. Context A of GpuVideoDecoder create new mailbox with new sync point
2. Context B in this code uses the mailbox after waiting the sync point
3. Context A deletes the mailbox.

When Context A deletes the texture bound to mailbox in GPU process, Context A
must make sure Context B is completed, but it's impossible without sync point.
I got it. Cool.

[Daniele Castagna, 2015/06/05 04:15:31]

Context A can delete the texture name whenever it wants as long as Context B
created one from the mailbox before that. Deleting the texture name doesn't
delete the texture object.
From the extension specification:
   "The texture object is deleted once all contexts have deleted the texture
name
    associated with the texture object, and detached it from all framebuffer
    objects as well as texture unit bindings."

Take a look at
https://code.google.com/p/chromium/codesearch#chromium/src/gpu/command_buffer...
that deletes the texture after all the texturerefs have been deleted.

Said so, we still need a syncpoint after we draw with the texture, since the
texture object might be reused, and we need to make sure we finish drawing
before reusing the texture storage.

+  GrBackendTextureDesc desc;
+  desc.fOrigin = kTopLeft_GrSurfaceOrigin;
+  desc.fWidth = video_frame->coded_size().width();
+  desc.fHeight = video_frame->coded_size().height();
+  desc.fConfig = kRGBA_8888_GrPixelConfig;
+  desc.fTextureHandle = *source_textures;
+  return SkImage::NewFromTexture(context_3d.gr_context, desc);
+}
+
 }  // anonymous namespace
 
 // Generates an RGB image from a VideoFrame. Convert YUV to RGB plain on GPU.
 class VideoImageGenerator : public SkImageGenerator {
  public:
   VideoImageGenerator(const scoped_refptr<VideoFrame>& frame)
       : SkImageGenerator(
             SkImageInfo::MakeN32Premul(frame->visible_rect().width(),
                                        frame->visible_rect().height()))
       , frame_(frame) {
@@ skipping 85 matching lines @@
     }
     return true;
   }
 
  private:
   scoped_refptr<VideoFrame> frame_;
 
   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoImageGenerator);
 };
 
-SkCanvasVideoRenderer::SkCanvasVideoRenderer()
-    : last_frame_timestamp_(media::kNoTimestamp()),
-      frame_deleting_timer_(
-          FROM_HERE,
-          base::TimeDelta::FromSeconds(kTemporaryResourceDeletionDelay),
-          this,
-          &SkCanvasVideoRenderer::ResetLastFrame),
-      accelerated_generator_(nullptr),
-      accelerated_last_frame_timestamp_(media::kNoTimestamp()),
-      accelerated_frame_deleting_timer_(
-          FROM_HERE,
-          base::TimeDelta::FromSeconds(kTemporaryResourceDeletionDelay),
-          this,
-          &SkCanvasVideoRenderer::ResetAcceleratedLastFrame) {
-  last_frame_.setIsVolatile(true);
+SkCanvasVideoRenderer::SkCanvasVideoRenderer() {
 }
 
-SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {}
+SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {
+}
 
 void SkCanvasVideoRenderer::Paint(const scoped_refptr<VideoFrame>& video_frame,
                                   SkCanvas* canvas,
                                   const gfx::RectF& dest_rect,
                                   uint8 alpha,
                                   SkXfermode::Mode mode,
                                   VideoRotation video_rotation,
                                   const Context3D& context_3d) {
   if (alpha == 0) {
     return;
   }
 
   SkRect dest;
   dest.set(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom());
 
   SkPaint paint;
   paint.setAlpha(alpha);
 
   // Paint black rectangle if there isn't a frame available or the
   // frame has an unexpected format.
   if (!video_frame.get() || video_frame->natural_size().IsEmpty() ||
       !IsYUVOrNative(video_frame->format())) {
     canvas->drawRect(dest, paint);
     canvas->flush();
     return;
   }
 
-  SkBitmap* target_frame = nullptr;
-
+  unsigned texture_id = 0;
+  SkImage* target_frame = nullptr;

[dshwang, 2015/05/28 15:54:17]

use skia::RefPtr<SkImage>

[Daniele Castagna, 2015/05/28 22:48:12]

Done.

   if (video_frame->format() == VideoFrame::NATIVE_TEXTURE) {
-    // Draw HW Video on both SW and HW Canvas.
-    // In SW Canvas case, rely on skia drawing Ganesh SkBitmap on SW SkCanvas.
-    if (accelerated_last_frame_.isNull() ||
-        video_frame->timestamp() != accelerated_last_frame_timestamp_) {
-      DCHECK(context_3d.gl);
-      DCHECK(context_3d.gr_context);
-      if (accelerated_generator_) {
-        // Reset SkBitmap used in SWVideo-to-HWCanvas path.
-        accelerated_last_frame_.reset();
-        accelerated_generator_ = nullptr;
-      }
-      if (!CopyVideoFrameTextureToSkBitmapTexture(
-              video_frame.get(), &accelerated_last_frame_, context_3d)) {
-        NOTREACHED();
-        return;
-      }
-      DCHECK(video_frame->visible_rect().width() ==
-                 accelerated_last_frame_.width() &&
-             video_frame->visible_rect().height() ==
-                 accelerated_last_frame_.height());
-
-      accelerated_last_frame_timestamp_ = video_frame->timestamp();
-    }
-    target_frame = &accelerated_last_frame_;
-    accelerated_frame_deleting_timer_.Reset();
-  } else if (canvas->getGrContext()) {
-    DCHECK(video_frame->format() != VideoFrame::NATIVE_TEXTURE);
-    if (accelerated_last_frame_.isNull() ||
-        video_frame->timestamp() != accelerated_last_frame_timestamp_) {
-      // Draw SW Video on HW Canvas.
-      if (!accelerated_generator_ && !accelerated_last_frame_.isNull()) {
-        // Reset SkBitmap used in HWVideo-to-HWCanvas path.
-        accelerated_last_frame_.reset();
-      }
-      accelerated_generator_ = new VideoImageGenerator(video_frame);
-
-      // Note: This takes ownership of |accelerated_generator_|.
-      if (!SkInstallDiscardablePixelRef(accelerated_generator_,
-                                        &accelerated_last_frame_)) {
-        NOTREACHED();
-        return;
-      }
-      DCHECK(video_frame->visible_rect().width() ==
-                 accelerated_last_frame_.width() &&
-             video_frame->visible_rect().height() ==
-                 accelerated_last_frame_.height());
-
-      accelerated_last_frame_timestamp_ = video_frame->timestamp();
-    } else if (accelerated_generator_) {
-      accelerated_generator_->set_frame(video_frame);
-    }
-    target_frame = &accelerated_last_frame_;
-    accelerated_frame_deleting_timer_.Reset();
+    DCHECK(context_3d.gr_context);
+    DCHECK(context_3d.gl);
+    target_frame =
+        NewSkImageFromVideoFrameNative(video_frame, context_3d, &texture_id);
   } else {
-    // Draw SW Video on SW Canvas.
-    DCHECK(video_frame->format() != VideoFrame::NATIVE_TEXTURE);
-    if (last_frame_.isNull() ||
-        video_frame->timestamp() != last_frame_timestamp_) {
-      // Check if |bitmap| needs to be (re)allocated.
-      if (last_frame_.isNull() ||
-          last_frame_.width() != video_frame->visible_rect().width() ||
-          last_frame_.height() != video_frame->visible_rect().height()) {
-        last_frame_.allocN32Pixels(video_frame->visible_rect().width(),
-                                   video_frame->visible_rect().height());
-        last_frame_.setIsVolatile(true);
-      }
-      last_frame_.lockPixels();
-      ConvertVideoFrameToRGBPixels(
-          video_frame, last_frame_.getPixels(), last_frame_.rowBytes());
-      last_frame_.notifyPixelsChanged();
-      last_frame_.unlockPixels();
-      last_frame_timestamp_ = video_frame->timestamp();
-    }
-    target_frame = &last_frame_;
-    frame_deleting_timer_.Reset();
+    VideoImageGenerator* generator = new VideoImageGenerator(video_frame);
+    target_frame = SkImage::NewFromGenerator(generator);
   }
 
   paint.setXfermodeMode(mode);
   paint.setFilterQuality(kLow_SkFilterQuality);
 
   bool need_transform =
       video_rotation != VIDEO_ROTATION_0 ||
       dest_rect.size() != video_frame->visible_rect().size() ||
       !dest_rect.origin().IsOrigin();
   if (need_transform) {
@@ skipping 22 matching lines @@
         video_rotation == VIDEO_ROTATION_270) {
       rotated_dest_size =
           gfx::SizeF(rotated_dest_size.height(), rotated_dest_size.width());
     }
     canvas->scale(
         SkFloatToScalar(rotated_dest_size.width() / target_frame->width()),
         SkFloatToScalar(rotated_dest_size.height() / target_frame->height()));
     canvas->translate(-SkFloatToScalar(target_frame->width() * 0.5f),
                       -SkFloatToScalar(target_frame->height() * 0.5f));
   }
-  canvas->drawBitmap(*target_frame, 0, 0, &paint);
+  canvas->drawImage(target_frame, 0, 0, &paint);
   if (need_transform)
     canvas->restore();
   canvas->flush();
-  // SkCanvas::flush() causes the generator to generate SkImage, so delete
-  // |video_frame| not to be outlived.
-  if (canvas->getGrContext() && accelerated_generator_)
-    accelerated_generator_->set_frame(nullptr);
+  SkSafeSetNull(target_frame);
+  // When target_frame is created from a backend texture, calling
+  // textureSkCanvas::flush() causes Skia to use all the resources backing the
+  // SkImage and the underlying texture can be delete.
+  // TODO(dcastagna): release the texture in a callback once Skia API exposes
+  // the functionality.
+  if (texture_id) {
+    DCHECK(!IsCanvasDrawingOnPicture(canvas));
+    DCHECK(context_3d.gl);
+    context_3d.gl->DeleteTextures(1, &texture_id);
+  }
 }
 
 void SkCanvasVideoRenderer::Copy(const scoped_refptr<VideoFrame>& video_frame,
                                  SkCanvas* canvas,
                                  const Context3D& context_3d) {
   Paint(video_frame, canvas, video_frame->visible_rect(), 0xff,
         SkXfermode::kSrc_Mode, media::VIDEO_ROTATION_0, context_3d);
 }
 
 // static
@@ skipping 166 matching lines @@
   gl->PixelStorei(GL_UNPACK_FLIP_Y_CHROMIUM, false);
   gl->PixelStorei(GL_UNPACK_PREMULTIPLY_ALPHA_CHROMIUM, false);
 
   gl->DeleteTextures(1, &source_texture);
   gl->Flush();
 
   SyncPointClientImpl client(gl);
   video_frame->UpdateReleaseSyncPoint(&client);
 }
 
-void SkCanvasVideoRenderer::ResetLastFrame() {
-  last_frame_.reset();
-  last_frame_timestamp_ = media::kNoTimestamp();
-}
-
-void SkCanvasVideoRenderer::ResetAcceleratedLastFrame() {
-  accelerated_last_frame_.reset();
-  accelerated_generator_ = nullptr;
-  accelerated_last_frame_timestamp_ = media::kNoTimestamp();
-}
-
 }  // namespace media