media: Refactor SkCanvasVideoRenderer to use SkImages and not SkBitmaps.

media/blink/skcanvas_video_renderer.cc

   // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/blink/skcanvas_video_renderer.h"
 
 #include "gpu/GLES2/gl2extchromium.h"
 #include "gpu/command_buffer/client/gles2_interface.h"
 #include "gpu/command_buffer/common/mailbox_holder.h"
 #include "media/base/video_frame.h"
 #include "media/base/yuv_convert.h"
 #include "skia/ext/refptr.h"
 #include "third_party/libyuv/include/libyuv.h"
 #include "third_party/skia/include/core/SkCanvas.h"
+#include "third_party/skia/include/core/SkImage.h"
 #include "third_party/skia/include/core/SkImageGenerator.h"
 #include "third_party/skia/include/gpu/GrContext.h"
 #include "third_party/skia/include/gpu/GrTextureProvider.h"
 #include "third_party/skia/include/gpu/SkGrPixelRef.h"
-#include "ui/gfx/skbitmap_operations.h"
 
 // Skia internal format depends on a platform. On Android it is ABGR, on others
 // it is ARGB.
 #if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \
     SK_A32_SHIFT == 24
 #define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB
 #define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB
 #elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \
     SK_A32_SHIFT == 24
 #define LIBYUV_I420_TO_ARGB libyuv::I420ToABGR
 #define LIBYUV_I422_TO_ARGB libyuv::I422ToABGR
 #else
 #error Unexpected Skia ARGB_8888 layout!
 #endif
 
 namespace media {
 
 namespace {
 
-// This class keeps two temporary resources; software bitmap, hardware bitmap.
-// If both bitmap are created and then only software bitmap is updated every
-// frame, hardware bitmap outlives until the media player dies. So we delete
-// a temporary resource if it is not used for 3 sec.
-const int kTemporaryResourceDeletionDelay = 3;  // Seconds;
+bool IsCanvasDrawingOnPicture(SkCanvas* canvas) {
+  SkImageInfo info;
+  size_t rowBytes;
+  return !canvas->getGrContext() && !canvas->peekPixels(&info, &rowBytes);
+}
 
 bool IsYUV(media::VideoFrame::Format format) {
   switch (format) {
     case VideoFrame::YV12:
     case VideoFrame::YV16:
     case VideoFrame::I420:
     case VideoFrame::YV12A:
     case VideoFrame::YV24:
     case VideoFrame::NV12:
       return true;
@@ skipping 14 matching lines @@
   int result;
   return video_frame->metadata()->GetInteger(
              media::VideoFrameMetadata::COLOR_SPACE, &result) &&
          result == color_space;
 }
 
 bool IsYUVOrNative(media::VideoFrame::Format format) {
   return IsYUV(format) || format == media::VideoFrame::NATIVE_TEXTURE;
 }
 
-bool IsSkBitmapProperlySizedTexture(const SkBitmap* bitmap,
-                                    const gfx::Size& size) {
-  return bitmap->getTexture() && bitmap->width() == size.width() &&
-         bitmap->height() == size.height();
-}
-
-bool AllocateSkBitmapTexture(GrContext* gr,
-                             SkBitmap* bitmap,
-                             const gfx::Size& size) {
-  DCHECK(gr);
-  GrTextureDesc desc;
-  // Use kRGBA_8888_GrPixelConfig, not kSkia8888_GrPixelConfig, to avoid
-  // RGBA to BGRA conversion.
-  desc.fConfig = kRGBA_8888_GrPixelConfig;
-  desc.fFlags = kRenderTarget_GrSurfaceFlag;
-  desc.fSampleCnt = 0;
-  desc.fOrigin = kTopLeft_GrSurfaceOrigin;
-  desc.fWidth = size.width();
-  desc.fHeight = size.height();
-  skia::RefPtr<GrTexture> texture = skia::AdoptRef(
-      gr->textureProvider()->refScratchTexture(
-          desc, GrTextureProvider::kExact_ScratchTexMatch));
-  if (!texture.get())
-    return false;
-
-  SkImageInfo info = SkImageInfo::MakeN32Premul(desc.fWidth, desc.fHeight);
-  SkGrPixelRef* pixel_ref = SkNEW_ARGS(SkGrPixelRef, (info, texture.get()));
-  if (!pixel_ref)
-    return false;
-  bitmap->setInfo(info);
-  bitmap->setPixelRef(pixel_ref)->unref();
-  return true;
-}
-
-bool CopyVideoFrameTextureToSkBitmapTexture(VideoFrame* video_frame,
-                                            SkBitmap* bitmap,
-                                            const Context3D& context_3d) {
-  // Check if we could reuse existing texture based bitmap.
-  // Otherwise, release existing texture based bitmap and allocate
-  // a new one based on video size.
-  if (!IsSkBitmapProperlySizedTexture(bitmap,
-                                      video_frame->visible_rect().size())) {
-    if (!AllocateSkBitmapTexture(context_3d.gr_context, bitmap,
-                                 video_frame->visible_rect().size())) {
-      return false;
-    }
-  }
-
-  unsigned texture_id =
-      static_cast<unsigned>((bitmap->getTexture())->getTextureHandle());
-  // If CopyVideoFrameTextureToGLTexture() changes the state of the
-  // |texture_id|, it's needed to invalidate the state cached in skia,
-  // but currently the state isn't changed.
-  SkCanvasVideoRenderer::CopyVideoFrameTextureToGLTexture(
-      context_3d.gl, video_frame, texture_id, GL_RGBA, GL_UNSIGNED_BYTE, true,
-      false);
-  bitmap->notifyPixelsChanged();
-  return true;
-}
-
 class SyncPointClientImpl : public VideoFrame::SyncPointClient {
  public:
   explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl) : gl_(gl) {}
   ~SyncPointClientImpl() override {}
   uint32 InsertSyncPoint() override { return gl_->InsertSyncPointCHROMIUM(); }
   void WaitSyncPoint(uint32 sync_point) override {
     gl_->WaitSyncPointCHROMIUM(sync_point);
   }
 
  private:
   gpu::gles2::GLES2Interface* gl_;
 
   DISALLOW_IMPLICIT_CONSTRUCTORS(SyncPointClientImpl);
 };
 
+// Creates a SkImage from a |video_frame| backed by native resources.
+// The SkImage will not take ownership of the underlying resources and the
+// caller will have to delete them after the SkImage is used.
+// |source_textures| is a pointer to an array that will be filled with
+// texture ids backing the returned image.
+skia::RefPtr<SkImage> NewSkImageFromVideoFrameNative(
+    const scoped_refptr<VideoFrame>& video_frame,
+    const Context3D& context_3d,
+    unsigned* source_textures) {
+  // TODO(dcastagna): Add support for YUV420.
+  DCHECK_EQ(VideoFrame::TEXTURE_RGBA, video_frame->texture_format());
+
+  // Get the texture from the mailbox and wrap it in a GrTexture.
+  const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(0);
+  DCHECK(mailbox_holder.texture_target == GL_TEXTURE_2D ||
+         mailbox_holder.texture_target == GL_TEXTURE_RECTANGLE_ARB ||
+         mailbox_holder.texture_target == GL_TEXTURE_EXTERNAL_OES)

[dshwang, 2015/05/29 08:49:47]

Can SkImage wrap RECTANBLE_ARB and EXTERNAL_OES also? +reed

[dshwang, 2015/05/29 17:30:30]

skia doesn't support RECTANBLE_ARB and EXTERNAL_OES for SkImage.
https://code.google.com/p/skia/issues/detail?id=3868
Those format texture should be copied to GL_TEXTURE_2D texture by
CopyVideoFrameTextureToSkBitmapTexture

[Daniele Castagna, 2015/06/05 04:15:31]

This happened to work for texture_rectangle. :/
Brian is working on supporting this in Skia in the right way.

I put back the copies for rectangle and external_oes. We'll have a significant
improvement once Brian implements the support for different texture targets.

+      << mailbox_holder.texture_target;
+  gpu::gles2::GLES2Interface* gl = context_3d.gl;
+  gl->WaitSyncPointCHROMIUM(mailbox_holder.sync_point);
+  *source_textures = gl->CreateAndConsumeTextureCHROMIUM(
+      mailbox_holder.texture_target, mailbox_holder.mailbox.name);
+  GrBackendTextureDesc desc;
+  desc.fOrigin = kTopLeft_GrSurfaceOrigin;
+  desc.fWidth = video_frame->coded_size().width();
+  desc.fHeight = video_frame->coded_size().height();
+  desc.fConfig = kRGBA_8888_GrPixelConfig;
+  desc.fTextureHandle = *source_textures;
+  return skia::AdoptRef(SkImage::NewFromTexture(context_3d.gr_context, desc));
+}
+
 }  // anonymous namespace
 
 // Generates an RGB image from a VideoFrame. Convert YUV to RGB plain on GPU.
 class VideoImageGenerator : public SkImageGenerator {
  public:
   VideoImageGenerator(const scoped_refptr<VideoFrame>& frame)
       : SkImageGenerator(
             SkImageInfo::MakeN32Premul(frame->visible_rect().width(),
                                        frame->visible_rect().height()))
       , frame_(frame) {
@@ skipping 85 matching lines @@
     }
     return true;
   }
 
  private:
   scoped_refptr<VideoFrame> frame_;
 
   DISALLOW_IMPLICIT_CONSTRUCTORS(VideoImageGenerator);
 };
 
-SkCanvasVideoRenderer::SkCanvasVideoRenderer()
-    : last_frame_timestamp_(media::kNoTimestamp()),
-      frame_deleting_timer_(
-          FROM_HERE,
-          base::TimeDelta::FromSeconds(kTemporaryResourceDeletionDelay),
-          this,
-          &SkCanvasVideoRenderer::ResetLastFrame),
-      accelerated_generator_(nullptr),
-      accelerated_last_frame_timestamp_(media::kNoTimestamp()),
-      accelerated_frame_deleting_timer_(
-          FROM_HERE,
-          base::TimeDelta::FromSeconds(kTemporaryResourceDeletionDelay),
-          this,
-          &SkCanvasVideoRenderer::ResetAcceleratedLastFrame) {
-  last_frame_.setIsVolatile(true);
+SkCanvasVideoRenderer::SkCanvasVideoRenderer() {
 }
 
-SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {}
+SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {
+}
 
 void SkCanvasVideoRenderer::Paint(const scoped_refptr<VideoFrame>& video_frame,
                                   SkCanvas* canvas,
                                   const gfx::RectF& dest_rect,
                                   uint8 alpha,
                                   SkXfermode::Mode mode,
                                   VideoRotation video_rotation,
                                   const Context3D& context_3d) {
   if (alpha == 0) {
     return;
   }
 
   SkRect dest;
   dest.set(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom());
 
   SkPaint paint;
   paint.setAlpha(alpha);
 
   // Paint black rectangle if there isn't a frame available or the
   // frame has an unexpected format.
   if (!video_frame.get() || video_frame->natural_size().IsEmpty() ||
       !IsYUVOrNative(video_frame->format())) {
     canvas->drawRect(dest, paint);
     canvas->flush();
     return;
   }
 
-  SkBitmap* target_frame = nullptr;
-
+  unsigned texture_id = 0;
+  skia::RefPtr<SkImage> target_frame;
   if (video_frame->format() == VideoFrame::NATIVE_TEXTURE) {
-    // Draw HW Video on both SW and HW Canvas.
-    // In SW Canvas case, rely on skia drawing Ganesh SkBitmap on SW SkCanvas.
-    if (accelerated_last_frame_.isNull() ||
-        video_frame->timestamp() != accelerated_last_frame_timestamp_) {
-      DCHECK(context_3d.gl);
-      DCHECK(context_3d.gr_context);
-      if (accelerated_generator_) {
-        // Reset SkBitmap used in SWVideo-to-HWCanvas path.
-        accelerated_last_frame_.reset();
-        accelerated_generator_ = nullptr;
-      }
-      if (!CopyVideoFrameTextureToSkBitmapTexture(
-              video_frame.get(), &accelerated_last_frame_, context_3d)) {
-        NOTREACHED();
-        return;
-      }
-      DCHECK(video_frame->visible_rect().width() ==
-                 accelerated_last_frame_.width() &&
-             video_frame->visible_rect().height() ==
-                 accelerated_last_frame_.height());
-
-      accelerated_last_frame_timestamp_ = video_frame->timestamp();
-    }
-    target_frame = &accelerated_last_frame_;
-    accelerated_frame_deleting_timer_.Reset();
-  } else if (canvas->getGrContext()) {
-    DCHECK(video_frame->format() != VideoFrame::NATIVE_TEXTURE);
-    if (accelerated_last_frame_.isNull() ||
-        video_frame->timestamp() != accelerated_last_frame_timestamp_) {
-      // Draw SW Video on HW Canvas.
-      if (!accelerated_generator_ && !accelerated_last_frame_.isNull()) {
-        // Reset SkBitmap used in HWVideo-to-HWCanvas path.
-        accelerated_last_frame_.reset();
-      }
-      accelerated_generator_ = new VideoImageGenerator(video_frame);
-
-      // Note: This takes ownership of |accelerated_generator_|.
-      if (!SkInstallDiscardablePixelRef(accelerated_generator_,
-                                        &accelerated_last_frame_)) {
-        NOTREACHED();
-        return;
-      }
-      DCHECK(video_frame->visible_rect().width() ==
-                 accelerated_last_frame_.width() &&
-             video_frame->visible_rect().height() ==
-                 accelerated_last_frame_.height());
-
-      accelerated_last_frame_timestamp_ = video_frame->timestamp();
-    } else if (accelerated_generator_) {
-      accelerated_generator_->set_frame(video_frame);
-    }
-    target_frame = &accelerated_last_frame_;
-    accelerated_frame_deleting_timer_.Reset();
+    DCHECK(context_3d.gr_context);
+    DCHECK(context_3d.gl);
+    target_frame =
+        NewSkImageFromVideoFrameNative(video_frame, context_3d, &texture_id);
   } else {
-    // Draw SW Video on SW Canvas.
-    DCHECK(video_frame->format() != VideoFrame::NATIVE_TEXTURE);
-    if (last_frame_.isNull() ||
-        video_frame->timestamp() != last_frame_timestamp_) {
-      // Check if |bitmap| needs to be (re)allocated.
-      if (last_frame_.isNull() ||
-          last_frame_.width() != video_frame->visible_rect().width() ||
-          last_frame_.height() != video_frame->visible_rect().height()) {
-        last_frame_.allocN32Pixels(video_frame->visible_rect().width(),
-                                   video_frame->visible_rect().height());
-        last_frame_.setIsVolatile(true);
-      }
-      last_frame_.lockPixels();
-      ConvertVideoFrameToRGBPixels(
-          video_frame, last_frame_.getPixels(), last_frame_.rowBytes());
-      last_frame_.notifyPixelsChanged();
-      last_frame_.unlockPixels();
-      last_frame_timestamp_ = video_frame->timestamp();
-    }
-    target_frame = &last_frame_;
-    frame_deleting_timer_.Reset();
+    VideoImageGenerator* generator = new VideoImageGenerator(video_frame);
+    target_frame = skia::AdoptRef(SkImage::NewFromGenerator(generator));
   }
 
   paint.setXfermodeMode(mode);
   paint.setFilterQuality(kLow_SkFilterQuality);
 
   bool need_transform =
       video_rotation != VIDEO_ROTATION_0 ||
       dest_rect.size() != video_frame->visible_rect().size() ||
       !dest_rect.origin().IsOrigin();
   if (need_transform) {
@@ skipping 22 matching lines @@
         video_rotation == VIDEO_ROTATION_270) {
       rotated_dest_size =
           gfx::SizeF(rotated_dest_size.height(), rotated_dest_size.width());
     }
     canvas->scale(
         SkFloatToScalar(rotated_dest_size.width() / target_frame->width()),
         SkFloatToScalar(rotated_dest_size.height() / target_frame->height()));
     canvas->translate(-SkFloatToScalar(target_frame->width() * 0.5f),
                       -SkFloatToScalar(target_frame->height() * 0.5f));
   }
-  canvas->drawBitmap(*target_frame, 0, 0, &paint);
+  canvas->drawImage(target_frame.get(), 0, 0, &paint);
   if (need_transform)
     canvas->restore();
   canvas->flush();
-  // SkCanvas::flush() causes the generator to generate SkImage, so delete
-  // |video_frame| not to be outlived.
-  if (canvas->getGrContext() && accelerated_generator_)
-    accelerated_generator_->set_frame(nullptr);
+  target_frame.clear();
+  // When target_frame is created from a backend texture, calling
+  // textureSkCanvas::flush() causes Skia to use all the resources backing the
+  // SkImage and the underlying texture can be delete.
+  // TODO(dcastagna): release the texture in a callback once Skia API exposes

[bsalomon, 2015/05/29 14:49:26]

Why don't we go ahead and add this functionality to Skia? The current test in
IsCanvasDrawingOnPicture is a bit of a hack and fragile.

[Daniele Castagna, 2015/06/05 04:15:31]

With the last patch texture_id is kept around for one more frame, so it's a
little bit less fragile.
Happy to change it as soon as we have the functionality in Skia though.

+  // the functionality.
+  if (texture_id) {
+    DCHECK(!IsCanvasDrawingOnPicture(canvas));
+    DCHECK(context_3d.gl);
+    context_3d.gl->DeleteTextures(1, &texture_id);
+  }
 }
 
 void SkCanvasVideoRenderer::Copy(const scoped_refptr<VideoFrame>& video_frame,
                                  SkCanvas* canvas,
                                  const Context3D& context_3d) {
   Paint(video_frame, canvas, video_frame->visible_rect(), 0xff,
         SkXfermode::kSrc_Mode, media::VIDEO_ROTATION_0, context_3d);
 }
 
 // static
@@ skipping 166 matching lines @@
   gl->PixelStorei(GL_UNPACK_FLIP_Y_CHROMIUM, false);
   gl->PixelStorei(GL_UNPACK_PREMULTIPLY_ALPHA_CHROMIUM, false);
 
   gl->DeleteTextures(1, &source_texture);
   gl->Flush();
 
   SyncPointClientImpl client(gl);
   video_frame->UpdateReleaseSyncPoint(&client);
 }
 
-void SkCanvasVideoRenderer::ResetLastFrame() {
-  last_frame_.reset();
-  last_frame_timestamp_ = media::kNoTimestamp();
-}
-
-void SkCanvasVideoRenderer::ResetAcceleratedLastFrame() {
-  accelerated_last_frame_.reset();
-  accelerated_generator_ = nullptr;
-  accelerated_last_frame_timestamp_ = media::kNoTimestamp();
-}
-
 }  // namespace media