media: SkCanvasVideoRenderer support for YUV native videoframes.

media/blink/skcanvas_video_renderer.cc

   // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/blink/skcanvas_video_renderer.h"
 
 #include "gpu/GLES2/gl2extchromium.h"
 #include "gpu/command_buffer/client/gles2_interface.h"
 #include "gpu/command_buffer/common/mailbox_holder.h"
 #include "media/base/video_frame.h"
 #include "media/base/yuv_convert.h"
 #include "skia/ext/refptr.h"
 #include "third_party/libyuv/include/libyuv.h"
 #include "third_party/skia/include/core/SkCanvas.h"
+#include "third_party/skia/include/core/SkImage.h"
 #include "third_party/skia/include/core/SkImageGenerator.h"
 #include "third_party/skia/include/gpu/GrContext.h"
+#include "third_party/skia/include/gpu/GrPaint.h"
+#include "third_party/skia/include/gpu/GrTexture.h"
 #include "third_party/skia/include/gpu/GrTextureProvider.h"
+#include "third_party/skia/include/gpu/SkGr.h"
 #include "third_party/skia/include/gpu/SkGrPixelRef.h"
 #include "ui/gfx/skbitmap_operations.h"
 
 // Skia internal format depends on a platform. On Android it is ABGR, on others
 // it is ARGB.
 #if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \
     SK_A32_SHIFT == 24
 #define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB
 #define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB
 #elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \
@@ skipping 49 matching lines @@
 
   SkImageInfo info = SkImageInfo::MakeN32Premul(desc.fWidth, desc.fHeight);
   SkGrPixelRef* pixel_ref = SkNEW_ARGS(SkGrPixelRef, (info, texture.get()));
   if (!pixel_ref)
     return false;
   bitmap->setInfo(info);
   bitmap->setPixelRef(pixel_ref)->unref();
   return true;
 }
 
-bool CopyVideoFrameTextureToSkBitmapTexture(VideoFrame* video_frame,
-                                            SkBitmap* bitmap,
-                                            const Context3D& context_3d) {
+class SyncPointClientImpl : public VideoFrame::SyncPointClient {
+ public:
+  explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl) : gl_(gl) {}
+  ~SyncPointClientImpl() override {}
+  uint32 InsertSyncPoint() override { return gl_->InsertSyncPointCHROMIUM(); }
+  void WaitSyncPoint(uint32 sync_point) override {
+    gl_->WaitSyncPointCHROMIUM(sync_point);
+  }
+
+ private:
+  gpu::gles2::GLES2Interface* gl_;
+
+  DISALLOW_IMPLICIT_CONSTRUCTORS(SyncPointClientImpl);
+};
+
+scoped_ptr<SkImage> CreateSkImageFromVideoFrameYUVTextures(
+    VideoFrame* video_frame,
+    const Context3D& context_3d) {
+  // Support only TEXTURE_YUV_420.
+  DCHECK(video_frame->storage_type() == VideoFrame::STORAGE_TEXTURE);
+  DCHECK_EQ(media::VideoFrame::I420, video_frame->format());
+  DCHECK_EQ(3u, media::VideoFrame::NumPlanes(video_frame->format()));
+
+  gpu::gles2::GLES2Interface* gl = context_3d.gl;
+  DCHECK(gl);
+  gfx::Size ya_tex_size = video_frame->coded_size();
+  gfx::Size uv_tex_size((ya_tex_size.width() + 1) / 2,
+                        (ya_tex_size.height() + 1) / 2);
+
+  unsigned source_textures[3] = {0};
+  for (size_t i = 0; i < media::VideoFrame::NumPlanes(video_frame->format());
+       ++i) {
+    // Get the texture from the mailbox and wrap it in a GrTexture.
+    const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(i);
+    DCHECK(mailbox_holder.texture_target == GL_TEXTURE_2D ||
+           mailbox_holder.texture_target == GL_TEXTURE_EXTERNAL_OES ||
+           mailbox_holder.texture_target == GL_TEXTURE_RECTANGLE_ARB);
+    gl->WaitSyncPointCHROMIUM(mailbox_holder.sync_point);
+    source_textures[i] = gl->CreateAndConsumeTextureCHROMIUM(
+        mailbox_holder.texture_target, mailbox_holder.mailbox.name);
+  }
+  GrBackendObject handles[3] = {
+      source_textures[0], source_textures[1], source_textures[2]};
+
+  SkISize yuvSizes[] = {
+      {ya_tex_size.width(), ya_tex_size.height()},
+      {uv_tex_size.width(), uv_tex_size.height()},
+      {uv_tex_size.width(), uv_tex_size.height()},
+  };
+  SkImage* img = SkImage::NewFromYUVTexturesCopy(
+      context_3d.gr_context, kRec601_SkYUVColorSpace, handles, yuvSizes,

[DaleCurtis, 2015/06/10 22:46:51]

Is this the right color space? VideoFrame should have a color space metadata
value you can use.

[Daniele Castagna, 2015/06/11 15:45:35]

We're using native YUV video frames only for I420, so I'd expect only REC601
color space.

Changed the code to get it from the metadata and added a DCHECK for the non
supported color space.

+      kTopLeft_GrSurfaceOrigin);
+  DCHECK(img);
+  gl->DeleteTextures(3, source_textures);
+  SyncPointClientImpl client(gl);
+  video_frame->UpdateReleaseSyncPoint(&client);
+  return make_scoped_ptr(img);
+}
+
+bool CopyVideoFrameSingleTextureToSkBitmap(VideoFrame* video_frame,
+                                           SkBitmap* bitmap,
+                                           const Context3D& context_3d) {
   // Check if we could reuse existing texture based bitmap.
   // Otherwise, release existing texture based bitmap and allocate
   // a new one based on video size.
   if (!IsSkBitmapProperlySizedTexture(bitmap,
                                       video_frame->visible_rect().size())) {
     if (!AllocateSkBitmapTexture(context_3d.gr_context, bitmap,
                                  video_frame->visible_rect().size())) {
       return false;
     }
   }
 
   unsigned texture_id =
       static_cast<unsigned>((bitmap->getTexture())->getTextureHandle());
-  // If CopyVideoFrameTextureToGLTexture() changes the state of the
+  // If CopyVideoFrameSingleTextureToGLTexture() changes the state of the
   // |texture_id|, it's needed to invalidate the state cached in skia,
   // but currently the state isn't changed.
-  SkCanvasVideoRenderer::CopyVideoFrameTextureToGLTexture(
+
+  SkCanvasVideoRenderer::CopyVideoFrameSingleTextureToGLTexture(
       context_3d.gl, video_frame, texture_id, GL_RGBA, GL_UNSIGNED_BYTE, true,
       false);
   bitmap->notifyPixelsChanged();
   return true;
 }
 
-class SyncPointClientImpl : public VideoFrame::SyncPointClient {
- public:
-  explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl) : gl_(gl) {}
-  ~SyncPointClientImpl() override {}
-  uint32 InsertSyncPoint() override { return gl_->InsertSyncPointCHROMIUM(); }
-  void WaitSyncPoint(uint32 sync_point) override {
-    gl_->WaitSyncPointCHROMIUM(sync_point);
-  }
-
- private:
-  gpu::gles2::GLES2Interface* gl_;
-
-  DISALLOW_IMPLICIT_CONSTRUCTORS(SyncPointClientImpl);
-};
-
 }  // anonymous namespace
 
 // Generates an RGB image from a VideoFrame. Convert YUV to RGB plain on GPU.
 class VideoImageGenerator : public SkImageGenerator {
  public:
   VideoImageGenerator(const scoped_refptr<VideoFrame>& frame)
       : SkImageGenerator(
             SkImageInfo::MakeN32Premul(frame->visible_rect().width(),
                                        frame->visible_rect().height()))
       , frame_(frame) {
@@ skipping 145 matching lines @@
     // In SW Canvas case, rely on skia drawing Ganesh SkBitmap on SW SkCanvas.
     if (accelerated_last_frame_.isNull() ||
         video_frame->timestamp() != accelerated_last_frame_timestamp_) {
       DCHECK(context_3d.gl);
       DCHECK(context_3d.gr_context);
       if (accelerated_generator_) {
         // Reset SkBitmap used in SWVideo-to-HWCanvas path.
         accelerated_last_frame_.reset();
         accelerated_generator_ = nullptr;
       }
-      if (!CopyVideoFrameTextureToSkBitmapTexture(
-              video_frame.get(), &accelerated_last_frame_, context_3d)) {
-        NOTREACHED();
-        return;
+
+      if (media::VideoFrame::NumPlanes(video_frame->format()) == 1) {
+        accelerated_last_image_.reset();
+        if (!CopyVideoFrameSingleTextureToSkBitmap(
+                video_frame.get(), &accelerated_last_frame_, context_3d)) {
+          NOTREACHED();
+          return;
+        }
+        DCHECK(video_frame->visible_rect().width() ==
+                   accelerated_last_frame_.width() &&
+               video_frame->visible_rect().height() ==
+                   accelerated_last_frame_.height());
+      } else {
+        accelerated_last_image_ = CreateSkImageFromVideoFrameYUVTextures(
+            video_frame.get(), context_3d);
+        DCHECK(accelerated_last_image_);
       }
-      DCHECK(video_frame->visible_rect().width() ==
-                 accelerated_last_frame_.width() &&
-             video_frame->visible_rect().height() ==
-                 accelerated_last_frame_.height());
-
       accelerated_last_frame_timestamp_ = video_frame->timestamp();
     }
     target_frame = &accelerated_last_frame_;
     accelerated_frame_deleting_timer_.Reset();
   } else if (canvas->getGrContext()) {
     if (accelerated_last_frame_.isNull() ||
         video_frame->timestamp() != accelerated_last_frame_timestamp_) {
       // Draw SW Video on HW Canvas.
       if (!accelerated_generator_ && !accelerated_last_frame_.isNull()) {
         // Reset SkBitmap used in HWVideo-to-HWCanvas path.
@@ skipping 75 matching lines @@
         video_rotation == VIDEO_ROTATION_270) {
       rotated_dest_size =
           gfx::SizeF(rotated_dest_size.height(), rotated_dest_size.width());
     }
     canvas->scale(
         SkFloatToScalar(rotated_dest_size.width() / target_frame->width()),
         SkFloatToScalar(rotated_dest_size.height() / target_frame->height()));
     canvas->translate(-SkFloatToScalar(target_frame->width() * 0.5f),
                       -SkFloatToScalar(target_frame->height() * 0.5f));
   }
-  canvas->drawBitmap(*target_frame, 0, 0, &paint);
+  if (accelerated_last_image_) {
+    canvas->drawImage(accelerated_last_image_.get(), 0, 0, &paint);
+  } else {
+    canvas->drawBitmap(*target_frame, 0, 0, &paint);
+  }
   if (need_transform)
     canvas->restore();
   canvas->flush();
   // SkCanvas::flush() causes the generator to generate SkImage, so delete
   // |video_frame| not to be outlived.
   if (canvas->getGrContext() && accelerated_generator_)
     accelerated_generator_->set_frame(nullptr);
 }
 
 void SkCanvasVideoRenderer::Copy(const scoped_refptr<VideoFrame>& video_frame,
@@ skipping 132 matching lines @@
     case VideoFrame::NV12:
 #endif
     case VideoFrame::ARGB:
     case VideoFrame::XRGB:
     case VideoFrame::UNKNOWN:
       NOTREACHED();
   }
 }
 
 // static
-void SkCanvasVideoRenderer::CopyVideoFrameTextureToGLTexture(
+void SkCanvasVideoRenderer::CopyVideoFrameSingleTextureToGLTexture(
     gpu::gles2::GLES2Interface* gl,
     VideoFrame* video_frame,
     unsigned int texture,
     unsigned int internal_format,
     unsigned int type,
     bool premultiply_alpha,
     bool flip_y) {
   DCHECK(video_frame);
   DCHECK_EQ(video_frame->storage_type(), VideoFrame::STORAGE_TEXTURE);
   DCHECK_EQ(1u, VideoFrame::NumPlanes(video_frame->format()));
@@ skipping 26 matching lines @@
   SyncPointClientImpl client(gl);
   video_frame->UpdateReleaseSyncPoint(&client);
 }
 
 void SkCanvasVideoRenderer::ResetLastFrame() {
   last_frame_.reset();
   last_frame_timestamp_ = media::kNoTimestamp();
 }
 
 void SkCanvasVideoRenderer::ResetAcceleratedLastFrame() {
+  accelerated_last_image_.reset();
   accelerated_last_frame_.reset();
   accelerated_generator_ = nullptr;
   accelerated_last_frame_timestamp_ = media::kNoTimestamp();
 }
 
 }  // namespace media