Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1201)

Unified Diff: media/filters/skcanvas_video_renderer.cc

Issue 445013002: media: Optimize HW Video to 2D Canvas copy. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Use scratch texture Created 6 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: media/filters/skcanvas_video_renderer.cc
diff --git a/media/filters/skcanvas_video_renderer.cc b/media/filters/skcanvas_video_renderer.cc
index 003c1951a9b4010e93e612a347d645b22aa3f66a..313bc79bcf508b9efe0e1c6244274d7ac995cfc1 100644
--- a/media/filters/skcanvas_video_renderer.cc
+++ b/media/filters/skcanvas_video_renderer.cc
@@ -5,10 +5,16 @@
#include "media/filters/skcanvas_video_renderer.h"
#include "base/logging.h"
+#include "gpu/GLES2/gl2extchromium.h"
+#include "gpu/command_buffer/client/gles2_interface.h"
+#include "gpu/command_buffer/common/mailbox_holder.h"
#include "media/base/video_frame.h"
#include "media/base/yuv_convert.h"
+#include "skia/ext/refptr.h"
#include "third_party/libyuv/include/libyuv.h"
#include "third_party/skia/include/core/SkCanvas.h"
+#include "third_party/skia/include/gpu/GrContext.h"
+#include "third_party/skia/include/gpu/SkGrPixelRef.h"
#include "ui/gfx/skbitmap_operations.h"
// Skia internal format depends on a platform. On Android it is ABGR, on others
@@ -27,7 +33,15 @@
namespace media {
-static bool IsYUV(media::VideoFrame::Format format) {
+namespace {
+
+// This class keeps two temporary resources; software bitmap, hardware bitmap.
+// If both bitmap are created and then only software bitmap is updated every
+// frame, hardware bitmap outlives until the media player dies. So we delete
+// a temporary resource if it is not used for 3 sec.
+const int kTemporaryResourceDeletionDelay = 3; // Seconds;
+
+bool IsYUV(media::VideoFrame::Format format) {
return format == media::VideoFrame::YV12 ||
format == media::VideoFrame::YV16 ||
format == media::VideoFrame::I420 ||
@@ -36,16 +50,15 @@ static bool IsYUV(media::VideoFrame::Format format) {
format == media::VideoFrame::YV24;
}
-static bool IsYUVOrNative(media::VideoFrame::Format format) {
+bool IsYUVOrNative(media::VideoFrame::Format format) {
return IsYUV(format) || format == media::VideoFrame::NATIVE_TEXTURE;
}
// Converts a VideoFrame containing YUV data to a SkBitmap containing RGB data.
//
// |bitmap| will be (re)allocated to match the dimensions of |video_frame|.
-static void ConvertVideoFrameToBitmap(
- const scoped_refptr<media::VideoFrame>& video_frame,
- SkBitmap* bitmap) {
+void ConvertVideoFrameToBitmap(media::VideoFrame* video_frame,
+ SkBitmap* bitmap) {
DCHECK(IsYUVOrNative(video_frame->format()))
<< video_frame->format();
if (IsYUV(video_frame->format())) {
@@ -180,18 +193,118 @@ static void ConvertVideoFrameToBitmap(
bitmap->unlockPixels();
}
+bool EnsureTextureBackedSkBitmap(GrContext* gr,
+ SkBitmap* bitmap,
+ const gfx::Size& size) {
+ if (!bitmap->getTexture() || bitmap->width() != size.width() ||
+ bitmap->height() != size.height()) {
+ if (!gr)
+ return false;
+ GrTextureDesc desc;
+ // Use kRGBA_8888_GrPixelConfig, not kSkia8888_GrPixelConfig, because
+ // glCopyTextureChromium doesn't support GL_BGRA_EXT as internal format.
+ desc.fConfig = kRGBA_8888_GrPixelConfig;
+ desc.fFlags = kRenderTarget_GrTextureFlagBit | kNoStencil_GrTextureFlagBit;
+ desc.fSampleCnt = 0;
+ desc.fOrigin = kTopLeft_GrSurfaceOrigin;
+ desc.fWidth = size.width();
+ desc.fHeight = size.height();
+ GrAutoScratchTexture scratch_texture(
+ gr, desc, GrContext::kExact_ScratchTexMatch);
+ skia::RefPtr<GrTexture> texture = skia::AdoptRef(scratch_texture.detach());
+ if (!texture.get())
+ return false;
+
+ SkImageInfo info = SkImageInfo::MakeN32Premul(desc.fWidth, desc.fHeight);
+ SkGrPixelRef* pixelRef = SkNEW_ARGS(SkGrPixelRef, (info, texture.get()));
+ if (!pixelRef)
+ return false;
+ bitmap->setInfo(info);
+ bitmap->setPixelRef(pixelRef)->unref();
+ }
+
+ return true;
+}
+
+bool ConvertVideoFrameToTexture(
+ VideoFrame* video_frame,
+ SkBitmap* bitmap,
+ SkCanvasVideoRenderer::Context3DProvider* context_provider) {
+ DCHECK(context_provider &&
+ video_frame->format() == VideoFrame::NATIVE_TEXTURE);
+ gpu::gles2::GLES2Interface* gl = context_provider->gl;
+ DCHECK(gl);
+
+ // Check if we could reuse existing texture based bitmap.
+ // Otherwise, release existing texture based bitmap and allocate
+ // a new one based on video size.
+ if (!EnsureTextureBackedSkBitmap(context_provider->gr_context,
+ bitmap,
+ video_frame->visible_rect().size())) {
+ return false;
+ }
+
+ unsigned textureId =
+ static_cast<unsigned>((bitmap->getTexture())->getTextureHandle());
+ // If CopyVideoFrameToTexture() changes the state of the 'textureId', it's
+ // needed to invalidate the state cached in skia, but currently the state
+ // isn't changed.
+ SkCanvasVideoRenderer::CopyVideoFrameToTexture(
+ gl, video_frame, textureId, 0, GL_RGBA, GL_UNSIGNED_BYTE, true, false);
+ return true;
+}
+
+void RotateBitmap(SkBitmap* bitmap, VideoRotation video_rotation) {
+ switch (video_rotation) {
+ case VIDEO_ROTATION_0:
+ break;
+ case VIDEO_ROTATION_90:
+ *bitmap = SkBitmapOperations::Rotate(*bitmap,
+ SkBitmapOperations::ROTATION_90_CW);
+ break;
+ case VIDEO_ROTATION_180:
+ *bitmap = SkBitmapOperations::Rotate(*bitmap,
+ SkBitmapOperations::ROTATION_180_CW);
+ break;
+ case VIDEO_ROTATION_270:
+ *bitmap = SkBitmapOperations::Rotate(*bitmap,
+ SkBitmapOperations::ROTATION_270_CW);
+ break;
+ }
+}
+
+class SyncPointClientImpl : public VideoFrame::SyncPointClient {
+ public:
+ explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl) : gl_(gl) {}
+ virtual ~SyncPointClientImpl() {}
+ virtual uint32 InsertSyncPoint() OVERRIDE {
+ return gl_->InsertSyncPointCHROMIUM();
+ }
+ virtual void WaitSyncPoint(uint32 sync_point) OVERRIDE {
+ gl_->WaitSyncPointCHROMIUM(sync_point);
+ }
+
+ private:
+ gpu::gles2::GLES2Interface* gl_;
+};
+
+} // anonymous namespace
+
SkCanvasVideoRenderer::SkCanvasVideoRenderer()
- : last_frame_timestamp_(media::kNoTimestamp()) {
+ : last_frame_timestamp_(media::kNoTimestamp()),
+ accelerated_last_frame_timestamp_(media::kNoTimestamp()) {
}
SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {}
-void SkCanvasVideoRenderer::Paint(media::VideoFrame* video_frame,
- SkCanvas* canvas,
- const gfx::RectF& dest_rect,
- uint8 alpha,
- SkXfermode::Mode mode,
- VideoRotation video_rotation) {
+void SkCanvasVideoRenderer::Paint(
+ media::VideoFrame* video_frame,
+ SkCanvas* canvas,
+ const gfx::RectF& dest_rect,
+ uint8 alpha,
+ SkXfermode::Mode mode,
+ VideoRotation video_rotation,
+ SkCanvasVideoRenderer::Context3DProvider* context_provider) {
if (alpha == 0) {
return;
}
@@ -204,41 +317,48 @@ void SkCanvasVideoRenderer::Paint(media::VideoFrame* video_frame,
// Paint black rectangle if there isn't a frame available or the
// frame has an unexpected format.
- if (!video_frame || !IsYUVOrNative(video_frame->format())) {
+ if (!video_frame || video_frame->natural_size().IsEmpty() ||
+ !IsYUVOrNative(video_frame->format())) {
canvas->drawRect(dest, paint);
return;
}
- // Check if we should convert and update |last_frame_|.
- if (last_frame_.isNull() ||
- video_frame->timestamp() != last_frame_timestamp_) {
- ConvertVideoFrameToBitmap(video_frame, &last_frame_);
-
- switch (video_rotation) {
- case VIDEO_ROTATION_0:
- break;
- case VIDEO_ROTATION_90:
- last_frame_ = SkBitmapOperations::Rotate(
- last_frame_, SkBitmapOperations::ROTATION_90_CW);
- break;
- case VIDEO_ROTATION_180:
- last_frame_ = SkBitmapOperations::Rotate(
- last_frame_, SkBitmapOperations::ROTATION_180_CW);
- break;
- case VIDEO_ROTATION_270:
- last_frame_ = SkBitmapOperations::Rotate(
- last_frame_, SkBitmapOperations::ROTATION_270_CW);
- break;
+ bool accelerated = false;
+ if (context_provider &&
+ video_frame->format() == media::VideoFrame::NATIVE_TEXTURE &&
+ canvas->getGrContext()) {
+ // TODO(dshwang): Android video decoder doesn't update the timestamp on a
+ // VideoFrame. To reduce redundant copy, Android should update the
+ // timestamp.
+ if (accelerated_last_frame_.isNull() ||
+ video_frame->timestamp() != accelerated_last_frame_timestamp_ ||
+ video_frame->timestamp() == base::TimeDelta()) {
+ accelerated = ConvertVideoFrameToTexture(
+ video_frame, &accelerated_last_frame_, context_provider);
+ if (accelerated) {
+ RotateBitmap(&accelerated_last_frame_, video_rotation);
+ accelerated_last_frame_timestamp_ = video_frame->timestamp();
+ }
+ } else {
+ DCHECK(accelerated_last_frame_.getTexture());
+ accelerated = true;
}
+ }
+ // Check if we should convert and update |last_frame_|.
+ if (!accelerated && (last_frame_.isNull() ||
+ video_frame->timestamp() != last_frame_timestamp_)) {
+ ConvertVideoFrameToBitmap(video_frame, &last_frame_);
+ RotateBitmap(&last_frame_, video_rotation);
last_frame_timestamp_ = video_frame->timestamp();
}
paint.setXfermodeMode(mode);
- // Paint using |last_frame_|.
- paint.setFilterLevel(SkPaint::kLow_FilterLevel);
- canvas->drawBitmapRect(last_frame_, NULL, dest, &paint);
+ canvas->drawBitmapRect(
+ accelerated ? accelerated_last_frame_ : last_frame_, NULL, dest, &paint);
+
+ CleanUpTemporaryBuffers();
}
void SkCanvasVideoRenderer::Copy(media::VideoFrame* video_frame,
@@ -248,7 +368,64 @@ void SkCanvasVideoRenderer::Copy(media::VideoFrame* video_frame,
video_frame->visible_rect(),
0xff,
SkXfermode::kSrc_Mode,
- media::VIDEO_ROTATION_0);
+ media::VIDEO_ROTATION_0,
+ NULL);
+}
+
+void SkCanvasVideoRenderer::CleanUpTemporaryBuffers() {
+ static const base::TimeDelta buffer_time =
+ base::TimeDelta::FromSeconds(kTemporaryResourceDeletionDelay);
+ // Find the latest time stamp.
+ base::TimeDelta last_timestamp =
+ accelerated_last_frame_timestamp_ > last_frame_timestamp_
+ ? accelerated_last_frame_timestamp_
+ : last_frame_timestamp_;
+ // Delete a too old resource.
+ if (last_timestamp > last_frame_timestamp_ + buffer_time &&
rileya (GONE FROM CHROMIUM) 2014/09/02 21:21:43 Multi-line conditionals need {}'s.
+ !last_frame_.isNull())
+ last_frame_.reset();
+ if (last_timestamp > accelerated_last_frame_timestamp_ + buffer_time &&
rileya (GONE FROM CHROMIUM) 2014/09/02 21:21:43 ditto
+ !accelerated_last_frame_.isNull())
+ accelerated_last_frame_.reset();
+}
+
+// static
+void SkCanvasVideoRenderer::CopyVideoFrameToTexture(
+ gpu::gles2::GLES2Interface* gl,
+ VideoFrame* video_frame,
+ unsigned int texture,
+ unsigned int level,
+ unsigned int internal_format,
+ unsigned int type,
+ bool premultiply_alpha,
+ bool flip_y) {
+ DCHECK(video_frame && video_frame->format() == VideoFrame::NATIVE_TEXTURE);
+ const gpu::MailboxHolder* mailbox_holder = video_frame->mailbox_holder();
+ DCHECK(mailbox_holder->texture_target == GL_TEXTURE_2D ||
+ mailbox_holder->texture_target == GL_TEXTURE_EXTERNAL_OES);
+
+ gl->WaitSyncPointCHROMIUM(mailbox_holder->sync_point);
+ uint32 source_texture = gl->CreateAndConsumeTextureCHROMIUM(
+ mailbox_holder->texture_target, mailbox_holder->mailbox.name);
+
+ // The video is stored in a unmultiplied format, so premultiply
+ // if necessary.
+ gl->PixelStorei(GL_UNPACK_PREMULTIPLY_ALPHA_CHROMIUM, premultiply_alpha);
+ // Application itself needs to take care of setting the right flip_y
rileya (GONE FROM CHROMIUM) 2014/09/02 21:21:43 In media/ we enclose variable names in ||'s, so th
+ // value down to get the expected result.
+ // flip_y==true means to reverse the video orientation while
+ // flip_y==false means to keep the intrinsic orientation.
+ gl->PixelStorei(GL_UNPACK_FLIP_Y_CHROMIUM, flip_y);
+ gl->CopyTextureCHROMIUM(
+ GL_TEXTURE_2D, source_texture, texture, level, internal_format, type);
+ gl->PixelStorei(GL_UNPACK_FLIP_Y_CHROMIUM, false);
+ gl->PixelStorei(GL_UNPACK_PREMULTIPLY_ALPHA_CHROMIUM, false);
+
+ gl->DeleteTextures(1, &source_texture);
+ gl->Flush();
+
+ SyncPointClientImpl client(gl);
+ video_frame->UpdateReleaseSyncPoint(&client);
}
} // namespace media

Powered by Google App Engine
This is Rietveld 408576698