OLD | NEW |
---|---|
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "media/filters/skcanvas_video_renderer.h" | 5 #include "media/filters/skcanvas_video_renderer.h" |
6 | 6 |
7 #include "base/logging.h" | 7 #include "base/logging.h" |
8 #include "gpu/GLES2/gl2extchromium.h" | |
9 #include "gpu/command_buffer/client/gles2_interface.h" | |
10 #include "gpu/command_buffer/common/mailbox_holder.h" | |
8 #include "media/base/video_frame.h" | 11 #include "media/base/video_frame.h" |
9 #include "media/base/yuv_convert.h" | 12 #include "media/base/yuv_convert.h" |
13 #include "skia/ext/refptr.h" | |
10 #include "third_party/libyuv/include/libyuv.h" | 14 #include "third_party/libyuv/include/libyuv.h" |
11 #include "third_party/skia/include/core/SkCanvas.h" | 15 #include "third_party/skia/include/core/SkCanvas.h" |
16 #include "third_party/skia/include/gpu/GrContext.h" | |
17 #include "third_party/skia/include/gpu/SkGrPixelRef.h" | |
12 #include "ui/gfx/skbitmap_operations.h" | 18 #include "ui/gfx/skbitmap_operations.h" |
13 | 19 |
14 // Skia internal format depends on a platform. On Android it is ABGR, on others | 20 // Skia internal format depends on a platform. On Android it is ABGR, on others |
15 // it is ARGB. | 21 // it is ARGB. |
16 #if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \ | 22 #if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \ |
17 SK_A32_SHIFT == 24 | 23 SK_A32_SHIFT == 24 |
18 #define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB | 24 #define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB |
19 #define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB | 25 #define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB |
20 #elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \ | 26 #elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \ |
21 SK_A32_SHIFT == 24 | 27 SK_A32_SHIFT == 24 |
22 #define LIBYUV_I420_TO_ARGB libyuv::I420ToABGR | 28 #define LIBYUV_I420_TO_ARGB libyuv::I420ToABGR |
23 #define LIBYUV_I422_TO_ARGB libyuv::I422ToABGR | 29 #define LIBYUV_I422_TO_ARGB libyuv::I422ToABGR |
24 #else | 30 #else |
25 #error Unexpected Skia ARGB_8888 layout! | 31 #error Unexpected Skia ARGB_8888 layout! |
26 #endif | 32 #endif |
27 | 33 |
28 namespace media { | 34 namespace media { |
29 | 35 |
30 static bool IsYUV(media::VideoFrame::Format format) { | 36 namespace { |
37 | |
38 bool IsYUV(media::VideoFrame::Format format) { | |
31 return format == media::VideoFrame::YV12 || | 39 return format == media::VideoFrame::YV12 || |
32 format == media::VideoFrame::YV16 || | 40 format == media::VideoFrame::YV16 || |
33 format == media::VideoFrame::I420 || | 41 format == media::VideoFrame::I420 || |
34 format == media::VideoFrame::YV12A || | 42 format == media::VideoFrame::YV12A || |
35 format == media::VideoFrame::YV12J || | 43 format == media::VideoFrame::YV12J || |
36 format == media::VideoFrame::YV24; | 44 format == media::VideoFrame::YV24; |
37 } | 45 } |
38 | 46 |
39 static bool IsYUVOrNative(media::VideoFrame::Format format) { | 47 bool IsYUVOrNative(media::VideoFrame::Format format) { |
40 return IsYUV(format) || format == media::VideoFrame::NATIVE_TEXTURE; | 48 return IsYUV(format) || format == media::VideoFrame::NATIVE_TEXTURE; |
41 } | 49 } |
42 | 50 |
43 // Converts a VideoFrame containing YUV data to a SkBitmap containing RGB data. | 51 // Converts a VideoFrame containing YUV data to a SkBitmap containing RGB data. |
44 // | 52 // |
45 // |bitmap| will be (re)allocated to match the dimensions of |video_frame|. | 53 // |bitmap| will be (re)allocated to match the dimensions of |video_frame|. |
46 static void ConvertVideoFrameToBitmap( | 54 void ConvertVideoFrameToBitmap(media::VideoFrame* video_frame, |
47 const scoped_refptr<media::VideoFrame>& video_frame, | 55 SkBitmap* bitmap) { |
48 SkBitmap* bitmap) { | |
49 DCHECK(IsYUVOrNative(video_frame->format())) | 56 DCHECK(IsYUVOrNative(video_frame->format())) |
50 << video_frame->format(); | 57 << video_frame->format(); |
51 if (IsYUV(video_frame->format())) { | 58 if (IsYUV(video_frame->format())) { |
52 DCHECK_EQ(video_frame->stride(media::VideoFrame::kUPlane), | 59 DCHECK_EQ(video_frame->stride(media::VideoFrame::kUPlane), |
53 video_frame->stride(media::VideoFrame::kVPlane)); | 60 video_frame->stride(media::VideoFrame::kVPlane)); |
54 } | 61 } |
55 | 62 |
56 // Check if |bitmap| needs to be (re)allocated. | 63 // Check if |bitmap| needs to be (re)allocated. |
57 if (bitmap->isNull() || | 64 if (bitmap->isNull() || |
58 bitmap->width() != video_frame->visible_rect().width() || | 65 bitmap->width() != video_frame->visible_rect().width() || |
(...skipping 114 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
173 break; | 180 break; |
174 | 181 |
175 default: | 182 default: |
176 NOTREACHED(); | 183 NOTREACHED(); |
177 break; | 184 break; |
178 } | 185 } |
179 bitmap->notifyPixelsChanged(); | 186 bitmap->notifyPixelsChanged(); |
180 bitmap->unlockPixels(); | 187 bitmap->unlockPixels(); |
181 } | 188 } |
182 | 189 |
190 bool EnsureTextureBackedSkBitmap(GrContext* gr, | |
191 SkBitmap* bitmap, | |
192 const gfx::Size& size, | |
193 GrSurfaceOrigin origin, | |
194 GrPixelConfig config) { | |
195 if (!bitmap->getTexture() || bitmap->width() != size.width() || | |
196 bitmap->height() != size.height()) { | |
197 if (!gr) | |
198 return false; | |
199 GrTextureDesc desc; | |
200 desc.fConfig = config; | |
201 desc.fFlags = kRenderTarget_GrTextureFlagBit | kNoStencil_GrTextureFlagBit; | |
202 desc.fSampleCnt = 0; | |
203 desc.fOrigin = origin; | |
204 desc.fWidth = size.width(); | |
205 desc.fHeight = size.height(); | |
206 skia::RefPtr<GrTexture> texture; | |
207 texture = skia::AdoptRef(gr->createUncachedTexture(desc, 0, 0)); | |
Justin Novosad
2014/08/28 17:23:18
Using a scratch texture would probably be better f
dshwang
2014/08/28 17:53:09
Yep, I'll update.
dshwang
2014/09/01 07:54:56
Done.
| |
208 if (!texture.get()) | |
209 return false; | |
210 | |
211 SkImageInfo info = SkImageInfo::MakeN32Premul(desc.fWidth, desc.fHeight); | |
212 SkGrPixelRef* pixelRef = SkNEW_ARGS(SkGrPixelRef, (info, texture.get())); | |
213 if (!pixelRef) | |
214 return false; | |
215 bitmap->setInfo(info); | |
216 bitmap->setPixelRef(pixelRef)->unref(); | |
217 } | |
218 | |
219 return true; | |
220 } | |
221 | |
222 bool ConvertVideoFrameToTexture( | |
223 VideoFrame* video_frame, | |
224 SkBitmap* bitmap, | |
225 SkCanvasVideoRenderer::Context3DProvider* context_provider) { | |
226 DCHECK(context_provider && | |
227 video_frame->format() == VideoFrame::NATIVE_TEXTURE); | |
228 gpu::gles2::GLES2Interface* gl = context_provider->gl; | |
229 DCHECK(gl); | |
230 | |
231 // Check if we could reuse existing texture based bitmap. | |
232 // Otherwise, release existing texture based bitmap and allocate | |
233 // a new one based on video size. | |
234 // Use kRGBA_8888_GrPixelConfig, not kSkia8888_GrPixelConfig, because | |
235 // glCopyTextureChromium doesn't support GL_BGRA_EXT as internal format. | |
236 if (!EnsureTextureBackedSkBitmap(context_provider->gr_context, | |
237 bitmap, | |
238 video_frame->visible_rect().size(), | |
239 kTopLeft_GrSurfaceOrigin, | |
240 kRGBA_8888_GrPixelConfig)) { | |
241 return false; | |
242 } | |
243 | |
244 unsigned textureId = | |
245 static_cast<unsigned>((bitmap->getTexture())->getTextureHandle()); | |
246 SkCanvasVideoRenderer::CopyVideoFrameToTexture( | |
247 gl, video_frame, textureId, 0, GL_RGBA, GL_UNSIGNED_BYTE, true, false); | |
248 return true; | |
249 } | |
250 | |
251 class SyncPointClientImpl : public VideoFrame::SyncPointClient { | |
252 public: | |
253 explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl) : gl_(gl) {} | |
254 virtual ~SyncPointClientImpl() {} | |
255 virtual uint32 InsertSyncPoint() OVERRIDE { | |
256 return gl_->InsertSyncPointCHROMIUM(); | |
257 } | |
258 virtual void WaitSyncPoint(uint32 sync_point) OVERRIDE { | |
259 gl_->WaitSyncPointCHROMIUM(sync_point); | |
260 } | |
261 | |
262 private: | |
263 gpu::gles2::GLES2Interface* gl_; | |
264 }; | |
265 | |
266 } // anonymous namespace | |
267 | |
183 SkCanvasVideoRenderer::SkCanvasVideoRenderer() | 268 SkCanvasVideoRenderer::SkCanvasVideoRenderer() |
184 : last_frame_timestamp_(media::kNoTimestamp()) { | 269 : last_frame_timestamp_(media::kNoTimestamp()), |
270 accelerated_last_frame_timestamp_(media::kNoTimestamp()) { | |
185 } | 271 } |
186 | 272 |
187 SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {} | 273 SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {} |
188 | 274 |
189 void SkCanvasVideoRenderer::Paint(media::VideoFrame* video_frame, | 275 void SkCanvasVideoRenderer::Paint( |
190 SkCanvas* canvas, | 276 media::VideoFrame* video_frame, |
191 const gfx::RectF& dest_rect, | 277 SkCanvas* canvas, |
192 uint8 alpha, | 278 const gfx::RectF& dest_rect, |
193 SkXfermode::Mode mode, | 279 uint8 alpha, |
194 VideoRotation video_rotation) { | 280 SkXfermode::Mode mode, |
281 VideoRotation video_rotation, | |
282 SkCanvasVideoRenderer::Context3DProvider* context_provider) { | |
195 if (alpha == 0) { | 283 if (alpha == 0) { |
196 return; | 284 return; |
197 } | 285 } |
198 | 286 |
199 SkRect dest; | 287 SkRect dest; |
200 dest.set(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom()); | 288 dest.set(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom()); |
201 | 289 |
202 SkPaint paint; | 290 SkPaint paint; |
203 paint.setAlpha(alpha); | 291 paint.setAlpha(alpha); |
204 | 292 |
205 // Paint black rectangle if there isn't a frame available or the | 293 // Paint black rectangle if there isn't a frame available or the |
206 // frame has an unexpected format. | 294 // frame has an unexpected format. |
207 if (!video_frame || !IsYUVOrNative(video_frame->format())) { | 295 if (!video_frame || video_frame->natural_size().IsEmpty() || |
296 !IsYUVOrNative(video_frame->format())) { | |
208 canvas->drawRect(dest, paint); | 297 canvas->drawRect(dest, paint); |
209 return; | 298 return; |
210 } | 299 } |
211 | 300 |
301 bool accelerated = false; | |
302 if (context_provider && | |
303 video_frame->format() == media::VideoFrame::NATIVE_TEXTURE && | |
304 canvas->getGrContext()) { | |
305 // TODO(dshwang): Android video decoder doesn't update the timestamp on a | |
306 // VideoFrame. To reduce redundant copy, Android should update the | |
307 // timestamp. | |
308 if (accelerated_last_frame_.isNull() || | |
309 video_frame->timestamp() != accelerated_last_frame_timestamp_ || | |
310 video_frame->timestamp() == base::TimeDelta()) { | |
311 accelerated = ConvertVideoFrameToTexture( | |
312 video_frame, &accelerated_last_frame_, context_provider); | |
313 if (accelerated) { | |
314 accelerated_last_frame_timestamp_ = video_frame->timestamp(); | |
315 } | |
316 } else { | |
317 DCHECK(accelerated_last_frame_.getTexture()); | |
318 accelerated = true; | |
319 } | |
320 } | |
321 | |
212 // Check if we should convert and update |last_frame_|. | 322 // Check if we should convert and update |last_frame_|. |
213 if (last_frame_.isNull() || | 323 if (!accelerated && (last_frame_.isNull() || |
rileya (GONE FROM CHROMIUM)
2014/08/27 20:30:50
It doesn't look like we address video rotation in
| |
214 video_frame->timestamp() != last_frame_timestamp_) { | 324 video_frame->timestamp() != last_frame_timestamp_)) { |
215 ConvertVideoFrameToBitmap(video_frame, &last_frame_); | 325 ConvertVideoFrameToBitmap(video_frame, &last_frame_); |
216 | 326 |
217 switch (video_rotation) { | 327 switch (video_rotation) { |
218 case VIDEO_ROTATION_0: | 328 case VIDEO_ROTATION_0: |
219 break; | 329 break; |
220 case VIDEO_ROTATION_90: | 330 case VIDEO_ROTATION_90: |
221 last_frame_ = SkBitmapOperations::Rotate( | 331 last_frame_ = SkBitmapOperations::Rotate( |
222 last_frame_, SkBitmapOperations::ROTATION_90_CW); | 332 last_frame_, SkBitmapOperations::ROTATION_90_CW); |
223 break; | 333 break; |
224 case VIDEO_ROTATION_180: | 334 case VIDEO_ROTATION_180: |
225 last_frame_ = SkBitmapOperations::Rotate( | 335 last_frame_ = SkBitmapOperations::Rotate( |
226 last_frame_, SkBitmapOperations::ROTATION_180_CW); | 336 last_frame_, SkBitmapOperations::ROTATION_180_CW); |
227 break; | 337 break; |
228 case VIDEO_ROTATION_270: | 338 case VIDEO_ROTATION_270: |
229 last_frame_ = SkBitmapOperations::Rotate( | 339 last_frame_ = SkBitmapOperations::Rotate( |
230 last_frame_, SkBitmapOperations::ROTATION_270_CW); | 340 last_frame_, SkBitmapOperations::ROTATION_270_CW); |
231 break; | 341 break; |
232 } | 342 } |
233 | 343 |
234 last_frame_timestamp_ = video_frame->timestamp(); | 344 last_frame_timestamp_ = video_frame->timestamp(); |
235 } | 345 } |
236 | 346 |
237 paint.setXfermodeMode(mode); | 347 paint.setXfermodeMode(mode); |
238 | 348 |
239 // Paint using |last_frame_|. | 349 canvas->drawBitmapRect( |
240 paint.setFilterLevel(SkPaint::kLow_FilterLevel); | 350 accelerated ? accelerated_last_frame_ : last_frame_, NULL, dest, &paint); |
241 canvas->drawBitmapRect(last_frame_, NULL, dest, &paint); | 351 |
352 CleanUpTemporaryBuffers(); | |
242 } | 353 } |
243 | 354 |
244 void SkCanvasVideoRenderer::Copy(media::VideoFrame* video_frame, | 355 void SkCanvasVideoRenderer::Copy(media::VideoFrame* video_frame, |
245 SkCanvas* canvas) { | 356 SkCanvas* canvas) { |
246 Paint(video_frame, | 357 Paint(video_frame, |
247 canvas, | 358 canvas, |
248 video_frame->visible_rect(), | 359 video_frame->visible_rect(), |
249 0xff, | 360 0xff, |
250 SkXfermode::kSrc_Mode, | 361 SkXfermode::kSrc_Mode, |
251 media::VIDEO_ROTATION_0); | 362 media::VIDEO_ROTATION_0, |
363 NULL); | |
364 } | |
365 | |
366 // If a buffer is not used by 3 sec, remove it. | |
367 void SkCanvasVideoRenderer::CleanUpTemporaryBuffers() { | |
368 static const base::TimeDelta buffer_time = base::TimeDelta::FromSeconds(3); | |
rileya (GONE FROM CHROMIUM)
2014/08/27 20:30:50
This seems extremely arbitrary. Can you explain th
| |
369 base::TimeDelta last_timestamp = | |
370 accelerated_last_frame_timestamp_ > last_frame_timestamp_ | |
371 ? accelerated_last_frame_timestamp_ | |
372 : last_frame_timestamp_; | |
373 if (last_timestamp > last_frame_timestamp_ + buffer_time && | |
374 !last_frame_.isNull()) | |
375 last_frame_.reset(); | |
376 if (last_timestamp > accelerated_last_frame_timestamp_ + buffer_time && | |
377 !accelerated_last_frame_.isNull()) | |
378 accelerated_last_frame_.reset(); | |
379 } | |
380 | |
381 // static | |
382 void SkCanvasVideoRenderer::CopyVideoFrameToTexture( | |
rileya (GONE FROM CHROMIUM)
2014/08/27 20:30:50
I'm not sure how I feel about putting all of this
| |
383 gpu::gles2::GLES2Interface* gl, | |
384 VideoFrame* video_frame, | |
385 unsigned int texture, | |
386 unsigned int level, | |
387 unsigned int internal_format, | |
388 unsigned int type, | |
389 bool premultiply_alpha, | |
390 bool flip_y) { | |
391 DCHECK(video_frame && video_frame->format() == VideoFrame::NATIVE_TEXTURE); | |
392 const gpu::MailboxHolder* mailbox_holder = video_frame->mailbox_holder(); | |
393 DCHECK(mailbox_holder->texture_target == GL_TEXTURE_2D || | |
394 mailbox_holder->texture_target == GL_TEXTURE_EXTERNAL_OES); | |
395 | |
396 gl->WaitSyncPointCHROMIUM(mailbox_holder->sync_point); | |
397 uint32 source_texture = gl->CreateAndConsumeTextureCHROMIUM( | |
398 mailbox_holder->texture_target, mailbox_holder->mailbox.name); | |
399 | |
400 // The video is stored in a unmultiplied format, so premultiply | |
401 // if necessary. | |
402 gl->PixelStorei(GL_UNPACK_PREMULTIPLY_ALPHA_CHROMIUM, premultiply_alpha); | |
403 // Application itself needs to take care of setting the right flip_y | |
404 // value down to get the expected result. | |
405 // flip_y==true means to reverse the video orientation while | |
406 // flip_y==false means to keep the intrinsic orientation. | |
407 gl->PixelStorei(GL_UNPACK_FLIP_Y_CHROMIUM, flip_y); | |
408 gl->CopyTextureCHROMIUM( | |
409 GL_TEXTURE_2D, source_texture, texture, level, internal_format, type); | |
410 gl->PixelStorei(GL_UNPACK_FLIP_Y_CHROMIUM, false); | |
411 gl->PixelStorei(GL_UNPACK_PREMULTIPLY_ALPHA_CHROMIUM, false); | |
412 | |
413 gl->DeleteTextures(1, &source_texture); | |
414 gl->Flush(); | |
415 | |
416 SyncPointClientImpl client(gl); | |
417 video_frame->UpdateReleaseSyncPoint(&client); | |
252 } | 418 } |
253 | 419 |
254 } // namespace media | 420 } // namespace media |
OLD | NEW |