OLD | NEW |
---|---|
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "media/filters/skcanvas_video_renderer.h" | 5 #include "media/filters/skcanvas_video_renderer.h" |
6 | 6 |
7 #include "base/logging.h" | 7 #include "base/logging.h" |
8 #include "gpu/GLES2/gl2extchromium.h" | |
9 #include "gpu/command_buffer/client/gles2_interface.h" | |
10 #include "gpu/command_buffer/common/mailbox_holder.h" | |
8 #include "media/base/video_frame.h" | 11 #include "media/base/video_frame.h" |
9 #include "media/base/yuv_convert.h" | 12 #include "media/base/yuv_convert.h" |
13 #include "skia/ext/refptr.h" | |
10 #include "third_party/libyuv/include/libyuv.h" | 14 #include "third_party/libyuv/include/libyuv.h" |
11 #include "third_party/skia/include/core/SkCanvas.h" | 15 #include "third_party/skia/include/core/SkCanvas.h" |
16 #include "third_party/skia/include/gpu/GrContext.h" | |
17 #include "third_party/skia/include/gpu/SkGrPixelRef.h" | |
12 #include "ui/gfx/skbitmap_operations.h" | 18 #include "ui/gfx/skbitmap_operations.h" |
13 | 19 |
14 // Skia internal format depends on a platform. On Android it is ABGR, on others | 20 // Skia internal format depends on a platform. On Android it is ABGR, on others |
15 // it is ARGB. | 21 // it is ARGB. |
16 #if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \ | 22 #if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \ |
17 SK_A32_SHIFT == 24 | 23 SK_A32_SHIFT == 24 |
18 #define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB | 24 #define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB |
19 #define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB | 25 #define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB |
20 #elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \ | 26 #elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \ |
21 SK_A32_SHIFT == 24 | 27 SK_A32_SHIFT == 24 |
22 #define LIBYUV_I420_TO_ARGB libyuv::I420ToABGR | 28 #define LIBYUV_I420_TO_ARGB libyuv::I420ToABGR |
23 #define LIBYUV_I422_TO_ARGB libyuv::I422ToABGR | 29 #define LIBYUV_I422_TO_ARGB libyuv::I422ToABGR |
24 #else | 30 #else |
25 #error Unexpected Skia ARGB_8888 layout! | 31 #error Unexpected Skia ARGB_8888 layout! |
26 #endif | 32 #endif |
27 | 33 |
28 namespace media { | 34 namespace media { |
29 | 35 |
30 static bool IsYUV(media::VideoFrame::Format format) { | 36 namespace { |
37 | |
38 // This class keeps two temporary resources; software bitmap, hardware bitmap. | |
39 // If both bitmap are created and then only software bitmap is updated every | |
40 // frame, hardware bitmap outlives until the media player dies. So we delete | |
41 // a temporary resource if it is not used for 3 sec. | |
42 const int kTemporaryResourceDeletionDelay = 3; // Seconds; | |
43 | |
44 bool IsYUV(media::VideoFrame::Format format) { | |
31 return format == media::VideoFrame::YV12 || | 45 return format == media::VideoFrame::YV12 || |
32 format == media::VideoFrame::YV16 || | 46 format == media::VideoFrame::YV16 || |
33 format == media::VideoFrame::I420 || | 47 format == media::VideoFrame::I420 || |
34 format == media::VideoFrame::YV12A || | 48 format == media::VideoFrame::YV12A || |
35 format == media::VideoFrame::YV12J || | 49 format == media::VideoFrame::YV12J || |
36 format == media::VideoFrame::YV24; | 50 format == media::VideoFrame::YV24; |
37 } | 51 } |
38 | 52 |
39 static bool IsYUVOrNative(media::VideoFrame::Format format) { | 53 bool IsYUVOrNative(media::VideoFrame::Format format) { |
40 return IsYUV(format) || format == media::VideoFrame::NATIVE_TEXTURE; | 54 return IsYUV(format) || format == media::VideoFrame::NATIVE_TEXTURE; |
41 } | 55 } |
42 | 56 |
43 // Converts a VideoFrame containing YUV data to a SkBitmap containing RGB data. | 57 // Converts a VideoFrame containing YUV data to a SkBitmap containing RGB data. |
44 // | 58 // |
45 // |bitmap| will be (re)allocated to match the dimensions of |video_frame|. | 59 // |bitmap| will be (re)allocated to match the dimensions of |video_frame|. |
46 static void ConvertVideoFrameToBitmap( | 60 void ConvertVideoFrameToBitmap(media::VideoFrame* video_frame, |
47 const scoped_refptr<media::VideoFrame>& video_frame, | 61 SkBitmap* bitmap) { |
48 SkBitmap* bitmap) { | |
49 DCHECK(IsYUVOrNative(video_frame->format())) | 62 DCHECK(IsYUVOrNative(video_frame->format())) |
50 << video_frame->format(); | 63 << video_frame->format(); |
51 if (IsYUV(video_frame->format())) { | 64 if (IsYUV(video_frame->format())) { |
52 DCHECK_EQ(video_frame->stride(media::VideoFrame::kUPlane), | 65 DCHECK_EQ(video_frame->stride(media::VideoFrame::kUPlane), |
53 video_frame->stride(media::VideoFrame::kVPlane)); | 66 video_frame->stride(media::VideoFrame::kVPlane)); |
54 } | 67 } |
55 | 68 |
56 // Check if |bitmap| needs to be (re)allocated. | 69 // Check if |bitmap| needs to be (re)allocated. |
57 if (bitmap->isNull() || | 70 if (bitmap->isNull() || |
58 bitmap->width() != video_frame->visible_rect().width() || | 71 bitmap->width() != video_frame->visible_rect().width() || |
(...skipping 114 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
173 break; | 186 break; |
174 | 187 |
175 default: | 188 default: |
176 NOTREACHED(); | 189 NOTREACHED(); |
177 break; | 190 break; |
178 } | 191 } |
179 bitmap->notifyPixelsChanged(); | 192 bitmap->notifyPixelsChanged(); |
180 bitmap->unlockPixels(); | 193 bitmap->unlockPixels(); |
181 } | 194 } |
182 | 195 |
196 bool EnsureTextureBackedSkBitmap(GrContext* gr, | |
197 SkBitmap* bitmap, | |
198 const gfx::Size& size) { | |
199 if (!bitmap->getTexture() || bitmap->width() != size.width() || | |
200 bitmap->height() != size.height()) { | |
201 if (!gr) | |
202 return false; | |
203 GrTextureDesc desc; | |
204 // Use kRGBA_8888_GrPixelConfig, not kSkia8888_GrPixelConfig, because | |
205 // glCopyTextureChromium doesn't support GL_BGRA_EXT as internal format. | |
206 desc.fConfig = kRGBA_8888_GrPixelConfig; | |
207 desc.fFlags = kRenderTarget_GrTextureFlagBit | kNoStencil_GrTextureFlagBit; | |
208 desc.fSampleCnt = 0; | |
209 desc.fOrigin = kTopLeft_GrSurfaceOrigin; | |
210 desc.fWidth = size.width(); | |
211 desc.fHeight = size.height(); | |
212 GrAutoScratchTexture scratch_texture( | |
213 gr, desc, GrContext::kExact_ScratchTexMatch); | |
214 skia::RefPtr<GrTexture> texture = skia::AdoptRef(scratch_texture.detach()); | |
215 if (!texture.get()) | |
216 return false; | |
217 | |
218 SkImageInfo info = SkImageInfo::MakeN32Premul(desc.fWidth, desc.fHeight); | |
219 SkGrPixelRef* pixelRef = SkNEW_ARGS(SkGrPixelRef, (info, texture.get())); | |
220 if (!pixelRef) | |
221 return false; | |
222 bitmap->setInfo(info); | |
223 bitmap->setPixelRef(pixelRef)->unref(); | |
224 } | |
225 | |
226 return true; | |
227 } | |
228 | |
229 bool ConvertVideoFrameToTexture( | |
230 VideoFrame* video_frame, | |
231 SkBitmap* bitmap, | |
232 SkCanvasVideoRenderer::Context3DProvider* context_provider) { | |
233 DCHECK(context_provider && | |
234 video_frame->format() == VideoFrame::NATIVE_TEXTURE); | |
235 gpu::gles2::GLES2Interface* gl = context_provider->gl; | |
236 DCHECK(gl); | |
237 | |
238 // Check if we could reuse existing texture based bitmap. | |
239 // Otherwise, release existing texture based bitmap and allocate | |
240 // a new one based on video size. | |
241 if (!EnsureTextureBackedSkBitmap(context_provider->gr_context, | |
242 bitmap, | |
243 video_frame->visible_rect().size())) { | |
244 return false; | |
245 } | |
246 | |
247 unsigned textureId = | |
248 static_cast<unsigned>((bitmap->getTexture())->getTextureHandle()); | |
249 // If CopyVideoFrameToTexture() changes the state of the 'textureId', it's | |
250 // needed to invalidate the state cached in skia, but currently the state | |
251 // isn't changed. | |
252 SkCanvasVideoRenderer::CopyVideoFrameToTexture( | |
253 gl, video_frame, textureId, 0, GL_RGBA, GL_UNSIGNED_BYTE, true, false); | |
254 return true; | |
255 } | |
256 | |
257 void RotateBitmap(SkBitmap* bitmap, VideoRotation video_rotation) { | |
258 switch (video_rotation) { | |
259 case VIDEO_ROTATION_0: | |
260 break; | |
261 case VIDEO_ROTATION_90: | |
262 *bitmap = SkBitmapOperations::Rotate(*bitmap, | |
263 SkBitmapOperations::ROTATION_90_CW); | |
264 break; | |
265 case VIDEO_ROTATION_180: | |
266 *bitmap = SkBitmapOperations::Rotate(*bitmap, | |
267 SkBitmapOperations::ROTATION_180_CW); | |
268 break; | |
269 case VIDEO_ROTATION_270: | |
270 *bitmap = SkBitmapOperations::Rotate(*bitmap, | |
271 SkBitmapOperations::ROTATION_270_CW); | |
272 break; | |
273 } | |
274 } | |
275 | |
276 class SyncPointClientImpl : public VideoFrame::SyncPointClient { | |
277 public: | |
278 explicit SyncPointClientImpl(gpu::gles2::GLES2Interface* gl) : gl_(gl) {} | |
279 virtual ~SyncPointClientImpl() {} | |
280 virtual uint32 InsertSyncPoint() OVERRIDE { | |
281 return gl_->InsertSyncPointCHROMIUM(); | |
282 } | |
283 virtual void WaitSyncPoint(uint32 sync_point) OVERRIDE { | |
284 gl_->WaitSyncPointCHROMIUM(sync_point); | |
285 } | |
286 | |
287 private: | |
288 gpu::gles2::GLES2Interface* gl_; | |
289 }; | |
290 | |
291 } // anonymous namespace | |
292 | |
183 SkCanvasVideoRenderer::SkCanvasVideoRenderer() | 293 SkCanvasVideoRenderer::SkCanvasVideoRenderer() |
184 : last_frame_timestamp_(media::kNoTimestamp()) { | 294 : last_frame_timestamp_(media::kNoTimestamp()), |
295 accelerated_last_frame_timestamp_(media::kNoTimestamp()) { | |
185 } | 296 } |
186 | 297 |
187 SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {} | 298 SkCanvasVideoRenderer::~SkCanvasVideoRenderer() {} |
188 | 299 |
189 void SkCanvasVideoRenderer::Paint(media::VideoFrame* video_frame, | 300 void SkCanvasVideoRenderer::Paint( |
190 SkCanvas* canvas, | 301 media::VideoFrame* video_frame, |
191 const gfx::RectF& dest_rect, | 302 SkCanvas* canvas, |
192 uint8 alpha, | 303 const gfx::RectF& dest_rect, |
193 SkXfermode::Mode mode, | 304 uint8 alpha, |
194 VideoRotation video_rotation) { | 305 SkXfermode::Mode mode, |
306 VideoRotation video_rotation, | |
307 SkCanvasVideoRenderer::Context3DProvider* context_provider) { | |
195 if (alpha == 0) { | 308 if (alpha == 0) { |
196 return; | 309 return; |
197 } | 310 } |
198 | 311 |
199 SkRect dest; | 312 SkRect dest; |
200 dest.set(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom()); | 313 dest.set(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom()); |
201 | 314 |
202 SkPaint paint; | 315 SkPaint paint; |
203 paint.setAlpha(alpha); | 316 paint.setAlpha(alpha); |
204 | 317 |
205 // Paint black rectangle if there isn't a frame available or the | 318 // Paint black rectangle if there isn't a frame available or the |
206 // frame has an unexpected format. | 319 // frame has an unexpected format. |
207 if (!video_frame || !IsYUVOrNative(video_frame->format())) { | 320 if (!video_frame || video_frame->natural_size().IsEmpty() || |
321 !IsYUVOrNative(video_frame->format())) { | |
208 canvas->drawRect(dest, paint); | 322 canvas->drawRect(dest, paint); |
209 return; | 323 return; |
210 } | 324 } |
211 | 325 |
326 bool accelerated = false; | |
327 if (context_provider && | |
328 video_frame->format() == media::VideoFrame::NATIVE_TEXTURE && | |
329 canvas->getGrContext()) { | |
330 // TODO(dshwang): Android video decoder doesn't update the timestamp on a | |
331 // VideoFrame. To reduce redundant copy, Android should update the | |
332 // timestamp. | |
333 if (accelerated_last_frame_.isNull() || | |
334 video_frame->timestamp() != accelerated_last_frame_timestamp_ || | |
335 video_frame->timestamp() == base::TimeDelta()) { | |
336 accelerated = ConvertVideoFrameToTexture( | |
337 video_frame, &accelerated_last_frame_, context_provider); | |
338 if (accelerated) { | |
339 RotateBitmap(&accelerated_last_frame_, video_rotation); | |
340 accelerated_last_frame_timestamp_ = video_frame->timestamp(); | |
341 } | |
342 } else { | |
343 DCHECK(accelerated_last_frame_.getTexture()); | |
344 accelerated = true; | |
345 } | |
346 } | |
347 | |
212 // Check if we should convert and update |last_frame_|. | 348 // Check if we should convert and update |last_frame_|. |
213 if (last_frame_.isNull() || | 349 if (!accelerated && (last_frame_.isNull() || |
214 video_frame->timestamp() != last_frame_timestamp_) { | 350 video_frame->timestamp() != last_frame_timestamp_)) { |
215 ConvertVideoFrameToBitmap(video_frame, &last_frame_); | 351 ConvertVideoFrameToBitmap(video_frame, &last_frame_); |
216 | 352 RotateBitmap(&last_frame_, video_rotation); |
217 switch (video_rotation) { | |
218 case VIDEO_ROTATION_0: | |
219 break; | |
220 case VIDEO_ROTATION_90: | |
221 last_frame_ = SkBitmapOperations::Rotate( | |
222 last_frame_, SkBitmapOperations::ROTATION_90_CW); | |
223 break; | |
224 case VIDEO_ROTATION_180: | |
225 last_frame_ = SkBitmapOperations::Rotate( | |
226 last_frame_, SkBitmapOperations::ROTATION_180_CW); | |
227 break; | |
228 case VIDEO_ROTATION_270: | |
229 last_frame_ = SkBitmapOperations::Rotate( | |
230 last_frame_, SkBitmapOperations::ROTATION_270_CW); | |
231 break; | |
232 } | |
233 | |
234 last_frame_timestamp_ = video_frame->timestamp(); | 353 last_frame_timestamp_ = video_frame->timestamp(); |
235 } | 354 } |
236 | 355 |
237 paint.setXfermodeMode(mode); | 356 paint.setXfermodeMode(mode); |
238 | 357 |
239 // Paint using |last_frame_|. | 358 canvas->drawBitmapRect( |
240 paint.setFilterLevel(SkPaint::kLow_FilterLevel); | 359 accelerated ? accelerated_last_frame_ : last_frame_, NULL, dest, &paint); |
241 canvas->drawBitmapRect(last_frame_, NULL, dest, &paint); | 360 |
361 CleanUpTemporaryBuffers(); | |
242 } | 362 } |
243 | 363 |
244 void SkCanvasVideoRenderer::Copy(media::VideoFrame* video_frame, | 364 void SkCanvasVideoRenderer::Copy(media::VideoFrame* video_frame, |
245 SkCanvas* canvas) { | 365 SkCanvas* canvas) { |
246 Paint(video_frame, | 366 Paint(video_frame, |
247 canvas, | 367 canvas, |
248 video_frame->visible_rect(), | 368 video_frame->visible_rect(), |
249 0xff, | 369 0xff, |
250 SkXfermode::kSrc_Mode, | 370 SkXfermode::kSrc_Mode, |
251 media::VIDEO_ROTATION_0); | 371 media::VIDEO_ROTATION_0, |
372 NULL); | |
373 } | |
374 | |
375 void SkCanvasVideoRenderer::CleanUpTemporaryBuffers() { | |
376 static const base::TimeDelta buffer_time = | |
377 base::TimeDelta::FromSeconds(kTemporaryResourceDeletionDelay); | |
378 // Find the latest time stamp. | |
379 base::TimeDelta last_timestamp = | |
380 accelerated_last_frame_timestamp_ > last_frame_timestamp_ | |
381 ? accelerated_last_frame_timestamp_ | |
382 : last_frame_timestamp_; | |
383 // Delete a too old resource. | |
384 if (last_timestamp > last_frame_timestamp_ + buffer_time && | |
rileya (GONE FROM CHROMIUM)
2014/09/02 21:21:43
Multi-line conditionals need {}'s.
| |
385 !last_frame_.isNull()) | |
386 last_frame_.reset(); | |
387 if (last_timestamp > accelerated_last_frame_timestamp_ + buffer_time && | |
rileya (GONE FROM CHROMIUM)
2014/09/02 21:21:43
ditto
| |
388 !accelerated_last_frame_.isNull()) | |
389 accelerated_last_frame_.reset(); | |
390 } | |
391 | |
392 // static | |
393 void SkCanvasVideoRenderer::CopyVideoFrameToTexture( | |
394 gpu::gles2::GLES2Interface* gl, | |
395 VideoFrame* video_frame, | |
396 unsigned int texture, | |
397 unsigned int level, | |
398 unsigned int internal_format, | |
399 unsigned int type, | |
400 bool premultiply_alpha, | |
401 bool flip_y) { | |
402 DCHECK(video_frame && video_frame->format() == VideoFrame::NATIVE_TEXTURE); | |
403 const gpu::MailboxHolder* mailbox_holder = video_frame->mailbox_holder(); | |
404 DCHECK(mailbox_holder->texture_target == GL_TEXTURE_2D || | |
405 mailbox_holder->texture_target == GL_TEXTURE_EXTERNAL_OES); | |
406 | |
407 gl->WaitSyncPointCHROMIUM(mailbox_holder->sync_point); | |
408 uint32 source_texture = gl->CreateAndConsumeTextureCHROMIUM( | |
409 mailbox_holder->texture_target, mailbox_holder->mailbox.name); | |
410 | |
411 // The video is stored in a unmultiplied format, so premultiply | |
412 // if necessary. | |
413 gl->PixelStorei(GL_UNPACK_PREMULTIPLY_ALPHA_CHROMIUM, premultiply_alpha); | |
414 // Application itself needs to take care of setting the right flip_y | |
rileya (GONE FROM CHROMIUM)
2014/09/02 21:21:43
In media/ we enclose variable names in ||'s, so th
| |
415 // value down to get the expected result. | |
416 // flip_y==true means to reverse the video orientation while | |
417 // flip_y==false means to keep the intrinsic orientation. | |
418 gl->PixelStorei(GL_UNPACK_FLIP_Y_CHROMIUM, flip_y); | |
419 gl->CopyTextureCHROMIUM( | |
420 GL_TEXTURE_2D, source_texture, texture, level, internal_format, type); | |
421 gl->PixelStorei(GL_UNPACK_FLIP_Y_CHROMIUM, false); | |
422 gl->PixelStorei(GL_UNPACK_PREMULTIPLY_ALPHA_CHROMIUM, false); | |
423 | |
424 gl->DeleteTextures(1, &source_texture); | |
425 gl->Flush(); | |
426 | |
427 SyncPointClientImpl client(gl); | |
428 video_frame->UpdateReleaseSyncPoint(&client); | |
252 } | 429 } |
253 | 430 |
254 } // namespace media | 431 } // namespace media |
OLD | NEW |