Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(5)

Side by Side Diff: media/filters/skcanvas_video_renderer.cc

Issue 289373011: Support for YUV 4:4:4 subsampling. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Software path. Created 6 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "media/filters/skcanvas_video_renderer.h" 5 #include "media/filters/skcanvas_video_renderer.h"
6 6
7 #include "base/logging.h" 7 #include "base/logging.h"
8 #include "media/base/video_frame.h" 8 #include "media/base/video_frame.h"
9 #include "media/base/yuv_convert.h" 9 #include "media/base/yuv_convert.h"
scherkus (not reviewing) 2014/05/22 23:45:11 is this still used?
sandersd (OOO until July 31) 2014/05/23 00:45:22 Yes, YV12J and YV12A still use it.
10 #include "third_party/libyuv/include/libyuv.h" 10 #include "third_party/libyuv/include/libyuv.h"
11 #include "third_party/skia/include/core/SkCanvas.h" 11 #include "third_party/skia/include/core/SkCanvas.h"
12 #include "third_party/skia/include/core/SkDevice.h" 12 #include "third_party/skia/include/core/SkDevice.h"
13 13
14 // Skia internal format depends on a platform. On Android it is ABGR, on others 14 // Skia internal format depends on a platform. On Android it is ABGR, on others
15 // it is ARGB. 15 // it is ARGB.
16 #if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \ 16 #if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \
17 SK_A32_SHIFT == 24 17 SK_A32_SHIFT == 24
18 #define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB 18 #define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB
19 #define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB 19 #define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB
20 #define LIBYUV_I444_TO_ARGB libyuv::I444ToARGB
20 #elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \ 21 #elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \
21 SK_A32_SHIFT == 24 22 SK_A32_SHIFT == 24
22 #define LIBYUV_I420_TO_ARGB libyuv::I420ToABGR 23 #define LIBYUV_I420_TO_ARGB libyuv::I420ToABGR
23 #define LIBYUV_I422_TO_ARGB libyuv::I422ToABGR 24 #define LIBYUV_I422_TO_ARGB libyuv::I422ToABGR
25 #define LIBYUV_I444_TO_ARGB libyuv::I444ToABGR
24 #else 26 #else
25 #error Unexpected Skia ARGB_8888 layout! 27 #error Unexpected Skia ARGB_8888 layout!
26 #endif 28 #endif
27 29
28 namespace media { 30 namespace media {
29 31
30 static bool IsYUV(media::VideoFrame::Format format) { 32 static bool IsYUV(media::VideoFrame::Format format) {
31 return format == media::VideoFrame::YV12 || 33 return format == media::VideoFrame::YV12 ||
34 format == media::VideoFrame::YV16 ||
32 format == media::VideoFrame::I420 || 35 format == media::VideoFrame::I420 ||
33 format == media::VideoFrame::YV16 || 36 format == media::VideoFrame::YV12A ||
34 format == media::VideoFrame::YV12J; 37 format == media::VideoFrame::YV12J ||
38 format == media::VideoFrame::I444;
35 } 39 }
36 40
37 static bool IsEitherYUVOrNative(media::VideoFrame::Format format) { 41 static bool IsYUVOrNative(media::VideoFrame::Format format) {
38 return IsYUV(format) || format == media::VideoFrame::NATIVE_TEXTURE; 42 return IsYUV(format) || format == media::VideoFrame::NATIVE_TEXTURE;
39 } 43 }
40 44
41 static bool IsEitherYUVOrYUVA(media::VideoFrame::Format format) {
42 return IsYUV(format) || format == media::VideoFrame::YV12A;
43 }
44
45 static bool IsEitherYUVOrYUVAOrNative(media::VideoFrame::Format format) {
46 return IsEitherYUVOrNative(format) || format == media::VideoFrame::YV12A;
47 }
48
49 // CanFastPaint is a helper method to determine the conditions for fast
50 // painting. The conditions are:
51 // 1. No skew in canvas matrix.
52 // 2. No flipping nor mirroring.
53 // 3. Canvas has pixel format ARGB8888.
54 // 4. Canvas is opaque.
55 // 5. Frame format is YV12, I420 or YV16.
56 //
57 // TODO(hclam): The fast paint method should support flipping and mirroring.
58 // Disable the flipping and mirroring checks once we have it.
59 static bool CanFastPaint(SkCanvas* canvas, uint8 alpha,
scherkus (not reviewing) 2014/05/22 23:59:40 fischman and I chatted that perhaps we should leav
sandersd (OOO until July 31) 2014/05/23 00:45:22 :-( I've restored this and added a check to skip
60 media::VideoFrame::Format format) {
61 if (alpha != 0xFF || !IsYUV(format))
62 return false;
63
64 const SkMatrix& total_matrix = canvas->getTotalMatrix();
65 // Perform the following checks here:
66 // 1. Check for skewing factors of the transformation matrix. They should be
67 // zero.
68 // 2. Check for mirroring and flipping. Make sure they are greater than zero.
69 if (SkScalarNearlyZero(total_matrix.getSkewX()) &&
70 SkScalarNearlyZero(total_matrix.getSkewY()) &&
71 total_matrix.getScaleX() > 0 &&
72 total_matrix.getScaleY() > 0) {
73 SkBaseDevice* device = canvas->getDevice();
74 const SkBitmap::Config config = device->config();
75
76 if (config == SkBitmap::kARGB_8888_Config && device->isOpaque()) {
77 return true;
78 }
79 }
80
81 return false;
82 }
83
84 // Fast paint does YUV => RGB, scaling, blitting all in one step into the
85 // canvas. It's not always safe and appropriate to perform fast paint.
86 // CanFastPaint() is used to determine the conditions.
87 static void FastPaint(
88 const scoped_refptr<media::VideoFrame>& video_frame,
89 SkCanvas* canvas,
90 const SkRect& dest_rect) {
91 DCHECK(IsYUV(video_frame->format())) << video_frame->format();
92 DCHECK_EQ(video_frame->stride(media::VideoFrame::kUPlane),
93 video_frame->stride(media::VideoFrame::kVPlane));
94
95 const SkBitmap& bitmap = canvas->getDevice()->accessBitmap(true);
96 media::YUVType yuv_type = media::YV16;
97 int y_shift = 0;
98 if (video_frame->format() == media::VideoFrame::YV12 ||
99 video_frame->format() == media::VideoFrame::I420 ||
100 video_frame->format() == media::VideoFrame::YV12A) {
101 yuv_type = media::YV12;
102 y_shift = 1;
103 }
104
105 if (video_frame->format() == media::VideoFrame::YV12J) {
106 yuv_type = media::YV12J;
107 y_shift = 1;
108 }
109
110 // Transform the destination rectangle to local coordinates.
111 const SkMatrix& local_matrix = canvas->getTotalMatrix();
112 SkRect local_dest_rect;
113 local_matrix.mapRect(&local_dest_rect, dest_rect);
114
115 // After projecting the destination rectangle to local coordinates, round
116 // the projected rectangle to integer values, this will give us pixel values
117 // of the rectangle.
118 SkIRect local_dest_irect, local_dest_irect_saved;
119 local_dest_rect.round(&local_dest_irect);
120 local_dest_rect.round(&local_dest_irect_saved);
121
122 // No point painting if the destination rect doesn't intersect with the
123 // clip rect.
124 SkIRect device_bounds;
125 if (!canvas->getClipDeviceBounds(&device_bounds))
126 return;
127 if (!local_dest_irect.intersect(device_bounds))
128 return;
129
130 // At this point |local_dest_irect| contains the rect that we should draw
131 // to within the clipping rect.
132
133 // Project the clip rect to the original video frame, obtains the
134 // dimensions of the projected clip rect, "left" and "top" of the rect.
135 // The math here are all integer math so we won't have rounding error and
136 // write outside of the canvas.
137 // We have the assumptions of dest_rect.width() and dest_rect.height()
138 // being non-zero, these are valid assumptions since finding intersection
139 // above rejects empty rectangle so we just do a DCHECK here.
140 DCHECK_NE(0, dest_rect.width());
141 DCHECK_NE(0, dest_rect.height());
142 size_t frame_clip_width = local_dest_irect.width() *
143 video_frame->visible_rect().width() /
144 local_dest_irect_saved.width();
145 size_t frame_clip_height = local_dest_irect.height() *
146 video_frame->visible_rect().height() /
147 local_dest_irect_saved.height();
148
149 // Project the "left" and "top" of the final destination rect to local
150 // coordinates of the video frame, use these values to find the offsets
151 // in the video frame to start reading.
152 size_t frame_clip_left =
153 video_frame->visible_rect().x() +
154 (local_dest_irect.fLeft - local_dest_irect_saved.fLeft) *
155 video_frame->visible_rect().width() / local_dest_irect_saved.width();
156 size_t frame_clip_top =
157 video_frame->visible_rect().y() +
158 (local_dest_irect.fTop - local_dest_irect_saved.fTop) *
159 video_frame->visible_rect().height() /
160 local_dest_irect_saved.height();
161
162 // Use the "left" and "top" of the destination rect to locate the offset
163 // in Y, U and V planes.
164 size_t y_offset =
165 (video_frame->stride(media::VideoFrame::kYPlane) * frame_clip_top) +
166 frame_clip_left;
167
168 // For format YV12, there is one U, V value per 2x2 block.
169 // For format YV16, there is one U, V value per 2x1 block.
170 size_t uv_offset = (video_frame->stride(media::VideoFrame::kUPlane) *
171 (frame_clip_top >> y_shift)) +
172 (frame_clip_left >> 1);
173
174 // Calculate the address for the top left corner of destination rect in
175 // the canvas that we will draw to. The address is obtained by the base
176 // address of the canvas shifted by "left" and "top" of the rect.
177 uint8* dest_rect_pointer = static_cast<uint8*>(bitmap.getPixels()) +
178 local_dest_irect.fTop * bitmap.rowBytes() +
179 local_dest_irect.fLeft * 4;
180
181 uint8* frame_clip_y =
182 video_frame->data(media::VideoFrame::kYPlane) + y_offset;
183 uint8* frame_clip_u =
184 video_frame->data(media::VideoFrame::kUPlane) + uv_offset;
185 uint8* frame_clip_v =
186 video_frame->data(media::VideoFrame::kVPlane) + uv_offset;
187
188 // TODO(hclam): do rotation and mirroring here.
189 // TODO(fbarchard): switch filtering based on performance.
190 bitmap.lockPixels();
191
192 // If there is no scaling and the frame format is supported, then use faster
193 // libyuv.
194 if (frame_clip_width == static_cast<size_t>(local_dest_irect.width()) &&
195 frame_clip_height == static_cast<size_t>(local_dest_irect.height())) {
196 switch (yuv_type) {
197 case media::YV12:
198 LIBYUV_I420_TO_ARGB(frame_clip_y,
199 video_frame->stride(media::VideoFrame::kYPlane),
200 frame_clip_u,
201 video_frame->stride(media::VideoFrame::kUPlane),
202 frame_clip_v,
203 video_frame->stride(media::VideoFrame::kVPlane),
204 dest_rect_pointer,
205 bitmap.rowBytes(),
206 local_dest_irect.width(),
207 local_dest_irect.height());
208 bitmap.unlockPixels();
209 return;
210
211 case media::YV16:
212 LIBYUV_I422_TO_ARGB(frame_clip_y,
213 video_frame->stride(media::VideoFrame::kYPlane),
214 frame_clip_u,
215 video_frame->stride(media::VideoFrame::kUPlane),
216 frame_clip_v,
217 video_frame->stride(media::VideoFrame::kVPlane),
218 dest_rect_pointer,
219 bitmap.rowBytes(),
220 local_dest_irect.width(),
221 local_dest_irect.height());
222 bitmap.unlockPixels();
223 return;
224
225 default:
226 break;
227 }
228 }
229
230 // Fallback to media, since the operation is not supported by libyuv.
231 media::ScaleYUVToRGB32(frame_clip_y,
232 frame_clip_u,
233 frame_clip_v,
234 dest_rect_pointer,
235 frame_clip_width,
236 frame_clip_height,
237 local_dest_irect.width(),
238 local_dest_irect.height(),
239 video_frame->stride(media::VideoFrame::kYPlane),
240 video_frame->stride(media::VideoFrame::kUPlane),
241 bitmap.rowBytes(),
242 yuv_type,
243 media::ROTATE_0,
244 media::FILTER_BILINEAR);
245 bitmap.unlockPixels();
246 }
247
248 // Converts a VideoFrame containing YUV data to a SkBitmap containing RGB data. 45 // Converts a VideoFrame containing YUV data to a SkBitmap containing RGB data.
249 // 46 //
250 // |bitmap| will be (re)allocated to match the dimensions of |video_frame|. 47 // |bitmap| will be (re)allocated to match the dimensions of |video_frame|.
251 static void ConvertVideoFrameToBitmap( 48 static void ConvertVideoFrameToBitmap(
252 const scoped_refptr<media::VideoFrame>& video_frame, 49 const scoped_refptr<media::VideoFrame>& video_frame,
253 SkBitmap* bitmap) { 50 SkBitmap* bitmap) {
254 DCHECK(IsEitherYUVOrYUVAOrNative(video_frame->format())) 51 DCHECK(IsYUVOrNative(video_frame->format()))
255 << video_frame->format(); 52 << video_frame->format();
256 if (IsEitherYUVOrYUVA(video_frame->format())) { 53 if (IsYUV(video_frame->format())) {
257 DCHECK_EQ(video_frame->stride(media::VideoFrame::kUPlane), 54 DCHECK_EQ(video_frame->stride(media::VideoFrame::kUPlane),
258 video_frame->stride(media::VideoFrame::kVPlane)); 55 video_frame->stride(media::VideoFrame::kVPlane));
259 } 56 }
260 57
261 // Check if |bitmap| needs to be (re)allocated. 58 // Check if |bitmap| needs to be (re)allocated.
262 if (bitmap->isNull() || 59 if (bitmap->isNull() ||
263 bitmap->width() != video_frame->visible_rect().width() || 60 bitmap->width() != video_frame->visible_rect().width() ||
264 bitmap->height() != video_frame->visible_rect().height()) { 61 bitmap->height() != video_frame->visible_rect().height()) {
265 bitmap->setConfig(SkBitmap::kARGB_8888_Config, 62 bitmap->setConfig(SkBitmap::kARGB_8888_Config,
266 video_frame->visible_rect().width(), 63 video_frame->visible_rect().width(),
267 video_frame->visible_rect().height()); 64 video_frame->visible_rect().height());
268 bitmap->allocPixels(); 65 bitmap->allocPixels();
269 bitmap->setIsVolatile(true); 66 bitmap->setIsVolatile(true);
270 } 67 }
271 68
272 bitmap->lockPixels(); 69 bitmap->lockPixels();
273 70
274 size_t y_offset = 0; 71 size_t y_offset = 0;
275 size_t uv_offset = 0; 72 size_t uv_offset = 0;
276 if (IsEitherYUVOrYUVA(video_frame->format())) { 73 if (IsYUV(video_frame->format())) {
277 int y_shift = (video_frame->format() == media::VideoFrame::YV16) ? 0 : 1; 74 int y_shift = (video_frame->format() == media::VideoFrame::YV16) ? 0 : 1;
278 // Use the "left" and "top" of the destination rect to locate the offset 75 // Use the "left" and "top" of the destination rect to locate the offset
279 // in Y, U and V planes. 76 // in Y, U and V planes.
280 y_offset = (video_frame->stride(media::VideoFrame::kYPlane) * 77 y_offset = (video_frame->stride(media::VideoFrame::kYPlane) *
281 video_frame->visible_rect().y()) + 78 video_frame->visible_rect().y()) +
282 video_frame->visible_rect().x(); 79 video_frame->visible_rect().x();
283 // For format YV12, there is one U, V value per 2x2 block. 80 // For format YV12, there is one U, V value per 2x2 block.
284 // For format YV16, there is one U, V value per 2x1 block. 81 // For format YV16, there is one U, V value per 2x1 block.
285 uv_offset = (video_frame->stride(media::VideoFrame::kUPlane) * 82 uv_offset = (video_frame->stride(media::VideoFrame::kUPlane) *
286 (video_frame->visible_rect().y() >> y_shift)) + 83 (video_frame->visible_rect().y() >> y_shift)) +
(...skipping 56 matching lines...) Expand 10 before | Expand all | Expand 10 after
343 static_cast<uint8*>(bitmap->getPixels()), 140 static_cast<uint8*>(bitmap->getPixels()),
344 video_frame->visible_rect().width(), 141 video_frame->visible_rect().width(),
345 video_frame->visible_rect().height(), 142 video_frame->visible_rect().height(),
346 video_frame->stride(media::VideoFrame::kYPlane), 143 video_frame->stride(media::VideoFrame::kYPlane),
347 video_frame->stride(media::VideoFrame::kUPlane), 144 video_frame->stride(media::VideoFrame::kUPlane),
348 video_frame->stride(media::VideoFrame::kAPlane), 145 video_frame->stride(media::VideoFrame::kAPlane),
349 bitmap->rowBytes(), 146 bitmap->rowBytes(),
350 media::YV12); 147 media::YV12);
351 break; 148 break;
352 149
150 case media::VideoFrame::I444:
151 LIBYUV_I444_TO_ARGB(
152 video_frame->data(media::VideoFrame::kYPlane) + y_offset,
153 video_frame->stride(media::VideoFrame::kYPlane),
154 video_frame->data(media::VideoFrame::kUPlane) + uv_offset,
155 video_frame->stride(media::VideoFrame::kUPlane),
156 video_frame->data(media::VideoFrame::kVPlane) + uv_offset,
157 video_frame->stride(media::VideoFrame::kVPlane),
158 static_cast<uint8*>(bitmap->getPixels()),
159 bitmap->rowBytes(),
160 video_frame->visible_rect().width(),
161 video_frame->visible_rect().height());
162 break;
163
353 case media::VideoFrame::NATIVE_TEXTURE: 164 case media::VideoFrame::NATIVE_TEXTURE:
354 DCHECK_EQ(video_frame->format(), media::VideoFrame::NATIVE_TEXTURE); 165 DCHECK_EQ(video_frame->format(), media::VideoFrame::NATIVE_TEXTURE);
355 video_frame->ReadPixelsFromNativeTexture(*bitmap); 166 video_frame->ReadPixelsFromNativeTexture(*bitmap);
356 break; 167 break;
357 168
358 default: 169 default:
359 NOTREACHED(); 170 NOTREACHED();
360 break; 171 break;
361 } 172 }
362 bitmap->notifyPixelsChanged(); 173 bitmap->notifyPixelsChanged();
(...skipping 15 matching lines...) Expand all
378 } 189 }
379 190
380 SkRect dest; 191 SkRect dest;
381 dest.set(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom()); 192 dest.set(dest_rect.x(), dest_rect.y(), dest_rect.right(), dest_rect.bottom());
382 193
383 SkPaint paint; 194 SkPaint paint;
384 paint.setAlpha(alpha); 195 paint.setAlpha(alpha);
385 196
386 // Paint black rectangle if there isn't a frame available or the 197 // Paint black rectangle if there isn't a frame available or the
387 // frame has an unexpected format. 198 // frame has an unexpected format.
388 if (!video_frame || !IsEitherYUVOrYUVAOrNative(video_frame->format())) { 199 if (!video_frame || !IsYUVOrNative(video_frame->format())) {
389 canvas->drawRect(dest, paint); 200 canvas->drawRect(dest, paint);
390 return; 201 return;
391 } 202 }
392 203
393 // Scale and convert to RGB in one step if we can.
394 if (CanFastPaint(canvas, alpha, video_frame->format())) {
395 FastPaint(video_frame, canvas, dest);
396 return;
397 }
398
399 // Check if we should convert and update |last_frame_|. 204 // Check if we should convert and update |last_frame_|.
400 if (last_frame_.isNull() || 205 if (last_frame_.isNull() ||
401 video_frame->timestamp() != last_frame_timestamp_) { 206 video_frame->timestamp() != last_frame_timestamp_) {
402 ConvertVideoFrameToBitmap(video_frame, &last_frame_); 207 ConvertVideoFrameToBitmap(video_frame, &last_frame_);
403 last_frame_timestamp_ = video_frame->timestamp(); 208 last_frame_timestamp_ = video_frame->timestamp();
404 } 209 }
405 210
406 // Do a slower paint using |last_frame_|. 211 // Do a slower paint using |last_frame_|.
scherkus (not reviewing) 2014/05/22 23:45:11 remove references to paints being slow(er)
sandersd (OOO until July 31) 2014/05/23 00:45:22 Done.
407 paint.setFilterLevel(SkPaint::kLow_FilterLevel); 212 paint.setFilterLevel(SkPaint::kLow_FilterLevel);
408 canvas->drawBitmapRect(last_frame_, NULL, dest, &paint); 213 canvas->drawBitmapRect(last_frame_, NULL, dest, &paint);
409 } 214 }
410 215
411 } // namespace media 216 } // namespace media
OLDNEW
« media/ffmpeg/ffmpeg_common.cc ('K') | « media/filters/ffmpeg_video_decoder.cc ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698