OLD | NEW |
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "content/browser/renderer_host/media/video_capture_controller.h" | 5 #include "content/browser/renderer_host/media/video_capture_controller.h" |
6 | 6 |
7 #include <set> | 7 #include <set> |
8 | 8 |
9 #include "base/bind.h" | 9 #include "base/bind.h" |
10 #include "base/debug/trace_event.h" | 10 #include "base/debug/trace_event.h" |
11 #include "base/memory/scoped_ptr.h" | 11 #include "base/memory/scoped_ptr.h" |
12 #include "base/stl_util.h" | 12 #include "base/stl_util.h" |
13 #include "content/browser/renderer_host/media/media_stream_manager.h" | 13 #include "content/browser/renderer_host/media/media_stream_manager.h" |
14 #include "content/browser/renderer_host/media/video_capture_manager.h" | 14 #include "content/browser/renderer_host/media/video_capture_manager.h" |
15 #include "content/public/browser/browser_thread.h" | 15 #include "content/public/browser/browser_thread.h" |
16 #include "media/base/video_frame.h" | 16 #include "media/base/video_frame.h" |
17 #include "media/base/video_util.h" | 17 #include "media/base/video_util.h" |
18 #include "media/base/yuv_convert.h" | 18 #include "media/base/yuv_convert.h" |
19 | 19 |
20 #if !defined(OS_IOS) && !defined(OS_ANDROID) | 20 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW) |
21 #include "third_party/libyuv/include/libyuv.h" | 21 #include "third_party/libyuv/include/libyuv.h" |
22 #endif | 22 #endif |
23 | 23 |
24 namespace { | |
25 | |
26 #if defined(OS_IOS) || defined(OS_ANDROID) | |
27 // TODO(wjia): Support stride. | |
28 void RotatePackedYV12Frame( | |
29 const uint8* src, | |
30 uint8* dest_yplane, | |
31 uint8* dest_uplane, | |
32 uint8* dest_vplane, | |
33 int width, | |
34 int height, | |
35 int rotation, | |
36 bool flip_vert, | |
37 bool flip_horiz) { | |
38 media::RotatePlaneByPixels( | |
39 src, dest_yplane, width, height, rotation, flip_vert, flip_horiz); | |
40 int y_size = width * height; | |
41 src += y_size; | |
42 media::RotatePlaneByPixels( | |
43 src, dest_uplane, width/2, height/2, rotation, flip_vert, flip_horiz); | |
44 src += y_size/4; | |
45 media::RotatePlaneByPixels( | |
46 src, dest_vplane, width/2, height/2, rotation, flip_vert, flip_horiz); | |
47 } | |
48 #endif // #if defined(OS_IOS) || defined(OS_ANDROID) | |
49 | |
50 } // namespace | |
51 | |
52 namespace content { | 24 namespace content { |
53 | 25 |
54 // The number of buffers that VideoCaptureBufferPool should allocate. | 26 // The number of buffers that VideoCaptureBufferPool should allocate. |
55 static const int kNoOfBuffers = 3; | 27 static const int kNoOfBuffers = 3; |
56 | 28 |
57 struct VideoCaptureController::ControllerClient { | 29 struct VideoCaptureController::ControllerClient { |
58 ControllerClient( | 30 ControllerClient( |
59 const VideoCaptureControllerID& id, | 31 const VideoCaptureControllerID& id, |
60 VideoCaptureControllerEventHandler* handler, | 32 VideoCaptureControllerEventHandler* handler, |
61 base::ProcessHandle render_process, | 33 base::ProcessHandle render_process, |
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
132 // The pool of shared-memory buffers used for capturing. | 104 // The pool of shared-memory buffers used for capturing. |
133 scoped_refptr<VideoCaptureBufferPool> buffer_pool_; | 105 scoped_refptr<VideoCaptureBufferPool> buffer_pool_; |
134 | 106 |
135 // Chopped pixels in width/height in case video capture device has odd | 107 // Chopped pixels in width/height in case video capture device has odd |
136 // numbers for width/height. | 108 // numbers for width/height. |
137 int chopped_width_; | 109 int chopped_width_; |
138 int chopped_height_; | 110 int chopped_height_; |
139 | 111 |
140 // Tracks the current frame format. | 112 // Tracks the current frame format. |
141 media::VideoCaptureCapability frame_info_; | 113 media::VideoCaptureCapability frame_info_; |
142 | |
143 // For NV21 we have to do color conversion into the intermediate buffer and | |
144 // from there the rotations. This variable won't be needed after | |
145 // http://crbug.com/292400 | |
146 #if defined(OS_IOS) || defined(OS_ANDROID) | |
147 scoped_ptr<uint8[]> i420_intermediate_buffer_; | |
148 #endif // #if defined(OS_IOS) || defined(OS_ANDROID) | |
149 }; | 114 }; |
150 | 115 |
151 VideoCaptureController::VideoCaptureController() | 116 VideoCaptureController::VideoCaptureController() |
152 : state_(VIDEO_CAPTURE_STATE_STARTED), | 117 : state_(VIDEO_CAPTURE_STATE_STARTED), |
153 weak_ptr_factory_(this) { | 118 weak_ptr_factory_(this) { |
154 memset(¤t_params_, 0, sizeof(current_params_)); | 119 memset(¤t_params_, 0, sizeof(current_params_)); |
155 } | 120 } |
156 | 121 |
157 VideoCaptureController::VideoCaptureDeviceClient::VideoCaptureDeviceClient( | 122 VideoCaptureController::VideoCaptureDeviceClient::VideoCaptureDeviceClient( |
158 const base::WeakPtr<VideoCaptureController>& controller) | 123 const base::WeakPtr<VideoCaptureController>& controller) |
(...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
269 buffer_pool_->RelinquishConsumerHold(buffer_id, 1); | 234 buffer_pool_->RelinquishConsumerHold(buffer_id, 1); |
270 } | 235 } |
271 | 236 |
272 scoped_refptr<media::VideoFrame> | 237 scoped_refptr<media::VideoFrame> |
273 VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer() { | 238 VideoCaptureController::VideoCaptureDeviceClient::ReserveOutputBuffer() { |
274 return buffer_pool_->ReserveI420VideoFrame(gfx::Size(frame_info_.width, | 239 return buffer_pool_->ReserveI420VideoFrame(gfx::Size(frame_info_.width, |
275 frame_info_.height), | 240 frame_info_.height), |
276 0); | 241 0); |
277 } | 242 } |
278 | 243 |
279 #if !defined(OS_IOS) && !defined(OS_ANDROID) | |
280 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame( | 244 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame( |
281 const uint8* data, | 245 const uint8* data, |
282 int length, | 246 int length, |
283 base::Time timestamp, | 247 base::Time timestamp, |
284 int rotation, | 248 int rotation, |
285 bool flip_vert, | 249 bool flip_vert, |
286 bool flip_horiz) { | 250 bool flip_horiz) { |
287 TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame"); | 251 TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame"); |
288 | 252 |
289 if (!buffer_pool_.get()) | 253 if (!buffer_pool_.get()) |
290 return; | 254 return; |
291 scoped_refptr<media::VideoFrame> dst = buffer_pool_->ReserveI420VideoFrame( | 255 scoped_refptr<media::VideoFrame> dst = buffer_pool_->ReserveI420VideoFrame( |
292 gfx::Size(frame_info_.width, frame_info_.height), rotation); | 256 gfx::Size(frame_info_.width, frame_info_.height), rotation); |
293 | 257 |
294 if (!dst.get()) | 258 if (!dst.get()) |
295 return; | 259 return; |
| 260 #if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW) |
296 | 261 |
297 uint8* yplane = dst->data(media::VideoFrame::kYPlane); | 262 uint8* yplane = dst->data(media::VideoFrame::kYPlane); |
298 uint8* uplane = dst->data(media::VideoFrame::kUPlane); | 263 uint8* uplane = dst->data(media::VideoFrame::kUPlane); |
299 uint8* vplane = dst->data(media::VideoFrame::kVPlane); | 264 uint8* vplane = dst->data(media::VideoFrame::kVPlane); |
300 int yplane_stride = frame_info_.width; | 265 int yplane_stride = frame_info_.width; |
301 int uv_plane_stride = (frame_info_.width + 1) / 2; | 266 int uv_plane_stride = (frame_info_.width + 1) / 2; |
302 int crop_x = 0; | 267 int crop_x = 0; |
303 int crop_y = 0; | 268 int crop_y = 0; |
| 269 int destination_width = frame_info_.width; |
| 270 int destination_height = frame_info_.height; |
304 libyuv::FourCC origin_colorspace = libyuv::FOURCC_ANY; | 271 libyuv::FourCC origin_colorspace = libyuv::FOURCC_ANY; |
305 // Assuming rotation happens first and flips next, we can consolidate both | 272 // Assuming rotation happens first and flips next, we can consolidate both |
306 // vertical and horizontal flips together with rotation into two variables: | 273 // vertical and horizontal flips together with rotation into two variables: |
307 // new_rotation = (rotation + 180 * vertical_flip) modulo 360 | 274 // new_rotation = (rotation + 180 * vertical_flip) modulo 360 |
308 // new_vertical_flip = horizontal_flip XOR vertical_flip | 275 // new_vertical_flip = horizontal_flip XOR vertical_flip |
309 int new_rotation_angle = (rotation + 180 * flip_vert) % 360; | 276 int new_rotation_angle = (rotation + 180 * flip_vert) % 360; |
310 libyuv::RotationMode rotation_mode = libyuv::kRotate0; | 277 libyuv::RotationMode rotation_mode = libyuv::kRotate0; |
311 if (new_rotation_angle == 90) | 278 if (new_rotation_angle == 90) |
312 rotation_mode = libyuv::kRotate90; | 279 rotation_mode = libyuv::kRotate90; |
313 else if (new_rotation_angle == 180) | 280 else if (new_rotation_angle == 180) |
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
369 media::ConvertRGB24ToYUV(rgb_src, | 336 media::ConvertRGB24ToYUV(rgb_src, |
370 yplane, | 337 yplane, |
371 uplane, | 338 uplane, |
372 vplane, | 339 vplane, |
373 frame_info_.width, | 340 frame_info_.width, |
374 frame_info_.height, | 341 frame_info_.height, |
375 rgb_stride, | 342 rgb_stride, |
376 yplane_stride, | 343 yplane_stride, |
377 uv_plane_stride); | 344 uv_plane_stride); |
378 } else { | 345 } else { |
| 346 if (new_rotation_angle==90 || new_rotation_angle==270){ |
| 347 // To be compatible with non-libyuv code in RotatePlaneByPixels, when |
| 348 // rotating by 90/270, only the maximum square portion located in the |
| 349 // center of the image is rotated. F.i. 640x480 pixels, only the central |
| 350 // 480 pixels would be rotated and the leftmost and rightmost 80 columns |
| 351 // would be ignored. This process is called letterboxing. |
| 352 int letterbox_thickness = abs(frame_info_.width - frame_info_.height) / 2; |
| 353 if (destination_width > destination_height) { |
| 354 yplane += letterbox_thickness; |
| 355 uplane += letterbox_thickness / 2; |
| 356 vplane += letterbox_thickness / 2; |
| 357 destination_width = destination_height; |
| 358 } else { |
| 359 yplane += letterbox_thickness * destination_width; |
| 360 uplane += (letterbox_thickness * destination_width) / 2; |
| 361 vplane += (letterbox_thickness * destination_width) / 2; |
| 362 destination_height = destination_width; |
| 363 } |
| 364 } |
379 libyuv::ConvertToI420( | 365 libyuv::ConvertToI420( |
380 data, | 366 data, length, |
381 length, | 367 yplane, yplane_stride, |
382 yplane, | 368 uplane, uv_plane_stride, |
383 yplane_stride, | 369 vplane, uv_plane_stride, |
384 uplane, | 370 crop_x, crop_y, |
385 uv_plane_stride, | |
386 vplane, | |
387 uv_plane_stride, | |
388 crop_x, | |
389 crop_y, | |
390 frame_info_.width + chopped_width_, | 371 frame_info_.width + chopped_width_, |
391 frame_info_.height * (flip_vert ^ flip_horiz ? -1 : 1), | 372 frame_info_.height * (flip_vert ^ flip_horiz ? -1 : 1), |
392 frame_info_.width, | 373 destination_width, |
393 frame_info_.height, | 374 destination_height, |
394 rotation_mode, | 375 rotation_mode, |
395 origin_colorspace); | 376 origin_colorspace); |
396 } | 377 } |
| 378 #else |
| 379 // Libyuv is not linked in for Android WebView builds, but video capture is |
| 380 // not used in those builds either. Whenever libyuv is added in that build, |
| 381 // address all these #ifdef parts, see http://crbug.com/299611 . |
| 382 NOTREACHED(); |
| 383 #endif // if !defined(AVOID_LIBYUV_FOR_ANDROID_WEBVIEW) |
397 BrowserThread::PostTask( | 384 BrowserThread::PostTask( |
398 BrowserThread::IO, | 385 BrowserThread::IO, |
399 FROM_HERE, | 386 FROM_HERE, |
400 base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread, | 387 base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread, |
401 controller_, | 388 controller_, |
402 dst, | 389 dst, |
403 timestamp)); | 390 timestamp)); |
404 } | 391 } |
405 #else | |
406 void VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedFrame( | |
407 const uint8* data, | |
408 int length, | |
409 base::Time timestamp, | |
410 int rotation, | |
411 bool flip_vert, | |
412 bool flip_horiz) { | |
413 DCHECK(frame_info_.color == media::PIXEL_FORMAT_I420 || | |
414 frame_info_.color == media::PIXEL_FORMAT_YV12 || | |
415 frame_info_.color == media::PIXEL_FORMAT_NV21 || | |
416 (rotation == 0 && !flip_vert && !flip_horiz)); | |
417 | |
418 TRACE_EVENT0("video", "VideoCaptureController::OnIncomingCapturedFrame"); | |
419 | |
420 if (!buffer_pool_) | |
421 return; | |
422 scoped_refptr<media::VideoFrame> dst = | |
423 buffer_pool_->ReserveI420VideoFrame(gfx::Size(frame_info_.width, | |
424 frame_info_.height), | |
425 rotation); | |
426 | |
427 if (!dst.get()) | |
428 return; | |
429 | |
430 uint8* yplane = dst->data(media::VideoFrame::kYPlane); | |
431 uint8* uplane = dst->data(media::VideoFrame::kUPlane); | |
432 uint8* vplane = dst->data(media::VideoFrame::kVPlane); | |
433 | |
434 // Do color conversion from the camera format to I420. | |
435 switch (frame_info_.color) { | |
436 case media::PIXEL_FORMAT_UNKNOWN: // Color format not set. | |
437 break; | |
438 case media::PIXEL_FORMAT_I420: | |
439 DCHECK(!chopped_width_ && !chopped_height_); | |
440 RotatePackedYV12Frame( | |
441 data, yplane, uplane, vplane, frame_info_.width, frame_info_.height, | |
442 rotation, flip_vert, flip_horiz); | |
443 break; | |
444 case media::PIXEL_FORMAT_YV12: | |
445 DCHECK(!chopped_width_ && !chopped_height_); | |
446 RotatePackedYV12Frame( | |
447 data, yplane, vplane, uplane, frame_info_.width, frame_info_.height, | |
448 rotation, flip_vert, flip_horiz); | |
449 break; | |
450 case media::PIXEL_FORMAT_NV21: { | |
451 DCHECK(!chopped_width_ && !chopped_height_); | |
452 int num_pixels = frame_info_.width * frame_info_.height; | |
453 media::ConvertNV21ToYUV(data, | |
454 &i420_intermediate_buffer_[0], | |
455 &i420_intermediate_buffer_[num_pixels], | |
456 &i420_intermediate_buffer_[num_pixels * 5 / 4], | |
457 frame_info_.width, | |
458 frame_info_.height); | |
459 RotatePackedYV12Frame( | |
460 i420_intermediate_buffer_.get(), yplane, uplane, vplane, | |
461 frame_info_.width, frame_info_.height, | |
462 rotation, flip_vert, flip_horiz); | |
463 break; | |
464 } | |
465 case media::PIXEL_FORMAT_YUY2: | |
466 DCHECK(!chopped_width_ && !chopped_height_); | |
467 if (frame_info_.width * frame_info_.height * 2 != length) { | |
468 // If |length| of |data| does not match the expected width and height | |
469 // we can't convert the frame to I420. YUY2 is 2 bytes per pixel. | |
470 break; | |
471 } | |
472 | |
473 media::ConvertYUY2ToYUV(data, yplane, uplane, vplane, frame_info_.width, | |
474 frame_info_.height); | |
475 break; | |
476 case media::PIXEL_FORMAT_RGB24: { | |
477 int ystride = frame_info_.width; | |
478 int uvstride = frame_info_.width / 2; | |
479 int rgb_stride = 3 * (frame_info_.width + chopped_width_); | |
480 const uint8* rgb_src = data; | |
481 media::ConvertRGB24ToYUV(rgb_src, yplane, uplane, vplane, | |
482 frame_info_.width, frame_info_.height, | |
483 rgb_stride, ystride, uvstride); | |
484 break; | |
485 } | |
486 case media::PIXEL_FORMAT_ARGB: | |
487 media::ConvertRGB32ToYUV(data, yplane, uplane, vplane, frame_info_.width, | |
488 frame_info_.height, | |
489 (frame_info_.width + chopped_width_) * 4, | |
490 frame_info_.width, frame_info_.width / 2); | |
491 break; | |
492 default: | |
493 NOTREACHED(); | |
494 } | |
495 | |
496 BrowserThread::PostTask(BrowserThread::IO, | |
497 FROM_HERE, | |
498 base::Bind(&VideoCaptureController::DoIncomingCapturedFrameOnIOThread, | |
499 controller_, dst, timestamp)); | |
500 } | |
501 #endif // #if !defined(OS_IOS) && !defined(OS_ANDROID) | |
502 | 392 |
503 void | 393 void |
504 VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedVideoFrame( | 394 VideoCaptureController::VideoCaptureDeviceClient::OnIncomingCapturedVideoFrame( |
505 const scoped_refptr<media::VideoFrame>& frame, | 395 const scoped_refptr<media::VideoFrame>& frame, |
506 base::Time timestamp) { | 396 base::Time timestamp) { |
507 if (!buffer_pool_) | 397 if (!buffer_pool_) |
508 return; | 398 return; |
509 | 399 |
510 // If this is a frame that belongs to the buffer pool, we can forward it | 400 // If this is a frame that belongs to the buffer pool, we can forward it |
511 // directly to the IO thread and be done. | 401 // directly to the IO thread and be done. |
(...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
629 chopped_width_ = 1; | 519 chopped_width_ = 1; |
630 } else { | 520 } else { |
631 chopped_width_ = 0; | 521 chopped_width_ = 0; |
632 } | 522 } |
633 if (info.height & 1) { | 523 if (info.height & 1) { |
634 --frame_info_.height; | 524 --frame_info_.height; |
635 chopped_height_ = 1; | 525 chopped_height_ = 1; |
636 } else { | 526 } else { |
637 chopped_height_ = 0; | 527 chopped_height_ = 0; |
638 } | 528 } |
639 #if defined(OS_IOS) || defined(OS_ANDROID) | |
640 if (frame_info_.color == media::PIXEL_FORMAT_NV21 && | |
641 !i420_intermediate_buffer_) { | |
642 i420_intermediate_buffer_.reset( | |
643 new uint8[frame_info_.width * frame_info_.height * 12 / 8]); | |
644 } | |
645 #endif // #if defined(OS_IOS) || defined(OS_ANDROID) | |
646 | 529 |
647 DCHECK(!buffer_pool_.get()); | 530 DCHECK(!buffer_pool_.get()); |
648 | 531 |
649 // TODO(nick): Give BufferPool the same lifetime as the controller, have it | 532 // TODO(nick): Give BufferPool the same lifetime as the controller, have it |
650 // support frame size changes, and stop checking it for NULL everywhere. | 533 // support frame size changes, and stop checking it for NULL everywhere. |
651 // http://crbug.com/266082 | 534 // http://crbug.com/266082 |
652 buffer_pool_ = new VideoCaptureBufferPool( | 535 buffer_pool_ = new VideoCaptureBufferPool( |
653 media::VideoFrame::AllocationSize( | 536 media::VideoFrame::AllocationSize( |
654 media::VideoFrame::I420, | 537 media::VideoFrame::I420, |
655 gfx::Size(frame_info_.width, frame_info_.height)), | 538 gfx::Size(frame_info_.width, frame_info_.height)), |
(...skipping 155 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
811 } | 694 } |
812 return NULL; | 695 return NULL; |
813 } | 696 } |
814 | 697 |
815 int VideoCaptureController::GetClientCount() { | 698 int VideoCaptureController::GetClientCount() { |
816 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); | 699 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO)); |
817 return controller_clients_.size(); | 700 return controller_clients_.size(); |
818 } | 701 } |
819 | 702 |
820 } // namespace content | 703 } // namespace content |
OLD | NEW |