OLD | NEW |
1 // Copyright 2015 The Chromium Authors. All rights reserved. | 1 // Copyright 2015 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "media/capture/video/video_capture_device_client.h" | 5 #include "media/capture/video/video_capture_device_client.h" |
6 | 6 |
7 #include <algorithm> | 7 #include <algorithm> |
8 #include <utility> | 8 #include <utility> |
9 | 9 |
10 #include "base/bind.h" | 10 #include "base/bind.h" |
11 #include "base/command_line.h" | 11 #include "base/command_line.h" |
12 #include "base/location.h" | 12 #include "base/location.h" |
13 #include "base/memory/ptr_util.h" | 13 #include "base/memory/ptr_util.h" |
14 #include "base/strings/stringprintf.h" | 14 #include "base/strings/stringprintf.h" |
15 #include "base/trace_event/trace_event.h" | 15 #include "base/trace_event/trace_event.h" |
16 #include "build/build_config.h" | 16 #include "build/build_config.h" |
17 #include "media/base/bind_to_current_loop.h" | 17 #include "media/base/bind_to_current_loop.h" |
18 #include "media/base/video_capture_types.h" | 18 #include "media/base/video_capture_types.h" |
19 #include "media/base/video_frame.h" | 19 #include "media/base/video_frame.h" |
20 #include "media/capture/video/video_capture_buffer_handle.h" | 20 #include "media/capture/video/video_capture_buffer_handle.h" |
21 #include "media/capture/video/video_capture_buffer_pool.h" | 21 #include "media/capture/video/video_capture_buffer_pool.h" |
22 #include "media/capture/video/video_capture_jpeg_decoder.h" | 22 #include "media/capture/video/video_capture_jpeg_decoder.h" |
23 #include "media/capture/video/video_frame_receiver.h" | 23 #include "media/capture/video/video_frame_receiver.h" |
24 #include "third_party/libyuv/include/libyuv.h" | 24 #include "third_party/libyuv/include/libyuv.h" |
25 | 25 |
26 using media::VideoCaptureFormat; | 26 using media::VideoCaptureFormat; |
27 using media::VideoFrame; | 27 using media::VideoFrame; |
28 using media::VideoFrameMetadata; | 28 using media::VideoFrameMetadata; |
29 | 29 |
| 30 namespace { |
| 31 |
| 32 bool IsFormatSupported(media::VideoPixelFormat pixel_format) { |
| 33 return (pixel_format == media::PIXEL_FORMAT_I420 || |
| 34 pixel_format == media::PIXEL_FORMAT_Y16); |
| 35 } |
| 36 |
| 37 } |
| 38 |
30 namespace media { | 39 namespace media { |
31 | 40 |
32 // Class combining a Client::Buffer interface implementation and a pool buffer | 41 // Class combining a Client::Buffer interface implementation and a pool buffer |
33 // implementation to guarantee proper cleanup on destruction on our side. | 42 // implementation to guarantee proper cleanup on destruction on our side. |
34 class AutoReleaseBuffer : public media::VideoCaptureDevice::Client::Buffer { | 43 class AutoReleaseBuffer : public media::VideoCaptureDevice::Client::Buffer { |
35 public: | 44 public: |
36 AutoReleaseBuffer(const scoped_refptr<VideoCaptureBufferPool>& pool, | 45 AutoReleaseBuffer(const scoped_refptr<VideoCaptureBufferPool>& pool, |
37 int buffer_id) | 46 int buffer_id) |
38 : id_(buffer_id), | 47 : id_(buffer_id), |
39 pool_(pool), | 48 pool_(pool), |
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
99 !external_jpeg_decoder_initialized_) { | 108 !external_jpeg_decoder_initialized_) { |
100 external_jpeg_decoder_initialized_ = true; | 109 external_jpeg_decoder_initialized_ = true; |
101 external_jpeg_decoder_ = jpeg_decoder_factory_callback_.Run(); | 110 external_jpeg_decoder_ = jpeg_decoder_factory_callback_.Run(); |
102 external_jpeg_decoder_->Initialize(); | 111 external_jpeg_decoder_->Initialize(); |
103 } | 112 } |
104 } | 113 } |
105 | 114 |
106 if (!frame_format.IsValid()) | 115 if (!frame_format.IsValid()) |
107 return; | 116 return; |
108 | 117 |
| 118 // The input |length| can be greater than the required buffer size because of |
| 119 // paddings and/or alignments, but it cannot be smaller. |
| 120 DCHECK_GE(static_cast<size_t>(length), frame_format.ImageAllocationSize()); |
| 121 |
| 122 const bool convert_to_I420 = |
| 123 frame_format.pixel_format != media::PIXEL_FORMAT_Y16; |
| 124 |
109 // |chopped_{width,height} and |new_unrotated_{width,height}| are the lowest | 125 // |chopped_{width,height} and |new_unrotated_{width,height}| are the lowest |
110 // bit decomposition of {width, height}, grabbing the odd and even parts. | 126 // bit decomposition of {width, height}, grabbing the odd and even parts. |
111 const int chopped_width = frame_format.frame_size.width() & 1; | 127 const int chopped_width = frame_format.frame_size.width() & 1; |
112 const int chopped_height = frame_format.frame_size.height() & 1; | 128 const int chopped_height = frame_format.frame_size.height() & 1; |
113 const int new_unrotated_width = frame_format.frame_size.width() & ~1; | 129 const int new_unrotated_width = convert_to_I420 |
114 const int new_unrotated_height = frame_format.frame_size.height() & ~1; | 130 ? (frame_format.frame_size.width() & ~1) |
| 131 : frame_format.frame_size.width(); |
| 132 const int new_unrotated_height = convert_to_I420 |
| 133 ? (frame_format.frame_size.height() & ~1) |
| 134 : frame_format.frame_size.height(); |
115 | 135 |
116 int destination_width = new_unrotated_width; | 136 int destination_width = new_unrotated_width; |
117 int destination_height = new_unrotated_height; | 137 int destination_height = new_unrotated_height; |
118 if (rotation == 90 || rotation == 270) | 138 if (rotation == 90 || rotation == 270) |
119 std::swap(destination_width, destination_height); | 139 std::swap(destination_width, destination_height); |
120 | 140 |
121 DCHECK_EQ(0, rotation % 90) << " Rotation must be a multiple of 90, now: " | 141 DCHECK_EQ(0, rotation % 90) << " Rotation must be a multiple of 90, now: " |
122 << rotation; | 142 << rotation; |
123 libyuv::RotationMode rotation_mode = libyuv::kRotate0; | 143 libyuv::RotationMode rotation_mode = libyuv::kRotate0; |
124 if (rotation == 90) | 144 if (rotation == 90) |
125 rotation_mode = libyuv::kRotate90; | 145 rotation_mode = libyuv::kRotate90; |
126 else if (rotation == 180) | 146 else if (rotation == 180) |
127 rotation_mode = libyuv::kRotate180; | 147 rotation_mode = libyuv::kRotate180; |
128 else if (rotation == 270) | 148 else if (rotation == 270) |
129 rotation_mode = libyuv::kRotate270; | 149 rotation_mode = libyuv::kRotate270; |
130 | 150 |
131 const gfx::Size dimensions(destination_width, destination_height); | 151 const gfx::Size dimensions(destination_width, destination_height); |
132 uint8_t *y_plane_data, *u_plane_data, *v_plane_data; | 152 uint8_t* y_plane_data = nullptr; |
| 153 uint8_t* u_plane_data = nullptr; |
| 154 uint8_t* v_plane_data = nullptr; |
| 155 |
133 std::unique_ptr<Buffer> buffer( | 156 std::unique_ptr<Buffer> buffer( |
134 ReserveI420OutputBuffer(dimensions, media::PIXEL_STORAGE_CPU, | 157 convert_to_I420 |
135 &y_plane_data, &u_plane_data, &v_plane_data)); | 158 ? ReserveI420OutputBuffer(dimensions, media::PIXEL_STORAGE_CPU, |
| 159 &y_plane_data, &u_plane_data, &v_plane_data) |
| 160 : ReserveOutputBuffer(dimensions, frame_format.pixel_format, |
| 161 media::PIXEL_STORAGE_CPU)); |
136 #if DCHECK_IS_ON() | 162 #if DCHECK_IS_ON() |
137 dropped_frame_counter_ = buffer.get() ? 0 : dropped_frame_counter_ + 1; | 163 dropped_frame_counter_ = buffer.get() ? 0 : dropped_frame_counter_ + 1; |
138 if (dropped_frame_counter_ >= kMaxDroppedFrames) | 164 if (dropped_frame_counter_ >= kMaxDroppedFrames) |
139 OnError(FROM_HERE, "Too many frames dropped"); | 165 OnError(FROM_HERE, "Too many frames dropped"); |
140 #endif | 166 #endif |
141 // Failed to reserve I420 output buffer, so drop the frame. | 167 // Failed to reserve output buffer, so drop the frame. |
142 if (!buffer.get()) | 168 if (!buffer.get()) |
143 return; | 169 return; |
144 | 170 |
145 const int yplane_stride = dimensions.width(); | 171 const int yplane_stride = dimensions.width(); |
146 const int uv_plane_stride = yplane_stride / 2; | 172 const int uv_plane_stride = yplane_stride / 2; |
147 int crop_x = 0; | 173 int crop_x = 0; |
148 int crop_y = 0; | 174 int crop_y = 0; |
149 libyuv::FourCC origin_colorspace = libyuv::FOURCC_ANY; | 175 libyuv::FourCC origin_colorspace = libyuv::FOURCC_ANY; |
150 | 176 |
151 bool flip = false; | 177 bool flip = false; |
(...skipping 49 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
201 // platforms. | 227 // platforms. |
202 #if defined(OS_WIN) | 228 #if defined(OS_WIN) |
203 flip = true; | 229 flip = true; |
204 #endif | 230 #endif |
205 case media::PIXEL_FORMAT_ARGB: | 231 case media::PIXEL_FORMAT_ARGB: |
206 origin_colorspace = libyuv::FOURCC_ARGB; | 232 origin_colorspace = libyuv::FOURCC_ARGB; |
207 break; | 233 break; |
208 case media::PIXEL_FORMAT_MJPEG: | 234 case media::PIXEL_FORMAT_MJPEG: |
209 origin_colorspace = libyuv::FOURCC_MJPG; | 235 origin_colorspace = libyuv::FOURCC_MJPG; |
210 break; | 236 break; |
| 237 case media::PIXEL_FORMAT_Y16: |
| 238 break; |
211 default: | 239 default: |
212 NOTREACHED(); | 240 NOTREACHED(); |
213 } | 241 } |
214 | 242 |
215 // The input |length| can be greater than the required buffer size because of | 243 // The input |length| can be greater than the required buffer size because of |
216 // paddings and/or alignments, but it cannot be smaller. | 244 // paddings and/or alignments, but it cannot be smaller. |
217 DCHECK_GE(static_cast<size_t>(length), frame_format.ImageAllocationSize()); | 245 DCHECK_GE(static_cast<size_t>(length), frame_format.ImageAllocationSize()); |
218 | 246 |
219 if (external_jpeg_decoder_) { | 247 if (external_jpeg_decoder_) { |
220 const VideoCaptureJpegDecoder::STATUS status = | 248 const VideoCaptureJpegDecoder::STATUS status = |
221 external_jpeg_decoder_->GetStatus(); | 249 external_jpeg_decoder_->GetStatus(); |
222 if (status == VideoCaptureJpegDecoder::FAILED) { | 250 if (status == VideoCaptureJpegDecoder::FAILED) { |
223 external_jpeg_decoder_.reset(); | 251 external_jpeg_decoder_.reset(); |
224 } else if (status == VideoCaptureJpegDecoder::INIT_PASSED && | 252 } else if (status == VideoCaptureJpegDecoder::INIT_PASSED && |
225 frame_format.pixel_format == media::PIXEL_FORMAT_MJPEG && | 253 frame_format.pixel_format == media::PIXEL_FORMAT_MJPEG && |
226 rotation == 0 && !flip) { | 254 rotation == 0 && !flip) { |
227 external_jpeg_decoder_->DecodeCapturedData(data, length, frame_format, | 255 external_jpeg_decoder_->DecodeCapturedData(data, length, frame_format, |
228 reference_time, timestamp, | 256 reference_time, timestamp, |
229 std::move(buffer)); | 257 std::move(buffer)); |
230 return; | 258 return; |
231 } | 259 } |
232 } | 260 } |
233 | 261 |
234 if (libyuv::ConvertToI420(data, length, y_plane_data, yplane_stride, | 262 VideoPixelFormat pixel_format = frame_format.pixel_format; |
235 u_plane_data, uv_plane_stride, v_plane_data, | 263 if (convert_to_I420) { |
236 uv_plane_stride, crop_x, crop_y, | 264 pixel_format = media::PIXEL_FORMAT_I420; |
237 frame_format.frame_size.width(), | 265 if (libyuv::ConvertToI420( |
238 (flip ? -1 : 1) * frame_format.frame_size.height(), | 266 data, length, y_plane_data, yplane_stride, u_plane_data, |
239 new_unrotated_width, new_unrotated_height, | 267 uv_plane_stride, v_plane_data, uv_plane_stride, crop_x, crop_y, |
240 rotation_mode, origin_colorspace) != 0) { | 268 frame_format.frame_size.width(), |
241 DLOG(WARNING) << "Failed to convert buffer's pixel format to I420 from " | 269 (flip ? -1 : 1) * frame_format.frame_size.height(), |
242 << media::VideoPixelFormatToString(frame_format.pixel_format); | 270 new_unrotated_width, new_unrotated_height, rotation_mode, |
243 return; | 271 origin_colorspace) != 0) { |
| 272 DLOG(WARNING) << "Failed to convert buffer's pixel format to I420 from " |
| 273 << media::VideoPixelFormatToString( |
| 274 frame_format.pixel_format); |
| 275 return; |
| 276 } |
| 277 } else { |
| 278 memcpy(buffer->data(), data, length); |
244 } | 279 } |
245 | 280 |
246 const VideoCaptureFormat output_format = | 281 const VideoCaptureFormat output_format = |
247 VideoCaptureFormat(dimensions, frame_format.frame_rate, | 282 VideoCaptureFormat(dimensions, frame_format.frame_rate, pixel_format, |
248 media::PIXEL_FORMAT_I420, media::PIXEL_STORAGE_CPU); | 283 media::PIXEL_STORAGE_CPU); |
249 OnIncomingCapturedBuffer(std::move(buffer), output_format, reference_time, | 284 OnIncomingCapturedBuffer(std::move(buffer), output_format, reference_time, |
250 timestamp); | 285 timestamp); |
251 } | 286 } |
252 | 287 |
253 std::unique_ptr<media::VideoCaptureDevice::Client::Buffer> | 288 std::unique_ptr<media::VideoCaptureDevice::Client::Buffer> |
254 VideoCaptureDeviceClient::ReserveOutputBuffer( | 289 VideoCaptureDeviceClient::ReserveOutputBuffer( |
255 const gfx::Size& frame_size, | 290 const gfx::Size& frame_size, |
256 media::VideoPixelFormat pixel_format, | 291 media::VideoPixelFormat pixel_format, |
257 media::VideoPixelStorage pixel_storage) { | 292 media::VideoPixelStorage pixel_storage) { |
258 DCHECK_GT(frame_size.width(), 0); | 293 DCHECK_GT(frame_size.width(), 0); |
259 DCHECK_GT(frame_size.height(), 0); | 294 DCHECK_GT(frame_size.height(), 0); |
260 // Currently, only I420 pixel format is supported. | 295 DCHECK(IsFormatSupported(pixel_format)); |
261 DCHECK_EQ(media::PIXEL_FORMAT_I420, pixel_format); | |
262 | 296 |
263 // TODO(mcasas): For PIXEL_STORAGE_GPUMEMORYBUFFER, find a way to indicate if | 297 // TODO(mcasas): For PIXEL_STORAGE_GPUMEMORYBUFFER, find a way to indicate if |
264 // it's a ShMem GMB or a DmaBuf GMB. | 298 // it's a ShMem GMB or a DmaBuf GMB. |
265 int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId; | 299 int buffer_id_to_drop = VideoCaptureBufferPool::kInvalidId; |
266 const int buffer_id = buffer_pool_->ReserveForProducer( | 300 const int buffer_id = buffer_pool_->ReserveForProducer( |
267 frame_size, pixel_format, pixel_storage, &buffer_id_to_drop); | 301 frame_size, pixel_format, pixel_storage, &buffer_id_to_drop); |
268 if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) | 302 if (buffer_id_to_drop != VideoCaptureBufferPool::kInvalidId) |
269 receiver_->OnBufferDestroyed(buffer_id_to_drop); | 303 receiver_->OnBufferDestroyed(buffer_id_to_drop); |
270 if (buffer_id == VideoCaptureBufferPool::kInvalidId) | 304 if (buffer_id == VideoCaptureBufferPool::kInvalidId) |
271 return nullptr; | 305 return nullptr; |
272 return base::WrapUnique<Buffer>( | 306 return base::WrapUnique<Buffer>( |
273 new AutoReleaseBuffer(buffer_pool_, buffer_id)); | 307 new AutoReleaseBuffer(buffer_pool_, buffer_id)); |
274 } | 308 } |
275 | 309 |
276 void VideoCaptureDeviceClient::OnIncomingCapturedBuffer( | 310 void VideoCaptureDeviceClient::OnIncomingCapturedBuffer( |
277 std::unique_ptr<Buffer> buffer, | 311 std::unique_ptr<Buffer> buffer, |
278 const VideoCaptureFormat& frame_format, | 312 const VideoCaptureFormat& frame_format, |
279 base::TimeTicks reference_time, | 313 base::TimeTicks reference_time, |
280 base::TimeDelta timestamp) { | 314 base::TimeDelta timestamp) { |
281 // Currently, only I420 pixel format is supported. | 315 DCHECK(IsFormatSupported(frame_format.pixel_format)); |
282 DCHECK_EQ(media::PIXEL_FORMAT_I420, frame_format.pixel_format); | |
283 DCHECK_EQ(media::PIXEL_STORAGE_CPU, frame_format.pixel_storage); | 316 DCHECK_EQ(media::PIXEL_STORAGE_CPU, frame_format.pixel_storage); |
284 | 317 |
285 scoped_refptr<VideoFrame> frame; | 318 scoped_refptr<VideoFrame> frame; |
286 if (buffer->IsBackedByVideoFrame()) { | 319 if (buffer->IsBackedByVideoFrame()) { |
287 frame = buffer->GetVideoFrame(); | 320 frame = buffer->GetVideoFrame(); |
288 frame->set_timestamp(timestamp); | 321 frame->set_timestamp(timestamp); |
289 } else { | 322 } else { |
290 frame = VideoFrame::WrapExternalSharedMemory( | 323 frame = VideoFrame::WrapExternalSharedMemory( |
291 media::PIXEL_FORMAT_I420, frame_format.frame_size, | 324 frame_format.pixel_format, frame_format.frame_size, |
292 gfx::Rect(frame_format.frame_size), frame_format.frame_size, | 325 gfx::Rect(frame_format.frame_size), frame_format.frame_size, |
293 reinterpret_cast<uint8_t*>(buffer->data()), | 326 reinterpret_cast<uint8_t*>(buffer->data()), |
294 VideoFrame::AllocationSize(media::PIXEL_FORMAT_I420, | 327 VideoFrame::AllocationSize(frame_format.pixel_format, |
295 frame_format.frame_size), | 328 frame_format.frame_size), |
296 base::SharedMemory::NULLHandle(), 0u, timestamp); | 329 base::SharedMemory::NULLHandle(), 0u, timestamp); |
297 } | 330 } |
298 if (!frame) | 331 if (!frame) |
299 return; | 332 return; |
300 frame->metadata()->SetDouble(media::VideoFrameMetadata::FRAME_RATE, | 333 frame->metadata()->SetDouble(media::VideoFrameMetadata::FRAME_RATE, |
301 frame_format.frame_rate); | 334 frame_format.frame_rate); |
302 frame->metadata()->SetTimeTicks(media::VideoFrameMetadata::REFERENCE_TIME, | 335 frame->metadata()->SetTimeTicks(media::VideoFrameMetadata::REFERENCE_TIME, |
303 reference_time); | 336 reference_time); |
304 OnIncomingCapturedVideoFrame(std::move(buffer), frame); | 337 OnIncomingCapturedVideoFrame(std::move(buffer), frame); |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
366 *u_plane_data = | 399 *u_plane_data = |
367 *y_plane_data + | 400 *y_plane_data + |
368 VideoFrame::PlaneSize(format, VideoFrame::kYPlane, dimensions).GetArea(); | 401 VideoFrame::PlaneSize(format, VideoFrame::kYPlane, dimensions).GetArea(); |
369 *v_plane_data = | 402 *v_plane_data = |
370 *u_plane_data + | 403 *u_plane_data + |
371 VideoFrame::PlaneSize(format, VideoFrame::kUPlane, dimensions).GetArea(); | 404 VideoFrame::PlaneSize(format, VideoFrame::kUPlane, dimensions).GetArea(); |
372 return buffer; | 405 return buffer; |
373 } | 406 } |
374 | 407 |
375 } // namespace media | 408 } // namespace media |
OLD | NEW |