OLD | NEW |
1 // Copyright 2014 The Chromium Authors. All rights reserved. | 1 // Copyright 2014 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "media/cast/sender/h264_vt_encoder.h" | 5 #include "media/cast/sender/h264_vt_encoder.h" |
6 | 6 |
7 #include <stddef.h> | 7 #include <stddef.h> |
8 | 8 |
9 #include <string> | 9 #include <string> |
10 #include <vector> | 10 #include <vector> |
11 | 11 |
12 #include "base/big_endian.h" | 12 #include "base/big_endian.h" |
13 #include "base/bind.h" | 13 #include "base/bind.h" |
14 #include "base/bind_helpers.h" | 14 #include "base/bind_helpers.h" |
15 #include "base/location.h" | 15 #include "base/location.h" |
16 #include "base/logging.h" | 16 #include "base/logging.h" |
17 #include "base/macros.h" | 17 #include "base/macros.h" |
18 #include "base/power_monitor/power_monitor.h" | 18 #include "base/power_monitor/power_monitor.h" |
19 #include "base/synchronization/lock.h" | 19 #include "base/synchronization/lock.h" |
20 #include "build/build_config.h" | 20 #include "build/build_config.h" |
21 #include "media/base/mac/corevideo_glue.h" | 21 #include "media/base/mac/corevideo_glue.h" |
22 #include "media/base/mac/video_frame_mac.h" | 22 #include "media/base/mac/video_frame_mac.h" |
| 23 #include "media/base/mac/videotoolbox_helpers.h" |
23 #include "media/cast/common/rtp_time.h" | 24 #include "media/cast/common/rtp_time.h" |
24 #include "media/cast/constants.h" | 25 #include "media/cast/constants.h" |
25 #include "media/cast/sender/video_frame_factory.h" | 26 #include "media/cast/sender/video_frame_factory.h" |
26 | 27 |
27 namespace media { | 28 namespace media { |
28 namespace cast { | 29 namespace cast { |
29 | 30 |
30 namespace { | 31 namespace { |
31 | 32 |
32 // Container for the associated data of a video frame being processed. | 33 // Container for the associated data of a video frame being processed. |
33 struct InProgressFrameEncode { | 34 struct InProgressFrameEncode { |
34 const RtpTimeTicks rtp_timestamp; | 35 const RtpTimeTicks rtp_timestamp; |
35 const base::TimeTicks reference_time; | 36 const base::TimeTicks reference_time; |
36 const VideoEncoder::FrameEncodedCallback frame_encoded_callback; | 37 const VideoEncoder::FrameEncodedCallback frame_encoded_callback; |
37 | 38 |
38 InProgressFrameEncode(RtpTimeTicks rtp, | 39 InProgressFrameEncode(RtpTimeTicks rtp, |
39 base::TimeTicks r_time, | 40 base::TimeTicks r_time, |
40 VideoEncoder::FrameEncodedCallback callback) | 41 VideoEncoder::FrameEncodedCallback callback) |
41 : rtp_timestamp(rtp), | 42 : rtp_timestamp(rtp), |
42 reference_time(r_time), | 43 reference_time(r_time), |
43 frame_encoded_callback(callback) {} | 44 frame_encoded_callback(callback) {} |
44 }; | 45 }; |
45 | 46 |
46 base::ScopedCFTypeRef<CFDictionaryRef> | |
47 DictionaryWithKeysAndValues(CFTypeRef* keys, CFTypeRef* values, size_t size) { | |
48 return base::ScopedCFTypeRef<CFDictionaryRef>(CFDictionaryCreate( | |
49 kCFAllocatorDefault, keys, values, size, &kCFTypeDictionaryKeyCallBacks, | |
50 &kCFTypeDictionaryValueCallBacks)); | |
51 } | |
52 | |
53 base::ScopedCFTypeRef<CFDictionaryRef> DictionaryWithKeyValue(CFTypeRef key, | |
54 CFTypeRef value) { | |
55 CFTypeRef keys[1] = {key}; | |
56 CFTypeRef values[1] = {value}; | |
57 return DictionaryWithKeysAndValues(keys, values, 1); | |
58 } | |
59 | |
60 base::ScopedCFTypeRef<CFArrayRef> ArrayWithIntegers(const int* v, size_t size) { | |
61 std::vector<CFNumberRef> numbers; | |
62 numbers.reserve(size); | |
63 for (const int* end = v + size; v < end; ++v) | |
64 numbers.push_back(CFNumberCreate(nullptr, kCFNumberSInt32Type, v)); | |
65 base::ScopedCFTypeRef<CFArrayRef> array(CFArrayCreate( | |
66 kCFAllocatorDefault, reinterpret_cast<const void**>(&numbers[0]), | |
67 numbers.size(), &kCFTypeArrayCallBacks)); | |
68 for (auto& number : numbers) { | |
69 CFRelease(number); | |
70 } | |
71 return array; | |
72 } | |
73 | |
74 template <typename NalSizeType> | |
75 void CopyNalsToAnnexB(char* avcc_buffer, | |
76 const size_t avcc_size, | |
77 std::string* annexb_buffer) { | |
78 static_assert(sizeof(NalSizeType) == 1 || sizeof(NalSizeType) == 2 || | |
79 sizeof(NalSizeType) == 4, | |
80 "NAL size type has unsupported size"); | |
81 static const char startcode_3[3] = {0, 0, 1}; | |
82 DCHECK(avcc_buffer); | |
83 DCHECK(annexb_buffer); | |
84 size_t bytes_left = avcc_size; | |
85 while (bytes_left > 0) { | |
86 DCHECK_GT(bytes_left, sizeof(NalSizeType)); | |
87 NalSizeType nal_size; | |
88 base::ReadBigEndian(avcc_buffer, &nal_size); | |
89 bytes_left -= sizeof(NalSizeType); | |
90 avcc_buffer += sizeof(NalSizeType); | |
91 | |
92 DCHECK_GE(bytes_left, nal_size); | |
93 annexb_buffer->append(startcode_3, sizeof(startcode_3)); | |
94 annexb_buffer->append(avcc_buffer, nal_size); | |
95 bytes_left -= nal_size; | |
96 avcc_buffer += nal_size; | |
97 } | |
98 } | |
99 | |
100 // Copy a H.264 frame stored in a CM sample buffer to an Annex B buffer. Copies | |
101 // parameter sets for keyframes before the frame data as well. | |
102 void CopySampleBufferToAnnexBBuffer(CoreMediaGlue::CMSampleBufferRef sbuf, | |
103 std::string* annexb_buffer, | |
104 bool keyframe) { | |
105 // Perform two pass, one to figure out the total output size, and another to | |
106 // copy the data after having performed a single output allocation. Note that | |
107 // we'll allocate a bit more because we'll count 4 bytes instead of 3 for | |
108 // video NALs. | |
109 | |
110 OSStatus status; | |
111 | |
112 // Get the sample buffer's block buffer and format description. | |
113 auto bb = CoreMediaGlue::CMSampleBufferGetDataBuffer(sbuf); | |
114 DCHECK(bb); | |
115 auto fdesc = CoreMediaGlue::CMSampleBufferGetFormatDescription(sbuf); | |
116 DCHECK(fdesc); | |
117 | |
118 size_t bb_size = CoreMediaGlue::CMBlockBufferGetDataLength(bb); | |
119 size_t total_bytes = bb_size; | |
120 | |
121 size_t pset_count; | |
122 int nal_size_field_bytes; | |
123 status = CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex( | |
124 fdesc, 0, nullptr, nullptr, &pset_count, &nal_size_field_bytes); | |
125 if (status == | |
126 CoreMediaGlue::kCMFormatDescriptionBridgeError_InvalidParameter) { | |
127 DLOG(WARNING) << " assuming 2 parameter sets and 4 bytes NAL length header"; | |
128 pset_count = 2; | |
129 nal_size_field_bytes = 4; | |
130 } else if (status != noErr) { | |
131 DLOG(ERROR) | |
132 << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: " | |
133 << status; | |
134 return; | |
135 } | |
136 | |
137 if (keyframe) { | |
138 const uint8_t* pset; | |
139 size_t pset_size; | |
140 for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) { | |
141 status = | |
142 CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex( | |
143 fdesc, pset_i, &pset, &pset_size, nullptr, nullptr); | |
144 if (status != noErr) { | |
145 DLOG(ERROR) | |
146 << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: " | |
147 << status; | |
148 return; | |
149 } | |
150 total_bytes += pset_size + nal_size_field_bytes; | |
151 } | |
152 } | |
153 | |
154 annexb_buffer->reserve(total_bytes); | |
155 | |
156 // Copy all parameter sets before keyframes. | |
157 if (keyframe) { | |
158 const uint8_t* pset; | |
159 size_t pset_size; | |
160 for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) { | |
161 status = | |
162 CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex( | |
163 fdesc, pset_i, &pset, &pset_size, nullptr, nullptr); | |
164 if (status != noErr) { | |
165 DLOG(ERROR) | |
166 << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: " | |
167 << status; | |
168 return; | |
169 } | |
170 static const char startcode_4[4] = {0, 0, 0, 1}; | |
171 annexb_buffer->append(startcode_4, sizeof(startcode_4)); | |
172 annexb_buffer->append(reinterpret_cast<const char*>(pset), pset_size); | |
173 } | |
174 } | |
175 | |
176 // Block buffers can be composed of non-contiguous chunks. For the sake of | |
177 // keeping this code simple, flatten non-contiguous block buffers. | |
178 base::ScopedCFTypeRef<CoreMediaGlue::CMBlockBufferRef> contiguous_bb( | |
179 bb, base::scoped_policy::RETAIN); | |
180 if (!CoreMediaGlue::CMBlockBufferIsRangeContiguous(bb, 0, 0)) { | |
181 contiguous_bb.reset(); | |
182 status = CoreMediaGlue::CMBlockBufferCreateContiguous( | |
183 kCFAllocatorDefault, bb, kCFAllocatorDefault, nullptr, 0, 0, 0, | |
184 contiguous_bb.InitializeInto()); | |
185 if (status != noErr) { | |
186 DLOG(ERROR) << " CMBlockBufferCreateContiguous failed: " << status; | |
187 return; | |
188 } | |
189 } | |
190 | |
191 // Copy all the NAL units. In the process convert them from AVCC format | |
192 // (length header) to AnnexB format (start code). | |
193 char* bb_data; | |
194 status = CoreMediaGlue::CMBlockBufferGetDataPointer(contiguous_bb, 0, nullptr, | |
195 nullptr, &bb_data); | |
196 if (status != noErr) { | |
197 DLOG(ERROR) << " CMBlockBufferGetDataPointer failed: " << status; | |
198 return; | |
199 } | |
200 | |
201 if (nal_size_field_bytes == 1) { | |
202 CopyNalsToAnnexB<uint8_t>(bb_data, bb_size, annexb_buffer); | |
203 } else if (nal_size_field_bytes == 2) { | |
204 CopyNalsToAnnexB<uint16_t>(bb_data, bb_size, annexb_buffer); | |
205 } else if (nal_size_field_bytes == 4) { | |
206 CopyNalsToAnnexB<uint32_t>(bb_data, bb_size, annexb_buffer); | |
207 } else { | |
208 NOTREACHED(); | |
209 } | |
210 } | |
211 | |
212 } // namespace | 47 } // namespace |
213 | 48 |
214 class H264VideoToolboxEncoder::VideoFrameFactoryImpl | 49 class H264VideoToolboxEncoder::VideoFrameFactoryImpl |
215 : public base::RefCountedThreadSafe<VideoFrameFactoryImpl>, | 50 : public base::RefCountedThreadSafe<VideoFrameFactoryImpl>, |
216 public VideoFrameFactory { | 51 public VideoFrameFactory { |
217 public: | 52 public: |
218 // Type that proxies the VideoFrameFactory interface to this class. | 53 // Type that proxies the VideoFrameFactory interface to this class. |
219 class Proxy; | 54 class Proxy; |
220 | 55 |
221 VideoFrameFactoryImpl(const base::WeakPtr<H264VideoToolboxEncoder>& encoder, | 56 VideoFrameFactoryImpl(const base::WeakPtr<H264VideoToolboxEncoder>& encoder, |
(...skipping 236 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
458 video_frame_factory_->Update(pool, frame_size_); | 293 video_frame_factory_->Update(pool, frame_size_); |
459 | 294 |
460 // Notify that reinitialization is done. | 295 // Notify that reinitialization is done. |
461 cast_environment_->PostTask( | 296 cast_environment_->PostTask( |
462 CastEnvironment::MAIN, FROM_HERE, | 297 CastEnvironment::MAIN, FROM_HERE, |
463 base::Bind(status_change_cb_, STATUS_INITIALIZED)); | 298 base::Bind(status_change_cb_, STATUS_INITIALIZED)); |
464 } | 299 } |
465 | 300 |
466 void H264VideoToolboxEncoder::ConfigureCompressionSession() { | 301 void H264VideoToolboxEncoder::ConfigureCompressionSession() { |
467 SetSessionProperty( | 302 SetSessionProperty( |
| 303 compression_session_, videotoolbox_glue_, |
468 videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(), | 304 videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(), |
469 videotoolbox_glue_->kVTProfileLevel_H264_Main_AutoLevel()); | 305 videotoolbox_glue_->kVTProfileLevel_H264_Main_AutoLevel()); |
470 SetSessionProperty(videotoolbox_glue_->kVTCompressionPropertyKey_RealTime(), | 306 SetSessionProperty(compression_session_, videotoolbox_glue_, |
| 307 videotoolbox_glue_->kVTCompressionPropertyKey_RealTime(), |
471 true); | 308 true); |
472 SetSessionProperty( | 309 SetSessionProperty( |
| 310 compression_session_, videotoolbox_glue_, |
473 videotoolbox_glue_->kVTCompressionPropertyKey_AllowFrameReordering(), | 311 videotoolbox_glue_->kVTCompressionPropertyKey_AllowFrameReordering(), |
474 false); | 312 false); |
475 SetSessionProperty( | 313 SetSessionProperty( |
| 314 compression_session_, videotoolbox_glue_, |
476 videotoolbox_glue_->kVTCompressionPropertyKey_MaxKeyFrameInterval(), 240); | 315 videotoolbox_glue_->kVTCompressionPropertyKey_MaxKeyFrameInterval(), 240); |
477 SetSessionProperty( | 316 SetSessionProperty( |
| 317 compression_session_, videotoolbox_glue_, |
478 videotoolbox_glue_ | 318 videotoolbox_glue_ |
479 ->kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration(), | 319 ->kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration(), |
480 240); | 320 240); |
481 // TODO(jfroy): implement better bitrate control | 321 // TODO(jfroy): implement better bitrate control |
482 // https://crbug.com/425352 | 322 // https://crbug.com/425352 |
483 SetSessionProperty( | 323 SetSessionProperty( |
| 324 compression_session_, videotoolbox_glue_, |
484 videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(), | 325 videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(), |
485 (video_config_.min_bitrate + video_config_.max_bitrate) / 2); | 326 (video_config_.min_bitrate + video_config_.max_bitrate) / 2); |
486 SetSessionProperty( | 327 SetSessionProperty( |
| 328 compression_session_, videotoolbox_glue_, |
487 videotoolbox_glue_->kVTCompressionPropertyKey_ExpectedFrameRate(), | 329 videotoolbox_glue_->kVTCompressionPropertyKey_ExpectedFrameRate(), |
488 video_config_.max_frame_rate); | 330 video_config_.max_frame_rate); |
489 // Keep these attachment settings in-sync with those in Initialize(). | 331 // Keep these attachment settings in-sync with those in Initialize(). |
490 SetSessionProperty( | 332 SetSessionProperty( |
| 333 compression_session_, videotoolbox_glue_, |
491 videotoolbox_glue_->kVTCompressionPropertyKey_ColorPrimaries(), | 334 videotoolbox_glue_->kVTCompressionPropertyKey_ColorPrimaries(), |
492 kCVImageBufferColorPrimaries_ITU_R_709_2); | 335 kCVImageBufferColorPrimaries_ITU_R_709_2); |
493 SetSessionProperty( | 336 SetSessionProperty( |
| 337 compression_session_, videotoolbox_glue_, |
494 videotoolbox_glue_->kVTCompressionPropertyKey_TransferFunction(), | 338 videotoolbox_glue_->kVTCompressionPropertyKey_TransferFunction(), |
495 kCVImageBufferTransferFunction_ITU_R_709_2); | 339 kCVImageBufferTransferFunction_ITU_R_709_2); |
496 SetSessionProperty( | 340 SetSessionProperty( |
| 341 compression_session_, videotoolbox_glue_, |
497 videotoolbox_glue_->kVTCompressionPropertyKey_YCbCrMatrix(), | 342 videotoolbox_glue_->kVTCompressionPropertyKey_YCbCrMatrix(), |
498 kCVImageBufferYCbCrMatrix_ITU_R_709_2); | 343 kCVImageBufferYCbCrMatrix_ITU_R_709_2); |
499 if (video_config_.max_number_of_video_buffers_used > 0) { | 344 if (video_config_.max_number_of_video_buffers_used > 0) { |
500 SetSessionProperty( | 345 SetSessionProperty( |
| 346 compression_session_, videotoolbox_glue_, |
501 videotoolbox_glue_->kVTCompressionPropertyKey_MaxFrameDelayCount(), | 347 videotoolbox_glue_->kVTCompressionPropertyKey_MaxFrameDelayCount(), |
502 video_config_.max_number_of_video_buffers_used); | 348 video_config_.max_number_of_video_buffers_used); |
503 } | 349 } |
504 } | 350 } |
505 | 351 |
506 void H264VideoToolboxEncoder::DestroyCompressionSession() { | 352 void H264VideoToolboxEncoder::DestroyCompressionSession() { |
507 DCHECK(thread_checker_.CalledOnValidThread()); | 353 DCHECK(thread_checker_.CalledOnValidThread()); |
508 | 354 |
509 // If the compression session exists, invalidate it. This blocks until all | 355 // If the compression session exists, invalidate it. This blocks until all |
510 // pending output callbacks have returned and any internal threads have | 356 // pending output callbacks have returned and any internal threads have |
(...skipping 155 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
666 // Reset the compression session only if the frame size is not zero (which | 512 // Reset the compression session only if the frame size is not zero (which |
667 // will obviously fail). It is possible for the frame size to be zero if no | 513 // will obviously fail). It is possible for the frame size to be zero if no |
668 // frame was submitted for encoding or requested from the video frame factory | 514 // frame was submitted for encoding or requested from the video frame factory |
669 // before suspension. | 515 // before suspension. |
670 if (!frame_size_.IsEmpty()) { | 516 if (!frame_size_.IsEmpty()) { |
671 VLOG(1) << "OnResume: Resetting compression session."; | 517 VLOG(1) << "OnResume: Resetting compression session."; |
672 ResetCompressionSession(); | 518 ResetCompressionSession(); |
673 } | 519 } |
674 } | 520 } |
675 | 521 |
676 bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key, | |
677 int32_t value) { | |
678 base::ScopedCFTypeRef<CFNumberRef> cfvalue( | |
679 CFNumberCreate(nullptr, kCFNumberSInt32Type, &value)); | |
680 return videotoolbox_glue_->VTSessionSetProperty(compression_session_, key, | |
681 cfvalue) == noErr; | |
682 } | |
683 | |
684 bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key, bool value) { | |
685 CFBooleanRef cfvalue = (value) ? kCFBooleanTrue : kCFBooleanFalse; | |
686 return videotoolbox_glue_->VTSessionSetProperty(compression_session_, key, | |
687 cfvalue) == noErr; | |
688 } | |
689 | |
690 bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key, | |
691 CFStringRef value) { | |
692 return videotoolbox_glue_->VTSessionSetProperty(compression_session_, key, | |
693 value) == noErr; | |
694 } | |
695 | |
696 void H264VideoToolboxEncoder::CompressionCallback(void* encoder_opaque, | 522 void H264VideoToolboxEncoder::CompressionCallback(void* encoder_opaque, |
697 void* request_opaque, | 523 void* request_opaque, |
698 OSStatus status, | 524 OSStatus status, |
699 VTEncodeInfoFlags info, | 525 VTEncodeInfoFlags info, |
700 CMSampleBufferRef sbuf) { | 526 CMSampleBufferRef sbuf) { |
701 auto encoder = reinterpret_cast<H264VideoToolboxEncoder*>(encoder_opaque); | 527 auto encoder = reinterpret_cast<H264VideoToolboxEncoder*>(encoder_opaque); |
702 const scoped_ptr<InProgressFrameEncode> request( | 528 const scoped_ptr<InProgressFrameEncode> request( |
703 reinterpret_cast<InProgressFrameEncode*>(request_opaque)); | 529 reinterpret_cast<InProgressFrameEncode*>(request_opaque)); |
704 bool keyframe = false; | 530 bool keyframe = false; |
705 bool has_frame_data = false; | 531 bool has_frame_data = false; |
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
760 encoded_frame->encode_completion_time = | 586 encoded_frame->encode_completion_time = |
761 encoder->cast_environment_->Clock()->NowTicks(); | 587 encoder->cast_environment_->Clock()->NowTicks(); |
762 encoder->cast_environment_->PostTask( | 588 encoder->cast_environment_->PostTask( |
763 CastEnvironment::MAIN, FROM_HERE, | 589 CastEnvironment::MAIN, FROM_HERE, |
764 base::Bind(request->frame_encoded_callback, | 590 base::Bind(request->frame_encoded_callback, |
765 base::Passed(&encoded_frame))); | 591 base::Passed(&encoded_frame))); |
766 } | 592 } |
767 | 593 |
768 } // namespace cast | 594 } // namespace cast |
769 } // namespace media | 595 } // namespace media |
OLD | NEW |