Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(49)

Side by Side Diff: media/cast/sender/h264_vt_encoder.cc

Issue 1636083003: H264 HW encode using VideoToolbox (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: miu@ comments. Created 4 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright 2014 The Chromium Authors. All rights reserved. 1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "media/cast/sender/h264_vt_encoder.h" 5 #include "media/cast/sender/h264_vt_encoder.h"
6 6
7 #include <stddef.h> 7 #include <stddef.h>
8 8
9 #include <string> 9 #include <string>
10 #include <vector> 10 #include <vector>
11 11
12 #include "base/big_endian.h" 12 #include "base/big_endian.h"
13 #include "base/bind.h" 13 #include "base/bind.h"
14 #include "base/bind_helpers.h" 14 #include "base/bind_helpers.h"
15 #include "base/location.h" 15 #include "base/location.h"
16 #include "base/logging.h" 16 #include "base/logging.h"
17 #include "base/macros.h" 17 #include "base/macros.h"
18 #include "base/power_monitor/power_monitor.h" 18 #include "base/power_monitor/power_monitor.h"
19 #include "base/synchronization/lock.h" 19 #include "base/synchronization/lock.h"
20 #include "build/build_config.h" 20 #include "build/build_config.h"
21 #include "media/base/mac/corevideo_glue.h" 21 #include "media/base/mac/corevideo_glue.h"
22 #include "media/base/mac/video_frame_mac.h" 22 #include "media/base/mac/video_frame_mac.h"
23 #include "media/base/mac/videotoolbox_helpers.h"
23 #include "media/cast/common/rtp_time.h" 24 #include "media/cast/common/rtp_time.h"
24 #include "media/cast/constants.h" 25 #include "media/cast/constants.h"
25 #include "media/cast/sender/video_frame_factory.h" 26 #include "media/cast/sender/video_frame_factory.h"
26 27
27 namespace media { 28 namespace media {
28 namespace cast { 29 namespace cast {
29 30
30 namespace { 31 namespace {
31 32
32 // Container for the associated data of a video frame being processed. 33 // Container for the associated data of a video frame being processed.
33 struct InProgressFrameEncode { 34 struct InProgressFrameEncode {
34 const RtpTimeTicks rtp_timestamp; 35 const RtpTimeTicks rtp_timestamp;
35 const base::TimeTicks reference_time; 36 const base::TimeTicks reference_time;
36 const VideoEncoder::FrameEncodedCallback frame_encoded_callback; 37 const VideoEncoder::FrameEncodedCallback frame_encoded_callback;
37 38
38 InProgressFrameEncode(RtpTimeTicks rtp, 39 InProgressFrameEncode(RtpTimeTicks rtp,
39 base::TimeTicks r_time, 40 base::TimeTicks r_time,
40 VideoEncoder::FrameEncodedCallback callback) 41 VideoEncoder::FrameEncodedCallback callback)
41 : rtp_timestamp(rtp), 42 : rtp_timestamp(rtp),
42 reference_time(r_time), 43 reference_time(r_time),
43 frame_encoded_callback(callback) {} 44 frame_encoded_callback(callback) {}
44 }; 45 };
45 46
46 base::ScopedCFTypeRef<CFDictionaryRef>
47 DictionaryWithKeysAndValues(CFTypeRef* keys, CFTypeRef* values, size_t size) {
48 return base::ScopedCFTypeRef<CFDictionaryRef>(CFDictionaryCreate(
49 kCFAllocatorDefault, keys, values, size, &kCFTypeDictionaryKeyCallBacks,
50 &kCFTypeDictionaryValueCallBacks));
51 }
52
53 base::ScopedCFTypeRef<CFDictionaryRef> DictionaryWithKeyValue(CFTypeRef key,
54 CFTypeRef value) {
55 CFTypeRef keys[1] = {key};
56 CFTypeRef values[1] = {value};
57 return DictionaryWithKeysAndValues(keys, values, 1);
58 }
59
60 base::ScopedCFTypeRef<CFArrayRef> ArrayWithIntegers(const int* v, size_t size) {
61 std::vector<CFNumberRef> numbers;
62 numbers.reserve(size);
63 for (const int* end = v + size; v < end; ++v)
64 numbers.push_back(CFNumberCreate(nullptr, kCFNumberSInt32Type, v));
65 base::ScopedCFTypeRef<CFArrayRef> array(CFArrayCreate(
66 kCFAllocatorDefault, reinterpret_cast<const void**>(&numbers[0]),
67 numbers.size(), &kCFTypeArrayCallBacks));
68 for (auto& number : numbers) {
69 CFRelease(number);
70 }
71 return array;
72 }
73
74 template <typename NalSizeType>
75 void CopyNalsToAnnexB(char* avcc_buffer,
76 const size_t avcc_size,
77 std::string* annexb_buffer) {
78 static_assert(sizeof(NalSizeType) == 1 || sizeof(NalSizeType) == 2 ||
79 sizeof(NalSizeType) == 4,
80 "NAL size type has unsupported size");
81 static const char startcode_3[3] = {0, 0, 1};
82 DCHECK(avcc_buffer);
83 DCHECK(annexb_buffer);
84 size_t bytes_left = avcc_size;
85 while (bytes_left > 0) {
86 DCHECK_GT(bytes_left, sizeof(NalSizeType));
87 NalSizeType nal_size;
88 base::ReadBigEndian(avcc_buffer, &nal_size);
89 bytes_left -= sizeof(NalSizeType);
90 avcc_buffer += sizeof(NalSizeType);
91
92 DCHECK_GE(bytes_left, nal_size);
93 annexb_buffer->append(startcode_3, sizeof(startcode_3));
94 annexb_buffer->append(avcc_buffer, nal_size);
95 bytes_left -= nal_size;
96 avcc_buffer += nal_size;
97 }
98 }
99
100 // Copy a H.264 frame stored in a CM sample buffer to an Annex B buffer. Copies
101 // parameter sets for keyframes before the frame data as well.
102 void CopySampleBufferToAnnexBBuffer(CoreMediaGlue::CMSampleBufferRef sbuf,
103 std::string* annexb_buffer,
104 bool keyframe) {
105 // Perform two pass, one to figure out the total output size, and another to
106 // copy the data after having performed a single output allocation. Note that
107 // we'll allocate a bit more because we'll count 4 bytes instead of 3 for
108 // video NALs.
109
110 OSStatus status;
111
112 // Get the sample buffer's block buffer and format description.
113 auto bb = CoreMediaGlue::CMSampleBufferGetDataBuffer(sbuf);
114 DCHECK(bb);
115 auto fdesc = CoreMediaGlue::CMSampleBufferGetFormatDescription(sbuf);
116 DCHECK(fdesc);
117
118 size_t bb_size = CoreMediaGlue::CMBlockBufferGetDataLength(bb);
119 size_t total_bytes = bb_size;
120
121 size_t pset_count;
122 int nal_size_field_bytes;
123 status = CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
124 fdesc, 0, nullptr, nullptr, &pset_count, &nal_size_field_bytes);
125 if (status ==
126 CoreMediaGlue::kCMFormatDescriptionBridgeError_InvalidParameter) {
127 DLOG(WARNING) << " assuming 2 parameter sets and 4 bytes NAL length header";
128 pset_count = 2;
129 nal_size_field_bytes = 4;
130 } else if (status != noErr) {
131 DLOG(ERROR)
132 << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: "
133 << status;
134 return;
135 }
136
137 if (keyframe) {
138 const uint8_t* pset;
139 size_t pset_size;
140 for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) {
141 status =
142 CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
143 fdesc, pset_i, &pset, &pset_size, nullptr, nullptr);
144 if (status != noErr) {
145 DLOG(ERROR)
146 << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: "
147 << status;
148 return;
149 }
150 total_bytes += pset_size + nal_size_field_bytes;
151 }
152 }
153
154 annexb_buffer->reserve(total_bytes);
155
156 // Copy all parameter sets before keyframes.
157 if (keyframe) {
158 const uint8_t* pset;
159 size_t pset_size;
160 for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) {
161 status =
162 CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
163 fdesc, pset_i, &pset, &pset_size, nullptr, nullptr);
164 if (status != noErr) {
165 DLOG(ERROR)
166 << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: "
167 << status;
168 return;
169 }
170 static const char startcode_4[4] = {0, 0, 0, 1};
171 annexb_buffer->append(startcode_4, sizeof(startcode_4));
172 annexb_buffer->append(reinterpret_cast<const char*>(pset), pset_size);
173 }
174 }
175
176 // Block buffers can be composed of non-contiguous chunks. For the sake of
177 // keeping this code simple, flatten non-contiguous block buffers.
178 base::ScopedCFTypeRef<CoreMediaGlue::CMBlockBufferRef> contiguous_bb(
179 bb, base::scoped_policy::RETAIN);
180 if (!CoreMediaGlue::CMBlockBufferIsRangeContiguous(bb, 0, 0)) {
181 contiguous_bb.reset();
182 status = CoreMediaGlue::CMBlockBufferCreateContiguous(
183 kCFAllocatorDefault, bb, kCFAllocatorDefault, nullptr, 0, 0, 0,
184 contiguous_bb.InitializeInto());
185 if (status != noErr) {
186 DLOG(ERROR) << " CMBlockBufferCreateContiguous failed: " << status;
187 return;
188 }
189 }
190
191 // Copy all the NAL units. In the process convert them from AVCC format
192 // (length header) to AnnexB format (start code).
193 char* bb_data;
194 status = CoreMediaGlue::CMBlockBufferGetDataPointer(contiguous_bb, 0, nullptr,
195 nullptr, &bb_data);
196 if (status != noErr) {
197 DLOG(ERROR) << " CMBlockBufferGetDataPointer failed: " << status;
198 return;
199 }
200
201 if (nal_size_field_bytes == 1) {
202 CopyNalsToAnnexB<uint8_t>(bb_data, bb_size, annexb_buffer);
203 } else if (nal_size_field_bytes == 2) {
204 CopyNalsToAnnexB<uint16_t>(bb_data, bb_size, annexb_buffer);
205 } else if (nal_size_field_bytes == 4) {
206 CopyNalsToAnnexB<uint32_t>(bb_data, bb_size, annexb_buffer);
207 } else {
208 NOTREACHED();
209 }
210 }
211
212 } // namespace 47 } // namespace
213 48
214 class H264VideoToolboxEncoder::VideoFrameFactoryImpl 49 class H264VideoToolboxEncoder::VideoFrameFactoryImpl
215 : public base::RefCountedThreadSafe<VideoFrameFactoryImpl>, 50 : public base::RefCountedThreadSafe<VideoFrameFactoryImpl>,
216 public VideoFrameFactory { 51 public VideoFrameFactory {
217 public: 52 public:
218 // Type that proxies the VideoFrameFactory interface to this class. 53 // Type that proxies the VideoFrameFactory interface to this class.
219 class Proxy; 54 class Proxy;
220 55
221 VideoFrameFactoryImpl(const base::WeakPtr<H264VideoToolboxEncoder>& encoder, 56 VideoFrameFactoryImpl(const base::WeakPtr<H264VideoToolboxEncoder>& encoder,
(...skipping 236 matching lines...) Expand 10 before | Expand all | Expand 10 after
458 video_frame_factory_->Update(pool, frame_size_); 293 video_frame_factory_->Update(pool, frame_size_);
459 294
460 // Notify that reinitialization is done. 295 // Notify that reinitialization is done.
461 cast_environment_->PostTask( 296 cast_environment_->PostTask(
462 CastEnvironment::MAIN, FROM_HERE, 297 CastEnvironment::MAIN, FROM_HERE,
463 base::Bind(status_change_cb_, STATUS_INITIALIZED)); 298 base::Bind(status_change_cb_, STATUS_INITIALIZED));
464 } 299 }
465 300
466 void H264VideoToolboxEncoder::ConfigureCompressionSession() { 301 void H264VideoToolboxEncoder::ConfigureCompressionSession() {
467 SetSessionProperty( 302 SetSessionProperty(
303 compression_session_, videotoolbox_glue_,
468 videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(), 304 videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(),
469 videotoolbox_glue_->kVTProfileLevel_H264_Main_AutoLevel()); 305 videotoolbox_glue_->kVTProfileLevel_H264_Main_AutoLevel());
470 SetSessionProperty(videotoolbox_glue_->kVTCompressionPropertyKey_RealTime(), 306 SetSessionProperty(compression_session_, videotoolbox_glue_,
307 videotoolbox_glue_->kVTCompressionPropertyKey_RealTime(),
471 true); 308 true);
472 SetSessionProperty( 309 SetSessionProperty(
310 compression_session_, videotoolbox_glue_,
473 videotoolbox_glue_->kVTCompressionPropertyKey_AllowFrameReordering(), 311 videotoolbox_glue_->kVTCompressionPropertyKey_AllowFrameReordering(),
474 false); 312 false);
475 SetSessionProperty( 313 SetSessionProperty(
314 compression_session_, videotoolbox_glue_,
476 videotoolbox_glue_->kVTCompressionPropertyKey_MaxKeyFrameInterval(), 240); 315 videotoolbox_glue_->kVTCompressionPropertyKey_MaxKeyFrameInterval(), 240);
477 SetSessionProperty( 316 SetSessionProperty(
317 compression_session_, videotoolbox_glue_,
478 videotoolbox_glue_ 318 videotoolbox_glue_
479 ->kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration(), 319 ->kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration(),
480 240); 320 240);
481 // TODO(jfroy): implement better bitrate control 321 // TODO(jfroy): implement better bitrate control
482 // https://crbug.com/425352 322 // https://crbug.com/425352
483 SetSessionProperty( 323 SetSessionProperty(
324 compression_session_, videotoolbox_glue_,
484 videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(), 325 videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(),
485 (video_config_.min_bitrate + video_config_.max_bitrate) / 2); 326 (video_config_.min_bitrate + video_config_.max_bitrate) / 2);
486 SetSessionProperty( 327 SetSessionProperty(
328 compression_session_, videotoolbox_glue_,
487 videotoolbox_glue_->kVTCompressionPropertyKey_ExpectedFrameRate(), 329 videotoolbox_glue_->kVTCompressionPropertyKey_ExpectedFrameRate(),
488 video_config_.max_frame_rate); 330 video_config_.max_frame_rate);
489 // Keep these attachment settings in-sync with those in Initialize(). 331 // Keep these attachment settings in-sync with those in Initialize().
490 SetSessionProperty( 332 SetSessionProperty(
333 compression_session_, videotoolbox_glue_,
491 videotoolbox_glue_->kVTCompressionPropertyKey_ColorPrimaries(), 334 videotoolbox_glue_->kVTCompressionPropertyKey_ColorPrimaries(),
492 kCVImageBufferColorPrimaries_ITU_R_709_2); 335 kCVImageBufferColorPrimaries_ITU_R_709_2);
493 SetSessionProperty( 336 SetSessionProperty(
337 compression_session_, videotoolbox_glue_,
494 videotoolbox_glue_->kVTCompressionPropertyKey_TransferFunction(), 338 videotoolbox_glue_->kVTCompressionPropertyKey_TransferFunction(),
495 kCVImageBufferTransferFunction_ITU_R_709_2); 339 kCVImageBufferTransferFunction_ITU_R_709_2);
496 SetSessionProperty( 340 SetSessionProperty(
341 compression_session_, videotoolbox_glue_,
497 videotoolbox_glue_->kVTCompressionPropertyKey_YCbCrMatrix(), 342 videotoolbox_glue_->kVTCompressionPropertyKey_YCbCrMatrix(),
498 kCVImageBufferYCbCrMatrix_ITU_R_709_2); 343 kCVImageBufferYCbCrMatrix_ITU_R_709_2);
499 if (video_config_.max_number_of_video_buffers_used > 0) { 344 if (video_config_.max_number_of_video_buffers_used > 0) {
500 SetSessionProperty( 345 SetSessionProperty(
346 compression_session_, videotoolbox_glue_,
501 videotoolbox_glue_->kVTCompressionPropertyKey_MaxFrameDelayCount(), 347 videotoolbox_glue_->kVTCompressionPropertyKey_MaxFrameDelayCount(),
502 video_config_.max_number_of_video_buffers_used); 348 video_config_.max_number_of_video_buffers_used);
503 } 349 }
504 } 350 }
505 351
506 void H264VideoToolboxEncoder::DestroyCompressionSession() { 352 void H264VideoToolboxEncoder::DestroyCompressionSession() {
507 DCHECK(thread_checker_.CalledOnValidThread()); 353 DCHECK(thread_checker_.CalledOnValidThread());
508 354
509 // If the compression session exists, invalidate it. This blocks until all 355 // If the compression session exists, invalidate it. This blocks until all
510 // pending output callbacks have returned and any internal threads have 356 // pending output callbacks have returned and any internal threads have
(...skipping 155 matching lines...) Expand 10 before | Expand all | Expand 10 after
666 // Reset the compression session only if the frame size is not zero (which 512 // Reset the compression session only if the frame size is not zero (which
667 // will obviously fail). It is possible for the frame size to be zero if no 513 // will obviously fail). It is possible for the frame size to be zero if no
668 // frame was submitted for encoding or requested from the video frame factory 514 // frame was submitted for encoding or requested from the video frame factory
669 // before suspension. 515 // before suspension.
670 if (!frame_size_.IsEmpty()) { 516 if (!frame_size_.IsEmpty()) {
671 VLOG(1) << "OnResume: Resetting compression session."; 517 VLOG(1) << "OnResume: Resetting compression session.";
672 ResetCompressionSession(); 518 ResetCompressionSession();
673 } 519 }
674 } 520 }
675 521
676 bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key,
677 int32_t value) {
678 base::ScopedCFTypeRef<CFNumberRef> cfvalue(
679 CFNumberCreate(nullptr, kCFNumberSInt32Type, &value));
680 return videotoolbox_glue_->VTSessionSetProperty(compression_session_, key,
681 cfvalue) == noErr;
682 }
683
684 bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key, bool value) {
685 CFBooleanRef cfvalue = (value) ? kCFBooleanTrue : kCFBooleanFalse;
686 return videotoolbox_glue_->VTSessionSetProperty(compression_session_, key,
687 cfvalue) == noErr;
688 }
689
690 bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key,
691 CFStringRef value) {
692 return videotoolbox_glue_->VTSessionSetProperty(compression_session_, key,
693 value) == noErr;
694 }
695
696 void H264VideoToolboxEncoder::CompressionCallback(void* encoder_opaque, 522 void H264VideoToolboxEncoder::CompressionCallback(void* encoder_opaque,
697 void* request_opaque, 523 void* request_opaque,
698 OSStatus status, 524 OSStatus status,
699 VTEncodeInfoFlags info, 525 VTEncodeInfoFlags info,
700 CMSampleBufferRef sbuf) { 526 CMSampleBufferRef sbuf) {
701 auto encoder = reinterpret_cast<H264VideoToolboxEncoder*>(encoder_opaque); 527 auto encoder = reinterpret_cast<H264VideoToolboxEncoder*>(encoder_opaque);
702 const scoped_ptr<InProgressFrameEncode> request( 528 const scoped_ptr<InProgressFrameEncode> request(
703 reinterpret_cast<InProgressFrameEncode*>(request_opaque)); 529 reinterpret_cast<InProgressFrameEncode*>(request_opaque));
704 bool keyframe = false; 530 bool keyframe = false;
705 bool has_frame_data = false; 531 bool has_frame_data = false;
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after
760 encoded_frame->encode_completion_time = 586 encoded_frame->encode_completion_time =
761 encoder->cast_environment_->Clock()->NowTicks(); 587 encoder->cast_environment_->Clock()->NowTicks();
762 encoder->cast_environment_->PostTask( 588 encoder->cast_environment_->PostTask(
763 CastEnvironment::MAIN, FROM_HERE, 589 CastEnvironment::MAIN, FROM_HERE,
764 base::Bind(request->frame_encoded_callback, 590 base::Bind(request->frame_encoded_callback,
765 base::Passed(&encoded_frame))); 591 base::Passed(&encoded_frame)));
766 } 592 }
767 593
768 } // namespace cast 594 } // namespace cast
769 } // namespace media 595 } // namespace media
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698