Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(284)

Side by Side Diff: media/base/mac/videotoolbox_helpers.cc

Issue 1636083003: H264 HW encode using VideoToolbox (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: jfroy@ comments. Created 4 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright 2014 The Chromium Authors. All rights reserved. 1 // Copyright 2016 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "media/cast/sender/h264_vt_encoder.h" 5 #include "media/base/mac/videotoolbox_helpers.h"
6 6
7 #include <stddef.h>
8
9 #include <string>
10 #include <vector> 7 #include <vector>
11 8
12 #include "base/big_endian.h" 9 #include "base/big_endian.h"
13 #include "base/bind.h" 10 #include "base/memory/scoped_ptr.h"
14 #include "base/bind_helpers.h"
15 #include "base/location.h"
16 #include "base/logging.h"
17 #include "base/macros.h"
18 #include "base/power_monitor/power_monitor.h"
19 #include "base/synchronization/lock.h"
20 #include "build/build_config.h"
21 #include "media/base/mac/corevideo_glue.h"
22 #include "media/base/mac/video_frame_mac.h"
23 #include "media/cast/common/rtp_time.h"
24 #include "media/cast/constants.h"
25 #include "media/cast/sender/video_frame_factory.h"
26 11
27 namespace media { 12 namespace media {
28 namespace cast {
29 13
30 namespace { 14 namespace video_toolbox {
31
32 // Container for the associated data of a video frame being processed.
33 struct InProgressFrameEncode {
34 const RtpTimeTicks rtp_timestamp;
35 const base::TimeTicks reference_time;
36 const VideoEncoder::FrameEncodedCallback frame_encoded_callback;
37
38 InProgressFrameEncode(RtpTimeTicks rtp,
39 base::TimeTicks r_time,
40 VideoEncoder::FrameEncodedCallback callback)
41 : rtp_timestamp(rtp),
42 reference_time(r_time),
43 frame_encoded_callback(callback) {}
44 };
45 15
46 base::ScopedCFTypeRef<CFDictionaryRef> 16 base::ScopedCFTypeRef<CFDictionaryRef>
47 DictionaryWithKeysAndValues(CFTypeRef* keys, CFTypeRef* values, size_t size) { 17 DictionaryWithKeysAndValues(CFTypeRef* keys, CFTypeRef* values, size_t size) {
48 return base::ScopedCFTypeRef<CFDictionaryRef>(CFDictionaryCreate( 18 return base::ScopedCFTypeRef<CFDictionaryRef>(CFDictionaryCreate(
49 kCFAllocatorDefault, keys, values, size, &kCFTypeDictionaryKeyCallBacks, 19 kCFAllocatorDefault, keys, values, size, &kCFTypeDictionaryKeyCallBacks,
50 &kCFTypeDictionaryValueCallBacks)); 20 &kCFTypeDictionaryValueCallBacks));
51 } 21 }
52 22
53 base::ScopedCFTypeRef<CFDictionaryRef> DictionaryWithKeyValue(CFTypeRef key, 23 base::ScopedCFTypeRef<CFDictionaryRef> DictionaryWithKeyValue(CFTypeRef key,
54 CFTypeRef value) { 24 CFTypeRef value) {
55 CFTypeRef keys[1] = {key}; 25 CFTypeRef keys[1] = {key};
56 CFTypeRef values[1] = {value}; 26 CFTypeRef values[1] = {value};
57 return DictionaryWithKeysAndValues(keys, values, 1); 27 return DictionaryWithKeysAndValues(keys, values, 1);
58 } 28 }
59 29
60 base::ScopedCFTypeRef<CFArrayRef> ArrayWithIntegers(const int* v, size_t size) { 30 base::ScopedCFTypeRef<CFArrayRef> ArrayWithIntegers(const int* v, size_t size) {
61 std::vector<CFNumberRef> numbers; 31 std::vector<CFNumberRef> numbers;
62 numbers.reserve(size); 32 numbers.reserve(size);
63 for (const int* end = v + size; v < end; ++v) 33 for (const int* end = v + size; v < end; ++v)
64 numbers.push_back(CFNumberCreate(nullptr, kCFNumberSInt32Type, v)); 34 numbers.push_back(CFNumberCreate(nullptr, kCFNumberSInt32Type, v));
65 base::ScopedCFTypeRef<CFArrayRef> array(CFArrayCreate( 35 base::ScopedCFTypeRef<CFArrayRef> array(CFArrayCreate(
66 kCFAllocatorDefault, reinterpret_cast<const void**>(&numbers[0]), 36 kCFAllocatorDefault, reinterpret_cast<const void**>(&numbers[0]),
67 numbers.size(), &kCFTypeArrayCallBacks)); 37 numbers.size(), &kCFTypeArrayCallBacks));
68 for (auto& number : numbers) { 38 for (auto& number : numbers) {
69 CFRelease(number); 39 CFRelease(number);
70 } 40 }
71 return array; 41 return array;
72 } 42 }
73 43
44 base::ScopedCFTypeRef<CFArrayRef> ArrayWithIntegerAndFloat(int int_val,
45 float float_val) {
46 std::vector<CFNumberRef> numbers;
miu 2016/03/08 22:36:05 This is a good place to use std::array: std::ar
emircan 2016/03/09 01:13:35 Done.
47 numbers.reserve(2);
48 numbers.push_back(CFNumberCreate(nullptr, kCFNumberSInt32Type, &int_val));
49 numbers.push_back(CFNumberCreate(nullptr, kCFNumberFloat32Type, &float_val));
50 base::ScopedCFTypeRef<CFArrayRef> array(CFArrayCreate(
51 kCFAllocatorDefault, reinterpret_cast<const void**>(&numbers[0]),
52 numbers.size(), &kCFTypeArrayCallBacks));
53 for (auto& number : numbers) {
54 CFRelease(number);
55 }
56 return array;
57 }
58
59 // Wrapper class for writing AnnexBBuffer output into.
60 class AnnexBBuffer {
61 public:
62 virtual bool Reserve(size_t size) = 0;
63 virtual void Append(const char* s, size_t n) = 0;
64 virtual size_t GetReservedSize() const = 0;
65 };
66
67 class RawAnnexBBuffer : public AnnexBBuffer {
68 public:
69 RawAnnexBBuffer(char* annexb_buffer, size_t annexb_buffer_size)
70 : annexb_buffer_(annexb_buffer),
71 annexb_buffer_size_(annexb_buffer_size),
72 annexb_buffer_offset_(0) {}
73 bool Reserve(size_t size) override {
74 reserved_size_ = size;
75 return size <= annexb_buffer_size_;
76 }
77 void Append(const char* s, size_t n) override {
78 memcpy(annexb_buffer_ + annexb_buffer_offset_, s, n);
79 annexb_buffer_offset_ += n;
80 DCHECK_GE(reserved_size_, annexb_buffer_offset_);
81 }
82 size_t GetReservedSize() const override { return reserved_size_; }
83
84 private:
85 char* annexb_buffer_;
86 size_t annexb_buffer_size_;
87 size_t annexb_buffer_offset_;
88 size_t reserved_size_;
89
90 DISALLOW_IMPLICIT_CONSTRUCTORS(RawAnnexBBuffer);
91 };
92
93 class StringAnnexBBuffer : public AnnexBBuffer {
94 public:
95 explicit StringAnnexBBuffer(std::string* str_annexb_buffer)
96 : str_annexb_buffer_(str_annexb_buffer) {}
97 bool Reserve(size_t size) override {
98 str_annexb_buffer_->reserve(size);
99 return true;
100 }
101 void Append(const char* s, size_t n) override {
102 str_annexb_buffer_->append(s, n);
103 }
104 size_t GetReservedSize() const override { return str_annexb_buffer_->size(); }
105
106 private:
107 std::string* str_annexb_buffer_;
108 DISALLOW_IMPLICIT_CONSTRUCTORS(StringAnnexBBuffer);
109 };
110
74 template <typename NalSizeType> 111 template <typename NalSizeType>
75 void CopyNalsToAnnexB(char* avcc_buffer, 112 void CopyNalsToAnnexB(char* avcc_buffer,
76 const size_t avcc_size, 113 const size_t avcc_size,
77 std::string* annexb_buffer) { 114 AnnexBBuffer* annexb_buffer) {
78 static_assert(sizeof(NalSizeType) == 1 || sizeof(NalSizeType) == 2 || 115 static_assert(sizeof(NalSizeType) == 1 || sizeof(NalSizeType) == 2 ||
79 sizeof(NalSizeType) == 4, 116 sizeof(NalSizeType) == 4,
80 "NAL size type has unsupported size"); 117 "NAL size type has unsupported size");
81 static const char startcode_3[3] = {0, 0, 1}; 118 static const char startcode_3[3] = {0, 0, 1};
82 DCHECK(avcc_buffer); 119 DCHECK(avcc_buffer);
83 DCHECK(annexb_buffer); 120 DCHECK(annexb_buffer);
84 size_t bytes_left = avcc_size; 121 size_t bytes_left = avcc_size;
85 while (bytes_left > 0) { 122 while (bytes_left > 0) {
86 DCHECK_GT(bytes_left, sizeof(NalSizeType)); 123 DCHECK_GT(bytes_left, sizeof(NalSizeType));
87 NalSizeType nal_size; 124 NalSizeType nal_size;
88 base::ReadBigEndian(avcc_buffer, &nal_size); 125 base::ReadBigEndian(avcc_buffer, &nal_size);
89 bytes_left -= sizeof(NalSizeType); 126 bytes_left -= sizeof(NalSizeType);
90 avcc_buffer += sizeof(NalSizeType); 127 avcc_buffer += sizeof(NalSizeType);
91 128
92 DCHECK_GE(bytes_left, nal_size); 129 DCHECK_GE(bytes_left, nal_size);
93 annexb_buffer->append(startcode_3, sizeof(startcode_3)); 130 annexb_buffer->Append(startcode_3, sizeof(startcode_3));
94 annexb_buffer->append(avcc_buffer, nal_size); 131 annexb_buffer->Append(avcc_buffer, nal_size);
95 bytes_left -= nal_size; 132 bytes_left -= nal_size;
96 avcc_buffer += nal_size; 133 avcc_buffer += nal_size;
97 } 134 }
98 } 135 }
99 136
100 // Copy a H.264 frame stored in a CM sample buffer to an Annex B buffer. Copies 137 bool CopySampleBufferToAnnexBBuffer(CoreMediaGlue::CMSampleBufferRef sbuf,
101 // parameter sets for keyframes before the frame data as well. 138 AnnexBBuffer* annexb_buffer,
102 void CopySampleBufferToAnnexBBuffer(CoreMediaGlue::CMSampleBufferRef sbuf,
103 std::string* annexb_buffer,
104 bool keyframe) { 139 bool keyframe) {
105 // Perform two pass, one to figure out the total output size, and another to 140 // Perform two pass, one to figure out the total output size, and another to
106 // copy the data after having performed a single output allocation. Note that 141 // copy the data after having performed a single output allocation. Note that
107 // we'll allocate a bit more because we'll count 4 bytes instead of 3 for 142 // we'll allocate a bit more because we'll count 4 bytes instead of 3 for
108 // video NALs. 143 // video NALs.
109
110 OSStatus status; 144 OSStatus status;
111 145
112 // Get the sample buffer's block buffer and format description. 146 // Get the sample buffer's block buffer and format description.
113 auto bb = CoreMediaGlue::CMSampleBufferGetDataBuffer(sbuf); 147 auto bb = CoreMediaGlue::CMSampleBufferGetDataBuffer(sbuf);
114 DCHECK(bb); 148 DCHECK(bb);
115 auto fdesc = CoreMediaGlue::CMSampleBufferGetFormatDescription(sbuf); 149 auto fdesc = CoreMediaGlue::CMSampleBufferGetFormatDescription(sbuf);
116 DCHECK(fdesc); 150 DCHECK(fdesc);
117 151
118 size_t bb_size = CoreMediaGlue::CMBlockBufferGetDataLength(bb); 152 size_t bb_size = CoreMediaGlue::CMBlockBufferGetDataLength(bb);
119 size_t total_bytes = bb_size; 153 size_t total_bytes = bb_size;
120 154
121 size_t pset_count; 155 size_t pset_count;
122 int nal_size_field_bytes; 156 int nal_size_field_bytes;
123 status = CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex( 157 status = CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
124 fdesc, 0, nullptr, nullptr, &pset_count, &nal_size_field_bytes); 158 fdesc, 0, nullptr, nullptr, &pset_count, &nal_size_field_bytes);
125 if (status == 159 if (status ==
126 CoreMediaGlue::kCMFormatDescriptionBridgeError_InvalidParameter) { 160 CoreMediaGlue::kCMFormatDescriptionBridgeError_InvalidParameter) {
127 DLOG(WARNING) << " assuming 2 parameter sets and 4 bytes NAL length header"; 161 DLOG(WARNING) << " assuming 2 parameter sets and 4 bytes NAL length header";
128 pset_count = 2; 162 pset_count = 2;
129 nal_size_field_bytes = 4; 163 nal_size_field_bytes = 4;
130 } else if (status != noErr) { 164 } else if (status != noErr) {
131 DLOG(ERROR) 165 DLOG(ERROR)
132 << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: " 166 << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: "
133 << status; 167 << status;
134 return; 168 return false;
135 } 169 }
136 170
137 if (keyframe) { 171 if (keyframe) {
138 const uint8_t* pset; 172 const uint8_t* pset;
139 size_t pset_size; 173 size_t pset_size;
140 for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) { 174 for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) {
141 status = 175 status =
142 CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex( 176 CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
143 fdesc, pset_i, &pset, &pset_size, nullptr, nullptr); 177 fdesc, pset_i, &pset, &pset_size, nullptr, nullptr);
144 if (status != noErr) { 178 if (status != noErr) {
145 DLOG(ERROR) 179 DLOG(ERROR)
146 << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: " 180 << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: "
147 << status; 181 << status;
148 return; 182 return false;
149 } 183 }
150 total_bytes += pset_size + nal_size_field_bytes; 184 total_bytes += pset_size + nal_size_field_bytes;
151 } 185 }
152 } 186 }
153 187
154 annexb_buffer->reserve(total_bytes); 188 if (!annexb_buffer->Reserve(total_bytes)) {
189 DLOG(ERROR) << "Cannot fit encode output into bitstream buffer. Requested:"
190 << total_bytes;
191 return false;
192 }
155 193
156 // Copy all parameter sets before keyframes. 194 // Copy all parameter sets before keyframes.
157 if (keyframe) { 195 if (keyframe) {
158 const uint8_t* pset; 196 const uint8_t* pset;
159 size_t pset_size; 197 size_t pset_size;
160 for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) { 198 for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) {
161 status = 199 status =
162 CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex( 200 CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
163 fdesc, pset_i, &pset, &pset_size, nullptr, nullptr); 201 fdesc, pset_i, &pset, &pset_size, nullptr, nullptr);
164 if (status != noErr) { 202 if (status != noErr) {
165 DLOG(ERROR) 203 DLOG(ERROR)
166 << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: " 204 << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: "
167 << status; 205 << status;
168 return; 206 return false;
169 } 207 }
170 static const char startcode_4[4] = {0, 0, 0, 1}; 208 static const char startcode_4[4] = {0, 0, 0, 1};
171 annexb_buffer->append(startcode_4, sizeof(startcode_4)); 209 annexb_buffer->Append(startcode_4, sizeof(startcode_4));
172 annexb_buffer->append(reinterpret_cast<const char*>(pset), pset_size); 210 annexb_buffer->Append(reinterpret_cast<const char*>(pset), pset_size);
173 } 211 }
174 } 212 }
175 213
176 // Block buffers can be composed of non-contiguous chunks. For the sake of 214 // Block buffers can be composed of non-contiguous chunks. For the sake of
177 // keeping this code simple, flatten non-contiguous block buffers. 215 // keeping this code simple, flatten non-contiguous block buffers.
178 base::ScopedCFTypeRef<CoreMediaGlue::CMBlockBufferRef> contiguous_bb( 216 base::ScopedCFTypeRef<CoreMediaGlue::CMBlockBufferRef> contiguous_bb(
179 bb, base::scoped_policy::RETAIN); 217 bb, base::scoped_policy::RETAIN);
180 if (!CoreMediaGlue::CMBlockBufferIsRangeContiguous(bb, 0, 0)) { 218 if (!CoreMediaGlue::CMBlockBufferIsRangeContiguous(bb, 0, 0)) {
181 contiguous_bb.reset(); 219 contiguous_bb.reset();
182 status = CoreMediaGlue::CMBlockBufferCreateContiguous( 220 status = CoreMediaGlue::CMBlockBufferCreateContiguous(
183 kCFAllocatorDefault, bb, kCFAllocatorDefault, nullptr, 0, 0, 0, 221 kCFAllocatorDefault, bb, kCFAllocatorDefault, nullptr, 0, 0, 0,
184 contiguous_bb.InitializeInto()); 222 contiguous_bb.InitializeInto());
185 if (status != noErr) { 223 if (status != noErr) {
186 DLOG(ERROR) << " CMBlockBufferCreateContiguous failed: " << status; 224 DLOG(ERROR) << " CMBlockBufferCreateContiguous failed: " << status;
187 return; 225 return false;
188 } 226 }
189 } 227 }
190 228
191 // Copy all the NAL units. In the process convert them from AVCC format 229 // Copy all the NAL units. In the process convert them from AVCC format
192 // (length header) to AnnexB format (start code). 230 // (length header) to AnnexB format (start code).
193 char* bb_data; 231 char* bb_data;
194 status = CoreMediaGlue::CMBlockBufferGetDataPointer(contiguous_bb, 0, nullptr, 232 status = CoreMediaGlue::CMBlockBufferGetDataPointer(contiguous_bb, 0, nullptr,
195 nullptr, &bb_data); 233 nullptr, &bb_data);
196 if (status != noErr) { 234 if (status != noErr) {
197 DLOG(ERROR) << " CMBlockBufferGetDataPointer failed: " << status; 235 DLOG(ERROR) << " CMBlockBufferGetDataPointer failed: " << status;
198 return; 236 return false;
199 } 237 }
200 238
201 if (nal_size_field_bytes == 1) { 239 if (nal_size_field_bytes == 1) {
202 CopyNalsToAnnexB<uint8_t>(bb_data, bb_size, annexb_buffer); 240 CopyNalsToAnnexB<uint8_t>(bb_data, bb_size, annexb_buffer);
203 } else if (nal_size_field_bytes == 2) { 241 } else if (nal_size_field_bytes == 2) {
204 CopyNalsToAnnexB<uint16_t>(bb_data, bb_size, annexb_buffer); 242 CopyNalsToAnnexB<uint16_t>(bb_data, bb_size, annexb_buffer);
205 } else if (nal_size_field_bytes == 4) { 243 } else if (nal_size_field_bytes == 4) {
206 CopyNalsToAnnexB<uint32_t>(bb_data, bb_size, annexb_buffer); 244 CopyNalsToAnnexB<uint32_t>(bb_data, bb_size, annexb_buffer);
207 } else { 245 } else {
208 NOTREACHED(); 246 NOTREACHED();
209 } 247 }
210 }
211
212 } // namespace
213
214 class H264VideoToolboxEncoder::VideoFrameFactoryImpl
215 : public base::RefCountedThreadSafe<VideoFrameFactoryImpl>,
216 public VideoFrameFactory {
217 public:
218 // Type that proxies the VideoFrameFactory interface to this class.
219 class Proxy;
220
221 VideoFrameFactoryImpl(const base::WeakPtr<H264VideoToolboxEncoder>& encoder,
222 const scoped_refptr<CastEnvironment>& cast_environment)
223 : encoder_(encoder), cast_environment_(cast_environment) {}
224
225 scoped_refptr<VideoFrame> MaybeCreateFrame(
226 const gfx::Size& frame_size,
227 base::TimeDelta timestamp) final {
228 if (frame_size.IsEmpty()) {
229 DVLOG(1) << "Rejecting empty video frame.";
230 return nullptr;
231 }
232
233 base::AutoLock auto_lock(lock_);
234
235 // If the pool size does not match, speculatively reset the encoder to use
236 // the new size and return null. Cache the new frame size right away and
237 // toss away the pixel buffer pool to avoid spurious tasks until the encoder
238 // is done resetting.
239 if (frame_size != pool_frame_size_) {
240 DVLOG(1) << "MaybeCreateFrame: Detected frame size change.";
241 cast_environment_->PostTask(
242 CastEnvironment::MAIN, FROM_HERE,
243 base::Bind(&H264VideoToolboxEncoder::UpdateFrameSize, encoder_,
244 frame_size));
245 pool_frame_size_ = frame_size;
246 pool_.reset();
247 return nullptr;
248 }
249
250 if (!pool_) {
251 DVLOG(1) << "MaybeCreateFrame: No pixel buffer pool.";
252 return nullptr;
253 }
254
255 // Allocate a pixel buffer from the pool and return a wrapper VideoFrame.
256 base::ScopedCFTypeRef<CVPixelBufferRef> buffer;
257 auto status = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool_,
258 buffer.InitializeInto());
259 if (status != kCVReturnSuccess) {
260 DLOG(ERROR) << "CVPixelBufferPoolCreatePixelBuffer failed: " << status;
261 return nullptr;
262 }
263
264 DCHECK(buffer);
265 return VideoFrame::WrapCVPixelBuffer(buffer, timestamp);
266 }
267
268 void Update(const base::ScopedCFTypeRef<CVPixelBufferPoolRef>& pool,
269 const gfx::Size& frame_size) {
270 base::AutoLock auto_lock(lock_);
271 pool_ = pool;
272 pool_frame_size_ = frame_size;
273 }
274
275 private:
276 friend class base::RefCountedThreadSafe<VideoFrameFactoryImpl>;
277 ~VideoFrameFactoryImpl() final {}
278
279 base::Lock lock_;
280 base::ScopedCFTypeRef<CVPixelBufferPoolRef> pool_;
281 gfx::Size pool_frame_size_;
282
283 // Weak back reference to the encoder and the cast envrionment so we can
284 // message the encoder when the frame size changes.
285 const base::WeakPtr<H264VideoToolboxEncoder> encoder_;
286 const scoped_refptr<CastEnvironment> cast_environment_;
287
288 DISALLOW_COPY_AND_ASSIGN(VideoFrameFactoryImpl);
289 };
290
291 class H264VideoToolboxEncoder::VideoFrameFactoryImpl::Proxy
292 : public VideoFrameFactory {
293 public:
294 explicit Proxy(
295 const scoped_refptr<VideoFrameFactoryImpl>& video_frame_factory)
296 : video_frame_factory_(video_frame_factory) {
297 DCHECK(video_frame_factory_);
298 }
299
300 scoped_refptr<VideoFrame> MaybeCreateFrame(
301 const gfx::Size& frame_size,
302 base::TimeDelta timestamp) final {
303 return video_frame_factory_->MaybeCreateFrame(frame_size, timestamp);
304 }
305
306 private:
307 ~Proxy() final {}
308
309 const scoped_refptr<VideoFrameFactoryImpl> video_frame_factory_;
310
311 DISALLOW_COPY_AND_ASSIGN(Proxy);
312 };
313
314 // static
315 bool H264VideoToolboxEncoder::IsSupported(
316 const VideoSenderConfig& video_config) {
317 return video_config.codec == CODEC_VIDEO_H264 && VideoToolboxGlue::Get();
318 }
319
320 H264VideoToolboxEncoder::H264VideoToolboxEncoder(
321 const scoped_refptr<CastEnvironment>& cast_environment,
322 const VideoSenderConfig& video_config,
323 const StatusChangeCallback& status_change_cb)
324 : cast_environment_(cast_environment),
325 videotoolbox_glue_(VideoToolboxGlue::Get()),
326 video_config_(video_config),
327 status_change_cb_(status_change_cb),
328 last_frame_id_(kFirstFrameId - 1),
329 encode_next_frame_as_keyframe_(false),
330 power_suspended_(false),
331 weak_factory_(this) {
332 DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
333 DCHECK(!status_change_cb_.is_null());
334
335 OperationalStatus operational_status =
336 H264VideoToolboxEncoder::IsSupported(video_config)
337 ? STATUS_INITIALIZED
338 : STATUS_UNSUPPORTED_CODEC;
339 cast_environment_->PostTask(
340 CastEnvironment::MAIN, FROM_HERE,
341 base::Bind(status_change_cb_, operational_status));
342
343 if (operational_status == STATUS_INITIALIZED) {
344 // Create the shared video frame factory. It persists for the combined
345 // lifetime of the encoder and all video frame factory proxies created by
346 // |CreateVideoFrameFactory| that reference it.
347 video_frame_factory_ =
348 scoped_refptr<VideoFrameFactoryImpl>(new VideoFrameFactoryImpl(
349 weak_factory_.GetWeakPtr(), cast_environment_));
350
351 // Register for power state changes.
352 auto power_monitor = base::PowerMonitor::Get();
353 if (power_monitor) {
354 power_monitor->AddObserver(this);
355 VLOG(1) << "Registered for power state changes.";
356 } else {
357 DLOG(WARNING) << "No power monitor. Process suspension will invalidate "
358 "the encoder.";
359 }
360 }
361 }
362
363 H264VideoToolboxEncoder::~H264VideoToolboxEncoder() {
364 DestroyCompressionSession();
365
366 // If video_frame_factory_ is not null, the encoder registered for power state
367 // changes in the ctor and it must now unregister.
368 if (video_frame_factory_) {
369 auto power_monitor = base::PowerMonitor::Get();
370 if (power_monitor)
371 power_monitor->RemoveObserver(this);
372 }
373 }
374
375 void H264VideoToolboxEncoder::ResetCompressionSession() {
376 DCHECK(thread_checker_.CalledOnValidThread());
377
378 // Ignore reset requests while power suspended.
379 if (power_suspended_)
380 return;
381
382 // Notify that we're resetting the encoder.
383 cast_environment_->PostTask(
384 CastEnvironment::MAIN, FROM_HERE,
385 base::Bind(status_change_cb_, STATUS_CODEC_REINIT_PENDING));
386
387 // Destroy the current session, if any.
388 DestroyCompressionSession();
389
390 // On OS X, allow the hardware encoder. Don't require it, it does not support
391 // all configurations (some of which are used for testing).
392 base::ScopedCFTypeRef<CFDictionaryRef> encoder_spec;
393 #if !defined(OS_IOS)
394 encoder_spec = DictionaryWithKeyValue(
395 videotoolbox_glue_
396 ->kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder() ,
397 kCFBooleanTrue);
398 #endif
399
400 // Force 420v so that clients can easily use these buffers as GPU textures.
401 const int format[] = {
402 CoreVideoGlue::kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange};
403
404 // Keep these attachment settings in-sync with those in ConfigureSession().
405 CFTypeRef attachments_keys[] = {kCVImageBufferColorPrimariesKey,
406 kCVImageBufferTransferFunctionKey,
407 kCVImageBufferYCbCrMatrixKey};
408 CFTypeRef attachments_values[] = {kCVImageBufferColorPrimaries_ITU_R_709_2,
409 kCVImageBufferTransferFunction_ITU_R_709_2,
410 kCVImageBufferYCbCrMatrix_ITU_R_709_2};
411 CFTypeRef buffer_attributes_keys[] = {kCVPixelBufferPixelFormatTypeKey,
412 kCVBufferPropagatedAttachmentsKey};
413 CFTypeRef buffer_attributes_values[] = {
414 ArrayWithIntegers(format, arraysize(format)).release(),
415 DictionaryWithKeysAndValues(attachments_keys, attachments_values,
416 arraysize(attachments_keys)).release()};
417 const base::ScopedCFTypeRef<CFDictionaryRef> buffer_attributes =
418 DictionaryWithKeysAndValues(buffer_attributes_keys,
419 buffer_attributes_values,
420 arraysize(buffer_attributes_keys));
421 for (auto& v : buffer_attributes_values)
422 CFRelease(v);
423
424 // Create the compression session.
425
426 // Note that the encoder object is given to the compression session as the
427 // callback context using a raw pointer. The C API does not allow us to use a
428 // smart pointer, nor is this encoder ref counted. However, this is still
429 // safe, because we 1) we own the compression session and 2) we tear it down
430 // safely. When destructing the encoder, the compression session is flushed
431 // and invalidated. Internally, VideoToolbox will join all of its threads
432 // before returning to the client. Therefore, when control returns to us, we
433 // are guaranteed that the output callback will not execute again.
434 OSStatus status = videotoolbox_glue_->VTCompressionSessionCreate(
435 kCFAllocatorDefault, frame_size_.width(), frame_size_.height(),
436 CoreMediaGlue::kCMVideoCodecType_H264, encoder_spec, buffer_attributes,
437 nullptr /* compressedDataAllocator */,
438 &H264VideoToolboxEncoder::CompressionCallback,
439 reinterpret_cast<void*>(this), compression_session_.InitializeInto());
440 if (status != noErr) {
441 DLOG(ERROR) << " VTCompressionSessionCreate failed: " << status;
442 // Notify that reinitialization has failed.
443 cast_environment_->PostTask(
444 CastEnvironment::MAIN, FROM_HERE,
445 base::Bind(status_change_cb_, STATUS_CODEC_INIT_FAILED));
446 return;
447 }
448
449 // Configure the session (apply session properties based on the current state
450 // of the encoder, experimental tuning and requirements).
451 ConfigureCompressionSession();
452
453 // Update the video frame factory.
454 base::ScopedCFTypeRef<CVPixelBufferPoolRef> pool(
455 videotoolbox_glue_->VTCompressionSessionGetPixelBufferPool(
456 compression_session_),
457 base::scoped_policy::RETAIN);
458 video_frame_factory_->Update(pool, frame_size_);
459
460 // Notify that reinitialization is done.
461 cast_environment_->PostTask(
462 CastEnvironment::MAIN, FROM_HERE,
463 base::Bind(status_change_cb_, STATUS_INITIALIZED));
464 }
465
466 void H264VideoToolboxEncoder::ConfigureCompressionSession() {
467 SetSessionProperty(
468 videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(),
469 videotoolbox_glue_->kVTProfileLevel_H264_Main_AutoLevel());
470 SetSessionProperty(videotoolbox_glue_->kVTCompressionPropertyKey_RealTime(),
471 true);
472 SetSessionProperty(
473 videotoolbox_glue_->kVTCompressionPropertyKey_AllowFrameReordering(),
474 false);
475 SetSessionProperty(
476 videotoolbox_glue_->kVTCompressionPropertyKey_MaxKeyFrameInterval(), 240);
477 SetSessionProperty(
478 videotoolbox_glue_
479 ->kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration(),
480 240);
481 // TODO(jfroy): implement better bitrate control
482 // https://crbug.com/425352
483 SetSessionProperty(
484 videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(),
485 (video_config_.min_bitrate + video_config_.max_bitrate) / 2);
486 SetSessionProperty(
487 videotoolbox_glue_->kVTCompressionPropertyKey_ExpectedFrameRate(),
488 video_config_.max_frame_rate);
489 // Keep these attachment settings in-sync with those in Initialize().
490 SetSessionProperty(
491 videotoolbox_glue_->kVTCompressionPropertyKey_ColorPrimaries(),
492 kCVImageBufferColorPrimaries_ITU_R_709_2);
493 SetSessionProperty(
494 videotoolbox_glue_->kVTCompressionPropertyKey_TransferFunction(),
495 kCVImageBufferTransferFunction_ITU_R_709_2);
496 SetSessionProperty(
497 videotoolbox_glue_->kVTCompressionPropertyKey_YCbCrMatrix(),
498 kCVImageBufferYCbCrMatrix_ITU_R_709_2);
499 if (video_config_.max_number_of_video_buffers_used > 0) {
500 SetSessionProperty(
501 videotoolbox_glue_->kVTCompressionPropertyKey_MaxFrameDelayCount(),
502 video_config_.max_number_of_video_buffers_used);
503 }
504 }
505
506 void H264VideoToolboxEncoder::DestroyCompressionSession() {
507 DCHECK(thread_checker_.CalledOnValidThread());
508
509 // If the compression session exists, invalidate it. This blocks until all
510 // pending output callbacks have returned and any internal threads have
511 // joined, ensuring no output callback ever sees a dangling encoder pointer.
512 //
513 // Before destroying the compression session, the video frame factory's pool
514 // is updated to null so that no thread will produce new video frames via the
515 // factory until a new compression session is created. The current frame size
516 // is passed to prevent the video frame factory from posting |UpdateFrameSize|
517 // tasks. Indeed, |DestroyCompressionSession| is either called from
518 // |ResetCompressionSession|, in which case a new pool and frame size will be
519 // set, or from callsites that require that there be no compression session
520 // (ex: the dtor).
521 if (compression_session_) {
522 video_frame_factory_->Update(
523 base::ScopedCFTypeRef<CVPixelBufferPoolRef>(nullptr), frame_size_);
524 videotoolbox_glue_->VTCompressionSessionInvalidate(compression_session_);
525 compression_session_.reset();
526 }
527 }
528
529 bool H264VideoToolboxEncoder::EncodeVideoFrame(
530 const scoped_refptr<media::VideoFrame>& video_frame,
531 const base::TimeTicks& reference_time,
532 const FrameEncodedCallback& frame_encoded_callback) {
533 DCHECK(thread_checker_.CalledOnValidThread());
534 DCHECK(!frame_encoded_callback.is_null());
535
536 // Reject empty video frames.
537 const gfx::Size frame_size = video_frame->visible_rect().size();
538 if (frame_size.IsEmpty()) {
539 DVLOG(1) << "Rejecting empty video frame.";
540 return false;
541 }
542
543 // Handle frame size changes. This will reset the compression session.
544 if (frame_size != frame_size_) {
545 DVLOG(1) << "EncodeVideoFrame: Detected frame size change.";
546 UpdateFrameSize(frame_size);
547 }
548
549 // Need a compression session to continue.
550 if (!compression_session_) {
551 DLOG(ERROR) << "No compression session.";
552 return false;
553 }
554
555 // Wrap the VideoFrame in a CVPixelBuffer. In all cases, no data will be
556 // copied. If the VideoFrame was created by this encoder's video frame
557 // factory, then the returned CVPixelBuffer will have been obtained from the
558 // compression session's pixel buffer pool. This will eliminate a copy of the
559 // frame into memory visible by the hardware encoder. The VideoFrame's
560 // lifetime is extended for the lifetime of the returned CVPixelBuffer.
561 auto pixel_buffer = media::WrapVideoFrameInCVPixelBuffer(*video_frame);
562 if (!pixel_buffer) {
563 DLOG(ERROR) << "WrapVideoFrameInCVPixelBuffer failed.";
564 return false;
565 }
566
567 // Convert the frame timestamp to CMTime.
568 auto timestamp_cm = CoreMediaGlue::CMTimeMake(
569 (reference_time - base::TimeTicks()).InMicroseconds(), USEC_PER_SEC);
570
571 // Wrap information we'll need after the frame is encoded in a heap object.
572 // We'll get the pointer back from the VideoToolbox completion callback.
573 scoped_ptr<InProgressFrameEncode> request(new InProgressFrameEncode(
574 RtpTimeTicks::FromTimeDelta(video_frame->timestamp(), kVideoFrequency),
575 reference_time, frame_encoded_callback));
576
577 // Build a suitable frame properties dictionary for keyframes.
578 base::ScopedCFTypeRef<CFDictionaryRef> frame_props;
579 if (encode_next_frame_as_keyframe_) {
580 frame_props = DictionaryWithKeyValue(
581 videotoolbox_glue_->kVTEncodeFrameOptionKey_ForceKeyFrame(),
582 kCFBooleanTrue);
583 encode_next_frame_as_keyframe_ = false;
584 }
585
586 // Submit the frame to the compression session. The function returns as soon
587 // as the frame has been enqueued.
588 OSStatus status = videotoolbox_glue_->VTCompressionSessionEncodeFrame(
589 compression_session_, pixel_buffer, timestamp_cm,
590 CoreMediaGlue::CMTime{0, 0, 0, 0}, frame_props,
591 reinterpret_cast<void*>(request.release()), nullptr);
592 if (status != noErr) {
593 DLOG(ERROR) << " VTCompressionSessionEncodeFrame failed: " << status;
594 return false;
595 }
596
597 return true; 248 return true;
598 } 249 }
599 250
600 void H264VideoToolboxEncoder::UpdateFrameSize(const gfx::Size& size_needed) { 251 bool CopySampleBufferToAnnexBBuffer(CoreMediaGlue::CMSampleBufferRef sbuf,
601 DCHECK(thread_checker_.CalledOnValidThread()); 252 bool keyframe,
602 253 std::string* annexb_buffer) {
603 // Our video frame factory posts a task to update the frame size when its 254 StringAnnexBBuffer buffer(annexb_buffer);
604 // cache of the frame size differs from what the client requested. To avoid 255 return CopySampleBufferToAnnexBBuffer(sbuf, &buffer, keyframe);
605 // spurious encoder resets, check again here.
606 if (size_needed == frame_size_) {
607 DCHECK(compression_session_);
608 return;
609 }
610
611 VLOG(1) << "Resetting compression session (for frame size change from "
612 << frame_size_.ToString() << " to " << size_needed.ToString() << ").";
613
614 // If there is an existing session, finish every pending frame.
615 if (compression_session_) {
616 EmitFrames();
617 }
618
619 // Store the new frame size.
620 frame_size_ = size_needed;
621
622 // Reset the compression session.
623 ResetCompressionSession();
624 } 256 }
625 257
626 void H264VideoToolboxEncoder::SetBitRate(int /*new_bit_rate*/) { 258 bool CopySampleBufferToAnnexBBuffer(CoreMediaGlue::CMSampleBufferRef sbuf,
627 DCHECK(thread_checker_.CalledOnValidThread()); 259 bool keyframe,
628 // VideoToolbox does not seem to support bitrate reconfiguration. 260 size_t annexb_buffer_size,
261 char* annexb_buffer,
262 size_t* used_buffer_size) {
263 RawAnnexBBuffer buffer(annexb_buffer, annexb_buffer_size);
264 const bool copy_rv = CopySampleBufferToAnnexBBuffer(sbuf, &buffer, keyframe);
265 *used_buffer_size = buffer.GetReservedSize();
266 return copy_rv;
629 } 267 }
630 268
631 void H264VideoToolboxEncoder::GenerateKeyFrame() { 269 SessionPropertySetter::SessionPropertySetter(
632 DCHECK(thread_checker_.CalledOnValidThread()); 270 base::ScopedCFTypeRef<VideoToolboxGlue::VTCompressionSessionRef> session,
633 encode_next_frame_as_keyframe_ = true; 271 const VideoToolboxGlue* const glue)
272 : session_(session), glue_(glue) {}
273
274 SessionPropertySetter::~SessionPropertySetter() {}
275
276 bool SessionPropertySetter::SetSessionProperty(CFStringRef key, int32_t value) {
277 DCHECK(session_);
278 DCHECK(glue_);
279 base::ScopedCFTypeRef<CFNumberRef> cfvalue(
280 CFNumberCreate(nullptr, kCFNumberSInt32Type, &value));
281 return glue_->VTSessionSetProperty(session_, key, cfvalue) == noErr;
634 } 282 }
635 283
636 scoped_ptr<VideoFrameFactory> 284 bool SessionPropertySetter::SetSessionProperty(CFStringRef key, bool value) {
637 H264VideoToolboxEncoder::CreateVideoFrameFactory() { 285 DCHECK(session_);
638 DCHECK(thread_checker_.CalledOnValidThread()); 286 DCHECK(glue_);
639 return scoped_ptr<VideoFrameFactory>( 287 CFBooleanRef cfvalue = (value) ? kCFBooleanTrue : kCFBooleanFalse;
640 new VideoFrameFactoryImpl::Proxy(video_frame_factory_)); 288 return glue_->VTSessionSetProperty(session_, key, cfvalue) == noErr;
641 } 289 }
642 290
643 void H264VideoToolboxEncoder::EmitFrames() { 291 bool SessionPropertySetter::SetSessionProperty(CFStringRef key,
644 DCHECK(thread_checker_.CalledOnValidThread()); 292 CFStringRef value) {
645 if (!compression_session_) 293 DCHECK(session_);
646 return; 294 DCHECK(glue_);
647 295 return glue_->VTSessionSetProperty(session_, key, value) == noErr;
648 OSStatus status = videotoolbox_glue_->VTCompressionSessionCompleteFrames(
649 compression_session_, CoreMediaGlue::CMTime{0, 0, 0, 0});
650 if (status != noErr) {
651 DLOG(ERROR) << " VTCompressionSessionCompleteFrames failed: " << status;
652 }
653 } 296 }
654 297
655 void H264VideoToolboxEncoder::OnSuspend() { 298 bool SessionPropertySetter::SetSessionProperty(CFStringRef key,
656 VLOG(1) 299 CFArrayRef value) {
657 << "OnSuspend: Emitting all frames and destroying compression session."; 300 DCHECK(session_);
658 EmitFrames(); 301 DCHECK(glue_);
659 DestroyCompressionSession(); 302 return glue_->VTSessionSetProperty(session_, key, value) == noErr;
660 power_suspended_ = true;
661 } 303 }
662 304
663 void H264VideoToolboxEncoder::OnResume() { 305 } // namespace video_toolbox
664 power_suspended_ = false;
665 306
666 // Reset the compression session only if the frame size is not zero (which
667 // will obviously fail). It is possible for the frame size to be zero if no
668 // frame was submitted for encoding or requested from the video frame factory
669 // before suspension.
670 if (!frame_size_.IsEmpty()) {
671 VLOG(1) << "OnResume: Resetting compression session.";
672 ResetCompressionSession();
673 }
674 }
675
676 bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key,
677 int32_t value) {
678 base::ScopedCFTypeRef<CFNumberRef> cfvalue(
679 CFNumberCreate(nullptr, kCFNumberSInt32Type, &value));
680 return videotoolbox_glue_->VTSessionSetProperty(compression_session_, key,
681 cfvalue) == noErr;
682 }
683
684 bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key, bool value) {
685 CFBooleanRef cfvalue = (value) ? kCFBooleanTrue : kCFBooleanFalse;
686 return videotoolbox_glue_->VTSessionSetProperty(compression_session_, key,
687 cfvalue) == noErr;
688 }
689
690 bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key,
691 CFStringRef value) {
692 return videotoolbox_glue_->VTSessionSetProperty(compression_session_, key,
693 value) == noErr;
694 }
695
696 void H264VideoToolboxEncoder::CompressionCallback(void* encoder_opaque,
697 void* request_opaque,
698 OSStatus status,
699 VTEncodeInfoFlags info,
700 CMSampleBufferRef sbuf) {
701 auto encoder = reinterpret_cast<H264VideoToolboxEncoder*>(encoder_opaque);
702 const scoped_ptr<InProgressFrameEncode> request(
703 reinterpret_cast<InProgressFrameEncode*>(request_opaque));
704 bool keyframe = false;
705 bool has_frame_data = false;
706
707 if (status != noErr) {
708 DLOG(ERROR) << " encoding failed: " << status;
709 encoder->cast_environment_->PostTask(
710 CastEnvironment::MAIN, FROM_HERE,
711 base::Bind(encoder->status_change_cb_, STATUS_CODEC_RUNTIME_ERROR));
712 } else if ((info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped)) {
713 DVLOG(2) << " frame dropped";
714 } else {
715 auto sample_attachments =
716 static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(
717 CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true),
718 0));
719
720 // If the NotSync key is not present, it implies Sync, which indicates a
721 // keyframe (at least I think, VT documentation is, erm, sparse). Could
722 // alternatively use kCMSampleAttachmentKey_DependsOnOthers == false.
723 keyframe = !CFDictionaryContainsKey(
724 sample_attachments,
725 CoreMediaGlue::kCMSampleAttachmentKey_NotSync());
726 has_frame_data = true;
727 }
728
729 // Increment the encoder-scoped frame id and assign the new value to this
730 // frame. VideoToolbox calls the output callback serially, so this is safe.
731 const uint32_t frame_id = ++encoder->last_frame_id_;
732
733 scoped_ptr<SenderEncodedFrame> encoded_frame(new SenderEncodedFrame());
734 encoded_frame->frame_id = frame_id;
735 encoded_frame->reference_time = request->reference_time;
736 encoded_frame->rtp_timestamp = request->rtp_timestamp;
737 if (keyframe) {
738 encoded_frame->dependency = EncodedFrame::KEY;
739 encoded_frame->referenced_frame_id = frame_id;
740 } else {
741 encoded_frame->dependency = EncodedFrame::DEPENDENT;
742 // H.264 supports complex frame reference schemes (multiple reference
743 // frames, slice references, backward and forward references, etc). Cast
744 // doesn't support the concept of forward-referencing frame dependencies or
745 // multiple frame dependencies; so pretend that all frames are only
746 // decodable after their immediately preceding frame is decoded. This will
747 // ensure a Cast receiver only attempts to decode the frames sequentially
748 // and in order. Furthermore, the encoder is configured to never use forward
749 // references (see |kVTCompressionPropertyKey_AllowFrameReordering|). There
750 // is no way to prevent multiple reference frames.
751 encoded_frame->referenced_frame_id = frame_id - 1;
752 }
753
754 if (has_frame_data)
755 CopySampleBufferToAnnexBBuffer(sbuf, &encoded_frame->data, keyframe);
756
757 // TODO(miu): Compute and populate the |deadline_utilization| and
758 // |lossy_utilization| performance metrics in |encoded_frame|.
759
760 encoded_frame->encode_completion_time =
761 encoder->cast_environment_->Clock()->NowTicks();
762 encoder->cast_environment_->PostTask(
763 CastEnvironment::MAIN, FROM_HERE,
764 base::Bind(request->frame_encoded_callback,
765 base::Passed(&encoded_frame)));
766 }
767
768 } // namespace cast
769 } // namespace media 307 } // namespace media
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698