OLD | NEW |
---|---|
(Empty) | |
1 // Copyright 2016 The Chromium Authors. All rights reserved. | |
2 // Use of this source code is governed by a BSD-style license that can be | |
3 // found in the LICENSE file. | |
4 | |
5 #include "content/common/gpu/media/vt_video_encode_accelerator_mac.h" | |
6 | |
7 #include "base/thread_task_runner_handle.h" | |
8 #include "media/base/mac/coremedia_glue.h" | |
9 #include "media/base/mac/corevideo_glue.h" | |
10 #include "media/base/mac/video_frame_mac.h" | |
11 | |
12 namespace content { | |
13 | |
14 namespace { | |
15 | |
16 // Subjectively chosen. | |
17 // TODO(emircan): Check if we can find the actual system capabilities via | |
18 // creating VTCompressionSessions with varying requirements. | |
19 // See crbug.com/584784. | |
20 const size_t kNumInputBuffers = 4; | |
21 const size_t kMaxFrameRateNumerator = 30; | |
22 const size_t kMaxFrameRateDenominator = 1; | |
23 const size_t kMaxResolutionWidth = 4096; | |
24 const size_t kMaxResolutionHeight = 2160; | |
25 const size_t kBitsPerByte = 8; | |
26 | |
27 } // namespace | |
28 | |
29 struct VTVideoEncodeAccelerator::InProgressFrameEncode { | |
30 InProgressFrameEncode(base::TimeDelta rtp_timestamp, base::TimeTicks ref_time) | |
31 : timestamp(rtp_timestamp), reference_time(ref_time) {} | |
32 const base::TimeDelta timestamp; | |
33 const base::TimeTicks reference_time; | |
34 | |
35 private: | |
36 DISALLOW_IMPLICIT_CONSTRUCTORS(InProgressFrameEncode); | |
37 }; | |
38 | |
39 struct VTVideoEncodeAccelerator::EncodeOutput { | |
40 EncodeOutput(VTEncodeInfoFlags info_flags, CMSampleBufferRef sbuf) | |
41 : info(info_flags), sample_buffer(sbuf) {} | |
42 const VTEncodeInfoFlags info; | |
43 const CMSampleBufferRef sample_buffer; | |
44 | |
45 private: | |
46 DISALLOW_IMPLICIT_CONSTRUCTORS(EncodeOutput); | |
47 }; | |
48 | |
49 struct VTVideoEncodeAccelerator::BitstreamBufferRef { | |
50 BitstreamBufferRef(int32_t id, | |
51 scoped_ptr<base::SharedMemory> shm, | |
52 size_t size) | |
53 : id(id), shm(std::move(shm)), size(size) {} | |
54 const int32_t id; | |
55 const scoped_ptr<base::SharedMemory> shm; | |
56 const size_t size; | |
57 | |
58 private: | |
59 DISALLOW_IMPLICIT_CONSTRUCTORS(BitstreamBufferRef); | |
60 }; | |
61 | |
62 VTVideoEncodeAccelerator::VTVideoEncodeAccelerator() | |
63 : set_data_rate_limit_(true), | |
64 client_task_runner_(base::ThreadTaskRunnerHandle::Get()) { | |
65 } | |
66 | |
67 VTVideoEncodeAccelerator::~VTVideoEncodeAccelerator() { | |
68 DCHECK(thread_checker_.CalledOnValidThread()); | |
69 } | |
70 | |
71 media::VideoEncodeAccelerator::SupportedProfiles | |
72 VTVideoEncodeAccelerator::GetSupportedProfiles() { | |
73 DVLOG(3) << __FUNCTION__; | |
74 DCHECK(thread_checker_.CalledOnValidThread()); | |
75 | |
76 SupportedProfiles profiles; | |
77 SupportedProfile profile; | |
78 profile.profile = media::H264PROFILE_BASELINE; | |
79 profile.max_framerate_numerator = kMaxFrameRateNumerator; | |
80 profile.max_framerate_denominator = kMaxFrameRateDenominator; | |
81 profile.max_resolution = gfx::Size(kMaxResolutionWidth, kMaxResolutionHeight); | |
82 profiles.push_back(profile); | |
83 return profiles; | |
84 } | |
85 | |
86 bool VTVideoEncodeAccelerator::Initialize( | |
87 media::VideoPixelFormat format, | |
88 const gfx::Size& input_visible_size, | |
89 media::VideoCodecProfile output_profile, | |
90 uint32_t initial_bitrate, | |
91 Client* client) { | |
92 DVLOG(3) << __FUNCTION__ | |
93 << ": input_format=" << media::VideoPixelFormatToString(format) | |
94 << ", input_visible_size=" << input_visible_size.ToString() | |
95 << ", output_profile=" << output_profile | |
96 << ", initial_bitrate=" << initial_bitrate; | |
97 DCHECK(thread_checker_.CalledOnValidThread()); | |
98 DCHECK(client); | |
99 | |
100 if (media::PIXEL_FORMAT_I420 != format) { | |
101 DLOG(ERROR) << "Input format not supported= " | |
102 << media::VideoPixelFormatToString(format); | |
103 return false; | |
104 } | |
105 if (media::H264PROFILE_BASELINE != output_profile) { | |
106 DLOG(ERROR) << "Output profile not supported= " | |
107 << output_profile; | |
108 return false; | |
109 } | |
110 | |
111 videotoolbox_glue_ = VideoToolboxGlue::Get(); | |
112 if (!videotoolbox_glue_) { | |
113 DLOG(ERROR) << "Failed creating VideoToolbox glue"; | |
114 return false; | |
115 } | |
116 | |
117 client_ptr_factory_.reset(new base::WeakPtrFactory<Client>(client)); | |
118 client_ = client_ptr_factory_->GetWeakPtr(); | |
119 input_visible_size_ = input_visible_size; | |
120 frame_rate_ = kMaxFrameRateNumerator / kMaxFrameRateDenominator; | |
121 target_bitrate_ = initial_bitrate; | |
122 bitstream_buffer_size_ = target_bitrate_ / kBitsPerByte; | |
123 | |
124 if (!ResetCompressionSession()) { | |
125 DLOG(ERROR) << "Failed creating compression session"; | |
126 return false; | |
127 } | |
128 | |
129 client_->RequireBitstreamBuffers(kNumInputBuffers, input_visible_size_, | |
130 bitstream_buffer_size_); | |
131 return true; | |
132 } | |
133 | |
134 void VTVideoEncodeAccelerator::Encode( | |
135 const scoped_refptr<media::VideoFrame>& frame, | |
136 bool force_keyframe) { | |
137 DVLOG(3) << __FUNCTION__; | |
138 DCHECK(thread_checker_.CalledOnValidThread()); | |
139 DCHECK(compression_session_); | |
140 DCHECK(frame); | |
141 | |
142 base::TimeTicks ref_time; | |
143 if (!frame->metadata()->GetTimeTicks( | |
144 media::VideoFrameMetadata::REFERENCE_TIME, &ref_time)) { | |
145 ref_time = base::TimeTicks::Now(); | |
146 } | |
147 auto timestamp_cm = CoreMediaGlue::CMTimeMake( | |
148 frame->timestamp().InMicroseconds(), USEC_PER_SEC); | |
149 // Wrap information we'll need after the frame is encoded in a heap object. | |
150 // We'll get the pointer back from the VideoToolbox completion callback. | |
151 scoped_ptr<InProgressFrameEncode> request(new InProgressFrameEncode( | |
Pawel Osciak
2016/02/18 11:16:14
Nit: please move above l.163.
emircan
2016/03/02 05:28:13
Done.
| |
152 frame->timestamp(), ref_time)); | |
153 | |
154 // TODO(emircan): See if we can eliminate a copy here by using | |
155 // CVPixelBufferPool for the allocation of incoming VideoFrames. | |
156 base::ScopedCFTypeRef<CVPixelBufferRef> pixel_buffer = | |
157 media::WrapVideoFrameInCVPixelBuffer(*frame); | |
158 base::ScopedCFTypeRef<CFDictionaryRef> frame_props = | |
159 media::video_toolbox::DictionaryWithKeyValue( | |
160 videotoolbox_glue_->kVTEncodeFrameOptionKey_ForceKeyFrame(), | |
161 force_keyframe ? kCFBooleanTrue : kCFBooleanFalse); | |
162 | |
163 OSStatus status = videotoolbox_glue_->VTCompressionSessionEncodeFrame( | |
164 compression_session_, pixel_buffer, timestamp_cm, | |
165 CoreMediaGlue::CMTime{0, 0, 0, 0}, frame_props, | |
166 reinterpret_cast<void*>(request.release()), nullptr); | |
167 if (status != noErr) { | |
168 DLOG(ERROR) << " VTCompressionSessionEncodeFrame failed: " << status; | |
169 client_->NotifyError(kPlatformFailureError); | |
170 } | |
171 } | |
172 | |
173 void VTVideoEncodeAccelerator::UseOutputBitstreamBuffer( | |
174 const media::BitstreamBuffer& buffer) { | |
175 DVLOG(3) << __FUNCTION__ << ": buffer size=" << buffer.size(); | |
176 DCHECK(thread_checker_.CalledOnValidThread()); | |
177 if (buffer.size() < bitstream_buffer_size_) { | |
178 DLOG(ERROR) << "Output BitstreamBuffer isn't big enough: " << buffer.size() | |
179 << " vs. " << bitstream_buffer_size_; | |
180 client_->NotifyError(kInvalidArgumentError); | |
181 return; | |
182 } | |
183 | |
184 scoped_ptr<base::SharedMemory> shm( | |
185 new base::SharedMemory(buffer.handle(), false)); | |
186 if (!shm->Map(buffer.size())) { | |
187 DLOG(ERROR) << "Failed mapping shared memory."; | |
188 client_->NotifyError(kPlatformFailureError); | |
189 return; | |
190 } | |
191 | |
192 scoped_ptr<BitstreamBufferRef> buffer_ref( | |
193 new BitstreamBufferRef(buffer.id(), std::move(shm), buffer.size())); | |
194 | |
195 // If there is already EncodeOutput waiting, copy its output first. | |
196 if (!encoder_output_queue_.empty()) { | |
197 scoped_ptr<VTVideoEncodeAccelerator::EncodeOutput> encode_output = | |
198 std::move(encoder_output_queue_.front()); | |
199 encoder_output_queue_.pop_front(); | |
200 ReturnBitstreamBuffer(encode_output->info, encode_output->sample_buffer, | |
201 std::move(buffer_ref)); | |
202 return; | |
203 } | |
204 | |
205 bitstream_buffer_queue_.push_back(std::move(buffer_ref)); | |
206 } | |
207 | |
208 void VTVideoEncodeAccelerator::RequestEncodingParametersChange( | |
209 uint32_t bitrate, | |
210 uint32_t framerate) { | |
211 DVLOG(3) << __FUNCTION__ << ": bitrate=" << bitrate | |
212 << ": framerate=" << framerate; | |
213 DCHECK(thread_checker_.CalledOnValidThread()); | |
214 | |
215 frame_rate_ = framerate > 1 ? framerate : 1; | |
216 target_bitrate_ = bitrate > 1 ? bitrate : 1; | |
217 | |
218 if (!compression_session_) { | |
219 client_->NotifyError(kPlatformFailureError); | |
220 return; | |
221 } | |
222 | |
223 media::video_toolbox::SessionPropertySetter session_property_setter( | |
224 compression_session_, videotoolbox_glue_); | |
225 | |
226 // TODO(emircan): See crbug.com/425352. | |
227 bool rv = session_property_setter.SetSessionProperty( | |
228 videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(), | |
229 target_bitrate_); | |
230 rv &= session_property_setter.SetSessionProperty( | |
231 videotoolbox_glue_->kVTCompressionPropertyKey_ExpectedFrameRate(), | |
232 frame_rate_); | |
233 DLOG_IF(ERROR, !rv) << "Couldn't change session encoding parameters."; | |
234 | |
235 // Some HW encoder platforms are missing this parameter. Try once to set it. | |
236 if (set_data_rate_limit_) { | |
237 const int data_rate_limits[] = {target_bitrate_ / kBitsPerByte, 1}; | |
238 set_data_rate_limit_ &= session_property_setter.SetSessionProperty( | |
Pawel Osciak
2016/02/18 11:16:14
I believe this would have the same effect?
if (s_
emircan
2016/03/02 05:28:13
Done.
| |
239 videotoolbox_glue_->kVTCompressionPropertyKey_DataRateLimits(), | |
240 media::video_toolbox::ArrayWithIntegers(data_rate_limits, | |
241 arraysize(data_rate_limits))); | |
242 } | |
243 } | |
244 | |
245 void VTVideoEncodeAccelerator::Destroy() { | |
246 DVLOG(3) << __FUNCTION__; | |
247 DCHECK(thread_checker_.CalledOnValidThread()); | |
248 | |
249 DestroyCompressionSession(); | |
250 delete this; | |
251 } | |
252 | |
253 // static | |
254 void VTVideoEncodeAccelerator::CompressionCallback(void* encoder_opaque, | |
255 void* request_opaque, | |
256 OSStatus status, | |
257 VTEncodeInfoFlags info, | |
258 CMSampleBufferRef sbuf) { | |
259 // This function may be called asynchronously, on a different thread from the | |
260 // one that calls VTCompressionSessionEncodeFrame. | |
261 DVLOG(3) << __FUNCTION__; | |
262 | |
263 auto encoder = reinterpret_cast<VTVideoEncodeAccelerator*>(encoder_opaque); | |
264 DCHECK(encoder); | |
265 | |
266 // Release InProgressFrameEncode, since we don't have support to return | |
267 // timestamps at this point. | |
268 scoped_ptr<InProgressFrameEncode> request( | |
269 reinterpret_cast<InProgressFrameEncode*>(request_opaque)); | |
270 request.reset(); | |
271 | |
272 if (status != noErr) { | |
273 DLOG(ERROR) << " encode failed: " << status; | |
274 encoder->client_task_runner_->PostTask( | |
Pawel Osciak
2016/02/18 11:16:14
Do we need to do anything in this method at all? C
emircan
2016/03/02 05:28:13
Done.
| |
275 FROM_HERE, base::Bind(&Client::NotifyError, encoder->client_, | |
276 kPlatformFailureError)); | |
277 return; | |
278 } | |
279 | |
280 // CFRetain is required to hold onto CMSampleBufferRef when posting task | |
281 // between threads. The object should be released later using CFRelease. | |
282 CFRetain(sbuf); | |
283 // This method is NOT called on |client_task_runner_|, so we still need to | |
284 // post a task back to it to reach |client_|. | |
285 encoder->client_task_runner_->PostTask( | |
286 FROM_HERE, | |
287 base::Bind(&VTVideoEncodeAccelerator::CompressionCallbackTask, | |
288 base::Unretained(encoder), info, sbuf)); | |
Pawel Osciak
2016/02/18 11:16:14
Please add a comment why it's safe to do Unretaine
emircan
2016/03/02 05:28:13
Actually, Unretained looks wrong to me now. If thi
| |
289 } | |
290 | |
291 void VTVideoEncodeAccelerator::CompressionCallbackTask(VTEncodeInfoFlags info, | |
292 CMSampleBufferRef sbuf) { | |
293 DVLOG(3) << __FUNCTION__; | |
294 DCHECK(thread_checker_.CalledOnValidThread()); | |
295 | |
296 // If there isn't any BitstreamBuffer to copy into, add it to a queue for | |
Pawel Osciak
2016/02/18 11:16:14
Could we perhaps have a TryReturnBitstreamBuffer()
emircan
2016/03/02 05:28:13
Here we are checking if bitstream_buffer_queue_ is
| |
297 // later use. | |
298 if (bitstream_buffer_queue_.empty()) { | |
299 scoped_ptr<EncodeOutput> encode_output(new EncodeOutput(info, sbuf)); | |
300 encoder_output_queue_.push_back(std::move(encode_output)); | |
301 return; | |
302 } | |
303 | |
304 scoped_ptr<VTVideoEncodeAccelerator::BitstreamBufferRef> buffer_ref = | |
305 std::move(bitstream_buffer_queue_.front()); | |
306 bitstream_buffer_queue_.pop_front(); | |
307 ReturnBitstreamBuffer(info, sbuf, std::move(buffer_ref)); | |
308 } | |
309 | |
310 void VTVideoEncodeAccelerator::ReturnBitstreamBuffer( | |
311 VTEncodeInfoFlags info, | |
312 CMSampleBufferRef sbuf, | |
313 scoped_ptr<VTVideoEncodeAccelerator::BitstreamBufferRef> buffer_ref) { | |
314 DVLOG(3) << __FUNCTION__; | |
315 DCHECK(thread_checker_.CalledOnValidThread()); | |
316 | |
317 if (info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped) { | |
318 DVLOG(2) << " frame dropped"; | |
319 CFRelease(sbuf); | |
320 client_->BitstreamBufferReady(buffer_ref->id, 0, false); | |
321 return; | |
322 } | |
323 | |
324 auto sample_attachments = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex( | |
325 CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true), 0)); | |
326 const bool keyframe = | |
327 !CFDictionaryContainsKey(sample_attachments, | |
328 CoreMediaGlue::kCMSampleAttachmentKey_NotSync()); | |
329 | |
330 size_t used_buffer_size = 0; | |
331 const bool copy_rv = media::video_toolbox::CopySampleBufferToAnnexBBuffer( | |
332 sbuf, keyframe, buffer_ref->size, | |
333 reinterpret_cast<char*>(buffer_ref->shm->memory()), &used_buffer_size); | |
334 CFRelease(sbuf); | |
335 if (!copy_rv) { | |
336 DLOG(ERROR) << "Cannot copy output from SampleBuffer to AnnexBBuffer."; | |
337 used_buffer_size = 0; | |
338 } | |
339 | |
340 client_->BitstreamBufferReady(buffer_ref->id, used_buffer_size, keyframe); | |
341 } | |
342 | |
343 bool VTVideoEncodeAccelerator::ResetCompressionSession() { | |
344 DCHECK(thread_checker_.CalledOnValidThread()); | |
345 | |
346 DestroyCompressionSession(); | |
347 | |
348 // Keep these in-sync with those in ConfigureCompressionSession(). | |
Pawel Osciak
2016/02/18 11:16:14
These keys below seem different from the ones in C
emircan
2016/03/02 05:28:13
Thanks for the notice, I am removing the comment s
| |
349 CFTypeRef attributes_keys[] = { | |
350 kCVPixelBufferOpenGLCompatibilityKey, | |
351 kCVPixelBufferIOSurfacePropertiesKey, | |
352 kCVPixelBufferPixelFormatTypeKey | |
353 }; | |
354 const int format[] = { | |
355 CoreVideoGlue::kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange}; | |
356 CFTypeRef attributes_values[] = { | |
357 kCFBooleanTrue, | |
358 media::video_toolbox::DictionaryWithKeysAndValues(nullptr, nullptr, 0) | |
359 .release(), | |
360 media::video_toolbox::ArrayWithIntegers(format, arraysize(format)) | |
361 .release()}; | |
362 const base::ScopedCFTypeRef<CFDictionaryRef> attributes = | |
363 media::video_toolbox::DictionaryWithKeysAndValues( | |
364 attributes_keys, attributes_values, arraysize(attributes_keys)); | |
365 for (auto& v : attributes_values) | |
366 CFRelease(v); | |
367 | |
368 bool session_rv = CreateCompressionSession(attributes, true); | |
369 if (!session_rv) { | |
370 // Try creating session again without forcing HW encode. | |
Pawel Osciak
2016/02/18 11:16:14
Are we interested in doing SW encode here at all?
emircan
2016/03/02 05:28:13
Eventually, we want to compare the performance of
| |
371 DestroyCompressionSession(); | |
372 session_rv = CreateCompressionSession(attributes, false); | |
373 if (!session_rv) { | |
374 DestroyCompressionSession(); | |
375 return false; | |
376 } | |
377 } | |
378 | |
379 const bool configure_rv = ConfigureCompressionSession(); | |
380 RequestEncodingParametersChange(target_bitrate_, frame_rate_); | |
381 return configure_rv; | |
382 } | |
383 | |
384 bool VTVideoEncodeAccelerator::CreateCompressionSession( | |
385 base::ScopedCFTypeRef<CFDictionaryRef> attributes, | |
386 bool require_hw_encoding) { | |
387 DCHECK(thread_checker_.CalledOnValidThread()); | |
388 | |
389 std::vector<CFTypeRef> encoder_keys; | |
390 std::vector<CFTypeRef> encoder_values; | |
391 encoder_keys.push_back(videotoolbox_glue_ | |
392 ->kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder()); | |
393 encoder_values.push_back(kCFBooleanTrue); | |
394 | |
395 if (require_hw_encoding) { | |
396 encoder_keys.push_back( | |
397 videotoolbox_glue_ | |
398 ->kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder()); | |
399 encoder_values.push_back(kCFBooleanTrue); | |
400 } | |
401 base::ScopedCFTypeRef<CFDictionaryRef> encoder_spec = | |
402 media::video_toolbox::DictionaryWithKeysAndValues( | |
403 encoder_keys.data(), encoder_values.data(), encoder_keys.size()); | |
404 | |
405 // Create the compression session. | |
406 OSStatus status = videotoolbox_glue_->VTCompressionSessionCreate( | |
407 kCFAllocatorDefault, | |
408 input_visible_size_.width(), | |
409 input_visible_size_.height(), | |
410 CoreMediaGlue::kCMVideoCodecType_H264, | |
411 encoder_spec, | |
412 attributes, | |
413 nullptr /* compressedDataAllocator */, | |
414 &VTVideoEncodeAccelerator::CompressionCallback, | |
415 reinterpret_cast<void*>(this), | |
Pawel Osciak
2016/02/18 11:16:14
Can we in any way guarantee CompressionCallback ca
emircan
2016/03/02 05:28:13
I followed the logic used by cast that you can fin
| |
416 compression_session_.InitializeInto()); | |
417 if (status != noErr) { | |
418 DLOG(ERROR) << " VTCompressionSessionCreate failed: " << status; | |
419 return false; | |
420 } | |
421 DVLOG(3) << " VTCompressionSession created with HW encode: " | |
422 << require_hw_encoding; | |
423 return true; | |
424 } | |
425 | |
426 bool VTVideoEncodeAccelerator::ConfigureCompressionSession() { | |
427 DCHECK(thread_checker_.CalledOnValidThread()); | |
428 DCHECK(compression_session_); | |
429 | |
430 media::video_toolbox::SessionPropertySetter session_property_setter( | |
431 compression_session_, videotoolbox_glue_); | |
432 bool rv = true; | |
433 rv &= session_property_setter.SetSessionProperty( | |
434 videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(), | |
435 videotoolbox_glue_->kVTProfileLevel_H264_Baseline_AutoLevel()); | |
436 rv &= session_property_setter.SetSessionProperty( | |
437 videotoolbox_glue_->kVTCompressionPropertyKey_RealTime(), true); | |
438 rv &= session_property_setter.SetSessionProperty( | |
439 videotoolbox_glue_->kVTCompressionPropertyKey_AllowFrameReordering(), | |
440 false); | |
441 DLOG_IF(ERROR, !rv) << " SetSessionProperty failed."; | |
442 return rv; | |
443 } | |
444 | |
445 void VTVideoEncodeAccelerator::DestroyCompressionSession() { | |
446 DCHECK(thread_checker_.CalledOnValidThread()); | |
447 | |
448 if (compression_session_) { | |
449 videotoolbox_glue_->VTCompressionSessionInvalidate(compression_session_); | |
450 compression_session_.reset(); | |
451 } | |
452 } | |
453 | |
454 } // namespace content | |
OLD | NEW |