H264 HW encode using VideoToolbox

content/common/gpu/media/vt_video_encode_accelerator_mac.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/common/gpu/media/vt_video_encode_accelerator_mac.h"
+
+#include "base/thread_task_runner_handle.h"
+#include "media/base/mac/coremedia_glue.h"
+#include "media/base/mac/corevideo_glue.h"
+#include "media/base/mac/video_frame_mac.h"
+
+namespace content {
+
+namespace {
+
+// Subjectively chosen.
+// TODO(emircan): Check if we can find the actual system capabilities via
+// creating VTCompressionSessions with varying requirements.
+// See crbug.com/584784.
+const size_t kNumInputBuffers = 1;

[jfroy, 2016/02/10 18:56:27]

This is somewhat configurable using
kVTCompressionPropertyKey_MaxFrameDelayCount. It's the maximum number of input
buffers the compression can buffer until it emits a frame. It defaults to
"unlimited", but you can set it to a known value and then report that as your
limit.

[emircan, 2016/02/11 09:22:09]

For me, setting kVTCompressionPropertyKey_MaxFrameDelayCount to any number
fails. I tried to 0-4 with no luck. I will look into this issue in a follow-up.
(OSX 10.11.3 Macbook Air 13)

+const size_t kMaxFrameRateNumerator = 30;
+const size_t kMaxFrameRateDenominator = 1;
+const size_t kMaxResolutionWidth = 4096;
+const size_t kMaxResolutionHeight = 2160;
+// The ratio of |input_visible_size| area to the max expected output
+// BitstreamBuffer size in bytes. VideoToolbox returns variable sized encoded
+// data whereas media::VideoEncodeAccelerator provides a uniform BitstreamBuffer
+// size to fill this data into. This ratio is used to determine a size that
+// would ideally be big enough to fit all frames.
+const size_t kOutputBufferSizeRatio = 10;
+const size_t kBitsPerByte = 8;
+
+}  // namespace
+
+struct VTVideoEncodeAccelerator::InProgressFrameEncode {
+  const base::TimeDelta timestamp;
+  const base::TimeTicks reference_time;
+
+  InProgressFrameEncode(base::TimeDelta rtp_timestamp, base::TimeTicks ref_time)
+      : timestamp(rtp_timestamp), reference_time(ref_time) {}
+
+ private:
+  DISALLOW_IMPLICIT_CONSTRUCTORS(InProgressFrameEncode);
+};
+
+struct VTVideoEncodeAccelerator::BitstreamBufferRef {
+  BitstreamBufferRef(int32_t id,
+                     scoped_ptr<base::SharedMemory> shm,
+                     size_t size)
+      : id(id), shm(std::move(shm)), size(size) {}
+  const int32_t id;
+  const scoped_ptr<base::SharedMemory> shm;
+  const size_t size;
+
+ private:
+  DISALLOW_IMPLICIT_CONSTRUCTORS(BitstreamBufferRef);
+};
+
+VTVideoEncodeAccelerator::VTVideoEncodeAccelerator()
+    : client_task_runner_(base::ThreadTaskRunnerHandle::Get()) {
+}
+
+VTVideoEncodeAccelerator::~VTVideoEncodeAccelerator() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+}
+
+media::VideoEncodeAccelerator::SupportedProfiles
+VTVideoEncodeAccelerator::GetSupportedProfiles() {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  SupportedProfiles profiles;
+  SupportedProfile profile;
+  profile.profile = media::H264PROFILE_BASELINE;
+  profile.max_framerate_numerator = kMaxFrameRateNumerator;
+  profile.max_framerate_denominator = kMaxFrameRateDenominator;
+  profile.max_resolution = gfx::Size(kMaxResolutionWidth, kMaxResolutionHeight);
+  profiles.push_back(profile);
+  return profiles;
+}
+
+bool VTVideoEncodeAccelerator::Initialize(
+    media::VideoPixelFormat format,
+    const gfx::Size& input_visible_size,
+    media::VideoCodecProfile output_profile,
+    uint32_t initial_bitrate,
+    Client* client) {
+  DVLOG(3) << __FUNCTION__
+           << ": input_format=" << media::VideoPixelFormatToString(format)
+           << ", input_visible_size=" << input_visible_size.ToString()
+           << ", output_profile=" << output_profile
+           << ", initial_bitrate=" << initial_bitrate;
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(client);
+
+  if (media::PIXEL_FORMAT_I420 != format) {

[jfroy, 2016/02/10 18:56:27]

You can also support NV12 pretty easily. I don't know if that's a common format
in Chrome.

[emircan, 2016/02/11 09:22:09]

I420 is the common format in Chrome HW encode.

+    DLOG(ERROR) << "Input format not supported= "
+                << media::VideoPixelFormatToString(format);
+    return false;
+  }
+  if (media::H264PROFILE_BASELINE != output_profile) {
+    DLOG(ERROR) << "Output profile not supported= "
+                << output_profile;
+    return false;
+  }
+
+  videotoolbox_glue_ = VideoToolboxGlue::Get();
+  if (!videotoolbox_glue_) {
+    DLOG(ERROR) << "Failed creating VideoToolbox glue";
+    return false;
+  }
+
+  client_ptr_factory_.reset(new base::WeakPtrFactory<Client>(client));
+  client_ = client_ptr_factory_->GetWeakPtr();
+  bitrate_ = initial_bitrate;
+  input_visible_size_ = input_visible_size;
+
+  if (!ResetCompressionSession()) {
+    DLOG(ERROR) << "Failed creating compression session";
+    return false;
+  }
+
+  client_->RequireBitstreamBuffers(
+      kNumInputBuffers, input_visible_size_,
+      std::max(input_visible_size_.GetArea() / kOutputBufferSizeRatio,
+               bitrate_ / kBitsPerByte));

[miu, 2016/02/10 21:04:56]

Sanity-check: Is the bitrate_ setting a "max bitrate" or a "target bitrate".  If
it's a target, there will be some variance where some frames take more or fewer
bytes than the average; and so you'd want the value here to be some multiple. 
If it's a max, do we know the encoder absolutely cannot exceed the max bitrate
in certain edge-case circumstances?

[jfroy, 2016/02/10 23:15:34]

Myself and lite@ did extensive tests of the iOS encoder's bitrate control. I
can't really speak to how VT will behave on OS X using QuickSync, but...

The AverageBitRate property (currently used by this implementation) really means
an average. The observed bitrate over a 1-5 seconds window was observed as much
as 200% higher than this average. This is especially true when the encoder is
configured for real-time use.

The DataRateLimits property (see crbug.com/425352) is better at setting the
maximum bitrate, within about 10% over a 1-5 seconds window.

[emircan, 2016/02/11 09:22:09]

I will go with jfroy@ suggestions. I will remove the use of 
kOutputBufferSizeRatio. Instead, I will define an average_bitrate_ and
max_bitrate_ params. I will take client input as average_bitrate_, and calculate
max_bitrate_ via a constant. WDYT?

[jfroy, 2016/02/11 18:27:40]

I would really run a few experiments to see how your rate control is working. VT
tries to be a uniform interface to widely different encoders, so those
parameters might work is a surprising and different way than on iOS.

For what it's worth, in cast we assume the bitrate specified by the client (i.e.
the target rate calculated by cast's rate control algorithm) is where it wants
the encoder to go and we set both the average bit rate and target rate
parameters to the same value (after converting to the right units -- the target
data rate takes bytes, not bits).

[miu, 2016/02/11 21:09:40]

I can't remember whether the VEA interface treats bitrate as a target or a max. 
Pawel would know for sure.

[emircan, 2016/02/12 03:40:55]

I did some further investigation as jfroy@ suggested. I found that when HW
encoder is actually used (see CreateCompressionSession()), setting
kVTCompressionPropertyKey_DataRateLimits parameter ends up giving an error: GVA
encoder warning: CFDataRateLimitType = 3 (DataRateLimitsSeconds) . Changing the
parameter doesn't make any difference  at that point. Googling
CFDataRateLimitType returns only a single result which isn't helpful, so I
cannot rely on the param as a max_bitrate_.

kVTCompressionPropertyKey_AverageBitRate is the only reliable one. Therefore, I
will use the given bitrate_ input there as the average, and on
kVTCompressionPropertyKey_DataRateLimits when it is available. Also, I will
decide BitstreamBuffer size based on (bitrate/8) rather than using other
subjective constants. It is big enough to cover most of the output cases. There
seems to be a frame once in ~5 seconds that is more than double the buffer size.

[Pawel Osciak, 2016/02/18 11:16:14]

It's not clearly specified in the docs, but it's expected to be the target
bitrate.

+  return true;
+}
+
+void VTVideoEncodeAccelerator::Encode(
+    const scoped_refptr<media::VideoFrame>& frame,
+    bool force_keyframe) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(compression_session_);
+  DCHECK(frame);
+
+  base::TimeTicks ref_time;
+  if (!frame->metadata()->GetTimeTicks(
+          media::VideoFrameMetadata::REFERENCE_TIME, &ref_time)) {
+    ref_time = base::TimeTicks::Now();
+  }
+  auto timestamp_cm = CoreMediaGlue::CMTimeMake(
+      frame->timestamp().InMicroseconds(), USEC_PER_SEC);
+  // Wrap information we'll need after the frame is encoded in a heap object.
+  // We'll get the pointer back from the VideoToolbox completion callback.
+  scoped_ptr<InProgressFrameEncode> request(new InProgressFrameEncode(
+      frame->timestamp(), ref_time));
+
+  // TODO(emircan): See if we can eliminate a copy here by using
+  // CVPixelBufferPool for the allocation of incoming VideoFrames.
+  base::ScopedCFTypeRef<CVPixelBufferRef> pixel_buffer =
+      media::WrapVideoFrameInCVPixelBuffer(*frame);
+  base::ScopedCFTypeRef<CFDictionaryRef> frame_props =
+      media::video_toolbox::DictionaryWithKeyValue(
+          videotoolbox_glue_->kVTEncodeFrameOptionKey_ForceKeyFrame(),
+          force_keyframe ? kCFBooleanTrue : kCFBooleanFalse);
+
+  OSStatus status = videotoolbox_glue_->VTCompressionSessionEncodeFrame(
+      compression_session_, pixel_buffer, timestamp_cm,
+      CoreMediaGlue::CMTime{0, 0, 0, 0}, frame_props,
+      reinterpret_cast<void*>(request.release()), nullptr);
+  if (status != noErr) {
+    DLOG(ERROR) << " VTCompressionSessionEncodeFrame failed: " << status;
+    client_->NotifyError(kPlatformFailureError);
+  }
+}
+
+void VTVideoEncodeAccelerator::UseOutputBitstreamBuffer(
+    const media::BitstreamBuffer& buffer) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+  if (buffer.size() < static_cast<size_t>(input_visible_size_.GetArea() /
+                                          kOutputBufferSizeRatio)) {
+    DLOG(ERROR) << "Output BitstreamBuffer isn't big enough: "
+                << buffer.size()
+                << " vs. "
+                << static_cast<size_t>(input_visible_size_.GetArea() /
+                                       kOutputBufferSizeRatio);
+    client_->NotifyError(kInvalidArgumentError);
+    return;
+  }
+
+  scoped_ptr<base::SharedMemory> shm(
+      new base::SharedMemory(buffer.handle(), false));
+  if (!shm->Map(buffer.size())) {
+    DLOG(ERROR) << "Failed mapping shared memory.";
+    client_->NotifyError(kPlatformFailureError);
+    return;
+  }
+
+  // If there are already CMSampleBufferRef waiting, copy their output first.
+  if (!encoder_output_sample_buffer_queue_.empty()) {
+    CMSampleBufferRef sbuf = encoder_output_sample_buffer_queue_.front();
+    encoder_output_sample_buffer_queue_.pop_front();
+
+    auto sample_attachments =
+        static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(
+            CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true),
+            0));
+    const bool keyframe = !CFDictionaryContainsKey(
+                              sample_attachments,
+                              CoreMediaGlue::kCMSampleAttachmentKey_NotSync());
+    size_t used_buffer_size = 0;
+    const bool copy_rv = media::video_toolbox::CopySampleBufferToAnnexBBuffer(
+        sbuf, keyframe, buffer.size(),
+        reinterpret_cast<uint8_t*>(shm->memory()), &used_buffer_size);
+    CFRelease(sbuf);
+    if (!copy_rv)
+      used_buffer_size = 0;
+    client_->BitstreamBufferReady(buffer.id(), used_buffer_size, keyframe);
+    return;
+  }
+
+  scoped_ptr<BitstreamBufferRef> buffer_ref(
+      new BitstreamBufferRef(buffer.id(), std::move(shm), buffer.size()));
+  encoder_output_queue_.push_back(std::move(buffer_ref));
+}
+
+void VTVideoEncodeAccelerator::RequestEncodingParametersChange(
+    uint32_t bitrate,
+    uint32_t framerate) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  bitrate_ = bitrate > 1 ? bitrate : 1;
+
+  if (!compression_session_) {
+    client_->NotifyError(kPlatformFailureError);
+    return;
+  }
+  // TODO(emircan): VideoToolbox does not seem to support bitrate

[jfroy, 2016/02/10 18:56:27]

See my update to that bug. You can actually control the bitrate at runtime using
2 properties.

[emircan, 2016/02/11 09:22:09]

Done.

+  // reconfiguration, see crbug.com/425352.
+  const bool rv = session_property_setter_->SetSessionProperty(

[miu, 2016/02/10 21:04:56]

Suggestion (to eliminate extra heap-allocated data member):

    media::video_toolbox::SessionPropertySetter(compression_session_,
videotoolbox_glue_)
        .SetSessionProperty(...);

See explanation in comment below...

[emircan, 2016/02/11 09:22:09]

Done. Keeping it as a class member on stack.

+      videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(),
+      static_cast<int32_t>(bitrate_));
+  if (!rv) {
+    DLOG(ERROR) << "Couldn't change session bitrate.";
+  }
+}
+
+void VTVideoEncodeAccelerator::Destroy() {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  DestroyCompressionSession();
+  delete this;
+}
+
+// static
+void VTVideoEncodeAccelerator::CompressionCallback(void* encoder_opaque,
+                                                   void* request_opaque,
+                                                   OSStatus status,
+                                                   VTEncodeInfoFlags info,
+                                                   CMSampleBufferRef sbuf) {
+  // This function may be called asynchronously, on a different thread from the
+  // one that calls VTCompressionSessionEncodeFrame.
+  DVLOG(3) << __FUNCTION__;
+
+  auto encoder = reinterpret_cast<VTVideoEncodeAccelerator*>(encoder_opaque);
+  DCHECK(encoder);
+
+  if (status != noErr) {
+    DLOG(ERROR) << " encode failed: " << status;
+    encoder->client_task_runner_->PostTask(
+        FROM_HERE, base::Bind(&Client::NotifyError, encoder->client_,
+                              kPlatformFailureError));
+    return;
+  }
+
+  // Release InProgressFrameEncode, since we don't have support to return
+  // timestamps at this point.
+  scoped_ptr<InProgressFrameEncode> request(

[miu, 2016/02/10 21:04:56]

This is happening after a possible return statement (above, on line 268).  In
order to avoid leaking memory, it should be moved to before the if-statement
above.

[emircan, 2016/02/11 09:22:09]

Done.

+      reinterpret_cast<InProgressFrameEncode*>(request_opaque));
+  request.reset();
+
+  // CFRetain is required to hold onto CMSampleBufferRef when posting task
+  // between threads. The object should be released later using CFRelease.
+  CFRetain(sbuf);
+  // This method is NOT called on |client_task_runner_|, so we still need to
+  // post a task back to it to reach |client_|.
+  encoder->client_task_runner_->PostTask(
+      FROM_HERE,
+      base::Bind(&VTVideoEncodeAccelerator::CompressionCallbackTask,
+                 base::Unretained(encoder), info, sbuf));
+}
+
+void VTVideoEncodeAccelerator::CompressionCallbackTask(VTEncodeInfoFlags info,
+                                                       CMSampleBufferRef sbuf) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  // If there isn't any BitstreamBuffer to copy into, add it to a queue for
+  // later use.
+  if (encoder_output_queue_.empty()) {
+    encoder_output_sample_buffer_queue_.push_back(sbuf);
+    return;

[miu, 2016/02/10 21:04:56]

Same problem w.r.t. early return.  The client should get a
BitstreamBufferReady() call with zero-byte payload so it knows which Encode()'s
failed.

[emircan, 2016/02/11 09:22:10]

I do not have a BitstreamBuffer to return at this case. This is Encoder()
followed by UseBitstreamBuffer() case that posciak@ mentioned. If I return
buffer->id=-1, this is same as kPlatformFailureError. So instead, I wait for a
buffer to arrive first.

[miu, 2016/02/11 21:09:40]

Acknowledged.  My misunderstanding.

+  }
+
+  bool frame_dropped = false;
+  if (info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped) {
+    DVLOG(2) << " frame dropped";
+    frame_dropped = true;

[miu, 2016/02/10 21:04:56]

Looks good here.  Just need to do this above too.

[emircan, 2016/02/11 09:22:09]

Done.

+  }
+
+  auto sample_attachments = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(

[jfroy, 2016/02/10 18:56:27]

This code seems duplicated from UseOutputBitstreamBuffer(). It would be good to
refactor it, since it's a bit subtle.

[emircan, 2016/02/11 09:22:09]

Done.

+      CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true), 0));
+  const bool keyframe =
+      !CFDictionaryContainsKey(sample_attachments,
+                               CoreMediaGlue::kCMSampleAttachmentKey_NotSync());
+
+  scoped_ptr<VTVideoEncodeAccelerator::BitstreamBufferRef> buffer_ref =
+      std::move(encoder_output_queue_.front());
+  encoder_output_queue_.pop_front();
+
+  size_t used_buffer_size = 0;
+  if (!frame_dropped) {
+    const bool copy_rv = media::video_toolbox::CopySampleBufferToAnnexBBuffer(
+        sbuf, keyframe, buffer_ref->size,
+        reinterpret_cast<uint8_t*>(buffer_ref->shm->memory()),
+        &used_buffer_size);
+    CFRelease(sbuf);
+    if (!copy_rv) {
+      DLOG(ERROR) << "Cannot copy output from SampleBuffer to AnnexBBuffer.";
+      used_buffer_size = 0;
+    }
+  }
+
+  client_->BitstreamBufferReady(buffer_ref->id, used_buffer_size, keyframe);
+}
+
+bool VTVideoEncodeAccelerator::ResetCompressionSession() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  DestroyCompressionSession();
+
+  base::ScopedCFTypeRef<CFDictionaryRef> encoder_spec =
+      media::video_toolbox::DictionaryWithKeyValue(videotoolbox_glue_
+        ->kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder(),

[jfroy, 2016/02/10 18:56:27]

Unless you want to fallback to Apple's *TERRIBLE* software encoder, I suggest
you also specify RequireHardwareAcceleratedVideoEncoder. If you can't use the
hardware encoder, you'd be much better served falling back on Chromium's
software encoder.

[emircan, 2016/02/11 09:22:09]

Thanks for the notice, I added RequireHardwareAcceleratedVideoEncoder. I will
compare this non-HW case against the SW implementation once it lands, and
revisit the fallback situation.

+      kCFBooleanTrue);
+
+  // Keep these in-sync with those in ConfigureCompressionSession().
+  CFTypeRef attributes_keys[] = {
+#if defined(OS_IOS)
+    kCVPixelBufferOpenGLESCompatibilityKey,
+#else
+    kCVPixelBufferOpenGLCompatibilityKey,
+#endif
+    kCVPixelBufferIOSurfacePropertiesKey,
+    kCVPixelBufferPixelFormatTypeKey
+  };
+  const int format[] = {
+      CoreVideoGlue::kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange};
+  CFTypeRef attributes_values[] = {
+      kCFBooleanTrue,
+      media::video_toolbox::DictionaryWithKeysAndValues(nullptr, nullptr, 0)
+          .release(),
+      media::video_toolbox::ArrayWithIntegers(format, arraysize(format))
+          .release()};
+  const base::ScopedCFTypeRef<CFDictionaryRef> attributes =
+      media::video_toolbox::DictionaryWithKeysAndValues(
+          attributes_keys, attributes_values, arraysize(attributes_keys));
+  for (auto& v : attributes_values)
+    CFRelease(v);
+
+  // Create the compression session.
+  OSStatus status = videotoolbox_glue_->VTCompressionSessionCreate(
+      kCFAllocatorDefault,
+      input_visible_size_.width(),
+      input_visible_size_.height(),
+      CoreMediaGlue::kCMVideoCodecType_H264,
+      encoder_spec,
+      attributes,
+      nullptr /* compressedDataAllocator */,
+      &VTVideoEncodeAccelerator::CompressionCallback,
+      reinterpret_cast<void*>(this),
+      compression_session_.InitializeInto());
+  if (status != noErr) {
+    DLOG(ERROR) << " VTCompressionSessionCreate failed: " << status;
+    return false;
+  }
+
+  return ConfigureCompressionSession();
+}
+
+bool VTVideoEncodeAccelerator::ConfigureCompressionSession() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(compression_session_);
+
+  session_property_setter_.reset(
+      new media::video_toolbox::SessionPropertySetter(compression_session_,
+                                                      videotoolbox_glue_));
+  bool rv = true;
+  rv &= session_property_setter_->SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(),
+      videotoolbox_glue_->kVTProfileLevel_H264_Baseline_AutoLevel());
+  rv &= session_property_setter_->SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_RealTime(), true);
+  rv &= session_property_setter_->SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(),
+      static_cast<int32_t>(bitrate_));
+  rv &= session_property_setter_->SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_AllowFrameReordering(),
+      false);
+  DLOG_IF(ERROR, !rv) << " SetSessionProperty failed.";
+  return rv;
+}
+
+void VTVideoEncodeAccelerator::DestroyCompressionSession() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  if (compression_session_) {
+    videotoolbox_glue_->VTCompressionSessionInvalidate(compression_session_);
+    compression_session_.reset();
+  }
+}
+
+} // namespace content