H264 HW encode using VideoToolbox

content/common/gpu/media/vt_video_encode_accelerator_mac.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/common/gpu/media/vt_video_encode_accelerator_mac.h"
+
+#include "base/thread_task_runner_handle.h"
+#include "media/base/mac/coremedia_glue.h"
+#include "media/base/mac/corevideo_glue.h"
+#include "media/base/mac/video_frame_mac.h"
+
+namespace content {
+
+namespace {
+
+// Subjectively chosen.
+// TODO(emircan): Check if we can find the actual system capabilities via
+// creating VTCompressionSessions with varying requirements.
+// See crbug.com/584784.
+const size_t kNumInputBuffers = 1;
+const size_t kMaxFrameRateNumerator = 30;
+const size_t kMaxFrameRateDenominator = 1;
+const size_t kMaxResolutionWidth = 4096;
+const size_t kMaxResolutionHeight = 2160;
+// The ratio of |input_visible_size| area to the max expected output
+// BitstreamBuffer size in bytes. VideoToolbox returns variable sized encoded
+// data whereas media::VideoEncodeAccelerator provides a uniform BitstreamBuffer
+// size to fill this data into. This ratio is used to determine a size that
+// would ideally be big enough to fit all frames.
+const size_t kOutputBufferSizeRatio = 10;

[miu, 2016/02/09 23:29:22]

Instead of a constant, shouldn't this depend on the configured bit rate?  e.g.,
2 Mbps versus 20 Mbps

[emircan, 2016/02/10 05:21:53]

I tried doing that but it would be very dependent on initial_bitrate, since we
request for Bitstream buffers once only in Initialize(). I will take max of
these two values to be safe. WDYT?

[miu, 2016/02/10 21:04:55]

Sounds fine.  I'll leave it to the VEA owners to work out the larger issue
(affecting all VEA's) in some future change.

+
+}  // namespace
+
+struct VTVideoEncodeAccelerator::InProgressFrameEncode {
+  const base::TimeDelta timestamp;
+  const base::TimeTicks reference_time;
+
+  InProgressFrameEncode(base::TimeDelta rtp_timestamp, base::TimeTicks ref_time)
+      : timestamp(rtp_timestamp), reference_time(ref_time) {}
+
+ private:
+  DISALLOW_IMPLICIT_CONSTRUCTORS(InProgressFrameEncode);
+};
+
+struct VTVideoEncodeAccelerator::BitstreamBufferRef {
+  BitstreamBufferRef(int32_t id,
+                     scoped_ptr<base::SharedMemory> shm,
+                     size_t size)
+      : id(id), shm(std::move(shm)), size(size) {}
+  const int32_t id;
+  const scoped_ptr<base::SharedMemory> shm;
+  const size_t size;
+
+ private:
+  DISALLOW_IMPLICIT_CONSTRUCTORS(BitstreamBufferRef);
+};
+
+VTVideoEncodeAccelerator::VTVideoEncodeAccelerator()
+    : client_task_runner_(base::ThreadTaskRunnerHandle::Get()) {
+}
+
+VTVideoEncodeAccelerator::~VTVideoEncodeAccelerator() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+}
+
+media::VideoEncodeAccelerator::SupportedProfiles
+VTVideoEncodeAccelerator::GetSupportedProfiles() {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  SupportedProfiles profiles;
+  SupportedProfile profile;
+  profile.profile = media::H264PROFILE_BASELINE;
+  profile.max_framerate_numerator = kMaxFrameRateNumerator;
+  profile.max_framerate_denominator = kMaxFrameRateDenominator;
+  profile.max_resolution = gfx::Size(kMaxResolutionWidth, kMaxResolutionHeight);
+  profiles.push_back(profile);
+  return profiles;
+}
+
+bool VTVideoEncodeAccelerator::Initialize(
+    media::VideoPixelFormat format,
+    const gfx::Size& input_visible_size,
+    media::VideoCodecProfile output_profile,
+    uint32_t initial_bitrate,
+    Client* client) {
+  DVLOG(3) << __FUNCTION__
+           << ": input_format=" << media::VideoPixelFormatToString(format)
+           << ", input_visible_size=" << input_visible_size.ToString()
+           << ", output_profile=" << output_profile
+           << ", initial_bitrate=" << initial_bitrate;
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(client);
+
+  if (media::PIXEL_FORMAT_I420 != format) {
+    DLOG(ERROR) << "Input format not supported= "
+                << media::VideoPixelFormatToString(format);
+    return false;
+  }
+  if (media::H264PROFILE_BASELINE != output_profile) {
+    DLOG(ERROR) << "Output profile not supported= "
+                << output_profile;
+    return false;
+  }
+
+  videotoolbox_glue_ = VideoToolboxGlue::Get();
+  if (!videotoolbox_glue_) {
+    DLOG(ERROR) << "Failed creating VideoToolbox glue";
+    return false;
+  }
+
+  client_ptr_factory_.reset(new base::WeakPtrFactory<Client>(client));
+  client_ = client_ptr_factory_->GetWeakPtr();
+  bitrate_ = initial_bitrate;
+  input_visible_size_ = input_visible_size;
+
+  if (!ResetCompressionSession()) {
+    DLOG(ERROR) << "Failed creating compression session";
+    return false;
+  }
+
+  client_->RequireBitstreamBuffers(
+      kNumInputBuffers, input_visible_size_,
+      input_visible_size_.GetArea() / kOutputBufferSizeRatio);
+  return true;
+}
+
+void VTVideoEncodeAccelerator::Encode(
+    const scoped_refptr<media::VideoFrame>& frame,
+    bool force_keyframe) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(compression_session_);
+  DCHECK(frame);
+
+  base::TimeTicks ref_time;
+  if (!frame->metadata()->GetTimeTicks(
+          media::VideoFrameMetadata::REFERENCE_TIME, &ref_time)) {
+    ref_time = base::TimeTicks::Now();
+  }
+  auto timestamp_cm = CoreMediaGlue::CMTimeMake(
+      frame->timestamp().InMicroseconds(), USEC_PER_SEC);
+  // Wrap information we'll need after the frame is encoded in a heap object.
+  // We'll get the pointer back from the VideoToolbox completion callback.
+  scoped_ptr<InProgressFrameEncode> request(new InProgressFrameEncode(
+      frame->timestamp(), ref_time));
+
+  // TODO(emircan): See if we can eliminate a copy here by using
+  // CVPixelBufferPool for the allocation of incoming VideoFrames.
+  base::ScopedCFTypeRef<CVPixelBufferRef> pixel_buffer =
+      media::WrapVideoFrameInCVPixelBuffer(*frame);
+  base::ScopedCFTypeRef<CFDictionaryRef> frame_props =
+      media::video_toolbox::DictionaryWithKeyValue(
+          videotoolbox_glue_->kVTEncodeFrameOptionKey_ForceKeyFrame(),
+          force_keyframe ? kCFBooleanTrue : kCFBooleanFalse);
+
+  OSStatus status = videotoolbox_glue_->VTCompressionSessionEncodeFrame(
+      compression_session_, pixel_buffer, timestamp_cm,
+      CoreMediaGlue::CMTime{0, 0, 0, 0}, frame_props,
+      reinterpret_cast<void*>(request.release()), nullptr);

[miu, 2016/02/09 23:29:22]

I could be mistaken, but it seems memory is being leaked here.  What
uses/destroys the InProgressFrameEncode object?

[emircan, 2016/02/10 05:21:53]

Thanks for pointing out. I will make a scoped_ptr to release it properly in the
callback.

+  if (status != noErr) {
+    DLOG(ERROR) << " VTCompressionSessionEncodeFrame failed: " << status;
+    client_->NotifyError(kPlatformFailureError);
+  }
+}
+
+void VTVideoEncodeAccelerator::UseOutputBitstreamBuffer(
+    const media::BitstreamBuffer& buffer) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+  if (buffer.size() < static_cast<size_t>(input_visible_size_.GetArea() /
+                                          kOutputBufferSizeRatio)) {
+    DLOG(ERROR) << "Output BitstreamBuffer isn't big enough: "
+                << buffer.size()
+                << " vs. "
+                << static_cast<size_t>(input_visible_size_.GetArea() /
+                                       kOutputBufferSizeRatio);
+    client_->NotifyError(kInvalidArgumentError);
+    return;
+  }
+
+  scoped_ptr<base::SharedMemory> shm(
+      new base::SharedMemory(buffer.handle(), false));
+  if (!shm->Map(buffer.size())) {
+    DLOG(ERROR) << "Failed mapping shared memory.";
+    client_->NotifyError(kPlatformFailureError);
+    return;
+  }
+
+  // If there are already CMSampleBufferRef waiting, copy their output first.
+  if (!encoder_output_sample_buffer_queue_.empty()) {
+    CMSampleBufferRef sbuf = encoder_output_sample_buffer_queue_.front();
+    encoder_output_sample_buffer_queue_.pop_front();
+
+    auto sample_attachments =
+        static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(
+            CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true),
+            0));
+    const bool keyframe = !CFDictionaryContainsKey(
+                              sample_attachments,
+                              CoreMediaGlue::kCMSampleAttachmentKey_NotSync());
+    size_t used_buffer_size = 0;
+    const bool copy_rv = media::video_toolbox::CopySampleBufferToAnnexBBuffer(
+        sbuf, reinterpret_cast<uint8_t*>(shm->memory()), buffer.size(),
+        keyframe, &used_buffer_size);
+    CFRelease(sbuf);
+    if (copy_rv) {
+      client_->BitstreamBufferReady(buffer.id(), used_buffer_size, keyframe);
+      return;
+    }
+  }
+
+  scoped_ptr<BitstreamBufferRef> buffer_ref(
+      new BitstreamBufferRef(buffer.id(), std::move(shm), buffer.size()));
+  encoder_output_queue_.push_back(std::move(buffer_ref));
+}
+
+void VTVideoEncodeAccelerator::RequestEncodingParametersChange(
+    uint32_t bitrate,
+    uint32_t framerate) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  bitrate_ = bitrate > 1 ? bitrate : 1;
+
+  if (!compression_session_) {
+    client_->NotifyError(kPlatformFailureError);
+    return;
+  }
+  // TODO(emircan): VideoToolbox does not seem to support bitrate
+  // reconfiguration, see crbug.com/425352.
+  const bool rv = session_property_setter_->SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(),
+      static_cast<int32_t>(bitrate_));
+  if (!rv) {
+    DLOG(ERROR) << "Couldn't change session bitrate.";
+  }
+}
+
+void VTVideoEncodeAccelerator::Destroy() {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  DestroyCompressionSession();
+  delete this;
+}
+
+// static
+void VTVideoEncodeAccelerator::CompressionCallback(void* encoder_opaque,
+                                                   void* request_opaque,
+                                                   OSStatus status,
+                                                   VTEncodeInfoFlags info,
+                                                   CMSampleBufferRef sbuf) {
+  // This function may be called asynchronously, on a different thread from the
+  // one that calls VTCompressionSessionEncodeFrame.
+  DVLOG(3) << __FUNCTION__;
+
+  auto encoder = reinterpret_cast<VTVideoEncodeAccelerator*>(encoder_opaque);
+  DCHECK(encoder);
+
+  if (status != noErr) {
+    DLOG(ERROR) << " encode failed: " << status;
+    encoder->client_task_runner_->PostTask(
+        FROM_HERE, base::Bind(&Client::NotifyError, encoder->client_,
+                              kPlatformFailureError));
+    return;
+  }
+
+  if (info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped) {
+    DVLOG(2) << " frame dropped";
+    return;
+  }
+
+  // CFRetain is required to hold onto CMSampleBufferRef when posting task
+  // between threads. The object should be released later using CFRelease.
+  CFRetain(sbuf);
+  // This method is NOT called on |client_task_runner_|, so we still need to
+  // post a task back to it to reach |client_|.
+  encoder->client_task_runner_->PostTask(
+      FROM_HERE,
+      base::Bind(&VTVideoEncodeAccelerator::CompressionCallbackTask,
+                 base::Unretained(encoder), sbuf));
+}
+
+void VTVideoEncodeAccelerator::CompressionCallbackTask(CMSampleBufferRef sbuf) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  // If there isn't any BitstreamBuffer to copy into, add it to a queue for
+  // later use.
+  if (encoder_output_queue_.empty()) {
+    encoder_output_sample_buffer_queue_.push_back(sbuf);
+    return;
+  }
+
+  auto sample_attachments = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(
+      CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true), 0));
+  const bool keyframe =
+      !CFDictionaryContainsKey(sample_attachments,
+                               CoreMediaGlue::kCMSampleAttachmentKey_NotSync());
+
+  scoped_ptr<VTVideoEncodeAccelerator::BitstreamBufferRef> buffer_ref =
+      std::move(encoder_output_queue_.front());
+  encoder_output_queue_.pop_front();
+
+  size_t used_buffer_size = 0;
+  const bool copy_rv = media::video_toolbox::CopySampleBufferToAnnexBBuffer(
+      sbuf, reinterpret_cast<uint8_t*>(buffer_ref->shm->memory()),
+      buffer_ref->size, keyframe, &used_buffer_size);
+  CFRelease(sbuf);
+  if (!copy_rv) {
+    DLOG(ERROR) << "Cannot copy output from SampleBuffer to AnnexBBuffer.";
+    encoder_output_queue_.push_back(std::move(buffer_ref));
+    return;
+  }
+
+  client_->BitstreamBufferReady(buffer_ref->id, used_buffer_size, keyframe);
+}
+
+bool VTVideoEncodeAccelerator::ResetCompressionSession() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  DestroyCompressionSession();
+
+  base::ScopedCFTypeRef<CFDictionaryRef> encoder_spec =
+      media::video_toolbox::DictionaryWithKeyValue(videotoolbox_glue_
+        ->kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder(),
+      kCFBooleanTrue);
+
+  // Keep these in-sync with those in ConfigureCompressionSession().
+  CFTypeRef attributes_keys[] = {
+#if defined(OS_IOS)
+    kCVPixelBufferOpenGLESCompatibilityKey,
+#else
+    kCVPixelBufferOpenGLCompatibilityKey,
+#endif
+    kCVPixelBufferIOSurfacePropertiesKey,
+    kCVPixelBufferPixelFormatTypeKey
+  };
+  const int format[] = {
+      CoreVideoGlue::kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange};
+  CFTypeRef attributes_values[] = {
+      kCFBooleanTrue,
+      media::video_toolbox::DictionaryWithKeysAndValues(nullptr, nullptr, 0)
+          .release(),
+      media::video_toolbox::ArrayWithIntegers(format, arraysize(format))
+          .release()};
+  const base::ScopedCFTypeRef<CFDictionaryRef> attributes =
+      media::video_toolbox::DictionaryWithKeysAndValues(
+          attributes_keys, attributes_values, arraysize(attributes_keys));
+  for (auto& v : attributes_values)
+    CFRelease(v);
+
+  // Create the compression session.
+  OSStatus status = videotoolbox_glue_->VTCompressionSessionCreate(
+      kCFAllocatorDefault,
+      input_visible_size_.width(),
+      input_visible_size_.height(),
+      CoreMediaGlue::kCMVideoCodecType_H264,
+      encoder_spec,
+      attributes,
+      nullptr /* compressedDataAllocator */,
+      &VTVideoEncodeAccelerator::CompressionCallback,
+      reinterpret_cast<void*>(this),
+      compression_session_.InitializeInto());
+  if (status != noErr) {
+    DLOG(ERROR) << " VTCompressionSessionCreate failed: " << status;
+    return false;
+  }
+
+  return ConfigureCompressionSession();
+}
+
+bool VTVideoEncodeAccelerator::ConfigureCompressionSession() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(compression_session_);
+
+  session_property_setter_.reset(

[miu, 2016/02/09 23:29:22]

This shouldn't be allocated for the lifetime of the class, since it's only used
briefly in this method (which is not being called very often).  Instead, just
make it a local stack variable:

  video_toolbox::SessionPropertySetter setter(...);
  setter.SetSessionProperty(...);
  ...

[emircan, 2016/02/10 05:21:53]

It is used on l.233 as well. Following example:
https://code.google.com/p/chromium/codesearch#chromium/src/third_party/webrtc...
.

[miu, 2016/02/10 21:04:55]

The point is that the optimizing compiler can do a lot to eliminate extra
storage, skip steps w.r.t. construction/destruction, etc. when the class is a
local variable.  When it's heap-allocated, it can't.  Also, by making this a
data member, memory is being wasted to hold an object that is only rarely and
briefly used.

+      new media::video_toolbox::SessionPropertySetter(compression_session_,
+                                                      videotoolbox_glue_));
+  bool rv = true;
+  rv &= session_property_setter_->SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(),
+      videotoolbox_glue_->kVTProfileLevel_H264_Baseline_AutoLevel());
+  rv &= session_property_setter_->SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_RealTime(), true);
+  rv &= session_property_setter_->SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(),
+      static_cast<int32_t>(bitrate_));
+  rv &= session_property_setter_->SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_AllowFrameReordering(),
+      false);
+  DLOG_IF(ERROR, !rv) << " SetSessionProperty failed.";
+  return rv;
+}
+
+void VTVideoEncodeAccelerator::DestroyCompressionSession() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  if (compression_session_) {
+    videotoolbox_glue_->VTCompressionSessionInvalidate(compression_session_);
+    compression_session_.reset();
+  }
+}
+
+} // namespace content