H264 HW encode using VideoToolbox

content/common/gpu/media/vt_video_encode_accelerator.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/common/gpu/media/vt_video_encode_accelerator.h"
+
+#include "base/thread_task_runner_handle.h"
+#include "media/base/mac/coremedia_glue.h"
+#include "media/base/mac/corevideo_glue.h"
+#include "media/base/mac/video_frame_mac.h"
+#include "media/base/mac/videotoolbox_helpers.h"
+
+namespace content {
+
+namespace {
+
+// Subjectively chosen.
+const size_t kNumInputBuffers = 4;
+const size_t kMaxFrameRateNumerator = 30;
+const size_t kMaxFrameRateDenominator = 1;
+const size_t kMaxResolutionWidth = 4096;
+const size_t kMaxResolutionHeight = 2160;
+const size_t kOutputBufferSizeRatio = 10;
+
+// Container for the associated data of a video frame being processed.
+struct InProgressFrameEncode {
+  const base::TimeDelta timestamp;
+  const base::TimeTicks reference_time;
+
+  InProgressFrameEncode(
+      base::TimeDelta rtp_timestamp,
+      base::TimeTicks ref_time)
+      : timestamp(rtp_timestamp),
+        reference_time(ref_time) {}
+};

[mcasas, 2016/02/03 20:01:31]

Recommend a 
private:
 DISALLOW_IMPLICIT_CONSTRUCTORS(InProgressFrameEncode);

[emircan, 2016/02/04 05:18:52]

Done.

+
+}  // namespace
+
+struct VTVideoEncodeAccelerator::BitstreamBufferRef {
+  BitstreamBufferRef(int32_t id,
+                     scoped_ptr<base::SharedMemory> shm,
+                     size_t size)
+      : id(id), shm(std::move(shm)), size(size) {}
+  const int32_t id;
+  const scoped_ptr<base::SharedMemory> shm;
+  const size_t size;
+};

[mcasas, 2016/02/03 20:01:31]

Same here, use DISALLOW_IMPLICIT_CONSTRUCTORS

[emircan, 2016/02/04 05:18:51]

Done.

+
+VTVideoEncodeAccelerator::VTVideoEncodeAccelerator()
+    : videotoolbox_glue_(VideoToolboxGlue::Get()),
+      child_task_runner_(base::ThreadTaskRunnerHandle::Get()) {
+}
+
+VTVideoEncodeAccelerator::~VTVideoEncodeAccelerator() {
+  DCHECK(child_task_runner_->BelongsToCurrentThread());
+}
+
+media::VideoEncodeAccelerator::SupportedProfiles
+VTVideoEncodeAccelerator::GetSupportedProfiles() {
+  DVLOG(3) << __FUNCTION__;
+
+  SupportedProfiles profiles;
+  SupportedProfile profile;
+  profile.profile = media::H264PROFILE_BASELINE;
+  profile.max_framerate_numerator = kMaxFrameRateNumerator;
+  profile.max_framerate_denominator = kMaxFrameRateDenominator;
+  profile.max_resolution = gfx::Size(kMaxResolutionWidth, kMaxResolutionHeight);
+  profiles.push_back(profile);
+  return profiles;
+}
+
+bool VTVideoEncodeAccelerator::Initialize(
+    media::VideoPixelFormat format,
+    const gfx::Size& input_visible_size,
+    media::VideoCodecProfile output_profile,
+    uint32_t initial_bitrate,
+    Client* client) {
+  DVLOG(3) << __FUNCTION__
+           << ": input_format=" << media::VideoPixelFormatToString(format)
+           << ", input_visible_size=" << input_visible_size.ToString()
+           << ", output_profile=" << output_profile
+           << ", initial_bitrate=" << initial_bitrate;
+  DCHECK(client);
+  DCHECK_EQ(media::PIXEL_FORMAT_I420, format);
+  DCHECK_EQ(media::H264PROFILE_BASELINE, output_profile);
+
+  bitrate_ = initial_bitrate;
+  input_visible_size_ = input_visible_size;
+
+  const bool session_created = ResetCompressionSession();

[mcasas, 2016/02/03 20:01:31]

No need for temporary?

[emircan, 2016/02/04 05:18:52]

Done. I was trying to be verbose.

+  if (!session_created) {
+    DLOG(ERROR) << "Failed creating compression session";
+    return false;
+  }
+
+  client_ptr_factory_.reset(new base::WeakPtrFactory<Client>(client));

[mcasas, 2016/02/03 20:01:31]

This |client_ptr_factory_| is only used here,
do you need to hold on to it?

[emircan, 2016/02/04 05:18:51]

I can invalidate the weakptrs in Destroy() using it as a class member.

+  client_ = client_ptr_factory_->GetWeakPtr();
+
+  child_task_runner_->PostTask(
+      FROM_HERE,
+      base::Bind(&Client::RequireBitstreamBuffers, client_, kNumInputBuffers,
+                 input_visible_size_,
+                 input_visible_size_.GetArea() / kOutputBufferSizeRatio));

[hbos_chromium, 2016/02/03 18:21:08]

About thread-safety, how is this class used across threads? Is
|child_task_runner_| not always the current thread? Is it used on multiple
threads and if not, why PostTask?

[emircan, 2016/02/04 05:18:52]

Thanks for pointing it out. Actually, I don't need to PostTask here. All the
threads in this class run on |child_task_runner_| except
VTVideoEncodeAccelerator::CompressionCallback() where I still need to post a
task. I realize I was inconsistent with thread_checker so consistently checking
it in every method as well.

+  return true;
+}
+
+void VTVideoEncodeAccelerator::Encode(
+    const scoped_refptr<media::VideoFrame>& frame,
+    bool force_keyframe) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(compression_session_);
+  DCHECK(frame);
+
+  base::TimeTicks ref_time;
+  if (!frame->metadata()->GetTimeTicks(
+          media::VideoFrameMetadata::REFERENCE_TIME, &ref_time)) {
+    ref_time = base::TimeTicks::Now();
+  }
+  auto timestamp_cm = CoreMediaGlue::CMTimeMake(
+      (ref_time - base::TimeTicks()).InMicroseconds(), USEC_PER_SEC);
+  // Wrap information we'll need after the frame is encoded in a heap object.
+  // We'll get the pointer back from the VideoToolbox completion callback.
+  scoped_ptr<InProgressFrameEncode> request(new InProgressFrameEncode(
+      frame->timestamp(), ref_time));
+
+  // TODO(emircan): See if we can eliminate a copy here by using
+  // CVPixelBufferPool for the allocation of incoming video frames.

[mcasas, 2016/02/03 20:01:31]

nit: s/video frames/VideoFrames/?

[emircan, 2016/02/04 05:18:52]

Done.

+  base::ScopedCFTypeRef<CVPixelBufferRef> pixel_buffer =
+      media::WrapVideoFrameInCVPixelBuffer(*frame);
+  base::ScopedCFTypeRef<CFDictionaryRef> frame_props =
+      media::DictionaryWithKeyValue(
+          videotoolbox_glue_->kVTEncodeFrameOptionKey_ForceKeyFrame(),
+          force_keyframe ? kCFBooleanTrue : kCFBooleanFalse);
+
+  OSStatus status = videotoolbox_glue_->VTCompressionSessionEncodeFrame(
+      compression_session_, pixel_buffer, timestamp_cm,
+      CoreMediaGlue::CMTime{0, 0, 0, 0}, frame_props,
+      reinterpret_cast<void*>(request.release()), nullptr);
+  DLOG_IF(ERROR, status != noErr)
+      << " VTCompressionSessionEncodeFrame failed: " << status;
+}
+
+void VTVideoEncodeAccelerator::UseOutputBitstreamBuffer(
+    const media::BitstreamBuffer& buffer) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(child_task_runner_->BelongsToCurrentThread());
+  DCHECK_GE(buffer.size(), static_cast<size_t>(input_visible_size_.GetArea() /
+                                               kOutputBufferSizeRatio));
+
+  scoped_ptr<base::SharedMemory> shm(
+      new base::SharedMemory(buffer.handle(), false));
+  if (!shm->Map(buffer.size())) {
+    DLOG(ERROR) << "Failed mapping shared memory.";
+    return;
+  }
+
+  scoped_ptr<BitstreamBufferRef> buffer_ref(
+      new BitstreamBufferRef(buffer.id(), std::move(shm), buffer.size()));
+  encoder_output_queue_.push_back(std::move(buffer_ref));
+}
+
+void VTVideoEncodeAccelerator::RequestEncodingParametersChange(
+    uint32_t bitrate,
+    uint32_t framerate) {
+  DVLOG(3) << __FUNCTION__;
+
+  bitrate_ = bitrate;
+  if (compression_session_) {

[hbos_chromium, 2016/02/03 18:21:08]

Does it make sense to DCHECK instead?

[mcasas, 2016/02/03 20:01:31]

nit: 
if (!compression_session_)
  return;
...

[emircan, 2016/02/04 05:18:52]

Done.

[emircan, 2016/02/04 05:18:52]

Acknowledged.

+    media::SetSessionProperty(
+        compression_session_, videotoolbox_glue_,
+        videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(),
+        static_cast<int32_t>(bitrate_));
+  }
+}
+
+void VTVideoEncodeAccelerator::Destroy() {
+  DVLOG(3) << __FUNCTION__;
+
+  client_ptr_factory_.reset();
+  DestroyCompressionSession();
+  delete this;
+}
+
+// static
+void VTVideoEncodeAccelerator::CompressionCallback(void* encoder_opaque,
+                                                   void* request_opaque,
+                                                   OSStatus status,
+                                                   VTEncodeInfoFlags info,
+                                                   CMSampleBufferRef sbuf) {
+  DVLOG(3) << __FUNCTION__;
+  DLOG_IF(ERROR, status != noErr) << " encode failed: " << status;
+
+  if (info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped) {
+    DVLOG(2) << " frame dropped";
+    return;
+  }
+
+  auto sample_attachments = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(
+      CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true), 0));
+  const bool keyframe =
+      !CFDictionaryContainsKey(sample_attachments,
+                               CoreMediaGlue::kCMSampleAttachmentKey_NotSync());
+
+  auto encoder = reinterpret_cast<VTVideoEncodeAccelerator*>(encoder_opaque);
+  if (encoder->encoder_output_queue_.empty()) {
+    DLOG(ERROR) << "No more bitstream buffer to encode into.";
+    return;
+  }
+  scoped_ptr<VTVideoEncodeAccelerator::BitstreamBufferRef> buffer_ref =
+      std::move(encoder->encoder_output_queue_.front());
+  encoder->encoder_output_queue_.pop_front();
+
+  std::string annexb_buffer;
+  media::CopySampleBufferToAnnexBBuffer(sbuf, &annexb_buffer, keyframe);
+  if (buffer_ref->size < annexb_buffer.size()) {
+    DLOG(ERROR) << "Cannot fit the encode output into bitstream buffer.";
+    return;
+  }
+  memcpy(buffer_ref->shm->memory(), annexb_buffer.data(),
+         annexb_buffer.size());
+
+  encoder->child_task_runner_->PostTask(
+      FROM_HERE, base::Bind(&Client::BitstreamBufferReady, encoder->client_,
+                            buffer_ref->id, annexb_buffer.size(), keyframe));
+}
+
+bool VTVideoEncodeAccelerator::ResetCompressionSession() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  DestroyCompressionSession();
+
+  // TODO(emircan): Investigate IOS constraints.

[mcasas, 2016/02/03 20:01:31]

?

[emircan, 2016/02/04 05:18:51]

Removed.

+  base::ScopedCFTypeRef<CFDictionaryRef> encoder_spec;
+  encoder_spec = media::DictionaryWithKeyValue(
+      videotoolbox_glue_
+          ->kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder(),
+      kCFBooleanTrue);
+
+  // Keep these in-sync with those in ConfigureSession().
+  CFTypeRef attributes_keys[] = {
+#if defined(OS_IOS)
+    kCVPixelBufferOpenGLESCompatibilityKey,
+#else
+    kCVPixelBufferOpenGLCompatibilityKey,
+#endif
+    kCVPixelBufferIOSurfacePropertiesKey,
+    kCVPixelBufferPixelFormatTypeKey
+  };
+  const int format[] = {
+      CoreVideoGlue::kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange};
+  CFTypeRef attributes_values[] = {
+      kCFBooleanTrue,
+      media::DictionaryWithKeysAndValues(nullptr, nullptr, 0).release(),
+      media::ArrayWithIntegers(format, arraysize(format)).release()};
+  const base::ScopedCFTypeRef<CFDictionaryRef> attributes =
+      media::DictionaryWithKeysAndValues(attributes_keys,
+                                  attributes_values,
+                                  arraysize(attributes_keys));
+  for (auto& v : attributes_values)
+    CFRelease(v);
+
+  // Create the compression session.
+  OSStatus status = videotoolbox_glue_->VTCompressionSessionCreate(
+      kCFAllocatorDefault,
+      input_visible_size_.width(),
+      input_visible_size_.height(),
+      CoreMediaGlue::kCMVideoCodecType_H264,
+      encoder_spec,
+      attributes,
+      nullptr /* compressedDataAllocator */,
+      &VTVideoEncodeAccelerator::CompressionCallback,
+      reinterpret_cast<void*>(this),
+      compression_session_.InitializeInto());
+  if (status != noErr) {
+    DLOG(ERROR) << " VTCompressionSessionCreate failed: " << status;
+    return false;
+  }
+  ConfigureCompressionSession();
+  return true;
+}
+
+void VTVideoEncodeAccelerator::ConfigureCompressionSession() {
+  DCHECK(compression_session_);
+
+  media::SetSessionProperty(
+      compression_session_, videotoolbox_glue_,
+      videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(),
+      videotoolbox_glue_->kVTProfileLevel_H264_Baseline_AutoLevel());
+  media::SetSessionProperty(
+      compression_session_, videotoolbox_glue_,
+      videotoolbox_glue_->kVTCompressionPropertyKey_RealTime(), true);
+  media::SetSessionProperty(
+      compression_session_, videotoolbox_glue_,
+      videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(),
+      static_cast<int32_t>(bitrate_));
+  media::SetSessionProperty(
+      compression_session_, videotoolbox_glue_,
+      videotoolbox_glue_->kVTCompressionPropertyKey_AllowFrameReordering(),
+      false);
+}
+
+void VTVideoEncodeAccelerator::DestroyCompressionSession() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  if (compression_session_) {
+    videotoolbox_glue_->VTCompressionSessionInvalidate(compression_session_);
+    compression_session_.reset();
+  }
+}
+
+} // namespace content