H264 HW encode using VideoToolbox

content/common/gpu/media/vt_video_encode_accelerator.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/common/gpu/media/vt_video_encode_accelerator.h"
+
+#include "base/thread_task_runner_handle.h"
+#include "media/base/mac/coremedia_glue.h"
+#include "media/base/mac/corevideo_glue.h"
+#include "media/base/mac/video_frame_mac.h"
+#include "media/base/mac/videotoolbox_helpers.h"
+
+namespace content {
+
+namespace {
+
+// Subjectively chosen.
+const size_t kNumInputBuffers = 4;

[miu, 2016/02/04 22:23:20]

IIRC, the VT encoder can hold onto several buffers at a time to help optimize
motion prediction.  I have no idea how many buffers it actually *will* hold onto
(1, 2, 8, or something else).  Suggest you consult documentation for this and/or
jfroy@.

[emircan, 2016/02/07 04:24:05]

I discussed with jfroy@ and added him as a reviewer. Unfortunately, in theory VT
can hold onto as much buffers as it decides to. I couldn't find any information
about it, and it doesn't look like something we can easily query.

+const size_t kMaxFrameRateNumerator = 30;

[miu, 2016/02/04 22:23:20]

Is 30 really the max for all hardware solutions?  I'd imagine the max framerate
to be machine-dependent, and also dependent on the resolution.  Furthermore,
there are likely some hardware encoders that cannot do 30 FPS at 4K resolution.

If anything, you should put a TODO+crbug here to query the platform for
supported profiles rather than hard-code these.

[emircan, 2016/02/07 04:24:05]

I added a TODO for querying platform capabilities. Unfortunately, there isn't an
easy way to query these. One way we discussed with jfroy@ is to create sessions
but that would definitely effect the initialize time. I will look out for better
solution and leave it as a TODO for now.

Coming to framerate and resolution; I agree that in theory there should be a
tradeoff. VTCompressionSession provides a parameter
kVTCompressionPropertyKey_ExpectedFrameRate, but there is nothing that I found
regarding resolution. Even if kVTCompressionPropertyKey_ExpectedFrameRate is
supported for 60, I am doubtful if it would do 4K 60 fps. So, I am just
hardcording max capabilities for now that we return in
VTVideoEncodeAccelerator::GetSupportedProfiles().

+const size_t kMaxFrameRateDenominator = 1;
+const size_t kMaxResolutionWidth = 4096;
+const size_t kMaxResolutionHeight = 2160;
+const size_t kOutputBufferSizeRatio = 10;

[miu, 2016/02/04 22:23:20]

Please document this.  What does 10 mean?

[emircan, 2016/02/07 04:24:05]

Done.

+
+// Container for the associated data of a video frame being processed.
+struct InProgressFrameEncode {
+  const base::TimeDelta timestamp;
+  const base::TimeTicks reference_time;
+
+  InProgressFrameEncode(base::TimeDelta rtp_timestamp, base::TimeTicks ref_time)
+      : timestamp(rtp_timestamp), reference_time(ref_time) {}
+
+ private:
+  DISALLOW_IMPLICIT_CONSTRUCTORS(InProgressFrameEncode);
+};
+
+}  // namespace
+
+struct VTVideoEncodeAccelerator::BitstreamBufferRef {
+  BitstreamBufferRef(int32_t id,
+                     scoped_ptr<base::SharedMemory> shm,
+                     size_t size)
+      : id(id), shm(std::move(shm)), size(size) {}
+  const int32_t id;
+  const scoped_ptr<base::SharedMemory> shm;
+  const size_t size;
+
+ private:
+  DISALLOW_IMPLICIT_CONSTRUCTORS(BitstreamBufferRef);
+};
+
+VTVideoEncodeAccelerator::VTVideoEncodeAccelerator()
+    : videotoolbox_glue_(VideoToolboxGlue::Get()),
+      client_task_runner_(base::ThreadTaskRunnerHandle::Get()) {
+}
+
+VTVideoEncodeAccelerator::~VTVideoEncodeAccelerator() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+}
+
+media::VideoEncodeAccelerator::SupportedProfiles
+VTVideoEncodeAccelerator::GetSupportedProfiles() {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  SupportedProfiles profiles;
+  SupportedProfile profile;
+  profile.profile = media::H264PROFILE_BASELINE;
+  profile.max_framerate_numerator = kMaxFrameRateNumerator;
+  profile.max_framerate_denominator = kMaxFrameRateDenominator;
+  profile.max_resolution = gfx::Size(kMaxResolutionWidth, kMaxResolutionHeight);
+  profiles.push_back(profile);
+  return profiles;
+}
+
+bool VTVideoEncodeAccelerator::Initialize(
+    media::VideoPixelFormat format,
+    const gfx::Size& input_visible_size,
+    media::VideoCodecProfile output_profile,
+    uint32_t initial_bitrate,
+    Client* client) {
+  DVLOG(3) << __FUNCTION__
+           << ": input_format=" << media::VideoPixelFormatToString(format)
+           << ", input_visible_size=" << input_visible_size.ToString()
+           << ", output_profile=" << output_profile
+           << ", initial_bitrate=" << initial_bitrate;
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(client);
+  DCHECK_EQ(media::PIXEL_FORMAT_I420, format);
+  DCHECK_EQ(media::H264PROFILE_BASELINE, output_profile);
+
+  bitrate_ = initial_bitrate;
+  input_visible_size_ = input_visible_size;
+
+  if (!ResetCompressionSession()) {
+    DLOG(ERROR) << "Failed creating compression session";
+    return false;
+  }
+
+  client_ptr_factory_.reset(new base::WeakPtrFactory<Client>(client));
+  client_ = client_ptr_factory_->GetWeakPtr();
+  client_->RequireBitstreamBuffers(
+      kNumInputBuffers, input_visible_size_,
+      input_visible_size_.GetArea() / kOutputBufferSizeRatio);
+  return true;
+}
+
+void VTVideoEncodeAccelerator::Encode(
+    const scoped_refptr<media::VideoFrame>& frame,
+    bool force_keyframe) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(compression_session_);
+  DCHECK(frame);
+
+  base::TimeTicks ref_time;
+  if (!frame->metadata()->GetTimeTicks(
+          media::VideoFrameMetadata::REFERENCE_TIME, &ref_time)) {
+    ref_time = base::TimeTicks::Now();
+  }
+  auto timestamp_cm = CoreMediaGlue::CMTimeMake(
+      (ref_time - base::TimeTicks()).InMicroseconds(), USEC_PER_SEC);

[miu, 2016/02/04 22:23:20]

IIUC, this should be frame->timestamp(), not (ref_time - base::TimeTicks()). 
Though, I see it's that way in the cast code as well.  Can you fix it there too?
 :)

[emircan, 2016/02/07 04:24:04]

Done.

+  // Wrap information we'll need after the frame is encoded in a heap object.
+  // We'll get the pointer back from the VideoToolbox completion callback.
+  scoped_ptr<InProgressFrameEncode> request(new InProgressFrameEncode(
+      frame->timestamp(), ref_time));
+
+  // TODO(emircan): See if we can eliminate a copy here by using
+  // CVPixelBufferPool for the allocation of incoming VideoFrames.
+  base::ScopedCFTypeRef<CVPixelBufferRef> pixel_buffer =
+      media::WrapVideoFrameInCVPixelBuffer(*frame);
+  base::ScopedCFTypeRef<CFDictionaryRef> frame_props =
+      media::DictionaryWithKeyValue(
+          videotoolbox_glue_->kVTEncodeFrameOptionKey_ForceKeyFrame(),
+          force_keyframe ? kCFBooleanTrue : kCFBooleanFalse);
+
+  OSStatus status = videotoolbox_glue_->VTCompressionSessionEncodeFrame(
+      compression_session_, pixel_buffer, timestamp_cm,
+      CoreMediaGlue::CMTime{0, 0, 0, 0}, frame_props,
+      reinterpret_cast<void*>(request.release()), nullptr);
+  DLOG_IF(ERROR, status != noErr)
+      << " VTCompressionSessionEncodeFrame failed: " << status;
+}
+
+void VTVideoEncodeAccelerator::UseOutputBitstreamBuffer(
+    const media::BitstreamBuffer& buffer) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK_GE(buffer.size(), static_cast<size_t>(input_visible_size_.GetArea() /
+                                               kOutputBufferSizeRatio));
+
+  scoped_ptr<base::SharedMemory> shm(
+      new base::SharedMemory(buffer.handle(), false));
+  if (!shm->Map(buffer.size())) {
+    DLOG(ERROR) << "Failed mapping shared memory.";
+    return;
+  }
+
+  scoped_ptr<BitstreamBufferRef> buffer_ref(
+      new BitstreamBufferRef(buffer.id(), std::move(shm), buffer.size()));
+  encoder_output_queue_.push_back(std::move(buffer_ref));
+}
+
+void VTVideoEncodeAccelerator::RequestEncodingParametersChange(
+    uint32_t bitrate,
+    uint32_t framerate) {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  bitrate_ = bitrate;
+  if (!compression_session_)
+    return;
+
+  media::SetSessionProperty(
+      compression_session_, videotoolbox_glue_,
+      videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(),
+      static_cast<int32_t>(bitrate_));
+}
+
+void VTVideoEncodeAccelerator::Destroy() {
+  DVLOG(3) << __FUNCTION__;
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  DestroyCompressionSession();
+  delete this;
+}
+
+// static
+void VTVideoEncodeAccelerator::CompressionCallback(void* encoder_opaque,
+                                                   void* request_opaque,
+                                                   OSStatus status,
+                                                   VTEncodeInfoFlags info,
+                                                   CMSampleBufferRef sbuf) {
+  DVLOG(3) << __FUNCTION__;
+
+  if (status != noErr)
+    DLOG(ERROR) << " encode failed: " << status;
+
+  if (info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped) {
+    DVLOG(2) << " frame dropped";
+    return;

[miu, 2016/02/04 22:23:20]

You can't return early anywhere in this function. 
Client::BitstreamBufferReady() must be called to release the buffer back to the
client.

[emircan, 2016/02/07 04:24:04]

I am holding onto a BitstreamBuffer from the queue on l.218. If we early return
here, the next callback can use it, and it should be fine?

[miu, 2016/02/09 23:29:22]

I could be misunderstanding the intended semantics of the VEA API, but if the
client sends a frame to VEA for encoding and the encoder decides to drop the
frame, shouldn't the client be made aware of that fact?  The client might need
to do something to account for that.

Also, the client may be assuming that for a sequence of calls to VEA::Encode(),
say for frame #1, #2, #3, etc.; the BitstreamBufferReady() method would be
invoked for those frames in the same ordering.  Meaning, the client would assume
the first BitstreamBufferReady() call is the result for frame #1, the next one
for frame #2, and so on.  FWIW, the media::cast::ExternalVideoEncoder (which
uses VEA) makes this assumption.

[emircan, 2016/02/10 05:21:53]

I see the sequence assumption. Cast VT code seems to continue and send back an
empty output buffer -payload_size=0- for that case, and I will follow that
logic.

https://code.google.com/p/chromium/codesearch#chromium/src/media/cast/sender/...

+  }
+
+  auto sample_attachments = static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(
+      CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true), 0));
+  const bool keyframe =
+      !CFDictionaryContainsKey(sample_attachments,
+                               CoreMediaGlue::kCMSampleAttachmentKey_NotSync());
+  auto encoder = reinterpret_cast<VTVideoEncodeAccelerator*>(encoder_opaque);
+  if (!encoder) {

[miu, 2016/02/04 22:23:20]

This should never be null.  DCHECK(!encoder) would suffice here.

[emircan, 2016/02/07 04:24:05]

Done.

+    DLOG(ERROR) << "No encoder reference.";
+    return;
+  }
+  if (encoder->encoder_output_queue_.empty()) {
+    DLOG(ERROR) << "No more bitstream buffer to encode into.";

[miu, 2016/02/04 22:23:20]

You should call Client::NotifyError() and either: 1) shutdown the encoder; or 2)
recover from this error by requiring a key frame be encoded next.

[emircan, 2016/02/07 04:24:05]

I added NotifyError call. However, I don't understand why we would need to
shutdown encoder or require a keyframe. AFAIU, other encoders aren't doing that:
https://code.google.com/p/chromium/codesearch#chromium/src/content/common/gpu...

[miu, 2016/02/09 23:29:22]

When decoding frames, there are implicit dependencies on prior frames.  So, if
the decoder is missing a P-frame or B-frame, it will not be able to continue
decoding until it receives an I-frame (a key frame).

One suggestion for solving this problem in your code here: Instead of calling
encoder_output_queue_.pop_front() here, perhaps you should make that call
*before* starting the encode (i.e., where you create the InProgressFrameEncode
struct).  That way, you know before you even start encoding the frame whether
you will have an output buffer available.  In other words, you'd be "reserving"
the output buffer for a frame upfront.

[emircan, 2016/02/10 05:21:53]

I changed this behavior after posciak@ comments. I am now pushing those to a
queue and waiting for the next BitstreamBuffer.

+    return;
+  }
+  scoped_ptr<VTVideoEncodeAccelerator::BitstreamBufferRef> buffer_ref =
+      std::move(encoder->encoder_output_queue_.front());
+  encoder->encoder_output_queue_.pop_front();
+
+  std::string annexb_buffer;
+  media::CopySampleBufferToAnnexBBuffer(sbuf, &annexb_buffer, keyframe);
+  if (buffer_ref->size < annexb_buffer.size()) {
+    DLOG(ERROR) << "Cannot fit the encode output into bitstream buffer.";
+    return;
+  }
+  memcpy(buffer_ref->shm->memory(), annexb_buffer.data(),

[miu, 2016/02/04 22:23:20]

Please remove this redundant memcpy().  Just provide buffer_ref->shm->memory()
as the 2nd argument in the call to media::CopySampleBufferToAnnexBBuffer()
above.

[emircan, 2016/02/07 04:24:05]

media::CopySampleBufferToAnnexBBuffer() expects a string input and reserves
buffer in the flights, so it isn't as simple. I was trying to leave it for a
follow-up patch, but since it came up I am doing it.

+         annexb_buffer.size());
+
+  // This method is NOT called on |client_task_runner_|, so we still need to
+  // post a task back to it to reach |client_|.
+  encoder->client_task_runner_->PostTask(
+      FROM_HERE, base::Bind(&Client::BitstreamBufferReady, encoder->client_,
+                            buffer_ref->id, annexb_buffer.size(), keyframe));
+}
+
+bool VTVideoEncodeAccelerator::ResetCompressionSession() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  DestroyCompressionSession();
+
+  base::ScopedCFTypeRef<CFDictionaryRef> encoder_spec;
+  encoder_spec = media::DictionaryWithKeyValue(
+      videotoolbox_glue_
+          ->kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder(),

[miu, 2016/02/04 22:23:20]

80 chars

[emircan, 2016/02/07 04:24:05]

Done.

+      kCFBooleanTrue);
+
+  // Keep these in-sync with those in ConfigureSession().
+  CFTypeRef attributes_keys[] = {
+#if defined(OS_IOS)
+    kCVPixelBufferOpenGLESCompatibilityKey,
+#else
+    kCVPixelBufferOpenGLCompatibilityKey,
+#endif
+    kCVPixelBufferIOSurfacePropertiesKey,
+    kCVPixelBufferPixelFormatTypeKey
+  };
+  const int format[] = {
+      CoreVideoGlue::kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange};
+  CFTypeRef attributes_values[] = {
+      kCFBooleanTrue,
+      media::DictionaryWithKeysAndValues(nullptr, nullptr, 0).release(),
+      media::ArrayWithIntegers(format, arraysize(format)).release()};
+  const base::ScopedCFTypeRef<CFDictionaryRef> attributes =
+      media::DictionaryWithKeysAndValues(attributes_keys,
+                                  attributes_values,
+                                  arraysize(attributes_keys));
+  for (auto& v : attributes_values)
+    CFRelease(v);
+
+  // Create the compression session.
+  OSStatus status = videotoolbox_glue_->VTCompressionSessionCreate(
+      kCFAllocatorDefault,
+      input_visible_size_.width(),
+      input_visible_size_.height(),
+      CoreMediaGlue::kCMVideoCodecType_H264,
+      encoder_spec,
+      attributes,
+      nullptr /* compressedDataAllocator */,
+      &VTVideoEncodeAccelerator::CompressionCallback,
+      reinterpret_cast<void*>(this),
+      compression_session_.InitializeInto());
+  if (status != noErr) {
+    DLOG(ERROR) << " VTCompressionSessionCreate failed: " << status;
+    return false;
+  }
+  ConfigureCompressionSession();
+  return true;
+}
+
+void VTVideoEncodeAccelerator::ConfigureCompressionSession() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(compression_session_);
+
+  media::SetSessionProperty(

[miu, 2016/02/04 22:23:20]

Suggestion: Instead of passing compression_session_ and videotoolbox_glue_ to
every function call, perhaps the SetSessionProperty() methods could be in a
simple class?  Example:

  media::video_toolbox::SessionPropertySetter setter(compression_session_,
videotoolbox_glue_);
  setter.Set(videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(),
             videotoolbox_glue_->kVTProfileLevel_H264_Baseline_AutoLevel());
  setter.Set(...);
  setter.Set(...);
  setter.Set(...);
  

[emircan, 2016/02/07 04:24:05]

Done.

+      compression_session_, videotoolbox_glue_,
+      videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(),
+      videotoolbox_glue_->kVTProfileLevel_H264_Baseline_AutoLevel());
+  media::SetSessionProperty(
+      compression_session_, videotoolbox_glue_,
+      videotoolbox_glue_->kVTCompressionPropertyKey_RealTime(), true);
+  media::SetSessionProperty(
+      compression_session_, videotoolbox_glue_,
+      videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(),
+      static_cast<int32_t>(bitrate_));
+  media::SetSessionProperty(
+      compression_session_, videotoolbox_glue_,
+      videotoolbox_glue_->kVTCompressionPropertyKey_AllowFrameReordering(),
+      false);
+}
+
+void VTVideoEncodeAccelerator::DestroyCompressionSession() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  if (compression_session_) {
+    videotoolbox_glue_->VTCompressionSessionInvalidate(compression_session_);
+    compression_session_.reset();
+  }
+}
+
+} // namespace content