VideoToolbox encoder for cast senders.

media/cast/sender/h264_vt_encoder.cc

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/cast/sender/h264_vt_encoder.h"
+
+#include <algorithm>
+#include <vector>
+
+#include "base/big_endian.h"
+#include "base/bind.h"
+#include "base/bind_helpers.h"
+#include "base/location.h"
+#include "base/logging.h"
+
+namespace media {
+namespace cast {
+
+namespace {
+
+bool SetSessionProperty(VTSessionRef session, CFStringRef key, uint32_t value) {
+  base::ScopedCFTypeRef<CFNumberRef> cfvalue(
+      CFNumberCreate(NULL, kCFNumberSInt32Type, &value));

[Robert Sesek, 2014/08/12 00:06:12]

You're converting a uint32 to an int32. That seems dangerous.

[jfroy, 2014/08/12 01:04:15]

There are no constants for unsigned types in CFNumber
(https://developer.apple.com/library/mac/documentation/CoreFoundation/Referenc...).
Presumably CFNumber copies the by-pointer value bit for bit without
interpretation.

[Robert Sesek, 2014/08/12 18:47:09]

Right, so either use kCFNumberSInt64Type or base/numerics/safe_conversions.h.

+  return VTSessionSetProperty(session, key, cfvalue) == noErr;
+}
+
+bool SetSessionProperty(VTSessionRef session, CFStringRef key, bool value) {
+  CFBooleanRef cfvalue = (value) ? kCFBooleanTrue : kCFBooleanFalse;
+  return VTSessionSetProperty(session, key, cfvalue) == noErr;
+}
+
+bool SetSessionProperty(VTSessionRef session,
+                        CFStringRef key,
+                        CFStringRef value) {
+  return VTSessionSetProperty(session, key, value) == noErr;
+}
+
+base::ScopedCFTypeRef<CFDictionaryRef> DictionaryWithKeyValue(CFTypeRef key,
+                                                              CFTypeRef value) {
+  CFTypeRef keys[1] = {key};
+  CFTypeRef values[1] = {value};
+  return base::ScopedCFTypeRef<CFDictionaryRef>(
+      CFDictionaryCreate(kCFAllocatorDefault,
+                         keys,
+                         values,
+                         1,
+                         &kCFTypeDictionaryKeyCallBacks,
+                         &kCFTypeDictionaryValueCallBacks));
+}
+
+struct FrameContext {
+  base::TimeTicks capture_time;
+  media::cast::VideoEncoder::FrameEncodedCallback frame_encoded_callback;
+};
+
+}  // namespace
+
+H264VideoToolboxEncoder::H264VideoToolboxEncoder(
+    scoped_refptr<CastEnvironment> cast_environment,
+    const VideoSenderConfig& video_config)
+    : cast_environment_(cast_environment),
+      cast_config_(video_config),
+      frame_id_(kStartFrameId),
+      last_keyframe_id_(kStartFrameId),
+      encode_next_frame_as_keyframe_(false) {
+  Initialize();
+}
+
+H264VideoToolboxEncoder::~H264VideoToolboxEncoder() {
+  Teardown();
+}
+
+CVPixelBufferPoolRef H264VideoToolboxEncoder::cv_pixel_buffer_pool() const {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(compression_session_);
+  return VTCompressionSessionGetPixelBufferPool(compression_session_);
+}
+
+void H264VideoToolboxEncoder::Initialize() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(!compression_session_);
+
+  // Note that the encoder object is given to the compression session as the
+  // callback context using a raw pointer. The C API does not allow us to use
+  // a smart pointer, nor is this encoder ref counted. However, this is still
+  // safe, because we 1) we own the compression session and 2) we tear it down
+  // safely. When destructing the encoder, the compression session is flushed
+  // and invalidated. Internally, VideoToolbox will join all of its threads
+  // before returning to the client. Therefore, when control returns to us, we
+  // are guaranteed that the output callback will not execute again.
+
+  // On OS X, allow the hardware encoder. Don't require it, it does not support
+  // all configurations (some of which are used for testing).
+  base::ScopedCFTypeRef<CFDictionaryRef> encoder_spec;
+#if !defined(OS_IOS)
+  encoder_spec = DictionaryWithKeyValue(
+      kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
+      kCFBooleanTrue);
+#endif
+
+  VTCompressionSessionRef session;
+  OSStatus status =
+      VTCompressionSessionCreate(kCFAllocatorDefault,
+                                 cast_config_.width,
+                                 cast_config_.height,
+                                 kCMVideoCodecType_H264,
+                                 encoder_spec,
+                                 NULL /* sourceImageBufferAttributes */,
+                                 NULL /* compressedDataAllocator */,
+                                 CompressionCallback,
+                                 reinterpret_cast<void*>(this),
+                                 &session);
+  if (status != noErr) {
+    DLOG(ERROR) << " VTCompressionSessionCreate failed: " << status;
+    return;
+  }
+  compression_session_.reset(session);
+
+  ConfigureSession();
+}
+
+void H264VideoToolboxEncoder::ConfigureSession() {
+  SetSessionProperty(compression_session_,
+                     kVTCompressionPropertyKey_ProfileLevel,
+                     kVTProfileLevel_H264_Main_AutoLevel);
+  SetSessionProperty(
+      compression_session_, kVTCompressionPropertyKey_RealTime, true);
+  SetSessionProperty(compression_session_,
+                     kVTCompressionPropertyKey_AllowFrameReordering,
+                     false);
+  SetSessionProperty(compression_session_,
+                     kVTCompressionPropertyKey_MaxKeyFrameInterval,
+                     240u);
+  SetSessionProperty(compression_session_,
+                     kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration,
+                     240u);
+  SetSessionProperty(compression_session_,
+                     kVTCompressionPropertyKey_AverageBitRate,
+                     static_cast<uint32_t>(cast_config_.start_bitrate));
+  SetSessionProperty(compression_session_,
+                     kVTCompressionPropertyKey_ExpectedFrameRate,
+                     static_cast<uint32_t>(cast_config_.max_frame_rate));
+  SetSessionProperty(compression_session_,
+                     kVTCompressionPropertyKey_ColorPrimaries,
+                     kCVImageBufferColorPrimaries_ITU_R_709_2);
+  SetSessionProperty(compression_session_,
+                     kVTCompressionPropertyKey_TransferFunction,
+                     kCVImageBufferTransferFunction_ITU_R_709_2);
+  SetSessionProperty(compression_session_,
+                     kVTCompressionPropertyKey_YCbCrMatrix,
+                     kCVImageBufferYCbCrMatrix_ITU_R_709_2);
+}
+
+void H264VideoToolboxEncoder::Teardown() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  // If the compression session exists, invalidate it. This blocks until all
+  // pending output callbacks have returned and any internal threads have
+  // joined, ensuring no output callback ever sees a dangling encoder pointer.
+  if (compression_session_) {
+    VTCompressionSessionInvalidate(compression_session_);
+    compression_session_.reset();
+  }
+}
+
+bool H264VideoToolboxEncoder::EncodeVideoFrame(
+    const scoped_refptr<media::VideoFrame>& video_frame,
+    const base::TimeTicks& capture_time,
+    const FrameEncodedCallback& frame_encoded_callback) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(!capture_time.is_null());
+
+  if (!compression_session_) {
+    DLOG(ERROR) << " compression session is null";
+    return false;
+  }
+
+  base::ScopedCFTypeRef<CVPixelBufferRef> pixel_buffer(
+      video_frame->cv_pixel_buffer(), base::scoped_policy::RETAIN);
+  if (!pixel_buffer) {
+    pixel_buffer = WrapVideoFrame(video_frame);
+    if (!pixel_buffer) {
+      return false;
+    }
+  }
+
+  CMTime timestamp_cm =
+      CMTimeMake(capture_time.ToInternalValue(), USEC_PER_SEC);
+
+  scoped_ptr<FrameContext> frame_context(new FrameContext());
+  frame_context->capture_time = capture_time;
+  frame_context->frame_encoded_callback = frame_encoded_callback;
+
+  base::ScopedCFTypeRef<CFDictionaryRef> frame_props;
+  if (encode_next_frame_as_keyframe_) {
+    frame_props = DictionaryWithKeyValue(kVTEncodeFrameOptionKey_ForceKeyFrame,
+                                         kCFBooleanTrue);
+    encode_next_frame_as_keyframe_ = false;
+  }
+
+  VTEncodeInfoFlags info;
+  OSStatus status = VTCompressionSessionEncodeFrame(
+      compression_session_,
+      pixel_buffer,
+      timestamp_cm,
+      kCMTimeInvalid,
+      frame_props,
+      reinterpret_cast<void*>(frame_context.release()),
+      &info);
+  if (status != noErr) {
+    DLOG(ERROR) << " VTCompressionSessionEncodeFrame failed: " << status;
+    return false;
+  }
+  if ((info & kVTEncodeInfo_FrameDropped)) {
+    DLOG(ERROR) << " frame dropped";
+    return false;
+  }
+
+  return true;
+}
+
+void H264VideoToolboxEncoder::SetBitRate(int new_bit_rate) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  // NOTE: VideoToolbox does not seem to support bitrate reconfiguration.

[Robert Sesek, 2014/08/12 00:06:12]

Remove "NOTE:" since this is a comment.

[jfroy, 2014/08/12 01:04:15]

Acknowledged.

+}
+
+void H264VideoToolboxEncoder::GenerateKeyFrame() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(compression_session_);
+
+  encode_next_frame_as_keyframe_ = true;
+}
+
+void H264VideoToolboxEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) {
+  // NOTE: Not supported by VideoToolbox in any meaningful manner.

[Robert Sesek, 2014/08/12 00:06:12]

Same.

[jfroy, 2014/08/12 01:04:15]

Acknowledged.

+}
+
+static void VideoFramePixelBufferReleaseCallback(void* frame_ref,
+                                                 const void* data,
+                                                 size_t size,
+                                                 size_t num_planes,
+                                                 const void* planes[]) {
+  free(const_cast<void*>(data));
+  reinterpret_cast<media::VideoFrame*>(frame_ref)->Release();
+}
+
+base::ScopedCFTypeRef<CVPixelBufferRef> H264VideoToolboxEncoder::WrapVideoFrame(
+    const scoped_refptr<media::VideoFrame>& frame) {
+  static const size_t MAX_PLANES = 3;
+
+  media::VideoFrame::Format format = frame->format();
+  size_t num_planes = media::VideoFrame::NumPlanes(format);
+  DCHECK_LE(num_planes, MAX_PLANES);
+  gfx::Size coded_size = frame->coded_size();
+
+  // media::VideoFrame only supports YUV formats, so there is no way to
+  // leverage VideoToolbox's ability to convert RGBA formats automatically. In
+  // addition, most of the media::VideoFrame formats are YVU, which VT does not
+  // support. Finally, media::VideoFrame formats do not carry any information
+  // about the color space, transform or any other colorimetric information
+  // that is generally needed to fully specify the input data. So essentially
+  // require that the input be YCbCr 4:2:0 (either planar or biplanar) and
+  // assume the standard video dynamic range for samples (although most modern
+  // HDTVs support full-range video these days).
+  OSType pixel_format;
+  if (format == media::VideoFrame::Format::I420) {
+    pixel_format = kCVPixelFormatType_420YpCbCr8Planar;
+  } else if (format == media::VideoFrame::Format::NV12) {
+    pixel_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
+  } else {
+    DLOG(ERROR) << " unsupported frame format: " << format;
+    return base::ScopedCFTypeRef<CVPixelBufferRef>(NULL);
+  }
+
+  // TODO(jfroy): Support extended pixels (i.e. padding).
+  if (frame->coded_size() != frame->visible_rect().size()) {
+    DLOG(ERROR) << " frame with extended pixels not supported: "
+                << " coded_size: " << coded_size.ToString()
+                << ", visible_rect: " << frame->visible_rect().ToString();
+    return base::ScopedCFTypeRef<CVPixelBufferRef>(NULL);
+  }
+
+  void* plane_ptrs[MAX_PLANES];
+  size_t plane_widths[MAX_PLANES];
+  size_t plane_heights[MAX_PLANES];
+  size_t plane_bytes_per_row[MAX_PLANES];
+  for (size_t plane_i = 0; plane_i < num_planes; ++plane_i) {
+    plane_ptrs[plane_i] = frame->data(plane_i);
+    gfx::Size plane_size =
+        media::VideoFrame::PlaneSize(format, plane_i, coded_size);
+    plane_widths[plane_i] = plane_size.width();
+    plane_heights[plane_i] = plane_size.height();
+    plane_bytes_per_row[plane_i] = frame->stride(plane_i);
+  }
+
+  // CVPixelBufferCreateWithPlanarBytes needs a dummy plane descriptor or the
+  // release callback will not execute. The descriptor is freed in the callback.
+  void* descriptor =
+      calloc(1,
+             std::max(sizeof(CVPlanarPixelBufferInfo_YCbCrPlanar),
+                      sizeof(CVPlanarPixelBufferInfo_YCbCrBiPlanar)));
+
+  // Wrap the frame's data in a CVPixelBuffer. Because this is a C API, we can't
+  // give it a smart pointer to the frame, so instead pass a raw pointer and
+  // increment the frame's reference count manually.
+  CVPixelBufferRef pixel_buffer;
+  CVReturn result =
+      CVPixelBufferCreateWithPlanarBytes(kCFAllocatorDefault,
+                                         coded_size.width(),
+                                         coded_size.height(),
+                                         format,
+                                         descriptor,
+                                         0,
+                                         num_planes,
+                                         plane_ptrs,
+                                         plane_widths,
+                                         plane_heights,
+                                         plane_bytes_per_row,
+                                         VideoFramePixelBufferReleaseCallback,
+                                         frame.get(),
+                                         NULL,
+                                         &pixel_buffer);
+  if (result != kCVReturnSuccess) {
+    DLOG(ERROR) << " CVPixelBufferCreateWithPlanarBytes failed: " << result;
+    return base::ScopedCFTypeRef<CVPixelBufferRef>(NULL);
+  }
+
+  // The CVPixelBuffer now references the data of the frame, so increment its
+  // reference count manually. The release callback set on the pixel buffer will
+  // release the frame.
+  frame->AddRef();
+
+  return base::ScopedCFTypeRef<CVPixelBufferRef>(pixel_buffer);
+}
+
+void H264VideoToolboxEncoder::CompressionCallback(void* encoder_opaque,
+                                                  void* frame_opaque,
+                                                  OSStatus status,
+                                                  VTEncodeInfoFlags info,
+                                                  CMSampleBufferRef sbuf) {
+  H264VideoToolboxEncoder* encoder =
+      reinterpret_cast<H264VideoToolboxEncoder*>(encoder_opaque);
+  scoped_ptr<FrameContext> frame_context(
+      reinterpret_cast<FrameContext*>(frame_opaque));
+
+  if (status != noErr) {
+    DLOG(ERROR) << " encoding failed: " << status;
+    return;
+  }
+  if ((info & kVTEncodeInfo_FrameDropped)) {
+    DVLOG(2) << " frame dropped";
+    return;
+  }
+  DCHECK_EQ(CMSampleBufferGetNumSamples(sbuf), 1);
+
+  CFDictionaryRef sample_attachments =
+      static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(
+          CMSampleBufferGetSampleAttachmentsArray(sbuf, true), 0));
+
+  // If the NotSync key is not present, it implies Sync, which indicates a
+  // keyframe (at least I think, VT documentation is, erm, sparse). Could
+  // alternatively use kCMSampleAttachmentKey_DependsOnOthers == false.
+  bool keyframe =
+      CFDictionaryContainsKey(sample_attachments,
+                              kCMSampleAttachmentKey_NotSync) == false;
+
+  // Generate a frame id and update the last keyframe id if needed.
+  // NOTE: VideoToolbox calls the output callback serially, so this is safe.
+  uint32 frame_id = ++encoder->frame_id_;
+  if (keyframe) {
+    encoder->last_keyframe_id_ = frame_id;
+  }
+
+  CMSampleTimingInfo timing_info;
+  CMSampleBufferGetSampleTimingInfo(sbuf, 0, &timing_info);
+
+  scoped_ptr<EncodedFrame> encoded_frame(new EncodedFrame());
+  encoded_frame->frame_id = frame_id;
+  encoded_frame->reference_time = frame_context->capture_time;
+  encoded_frame->rtp_timestamp =
+      GetVideoRtpTimestamp(frame_context->capture_time);
+  if (keyframe) {
+    encoded_frame->dependency = EncodedFrame::KEY;
+    encoded_frame->referenced_frame_id = frame_id;
+  } else {
+    encoded_frame->dependency = EncodedFrame::DEPENDENT;
+    // NOTE: Technically wrong, but without parsing the NALs our best guess is
+    // the last keyframe.
+    encoded_frame->referenced_frame_id = encoder->last_keyframe_id_;
+  }
+
+  CopySampleBufferToAnnexBBuffer(sbuf, &encoded_frame->data, keyframe);
+
+  encoder->cast_environment_->PostTask(
+      CastEnvironment::MAIN,
+      FROM_HERE,
+      base::Bind(frame_context->frame_encoded_callback,
+                 base::Passed(&encoded_frame)));
+}
+
+template <typename NalSizeType>
+static void CopyNalsToAnnexB(char* avcc_buffer,
+                             const size_t avcc_size,
+                             std::string* annexb_buffer) {
+  COMPILE_ASSERT(sizeof(NalSizeType) == 1 || sizeof(NalSizeType) == 2 ||
+                     sizeof(NalSizeType) == 4,
+                 "NAL size type has unsupported size");
+  static const char startcode_3[3] = {0, 0, 1};
+  DCHECK(avcc_buffer);
+  DCHECK(annexb_buffer);
+  size_t bytes_left = avcc_size;
+  while (bytes_left > 0) {
+    DCHECK_GT(bytes_left, sizeof(NalSizeType));
+    NalSizeType nal_size;
+    base::ReadBigEndian(avcc_buffer, &nal_size);
+    bytes_left -= sizeof(NalSizeType);
+    avcc_buffer += sizeof(NalSizeType);
+
+    DCHECK_GE(bytes_left, nal_size);
+    annexb_buffer->append(startcode_3, sizeof(startcode_3));
+    annexb_buffer->append(avcc_buffer, nal_size);
+    bytes_left -= nal_size;
+    avcc_buffer += nal_size;
+  }
+}
+
+void H264VideoToolboxEncoder::CopySampleBufferToAnnexBBuffer(
+    CMSampleBufferRef sbuf,
+    std::string* annexb_buffer,
+    bool keyframe) {
+  // Perform two pass, one to figure out the total output size, and another to
+  // copy the data after having performed a single output allocation. Note that
+  // we'll allocate a bit more because we'll count 4 bytes instead of 3 for
+  // video NALs.
+
+  // TODO(jfroy): There is a bug in
+  // CMVideoFormatDescriptionGetH264ParameterSetAtIndex, iterate until fail.
+
+  OSStatus status;
+
+  // Get the sample buffer's block buffer and format description.
+  CMBlockBufferRef bb = CMSampleBufferGetDataBuffer(sbuf);
+  DCHECK(bb);
+  CMFormatDescriptionRef fdesc = CMSampleBufferGetFormatDescription(sbuf);
+  DCHECK(fdesc);
+
+  size_t bb_size = CMBlockBufferGetDataLength(bb);
+  size_t total_bytes = bb_size;
+
+  size_t pset_count;
+  int nal_size_field_bytes;
+  status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
+      fdesc, 0, NULL, NULL, &pset_count, &nal_size_field_bytes);
+  if (status == kCMFormatDescriptionBridgeError_InvalidParameter) {
+    DLOG(WARNING) << " assuming 2 parameter sets and 4 bytes NAL length header";
+    pset_count = 2;
+    nal_size_field_bytes = 4;
+  } else if (status != noErr) {
+    DLOG(ERROR)
+        << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: "
+        << status;
+    return;
+  }
+
+  if (keyframe) {
+    const uint8_t* pset;
+    size_t pset_size;
+    for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) {
+      status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
+          fdesc, pset_i, &pset, &pset_size, NULL, NULL);
+      if (status != noErr) {
+        DLOG(ERROR)
+            << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: "
+            << status;
+        return;
+      }
+      total_bytes += pset_size + nal_size_field_bytes;
+    }
+  }
+
+  annexb_buffer->reserve(total_bytes);
+
+  // Copy all parameter sets before keyframes.
+  if (keyframe) {
+    const uint8_t* pset;
+    size_t pset_size;
+    for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) {
+      status = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
+          fdesc, pset_i, &pset, &pset_size, NULL, NULL);
+      if (status != noErr) {
+        DLOG(ERROR)
+            << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: "
+            << status;
+        return;
+      }
+      static const char startcode_4[4] = {0, 0, 0, 1};
+      annexb_buffer->append(startcode_4, sizeof(startcode_4));
+      annexb_buffer->append(reinterpret_cast<const char*>(pset), pset_size);
+    }
+  }
+
+  // Block buffers can be composed of non-contiguous chunks. For the sake of
+  // keeping this code simple, flatten non-contiguous block buffers.
+  base::ScopedCFTypeRef<CMBlockBufferRef> contiguous_bb(
+      bb, base::scoped_policy::RETAIN);
+  if (!CMBlockBufferIsRangeContiguous(bb, 0, 0)) {
+    contiguous_bb.reset();
+    status = CMBlockBufferCreateContiguous(kCFAllocatorDefault,
+                                           bb,
+                                           kCFAllocatorDefault,
+                                           NULL,
+                                           0,
+                                           0,
+                                           0,
+                                           contiguous_bb.InitializeInto());
+    if (status != noErr) {
+      DLOG(ERROR) << " CMBlockBufferCreateContiguous failed: " << status;
+      return;
+    }
+  }
+
+  // Copy all the NAL units. In the process convert them from AVCC format
+  // (length header) to AnnexB format (start code).
+  char* bb_data;
+  status = CMBlockBufferGetDataPointer(contiguous_bb, 0, NULL, NULL, &bb_data);
+  if (status != noErr) {
+    DLOG(ERROR) << " CMBlockBufferGetDataPointer failed: " << status;
+    return;
+  }
+
+  if (nal_size_field_bytes == 1) {
+    CopyNalsToAnnexB<uint8_t>(bb_data, bb_size, annexb_buffer);
+  } else if (nal_size_field_bytes == 2) {
+    CopyNalsToAnnexB<uint16_t>(bb_data, bb_size, annexb_buffer);
+  } else if (nal_size_field_bytes == 4) {
+    CopyNalsToAnnexB<uint32_t>(bb_data, bb_size, annexb_buffer);
+  }
+}
+
+}  // namespace cast
+}  // namespace media