VideoToolbox encoder for cast senders.

media/cast/sender/h264_vt_encoder.cc

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/cast/sender/h264_vt_encoder.h"
+
+#include <algorithm>
+
+#include "base/big_endian.h"
+#include "base/bind.h"
+#include "base/bind_helpers.h"
+#include "base/location.h"
+#include "base/logging.h"
+#include "media/base/mac/corevideo_glue.h"
+
+namespace media {
+namespace cast {
+
+namespace {
+
+struct FrameContext {
+  base::TimeTicks capture_time;
+  media::cast::VideoEncoder::FrameEncodedCallback frame_encoded_callback;
+};
+
+base::ScopedCFTypeRef<CFDictionaryRef> DictionaryWithKeyValue(CFTypeRef key,
+                                                              CFTypeRef value) {
+  CFTypeRef keys[1] = {key};
+  CFTypeRef values[1] = {value};
+  return base::ScopedCFTypeRef<CFDictionaryRef>(
+      CFDictionaryCreate(kCFAllocatorDefault,
+                         keys,
+                         values,
+                         1,
+                         &kCFTypeDictionaryKeyCallBacks,
+                         &kCFTypeDictionaryValueCallBacks));
+}
+
+template <typename NalSizeType>
+void CopyNalsToAnnexB(char* avcc_buffer,
+                      const size_t avcc_size,
+                      std::string* annexb_buffer) {
+  COMPILE_ASSERT(sizeof(NalSizeType) == 1 || sizeof(NalSizeType) == 2 ||
+                     sizeof(NalSizeType) == 4,
+                 "NAL size type has unsupported size");
+  static const char startcode_3[3] = {0, 0, 1};
+  DCHECK(avcc_buffer);
+  DCHECK(annexb_buffer);
+  size_t bytes_left = avcc_size;
+  while (bytes_left > 0) {
+    DCHECK_GT(bytes_left, sizeof(NalSizeType));
+    NalSizeType nal_size;
+    base::ReadBigEndian(avcc_buffer, &nal_size);
+    bytes_left -= sizeof(NalSizeType);
+    avcc_buffer += sizeof(NalSizeType);
+
+    DCHECK_GE(bytes_left, nal_size);
+    annexb_buffer->append(startcode_3, sizeof(startcode_3));
+    annexb_buffer->append(avcc_buffer, nal_size);
+    bytes_left -= nal_size;
+    avcc_buffer += nal_size;
+  }
+}
+
+// Copy a H.264 frame stored in a CM sample buffer to an Annex B buffer. Copies
+// parameter sets for keyframes before the frame data as well.
+void CopySampleBufferToAnnexBBuffer(CoreMediaGlue::CMSampleBufferRef sbuf,
+                                    std::string* annexb_buffer,
+                                    bool keyframe) {
+  // Perform two pass, one to figure out the total output size, and another to
+  // copy the data after having performed a single output allocation. Note that
+  // we'll allocate a bit more because we'll count 4 bytes instead of 3 for
+  // video NALs.
+
+  OSStatus status;
+
+  // Get the sample buffer's block buffer and format description.
+  CoreMediaGlue::CMBlockBufferRef bb =
+      CoreMediaGlue::CMSampleBufferGetDataBuffer(sbuf);
+  DCHECK(bb);
+  CoreMediaGlue::CMFormatDescriptionRef fdesc =
+      CoreMediaGlue::CMSampleBufferGetFormatDescription(sbuf);
+  DCHECK(fdesc);
+
+  size_t bb_size = CoreMediaGlue::CMBlockBufferGetDataLength(bb);
+  size_t total_bytes = bb_size;
+
+  size_t pset_count;
+  int nal_size_field_bytes;
+  status = CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
+      fdesc, 0, NULL, NULL, &pset_count, &nal_size_field_bytes);
+  if (status ==
+      CoreMediaGlue::kCMFormatDescriptionBridgeError_InvalidParameter) {
+    DLOG(WARNING) << " assuming 2 parameter sets and 4 bytes NAL length header";
+    pset_count = 2;
+    nal_size_field_bytes = 4;
+  } else if (status != noErr) {
+    DLOG(ERROR)
+        << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: "
+        << status;
+    return;
+  }
+
+  if (keyframe) {
+    const uint8_t* pset;
+    size_t pset_size;
+    for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) {
+      status =
+          CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
+              fdesc, pset_i, &pset, &pset_size, NULL, NULL);
+      if (status != noErr) {
+        DLOG(ERROR)
+            << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: "
+            << status;
+        return;
+      }
+      total_bytes += pset_size + nal_size_field_bytes;
+    }
+  }
+
+  annexb_buffer->reserve(total_bytes);
+
+  // Copy all parameter sets before keyframes.
+  if (keyframe) {
+    const uint8_t* pset;
+    size_t pset_size;
+    for (size_t pset_i = 0; pset_i < pset_count; ++pset_i) {
+      status =
+          CoreMediaGlue::CMVideoFormatDescriptionGetH264ParameterSetAtIndex(
+              fdesc, pset_i, &pset, &pset_size, NULL, NULL);
+      if (status != noErr) {
+        DLOG(ERROR)
+            << " CMVideoFormatDescriptionGetH264ParameterSetAtIndex failed: "
+            << status;
+        return;
+      }
+      static const char startcode_4[4] = {0, 0, 0, 1};
+      annexb_buffer->append(startcode_4, sizeof(startcode_4));
+      annexb_buffer->append(reinterpret_cast<const char*>(pset), pset_size);
+    }
+  }
+
+  // Block buffers can be composed of non-contiguous chunks. For the sake of
+  // keeping this code simple, flatten non-contiguous block buffers.
+  base::ScopedCFTypeRef<CoreMediaGlue::CMBlockBufferRef> contiguous_bb(
+      bb, base::scoped_policy::RETAIN);
+  if (!CoreMediaGlue::CMBlockBufferIsRangeContiguous(bb, 0, 0)) {
+    contiguous_bb.reset();
+    status = CoreMediaGlue::CMBlockBufferCreateContiguous(
+        kCFAllocatorDefault,
+        bb,
+        kCFAllocatorDefault,
+        NULL,
+        0,
+        0,
+        0,
+        contiguous_bb.InitializeInto());
+    if (status != noErr) {
+      DLOG(ERROR) << " CMBlockBufferCreateContiguous failed: " << status;
+      return;
+    }
+  }
+
+  // Copy all the NAL units. In the process convert them from AVCC format
+  // (length header) to AnnexB format (start code).
+  char* bb_data;
+  status = CoreMediaGlue::CMBlockBufferGetDataPointer(
+      contiguous_bb, 0, NULL, NULL, &bb_data);
+  if (status != noErr) {
+    DLOG(ERROR) << " CMBlockBufferGetDataPointer failed: " << status;
+    return;
+  }
+
+  if (nal_size_field_bytes == 1) {
+    CopyNalsToAnnexB<uint8_t>(bb_data, bb_size, annexb_buffer);
+  } else if (nal_size_field_bytes == 2) {
+    CopyNalsToAnnexB<uint16_t>(bb_data, bb_size, annexb_buffer);
+  } else if (nal_size_field_bytes == 4) {
+    CopyNalsToAnnexB<uint32_t>(bb_data, bb_size, annexb_buffer);
+  }

[miu, 2014/09/24 00:10:13]

For safety:

  } else {
    NOTREACHED();
  }

Or, are there valid cases other than 1/2/4 bytes where they shouldn't be copied?

[jfroy, 2014/09/24 01:00:58]

No those are the only valid values per the H264 spec. I'll add the NOTREACHED
branch.

+}
+
+}  // namespace
+
+H264VideoToolboxEncoder::H264VideoToolboxEncoder(
+    scoped_refptr<CastEnvironment> cast_environment,
+    const VideoSenderConfig& video_config,
+    const CastInitializationCallback& initialization_cb)
+    : cast_environment_(cast_environment),
+      videotoolbox_glue_(VideoToolboxGlue::Get()),
+      frame_id_(kStartFrameId),
+      encode_next_frame_as_keyframe_(false) {
+  DCHECK(!initialization_cb.is_null());
+  CastInitializationStatus initialization_status;
+  if (videotoolbox_glue_) {
+    initialization_status = (Initialize(video_config))
+                                ? STATUS_VIDEO_INITIALIZED
+                                : STATUS_INVALID_VIDEO_CONFIGURATION;
+  } else {
+    LOG(ERROR) << " VideoToolbox is not available";
+    initialization_status = STATUS_HW_VIDEO_ENCODER_NOT_SUPPORTED;
+  }
+  cast_environment_->PostTask(
+      CastEnvironment::MAIN,
+      FROM_HERE,
+      base::Bind(initialization_cb, initialization_status));
+}
+
+H264VideoToolboxEncoder::~H264VideoToolboxEncoder() {
+  Teardown();
+}
+
+CVPixelBufferPoolRef H264VideoToolboxEncoder::cv_pixel_buffer_pool() const {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(compression_session_);
+  return videotoolbox_glue_->VTCompressionSessionGetPixelBufferPool(
+      compression_session_);
+}
+
+bool H264VideoToolboxEncoder::Initialize(
+    const VideoSenderConfig& video_config) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(!compression_session_);
+
+  // Note that the encoder object is given to the compression session as the
+  // callback context using a raw pointer. The C API does not allow us to use
+  // a smart pointer, nor is this encoder ref counted. However, this is still
+  // safe, because we 1) we own the compression session and 2) we tear it down
+  // safely. When destructing the encoder, the compression session is flushed
+  // and invalidated. Internally, VideoToolbox will join all of its threads
+  // before returning to the client. Therefore, when control returns to us, we
+  // are guaranteed that the output callback will not execute again.
+
+  // On OS X, allow the hardware encoder. Don't require it, it does not support
+  // all configurations (some of which are used for testing).
+  base::ScopedCFTypeRef<CFDictionaryRef> encoder_spec;
+#if !defined(OS_IOS)
+  encoder_spec = DictionaryWithKeyValue(
+      videotoolbox_glue_
+          ->kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder(),
+      kCFBooleanTrue);
+#endif
+
+  VTCompressionSessionRef session;
+  OSStatus status = videotoolbox_glue_->VTCompressionSessionCreate(
+      kCFAllocatorDefault,
+      video_config.width,
+      video_config.height,
+      CoreMediaGlue::kCMVideoCodecType_H264,
+      encoder_spec,
+      NULL /* sourceImageBufferAttributes */,
+      NULL /* compressedDataAllocator */,
+      CompressionCallback,

[miu, 2014/09/24 00:10:13]

Style nit (function pointer):

  &H264VideoToolboxEncoder::CompressionCallback

+      reinterpret_cast<void*>(this),
+      &session);
+  if (status != noErr) {
+    DLOG(ERROR) << " VTCompressionSessionCreate failed: " << status;
+    return false;
+  }
+  compression_session_.reset(session);
+
+  ConfigureSession(video_config);
+
+  return true;
+}
+
+void H264VideoToolboxEncoder::ConfigureSession(
+    const VideoSenderConfig& video_config) {
+  SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_ProfileLevel(),
+      videotoolbox_glue_->kVTProfileLevel_H264_Main_AutoLevel());
+  SetSessionProperty(videotoolbox_glue_->kVTCompressionPropertyKey_RealTime(),
+                     true);
+  SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_AllowFrameReordering(),
+      false);
+  SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_MaxKeyFrameInterval(),
+      240u);
+  SetSessionProperty(
+      videotoolbox_glue_
+          ->kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration(),
+      240u);
+  SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_AverageBitRate(),
+      static_cast<uint32_t>(video_config.start_bitrate));

[miu, 2014/09/24 00:10:13]

Since bit rate change is not supported, should this be:

  (video_config.min_bitrate + video_config.max_bitrate) / 2

[jfroy, 2014/09/24 01:00:58]

Yeah probably. My producer code currently has min = max = start, so I hadn't
noticed that. This implementation will eventually need better bitrate control
(because the encoder needs to be torn down, I'm toying with the idea of bitrate
bands, maybe 3-4, based on the video config values).

+  SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_ExpectedFrameRate(),
+      static_cast<uint32_t>(video_config.max_frame_rate));
+  SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_ColorPrimaries(),
+      kCVImageBufferColorPrimaries_ITU_R_709_2);
+  SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_TransferFunction(),
+      kCVImageBufferTransferFunction_ITU_R_709_2);
+  SetSessionProperty(
+      videotoolbox_glue_->kVTCompressionPropertyKey_YCbCrMatrix(),
+      kCVImageBufferYCbCrMatrix_ITU_R_709_2);
+}
+
+void H264VideoToolboxEncoder::Teardown() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+
+  // If the compression session exists, invalidate it. This blocks until all
+  // pending output callbacks have returned and any internal threads have
+  // joined, ensuring no output callback ever sees a dangling encoder pointer.
+  if (compression_session_) {
+    videotoolbox_glue_->VTCompressionSessionInvalidate(compression_session_);
+    compression_session_.reset();
+  }
+}
+
+bool H264VideoToolboxEncoder::EncodeVideoFrame(
+    const scoped_refptr<media::VideoFrame>& video_frame,
+    const base::TimeTicks& capture_time,
+    const FrameEncodedCallback& frame_encoded_callback) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(!capture_time.is_null());
+
+  if (!compression_session_) {
+    DLOG(ERROR) << " compression session is null";
+    return false;
+  }
+
+  // Clients can opt-in to an optimization whereby frames are stored in pixel
+  // buffers owned by the encoder. This can eliminate a data copy on some
+  // hardware configurations. When this optimization is used, the VideoFrame
+  // will wrap a CVPixelBuffer which we attempt to get first. If that fails,
+  // then we must wrap the VideoFrame in a CVPixelBuffer as an adapter to the
+  // VideoToolbox API. The WrapVideoFrame function performs this operation. The
+  // VideoFrame reference count is incremented and the resulting CVPixelBuffer
+  // will release the VideoFrame when it itself is destroyed. Because encoding
+  // is asynchronous and the encoder can reference CVPixelBuffers for a period
+  // of time in order to perform inter-frame compression, CVPixelBuffers must
+  // "own" the VideoFrame they wrap, not the other way around.
+  base::ScopedCFTypeRef<CVPixelBufferRef> pixel_buffer(
+      video_frame->cv_pixel_buffer(), base::scoped_policy::RETAIN);
+  if (!pixel_buffer) {
+    pixel_buffer = WrapVideoFrame(video_frame);
+    if (!pixel_buffer) {
+      return false;
+    }
+  }
+
+  CoreMediaGlue::CMTime timestamp_cm = CoreMediaGlue::CMTimeMake(
+      (capture_time - base::TimeTicks()).InMicroseconds(), USEC_PER_SEC);
+
+  scoped_ptr<FrameContext> frame_context(new FrameContext());
+  frame_context->capture_time = capture_time;
+  frame_context->frame_encoded_callback = frame_encoded_callback;
+
+  base::ScopedCFTypeRef<CFDictionaryRef> frame_props;
+  if (encode_next_frame_as_keyframe_) {
+    frame_props = DictionaryWithKeyValue(
+        videotoolbox_glue_->kVTEncodeFrameOptionKey_ForceKeyFrame(),
+        kCFBooleanTrue);
+    encode_next_frame_as_keyframe_ = false;
+  }
+
+  VTEncodeInfoFlags info;
+  OSStatus status = videotoolbox_glue_->VTCompressionSessionEncodeFrame(
+      compression_session_,
+      pixel_buffer,
+      timestamp_cm,
+      CoreMediaGlue::CMTime{0, 0, 0, 0},
+      frame_props,
+      reinterpret_cast<void*>(frame_context.release()),
+      &info);
+  if (status != noErr) {
+    DLOG(ERROR) << " VTCompressionSessionEncodeFrame failed: " << status;
+    return false;
+  }
+  if ((info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped)) {
+    DLOG(ERROR) << " frame dropped";
+    return false;
+  }
+
+  return true;
+}
+
+void H264VideoToolboxEncoder::SetBitRate(int new_bit_rate) {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  // VideoToolbox does not seem to support bitrate reconfiguration.
+}
+
+void H264VideoToolboxEncoder::GenerateKeyFrame() {
+  DCHECK(thread_checker_.CalledOnValidThread());
+  DCHECK(compression_session_);
+
+  encode_next_frame_as_keyframe_ = true;
+}
+
+void H264VideoToolboxEncoder::LatestFrameIdToReference(uint32 /*frame_id*/) {
+  // Not supported by VideoToolbox in any meaningful manner.
+}
+
+static void VideoFramePixelBufferReleaseCallback(void* frame_ref,
+                                                 const void* data,
+                                                 size_t size,
+                                                 size_t num_planes,
+                                                 const void* planes[]) {
+  free(const_cast<void*>(data));
+  reinterpret_cast<media::VideoFrame*>(frame_ref)->Release();
+}
+
+base::ScopedCFTypeRef<CVPixelBufferRef> H264VideoToolboxEncoder::WrapVideoFrame(
+    const scoped_refptr<media::VideoFrame>& frame) {
+  static const size_t MAX_PLANES = 3;
+
+  media::VideoFrame::Format format = frame->format();
+  size_t num_planes = media::VideoFrame::NumPlanes(format);
+  DCHECK_LE(num_planes, MAX_PLANES);
+  gfx::Size coded_size = frame->coded_size();
+
+  // media::VideoFrame only supports YUV formats, so there is no way to
+  // leverage VideoToolbox's ability to convert RGBA formats automatically. In
+  // addition, most of the media::VideoFrame formats are YVU, which VT does not
+  // support. Finally, media::VideoFrame formats do not carry any information
+  // about the color space, transform or any other colorimetric information
+  // that is generally needed to fully specify the input data. So essentially
+  // require that the input be YCbCr 4:2:0 (either planar or biplanar) and
+  // assume the standard video dynamic range for samples (although most modern
+  // HDTVs support full-range video these days).
+  OSType pixel_format;
+  if (format == media::VideoFrame::Format::I420) {
+    pixel_format = kCVPixelFormatType_420YpCbCr8Planar;
+  } else if (format == media::VideoFrame::Format::NV12) {
+    // TODO(jfroy): Use kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange when the
+    // minimum OS X and iOS SDKs permits it.
+    pixel_format = '420v';
+  } else {
+    DLOG(ERROR) << " unsupported frame format: " << format;
+    return base::ScopedCFTypeRef<CVPixelBufferRef>(NULL);
+  }
+
+  // TODO(jfroy): Support extended pixels (i.e. padding).
+  if (frame->coded_size() != frame->visible_rect().size()) {
+    DLOG(ERROR) << " frame with extended pixels not supported: "
+                << " coded_size: " << coded_size.ToString()
+                << ", visible_rect: " << frame->visible_rect().ToString();
+    return base::ScopedCFTypeRef<CVPixelBufferRef>(NULL);
+  }
+
+  void* plane_ptrs[MAX_PLANES];
+  size_t plane_widths[MAX_PLANES];
+  size_t plane_heights[MAX_PLANES];
+  size_t plane_bytes_per_row[MAX_PLANES];
+  for (size_t plane_i = 0; plane_i < num_planes; ++plane_i) {
+    plane_ptrs[plane_i] = frame->data(plane_i);
+    gfx::Size plane_size =
+        media::VideoFrame::PlaneSize(format, plane_i, coded_size);
+    plane_widths[plane_i] = plane_size.width();
+    plane_heights[plane_i] = plane_size.height();
+    plane_bytes_per_row[plane_i] = frame->stride(plane_i);
+  }
+
+  // CVPixelBufferCreateWithPlanarBytes needs a dummy plane descriptor or the
+  // release callback will not execute. The descriptor is freed in the callback.
+  void* descriptor = calloc(
+      1,
+      std::max(sizeof(CVPlanarPixelBufferInfo_YCbCrPlanar),
+               sizeof(CoreVideoGlue::CVPlanarPixelBufferInfo_YCbCrBiPlanar)));
+
+  // Wrap the frame's data in a CVPixelBuffer. Because this is a C API, we can't
+  // give it a smart pointer to the frame, so instead pass a raw pointer and
+  // increment the frame's reference count manually.
+  CVPixelBufferRef pixel_buffer;
+  CVReturn result =
+      CVPixelBufferCreateWithPlanarBytes(kCFAllocatorDefault,
+                                         coded_size.width(),
+                                         coded_size.height(),
+                                         format,

[miu, 2014/09/24 00:10:13]

Should this be pixel_format instead?

[jfroy, 2014/09/24 01:00:58]

Wow yeah. Good catch. Thanks for nothing, compiler...

+                                         descriptor,
+                                         0,
+                                         num_planes,
+                                         plane_ptrs,
+                                         plane_widths,
+                                         plane_heights,
+                                         plane_bytes_per_row,
+                                         VideoFramePixelBufferReleaseCallback,

[miu, 2014/09/24 00:10:13]

nit: &VideoFramePixelBufferReleaseCallback
    ^^^

+                                         frame.get(),
+                                         NULL,
+                                         &pixel_buffer);
+  if (result != kCVReturnSuccess) {
+    DLOG(ERROR) << " CVPixelBufferCreateWithPlanarBytes failed: " << result;
+    return base::ScopedCFTypeRef<CVPixelBufferRef>(NULL);
+  }
+
+  // The CVPixelBuffer now references the data of the frame, so increment its
+  // reference count manually. The release callback set on the pixel buffer will
+  // release the frame.
+  frame->AddRef();
+
+  return base::ScopedCFTypeRef<CVPixelBufferRef>(pixel_buffer);
+}
+
+bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key,
+                                                 uint32_t value) {

[miu, 2014/09/24 00:10:12]

This should probably be a signed int32_t instead, since this is how
CFNumberCreate() will interpret it.  Also, the callers of SetSessionProperty()
wouldn't need the uint32 cast anymore.

[jfroy, 2014/09/24 01:00:58]

Fair enough.

+  base::ScopedCFTypeRef<CFNumberRef> cfvalue(
+      CFNumberCreate(NULL, kCFNumberSInt32Type, &value));
+  return videotoolbox_glue_->VTSessionSetProperty(
+             compression_session_, key, cfvalue) == noErr;
+}
+
+bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key, bool value) {
+  CFBooleanRef cfvalue = (value) ? kCFBooleanTrue : kCFBooleanFalse;
+  return videotoolbox_glue_->VTSessionSetProperty(
+             compression_session_, key, cfvalue) == noErr;
+}
+
+bool H264VideoToolboxEncoder::SetSessionProperty(CFStringRef key,
+                                                 CFStringRef value) {
+  return videotoolbox_glue_->VTSessionSetProperty(
+             compression_session_, key, value) == noErr;
+}
+
+void H264VideoToolboxEncoder::CompressionCallback(void* encoder_opaque,
+                                                  void* frame_opaque,
+                                                  OSStatus status,
+                                                  VTEncodeInfoFlags info,
+                                                  CMSampleBufferRef sbuf) {
+  H264VideoToolboxEncoder* encoder =
+      reinterpret_cast<H264VideoToolboxEncoder*>(encoder_opaque);
+  scoped_ptr<FrameContext> frame_context(
+      reinterpret_cast<FrameContext*>(frame_opaque));
+
+  if (status != noErr) {
+    DLOG(ERROR) << " encoding failed: " << status;
+    return;
+  }
+  if ((info & VideoToolboxGlue::kVTEncodeInfo_FrameDropped)) {
+    DVLOG(2) << " frame dropped";
+    return;
+  }
+
+  CFDictionaryRef sample_attachments =
+      static_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(
+          CoreMediaGlue::CMSampleBufferGetSampleAttachmentsArray(sbuf, true),
+          0));
+
+  // If the NotSync key is not present, it implies Sync, which indicates a
+  // keyframe (at least I think, VT documentation is, erm, sparse). Could
+  // alternatively use kCMSampleAttachmentKey_DependsOnOthers == false.
+  bool keyframe = CFDictionaryContainsKey(

[miu, 2014/09/24 00:10:13]

nit: bool keyframe = !CFDictionaryContainsKey(...);

+                      sample_attachments,
+                      CoreMediaGlue::kCMSampleAttachmentKey_NotSync()) == false;
+
+  // Increment the encoder-scoped frame id and assign the new value to this
+  // frame. VideoToolbox calls the output callback serially, so this is safe.
+  uint32 frame_id = ++encoder->frame_id_;
+
+  scoped_ptr<EncodedFrame> encoded_frame(new EncodedFrame());
+  encoded_frame->frame_id = frame_id;
+  encoded_frame->reference_time = frame_context->capture_time;
+  encoded_frame->rtp_timestamp =
+      GetVideoRtpTimestamp(frame_context->capture_time);
+  if (keyframe) {
+    encoded_frame->dependency = EncodedFrame::KEY;
+    encoded_frame->referenced_frame_id = frame_id;
+  } else {
+    encoded_frame->dependency = EncodedFrame::DEPENDENT;
+    // H.264 supports complex frame reference schemes (multiple reference
+    // frames, slice references, backward and forward references, etc). This
+    // implementation compromises by setting the referenced frame ID to that of

[miu, 2014/09/24 00:10:13]

IMO, for clarity, the last sentence in this comment should be:

"Cast doesn't support the concept of forward-referencing frame dependencies; so
pretend that all frames are only decodable after their immediately preceding
frame is decoded.  This will ensure a Cast receiver only attempts to decode the
frames sequentially and in order."

[jfroy, 2014/09/24 01:00:58]

Alright. I'm also going to mention that forward references are explicitly
disabled and that the encoder will emit frames essentially in presentation order
(there can still be multiple backward references, but I think the profile and
level I'm using restricts ref frames to 1).

+    // the previous frame.
+    encoded_frame->referenced_frame_id = frame_id - 1;
+  }
+
+  CopySampleBufferToAnnexBBuffer(sbuf, &encoded_frame->data, keyframe);
+
+  encoder->cast_environment_->PostTask(
+      CastEnvironment::MAIN,
+      FROM_HERE,
+      base::Bind(frame_context->frame_encoded_callback,
+                 base::Passed(&encoded_frame)));
+}
+
+}  // namespace cast
+}  // namespace media