Implement actually decoding frames in VTVideoDecodeAccelerator.

content/common/gpu/media/vt_video_decode_accelerator.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <CoreVideo/CoreVideo.h>
 #include <OpenGL/CGLIOSurface.h>
 
 #include "base/bind.h"
 #include "base/thread_task_runner_handle.h"
 #include "content/common/gpu/media/vt_video_decode_accelerator.h"
 #include "media/filters/h264_parser.h"
 
 using content_common_gpu_media::kModuleVt;
 using content_common_gpu_media::InitializeStubs;
 using content_common_gpu_media::IsVtInitialized;
 using content_common_gpu_media::StubPathMap;
 
 namespace content {
 
-// Size of length headers prepended to NALUs in MPEG-4 framing. (1, 2, or 4.)
+// Size of NALU length headers in AVCC/MPEG-4 format (can be 1, 2, or 4).
 static const int kNALUHeaderLength = 4;
 
+// We only request 5 picture buffers from the client which are used to hold the
+// decoded samples. These buffers are then reused when the client tells us that
+// it is done with the buffer.
+static const int kNumPictureBuffers = 5;
+
 // Route decoded frame callbacks back into the VTVideoDecodeAccelerator.
 static void OutputThunk(
     void* decompression_output_refcon,
     void* source_frame_refcon,
     OSStatus status,
     VTDecodeInfoFlags info_flags,
     CVImageBufferRef image_buffer,
     CMTime presentation_time_stamp,
     CMTime presentation_duration) {
+  // TODO(sandersd): Implement flush-before-delete to guarantee validity.
   VTVideoDecodeAccelerator* vda =
       reinterpret_cast<VTVideoDecodeAccelerator*>(decompression_output_refcon);
-  int32_t* bitstream_id_ptr = reinterpret_cast<int32_t*>(source_frame_refcon);
-  int32_t bitstream_id = *bitstream_id_ptr;
-  delete bitstream_id_ptr;
-  CFRetain(image_buffer);
-  vda->Output(bitstream_id, status, info_flags, image_buffer);
+  intptr_t bitstream_id = reinterpret_cast<intptr_t>(source_frame_refcon);
+  vda->Output(bitstream_id, status, image_buffer);
+}
+
+VTVideoDecodeAccelerator::DecodedFrame::DecodedFrame(
+    uint32_t bitstream_id,
+    CVImageBufferRef image_buffer)
+    : bitstream_id(bitstream_id),
+      image_buffer(image_buffer) {
+}
+
+VTVideoDecodeAccelerator::DecodedFrame::~DecodedFrame() {
 }
 
 VTVideoDecodeAccelerator::VTVideoDecodeAccelerator(CGLContextObj cgl_context)
     : cgl_context_(cgl_context),
       client_(NULL),
-      decoder_thread_("VTDecoderThread"),
       format_(NULL),
       session_(NULL),
-      weak_this_factory_(this) {
+      gpu_task_runner_(base::ThreadTaskRunnerHandle::Get()),
+      weak_this_factory_(this),
+      decoder_thread_("VTDecoderThread") {
   callback_.decompressionOutputCallback = OutputThunk;
   callback_.decompressionOutputRefCon = this;
 }
 
 VTVideoDecodeAccelerator::~VTVideoDecodeAccelerator() {
 }
 
 bool VTVideoDecodeAccelerator::Initialize(
     media::VideoCodecProfile profile,
     Client* client) {
   DCHECK(CalledOnValidThread());
+
   client_ = client;
+  weak_client_factory_.reset(
+      new base::WeakPtrFactory<media::VideoDecodeAccelerator::Client>(client));
 
   // Only H.264 is supported.
   if (profile < media::H264PROFILE_MIN || profile > media::H264PROFILE_MAX)
     return false;
 
   // TODO(sandersd): Move VideoToolbox library loading to sandbox startup;
   // until then, --no-sandbox is required.
   if (!IsVtInitialized()) {
     StubPathMap paths;
     // CoreVideo is also required, but the loader stops after the first
@@ skipping 11 matching lines @@
     return false;
 
   // Note that --ignore-gpu-blacklist is still required to get here.
   return true;
 }
 
 // TODO(sandersd): Proper error reporting instead of CHECKs.
 void VTVideoDecodeAccelerator::ConfigureDecoder(
     const std::vector<const uint8_t*>& nalu_data_ptrs,
     const std::vector<size_t>& nalu_data_sizes) {
+  DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
+  // Construct a new format description from the parameter sets.
+  // TODO(sandersd): Replace this with custom code to support OS X < 10.9.
   format_.reset();
   CHECK(!CMVideoFormatDescriptionCreateFromH264ParameterSets(
       kCFAllocatorDefault,
       nalu_data_ptrs.size(),      // parameter_set_count
       &nalu_data_ptrs.front(),    // &parameter_set_pointers
       &nalu_data_sizes.front(),   // &parameter_set_sizes
       kNALUHeaderLength,          // nal_unit_header_length
-      format_.InitializeInto()
-  ));
+      format_.InitializeInto()));
+  CMVideoDimensions coded_dimensions =
+      CMVideoFormatDescriptionGetDimensions(format_);
 
-  // TODO(sandersd): Check if the size has changed and handle picture requests.
-  CMVideoDimensions coded_size = CMVideoFormatDescriptionGetDimensions(format_);
-  coded_size_.SetSize(coded_size.width, coded_size.height);
-
+  // Prepare VideoToolbox configuration dictionaries.
   base::ScopedCFTypeRef<CFMutableDictionaryRef> decoder_config(
       CFDictionaryCreateMutable(
           kCFAllocatorDefault,
           1,  // capacity
           &kCFTypeDictionaryKeyCallBacks,
           &kCFTypeDictionaryValueCallBacks));
 
   CFDictionarySetValue(
       decoder_config,
       // kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder
       CFSTR("EnableHardwareAcceleratedVideoDecoder"),
       kCFBooleanTrue);
 
   base::ScopedCFTypeRef<CFMutableDictionaryRef> image_config(
       CFDictionaryCreateMutable(
           kCFAllocatorDefault,
           4,  // capacity
           &kCFTypeDictionaryKeyCallBacks,
           &kCFTypeDictionaryValueCallBacks));
 
-  // TODO(sandersd): ARGB for video that is not 4:2:0.
-  int32_t pixel_format = '2vuy';
 #define CFINT(i) CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &i)
+  // TODO(sandersd): RGBA option for 4:4:4 video.
+  int32_t pixel_format = kCVPixelFormatType_422YpCbCr8;
   base::ScopedCFTypeRef<CFNumberRef> cf_pixel_format(CFINT(pixel_format));
-  base::ScopedCFTypeRef<CFNumberRef> cf_width(CFINT(coded_size.width));
-  base::ScopedCFTypeRef<CFNumberRef> cf_height(CFINT(coded_size.height));
+  base::ScopedCFTypeRef<CFNumberRef> cf_width(CFINT(coded_dimensions.width));
+  base::ScopedCFTypeRef<CFNumberRef> cf_height(CFINT(coded_dimensions.height));
 #undef CFINT
   CFDictionarySetValue(
       image_config, kCVPixelBufferPixelFormatTypeKey, cf_pixel_format);
   CFDictionarySetValue(image_config, kCVPixelBufferWidthKey, cf_width);
   CFDictionarySetValue(image_config, kCVPixelBufferHeightKey, cf_height);
   CFDictionarySetValue(
       image_config, kCVPixelBufferOpenGLCompatibilityKey, kCFBooleanTrue);
 
-  // TODO(sandersd): Skip if the session is compatible.
-  // TODO(sandersd): Flush frames when resetting.
+  // TODO(sandersd): Check if the session is already compatible.
+  // TODO(sandersd): Flush.
   session_.reset();

[scherkus (not reviewing), 2014/07/17 21:17:26]

shouldn't this already be NULL / DCHECK it is?

[sandersd (OOO until July 31), 2014/07/17 23:22:46]

That's true now that I've changed DecodeTask to only call once, but that is
temporary. It turns out that the previous SPS/PPS may need to be saved because
it's possible to update only one of them (but I need to read the spec to be sure
I get this right), but the code was assuming they always come together.

   CHECK(!VTDecompressionSessionCreate(
       kCFAllocatorDefault,
       format_,              // video_format_description
       decoder_config,       // video_decoder_specification
       image_config,         // destination_image_buffer_attributes
       &callback_,           // output_callback
-      session_.InitializeInto()
-  ));
-  DVLOG(2) << "Created VTDecompressionSession";
+      session_.InitializeInto()));
+
+  // If the size has changed, request new picture buffers.
+  gfx::Size new_coded_size(coded_dimensions.width, coded_dimensions.height);
+  if (coded_size_ != new_coded_size) {

[scherkus (not reviewing), 2014/07/17 21:17:26]

FYI coded_size_ is being read/written to from multiple threads

also, is this a code path you hit today? it seems like this code can only be
called once to initialize session_ and then never again

perhaps you'd be better off:
  1) removing weak_client_factory_ entirely
  2) posting a task to a helper method on the gpu thread (e.g.,
CodedSizeChanged(...) or DecoderConfigured(...) or what have you)
  3) updating coded_size_ on the GPU thread
  4) calling client_->ProvidePictureBuffers(...) as needed


that eliminates reading/writing coded_size_ on multiple threads and keeps us
using a single weak pointer

[sandersd (OOO until July 31), 2014/07/17 23:22:46]

Good catch on the data race.

This is a tricky operation, because we have to make sure that the current
pictures are all released before we send the current frame to be decoded. This
is complicated by the fact that we can't immediately release pictures that are
backed by images (that can't happen safely until they are passed back in
ReusePictureBuffer).

I've changed this to use SizeChangedTask, which I believe will be safe with the
addition of a SIZE_CHANGED state (states are part of the flushing logic coming
next).

+    coded_size_ = new_coded_size;
+    // TODO(sandersd): Dismiss existing picture buffers.
+    gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
+        &media::VideoDecodeAccelerator::Client::ProvidePictureBuffers,
+        weak_client_factory_->GetWeakPtr(),
+        kNumPictureBuffers,
+        coded_size_,
+        GL_TEXTURE_RECTANGLE_ARB));
+  }
 }
 
 void VTVideoDecodeAccelerator::Decode(const media::BitstreamBuffer& bitstream) {
   DCHECK(CalledOnValidThread());
   decoder_thread_.message_loop_proxy()->PostTask(FROM_HERE, base::Bind(
       &VTVideoDecodeAccelerator::DecodeTask, base::Unretained(this),
       bitstream));
 }
 
 void VTVideoDecodeAccelerator::DecodeTask(
     const media::BitstreamBuffer bitstream) {
   DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
 
   // Map the bitstream buffer.
   base::SharedMemory memory(bitstream.handle(), true);
   size_t size = bitstream.size();
   CHECK(memory.Map(size));
   const uint8_t* buf = static_cast<uint8_t*>(memory.memory());
 
-  // Locate relevant NALUs in the buffer.
+  // NALUs are stored with Annex B format in the bitstream buffer (3-byte start
+  // codes), but VideoToolbox expects AVCC/MPEG-4 format (length headers), so we
+  // must to rewrite the data.
+  //
+  // 1. Locate relevant NALUs and compute the size of the translated data.
+  //    Also record any parameter sets for VideoToolbox initialization.
   size_t data_size = 0;
   std::vector<media::H264NALU> nalus;
   std::vector<const uint8_t*> config_nalu_data_ptrs;
   std::vector<size_t> config_nalu_data_sizes;
   parser_.SetStream(buf, size);
   media::H264NALU nalu;
   while (true) {
     media::H264Parser::Result result = parser_.AdvanceToNextNALU(&nalu);
     if (result == media::H264Parser::kEOStream)
       break;
     CHECK_EQ(result, media::H264Parser::kOk);
+    // TODO(sandersd): Check that these are only at the start.
     if (nalu.nal_unit_type == media::H264NALU::kSPS ||
         nalu.nal_unit_type == media::H264NALU::kPPS ||
         nalu.nal_unit_type == media::H264NALU::kSPSExt) {
+      DVLOG(2) << "Parameter set " << nalu.nal_unit_type;
       config_nalu_data_ptrs.push_back(nalu.data);
       config_nalu_data_sizes.push_back(nalu.size);
+    } else {
+      nalus.push_back(nalu);
+      data_size += kNALUHeaderLength + nalu.size;
     }
-    nalus.push_back(nalu);
-    // Each NALU will have a 4-byte length header prepended.
-    data_size += kNALUHeaderLength + nalu.size;
   }
 
-  if (!config_nalu_data_ptrs.empty())
+  // 2. Initialize VideoToolbox.
+  // TODO(sandersd): Reinitialize when there are new parameter sets.
+  if (!session_)
     ConfigureDecoder(config_nalu_data_ptrs, config_nalu_data_sizes);
 
-  // TODO(sandersd): Rewrite slice NALU headers and send for decoding.
+  // 3. Allocate a memory-backed CMBlockBuffer for the translated data.
+  base::ScopedCFTypeRef<CMBlockBufferRef> data;
+  CHECK(!CMBlockBufferCreateWithMemoryBlock(
+      kCFAllocatorDefault,
+      NULL,                 // &memory_block
+      data_size,            // block_length
+      kCFAllocatorDefault,  // block_allocator
+      NULL,                 // &custom_block_source
+      0,                    // offset_to_data
+      data_size,            // data_length
+      0,                    // flags
+      data.InitializeInto()));
+
+  // 4. Copy NALU data, inserting length headers.
+  size_t offset = 0;
+  for (size_t i = 0; i < nalus.size(); i++) {
+    media::H264NALU& nalu = nalus[i];
+    uint8_t header[4] = {0xff & nalu.size >> 24,
+                         0xff & nalu.size >> 16,
+                         0xff & nalu.size >> 8,
+                         0xff & nalu.size};
+    CHECK(!CMBlockBufferReplaceDataBytes(header, data, offset, 4));
+    offset += 4;
+    CHECK(!CMBlockBufferReplaceDataBytes(nalu.data, data, offset, nalu.size));
+    offset += nalu.size;
+  }
+
+  // 5. Package the data for VideoToolbox and request decoding.
+  base::ScopedCFTypeRef<CMSampleBufferRef> frame;
+  CHECK(!CMSampleBufferCreate(
+      kCFAllocatorDefault,
+      data,                 // data_buffer
+      true,                 // data_ready
+      NULL,                 // make_data_ready_callback
+      NULL,                 // make_data_ready_refcon
+      format_,              // format_description
+      1,                    // num_samples
+      0,                    // num_sample_timing_entries
+      NULL,                 // &sample_timing_array
+      0,                    // num_sample_size_entries
+      NULL,                 // &sample_size_array
+      frame.InitializeInto()));
+
+  VTDecodeFrameFlags decode_flags =
+      kVTDecodeFrame_EnableAsynchronousDecompression |
+      kVTDecodeFrame_EnableTemporalProcessing;
+
+  intptr_t bitstream_id = bitstream.id();
+  CHECK(!VTDecompressionSessionDecodeFrame(
+      session_,
+      frame,                                  // sample_buffer
+      decode_flags,                           // decode_flags
+      reinterpret_cast<void*>(bitstream_id),  // source_frame_refcon
+      NULL));                                 // &info_flags_out
 }
 
 // This method may be called on any VideoToolbox thread.
 void VTVideoDecodeAccelerator::Output(
     int32_t bitstream_id,
     OSStatus status,
-    VTDecodeInfoFlags info_flags,
     CVImageBufferRef image_buffer) {
-  // TODO(sandersd): Store the frame in a queue.
-  CFRelease(image_buffer);
+  CHECK(!status);
+  CHECK_EQ(CFGetTypeID(image_buffer), CVPixelBufferGetTypeID());
+  CFRetain(image_buffer);
+  gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
+      &VTVideoDecodeAccelerator::OutputTask,
+      weak_this_factory_.GetWeakPtr(),
+      DecodedFrame(bitstream_id, image_buffer)));
+}
+
+void VTVideoDecodeAccelerator::OutputTask(DecodedFrame frame) {
+  DCHECK(CalledOnValidThread());
+  decoded_frames_.push(frame);
+  SendPictures();
 }
 
 void VTVideoDecodeAccelerator::AssignPictureBuffers(
     const std::vector<media::PictureBuffer>& pictures) {
   DCHECK(CalledOnValidThread());
+
+  for (size_t i = 0; i < pictures.size(); i++) {
+    picture_ids_.push(pictures[i].id());
+    texture_ids_[pictures[i].id()] = pictures[i].texture_id();
+  }
+
+  // Pictures are not marked as uncleared until this method returns. They will
+  // become broken if they are used before that happens.

[scherkus (not reviewing), 2014/07/17 21:17:26]

to confirm -- we need the post task here, right?

[sandersd (OOO until July 31), 2014/07/17 23:22:46]

Yes.

+  gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
+      &VTVideoDecodeAccelerator::SendPictures,
+      weak_this_factory_.GetWeakPtr()));
 }
 
 void VTVideoDecodeAccelerator::ReusePictureBuffer(int32_t picture_id) {
   DCHECK(CalledOnValidThread());
+  DCHECK_EQ(CFGetRetainCount(picture_bindings_[picture_id]), 1);
+  picture_bindings_.erase(picture_id);
+  picture_ids_.push(picture_id);
+  SendPictures();
+}
+
+void VTVideoDecodeAccelerator::SendPictures() {
+  DCHECK(CalledOnValidThread());
+  if (picture_ids_.empty() || decoded_frames_.empty())
+    return;
+
+  CGLContextObj prev_context = CGLGetCurrentContext();
+  CHECK(!CGLSetCurrentContext(cgl_context_));
+  glEnable(GL_TEXTURE_RECTANGLE_ARB);
+
+  while (!picture_ids_.empty() && !decoded_frames_.empty()) {
+    int32_t picture_id = picture_ids_.front();
+    picture_ids_.pop();
+    DecodedFrame frame = decoded_frames_.front();
+    decoded_frames_.pop();
+    IOSurfaceRef surface = CVPixelBufferGetIOSurface(frame.image_buffer);
+
+    glBindTexture(GL_TEXTURE_RECTANGLE_ARB, texture_ids_[picture_id]);
+    CHECK(!CGLTexImageIOSurface2D(
+        cgl_context_,                 // ctx
+        GL_TEXTURE_RECTANGLE_ARB,     // target
+        GL_RGB,                       // internal_format
+        coded_size_.width(),          // width
+        coded_size_.height(),         // height
+        GL_YCBCR_422_APPLE,           // format
+        GL_UNSIGNED_SHORT_8_8_APPLE,  // type
+        surface,                      // io_surface
+        0));                          // plane
+    glBindTexture(GL_TEXTURE_RECTANGLE_ARB, 0);
+
+    picture_bindings_[picture_id] = frame.image_buffer;
+    client_->PictureReady(media::Picture(picture_id, frame.bitstream_id));
+    client_->NotifyEndOfBitstreamBuffer(frame.bitstream_id);
+  }
+
+  glDisable(GL_TEXTURE_RECTANGLE_ARB);
+  CHECK(!CGLSetCurrentContext(prev_context));
 }
 
 void VTVideoDecodeAccelerator::Flush() {
   DCHECK(CalledOnValidThread());
   // TODO(sandersd): Trigger flush, sending frames.
 }
 
 void VTVideoDecodeAccelerator::Reset() {
   DCHECK(CalledOnValidThread());
   // TODO(sandersd): Trigger flush, discarding frames.
 }
 
 void VTVideoDecodeAccelerator::Destroy() {
   DCHECK(CalledOnValidThread());
   // TODO(sandersd): Trigger flush, discarding frames, and wait for them.
   delete this;
 }
 
 bool VTVideoDecodeAccelerator::CanDecodeOnIOThread() {
   return false;
 }
 
 }  // namespace content