Use PPS/SPS NALUs to initialize a VTDecompressionSession.

content/common/gpu/media/vt_video_decode_accelerator.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+#include <CoreVideo/CoreVideo.h>
+#include <OpenGL/CGLIOSurface.h>
+
+#include "base/bind.h"
+#include "base/thread_task_runner_handle.h"
 #include "content/common/gpu/media/vt_video_decode_accelerator.h"
+#include "media/filters/h264_parser.h"
+
+using content_common_gpu_media::kModuleVt;
+using content_common_gpu_media::InitializeStubs;
+using content_common_gpu_media::IsVtInitialized;
+using content_common_gpu_media::StubPathMap;
 
 namespace content {
 
+// Route decoded frame callbacks back into the VTVideoDecodeAccelerator.
+static void OutputThunk(
+    void* decompression_output_refcon,
+    void* source_frame_refcon,
+    OSStatus status,
+    VTDecodeInfoFlags info_flags,
+    CVImageBufferRef image_buffer,
+    CMTime presentation_time_stamp,
+    CMTime presentation_duration) {
+  VTVideoDecodeAccelerator* vda =
+      reinterpret_cast<VTVideoDecodeAccelerator*>(decompression_output_refcon);
+  int32_t* bitstream_id_ptr = reinterpret_cast<int32_t*>(source_frame_refcon);
+  int32_t bitstream_id = *bitstream_id_ptr;
+  delete bitstream_id_ptr;
+  CFRetain(image_buffer);
+  vda->Output(bitstream_id, status, info_flags, image_buffer);
+}
+
 VTVideoDecodeAccelerator::VTVideoDecodeAccelerator(CGLContextObj cgl_context)
-    : loop_proxy_(base::MessageLoopProxy::current()),
+    : gpu_task_runner_(base::ThreadTaskRunnerHandle::Get()),
       cgl_context_(cgl_context),
       client_(NULL),
+      decoder_thread_("VTDecoderThread"),
+      format_(NULL),
+      session_(NULL),
+      coded_width_(0),
+      coded_height_(0),
       weak_this_factory_(this) {
+  callback_.decompressionOutputCallback = OutputThunk;
+  callback_.decompressionOutputRefCon = this;
 }
 
 VTVideoDecodeAccelerator::~VTVideoDecodeAccelerator() {
 }
 
 bool VTVideoDecodeAccelerator::Initialize(
     media::VideoCodecProfile profile,
     Client* client) {
   DCHECK(CalledOnValidThread());
-  DVLOG(2) << __FUNCTION__;
   client_ = client;
 
   // Only H.264 is supported.
   if (profile < media::H264PROFILE_MIN || profile > media::H264PROFILE_MAX)
     return false;
 
-  // Prevent anyone from using VTVideoDecoder for now. http://crbug.com/133828
-  return false;
+  // TODO(sandersd): Move VideoToolbox library loading to sandbox startup;
+  // until then, --no-sandbox is required.
+  if (!IsVtInitialized()) {
+    StubPathMap paths;
+    // CoreVideo is also required, but the loader stops after the first
+    // path is loaded. Instead we rely on the transitive dependency from
+    // VideoToolbox to CoreVideo.
+    // TODO(sandersd): Fallback to PrivateFrameworks for VideoToolbox.
+    paths[kModuleVt].push_back(FILE_PATH_LITERAL(
+        "/System/Library/Frameworks/VideoToolbox.framework/VideoToolbox"));
+    if (!InitializeStubs(paths))
+      return false;
+  }
+
+  // Spawn a thread to handle parsing and calling VideoToolbox.
+  if (!decoder_thread_.Start())
+    return false;
+
+  // Note that --ignore-gpu-blacklist is still required to get here.
+  return true;
+}
+
+// TODO(sandersd): Proper error reporting instead of CHECKs.
+void VTVideoDecodeAccelerator::ConfigureDecoder(
+    const std::vector<const uint8_t*>& nalu_data_ptrs,
+    const std::vector<size_t>& nalu_data_sizes) {
+  format_.reset();
+  CHECK(!CMVideoFormatDescriptionCreateFromH264ParameterSets(
+      kCFAllocatorDefault,
+      nalu_data_ptrs.size(),      // parameter_set_count
+      &nalu_data_ptrs.front(),    // &parameter_set_pointers
+      &nalu_data_sizes.front(),   // &parameter_set_sizes
+      4,                          // nal_unit_header_length
+      format_.InitializeInto()
+  ));
+
+  // TODO(sandersd): Check if the size has changed and handle picture requests.
+  CMVideoDimensions coded_size = CMVideoFormatDescriptionGetDimensions(format_);
+  coded_width_ = coded_size.width;
+  coded_height_ = coded_size.height;
+
+  base::ScopedCFTypeRef<CFMutableDictionaryRef> decoder_config(
+      CFDictionaryCreateMutable(
+          kCFAllocatorDefault,
+          1,  // capacity
+          &kCFTypeDictionaryKeyCallBacks,
+          &kCFTypeDictionaryValueCallBacks));
+
+  CFDictionarySetValue(
+      decoder_config,
+      // kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder

[scherkus (not reviewing), 2014/07/09 20:37:44]

q: can we not reference this directly?

[sandersd (OOO until July 31), 2014/07/09 20:52:50]

I didn't find a way to have the stub generator link non-function symbols.

+      CFSTR("EnableHardwareAcceleratedVideoDecoder"),
+      kCFBooleanTrue);
+
+  base::ScopedCFTypeRef<CFMutableDictionaryRef> image_config(
+      CFDictionaryCreateMutable(
+          kCFAllocatorDefault,
+          4,  // capacity
+          &kCFTypeDictionaryKeyCallBacks,
+          &kCFTypeDictionaryValueCallBacks));
+
+  // TODO(sandersd): ARGB for video that is not 4:2:0.
+  int32_t pixel_format = '2vuy';
+#define CFINT(i) CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &i)
+  base::ScopedCFTypeRef<CFNumberRef> cf_pixel_format(CFINT(pixel_format));
+  base::ScopedCFTypeRef<CFNumberRef> cf_width(CFINT(coded_width_));
+  base::ScopedCFTypeRef<CFNumberRef> cf_height(CFINT(coded_height_));
+  CFDictionarySetValue(
+      image_config, kCVPixelBufferPixelFormatTypeKey, cf_pixel_format);
+  CFDictionarySetValue(image_config, kCVPixelBufferWidthKey, cf_width);
+  CFDictionarySetValue(image_config, kCVPixelBufferHeightKey, cf_height);
+  CFDictionarySetValue(
+      image_config, kCVPixelBufferOpenGLCompatibilityKey, kCFBooleanTrue);
+
+  // TODO(sandersd): Skip if the session is compatable.
+  // TODO(sandersd): Flush frames when resetting.
+  session_.reset();
+  CHECK(!VTDecompressionSessionCreate(
+      kCFAllocatorDefault,
+      format_,              // video_format_description
+      decoder_config,       // video_decoder_specification
+      image_config,         // destination_image_buffer_attributes
+      &callback_,           // output_callback
+      session_.InitializeInto()
+  ));
+  DVLOG(2) << "Created VTDecompressionSession";
 }
 
 void VTVideoDecodeAccelerator::Decode(const media::BitstreamBuffer& bitstream) {
   DCHECK(CalledOnValidThread());
+  decoder_thread_.message_loop_proxy()->PostTask(FROM_HERE, base::Bind(
+      &VTVideoDecodeAccelerator::DecodeTask, weak_this_factory_.GetWeakPtr(),

[scherkus (not reviewing), 2014/07/09 20:37:44]

wouldn't this DCHECK() when running?

weak_this_factory_ would be bound to the gpu thread and should DCHECK() when ran
on a different thread

[sandersd (OOO until July 31), 2014/07/09 20:52:50]

My understanding of the WeakPtrFactory is that only dereferencing and
invalidation are bound to a particular thread. The only alternative I see is
saving a weak pointer in the class, which would then need to be copied by Bind,
and I don't see how that would be different.

The code has a few layers of indirection though, I could easily be
misunderstanding.

[scherkus (not reviewing), 2014/07/09 22:04:45]

There's a bit of subtleness with how WeakPtr{Factory} work, namely that they get
bound to the thread where the first dereference happens:
https://code.google.com/p/chromium/codesearch#chromium/src/base/memory/weak_p...

The most typical usage is to hand out a WeakPtr-using-callback to a different
thread that will only get run back on the *original thread* where the PostTask
was made.

In your case, the WeakPtrFactory's first dereference is on the
|decoder_thread_|, which isn't the right thread we want our WeakPtrs to be bound
to. When VTVDA gets deleted on the GPU thread, it'll invoke ~WeakPtrFactory()
... but it's been bound to the |decoder_thread_| and we'll DCHECK.

Here's a code sample that hopefully illustrates the problem:

class Bar {
 public:
  Bar() {}

  void TestFirstDereferenceAfterPosting() {
    base::WeakPtrFactory<Bar> weak_factory(this);
    base::WeakPtr<Bar> weak_this = weak_factory.GetWeakPtr();                   
                                                                              

    base::Thread thread("BarThread");                                           
                                                                              
    CHECK(thread.Start());                                                      
                                                                              
    base::WaitableEvent event(false, false);
    thread.message_loop_proxy()->PostTask(
        FROM_HERE, base::Bind(&Bar::Signal, weak_this, &event));
    event.Wait();                                                               
                                                                              

    // |weak_factory| has now been bound to |thread|.

    // Simulate ~WeakPtrFactory<T>(). This will DCHECK().
    weak_factory.InvalidateWeakPtrs();
  }

  // This will cause WeakPtrFactory to get bound to wrong thread.
  void Signal(base::WaitableEvent* event) {
    event->Signal();
  }

 private:
  DISALLOW_COPY_AND_ASSIGN(Bar);
};


The solution here is to use base::Unretained(this) when posting work to
|decoder_thread_|. Since VTVDA owns |decoder_thread_| and stops it inside of
~VTVDA(), you can guarantee that all work posted to that thread will complete
before VTVDA is destroyed and we won't use-after-free.

Do note that you'll still need to use WeakPtrs when posting work from
|decoder_thread_| *back* to the GPU thread ... but as far as this CL is
concerned you don't need to worry about that just yet.

[sandersd (OOO until July 31), 2014/07/09 22:27:37]

Done.

Blindingly obvious now.

[scherkus (not reviewing), 2014/07/09 22:39:24]

haha yeah it's all a matter of learning the in's-and-out's of Chromium's
threading/callback primitives ... it can take some time :)

+      bitstream));
+}
+
+void VTVideoDecodeAccelerator::DecodeTask(
+    const media::BitstreamBuffer bitstream) {
+  DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
+
+  // Map the bitstream buffer.
+  base::SharedMemory memory(bitstream.handle(), true);
+  size_t size = bitstream.size();
+  CHECK(memory.Map(size));
+  const uint8_t* buf = static_cast<uint8_t*>(memory.memory());
+
+  // Locate relevant NALUs in the buffer.
+  size_t data_size = 0;
+  std::vector<media::H264NALU> nalus;
+  std::vector<const uint8_t*> config_nalu_data_ptrs;
+  std::vector<size_t> config_nalu_data_sizes;
+  parser_.SetStream(buf, size);
+  media::H264NALU nalu;
+  while (true) {
+    media::H264Parser::Result result = parser_.AdvanceToNextNALU(&nalu);
+    if (result == media::H264Parser::kEOStream)
+      break;
+    CHECK_EQ(result, media::H264Parser::kOk);
+    if (nalu.nal_unit_type == media::H264NALU::kSPS ||
+        nalu.nal_unit_type == media::H264NALU::kPPS ||
+        nalu.nal_unit_type == media::H264NALU::kSPSExt) {
+      config_nalu_data_ptrs.push_back(nalu.data);
+      config_nalu_data_sizes.push_back(nalu.size);
+    }
+    nalus.push_back(nalu);
+    data_size += 4 + nalu.size;
+  }
+
+  if (!config_nalu_data_ptrs.empty())
+    ConfigureDecoder(config_nalu_data_ptrs, config_nalu_data_sizes);
+
+  // TODO(sandersd): Rewrite slice NALU headers and send for decoding.
+}
+
+// This method may be called on any VideoToolbox thread.
+void VTVideoDecodeAccelerator::Output(
+    int32_t bitstream_id,
+    OSStatus status,
+    VTDecodeInfoFlags info_flags,
+    CVImageBufferRef image_buffer) {
+  // TODO(sandersd): Store the frame in a queue.
+  CFRelease(image_buffer);
 }
 
 void VTVideoDecodeAccelerator::AssignPictureBuffers(
     const std::vector<media::PictureBuffer>& pictures) {
   DCHECK(CalledOnValidThread());
 }
 
 void VTVideoDecodeAccelerator::ReusePictureBuffer(int32_t picture_id) {
   DCHECK(CalledOnValidThread());
 }
 
 void VTVideoDecodeAccelerator::Flush() {
   DCHECK(CalledOnValidThread());
+  // TODO(sandersd): Trigger flush, sending frames.
 }
 
 void VTVideoDecodeAccelerator::Reset() {
   DCHECK(CalledOnValidThread());
+  // TODO(sandersd): Trigger flush, discarding frames.
 }
 
 void VTVideoDecodeAccelerator::Destroy() {
   DCHECK(CalledOnValidThread());
+  // TODO(sandersd): Trigger flush, discarding frames, and wait for them.
+  delete this;
 }
 
 bool VTVideoDecodeAccelerator::CanDecodeOnIOThread() {
   return false;
 }
 
 }  // namespace content