Implement flushing in VTVideoDecodeAccelerator.

content/common/gpu/media/vt_video_decode_accelerator.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <CoreVideo/CoreVideo.h>
 #include <OpenGL/CGLIOSurface.h>
 
 #include "base/bind.h"
 #include "base/command_line.h"
 #include "base/sys_byteorder.h"
@@ skipping 21 matching lines @@
 
 // Route decoded frame callbacks back into the VTVideoDecodeAccelerator.
 static void OutputThunk(
     void* decompression_output_refcon,
     void* source_frame_refcon,
     OSStatus status,
     VTDecodeInfoFlags info_flags,
     CVImageBufferRef image_buffer,
     CMTime presentation_time_stamp,
     CMTime presentation_duration) {
-  // TODO(sandersd): Implement flush-before-delete to guarantee validity.
   VTVideoDecodeAccelerator* vda =
       reinterpret_cast<VTVideoDecodeAccelerator*>(decompression_output_refcon);
   int32_t bitstream_id = reinterpret_cast<intptr_t>(source_frame_refcon);
   vda->Output(bitstream_id, status, image_buffer);
 }
 
 VTVideoDecodeAccelerator::DecodedFrame::DecodedFrame(
     int32_t bitstream_id,
     CVImageBufferRef image_buffer)
     : bitstream_id(bitstream_id),
       image_buffer(image_buffer) {
 }
 
 VTVideoDecodeAccelerator::DecodedFrame::~DecodedFrame() {
 }
 
 VTVideoDecodeAccelerator::VTVideoDecodeAccelerator(CGLContextObj cgl_context)
     : cgl_context_(cgl_context),
       client_(NULL),
+      state_(NORMAL),
+      frames_pending_decode_(0),
       format_(NULL),
       session_(NULL),
       gpu_task_runner_(base::ThreadTaskRunnerHandle::Get()),
       weak_this_factory_(this),
       decoder_thread_("VTDecoderThread") {
   callback_.decompressionOutputCallback = OutputThunk;
   callback_.decompressionOutputRefCon = this;
 }
 
 VTVideoDecodeAccelerator::~VTVideoDecodeAccelerator() {
@@ skipping 80 matching lines @@
   base::ScopedCFTypeRef<CFNumberRef> cf_height(CFINT(coded_dimensions.height));
 #undef CFINT
   CFDictionarySetValue(
       image_config, kCVPixelBufferPixelFormatTypeKey, cf_pixel_format);
   CFDictionarySetValue(image_config, kCVPixelBufferWidthKey, cf_width);
   CFDictionarySetValue(image_config, kCVPixelBufferHeightKey, cf_height);
   CFDictionarySetValue(
       image_config, kCVPixelBufferOpenGLCompatibilityKey, kCFBooleanTrue);
 
   // TODO(sandersd): Check if the session is already compatible.
-  // TODO(sandersd): Flush.
   session_.reset();
   CHECK(!VTDecompressionSessionCreate(
       kCFAllocatorDefault,
       format_,              // video_format_description
       decoder_config,       // video_decoder_specification
       image_config,         // destination_image_buffer_attributes
       &callback_,           // output_callback
       session_.InitializeInto()));
 
   // If the size has changed, trigger a request for new picture buffers.
   gfx::Size new_coded_size(coded_dimensions.width, coded_dimensions.height);
   if (coded_size_ != new_coded_size) {
     coded_size_ = new_coded_size;
     gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
         &VTVideoDecodeAccelerator::SizeChangedTask,
         weak_this_factory_.GetWeakPtr(),
         coded_size_));;
   }
 }
 
 void VTVideoDecodeAccelerator::Decode(const media::BitstreamBuffer& bitstream) {
   DCHECK(CalledOnValidThread());
-  // TODO(sandersd): Test what happens if bitstream buffers are passed to VT out
-  // of order.
+  frames_pending_decode_++;

[Pawel Osciak, 2014/08/27 08:25:20]

Client will keep calling Decode()s after it calls Reset(), and before it gets
NotifyResetDone(), for the frames to be decoded after Reset(). So this is not
precise I think, or may not even work at all, because this may never drop down
to 0... Am I missing something?

[sandersd (OOO until July 31), 2014/08/27 21:40:43]

video_decoder.h specifies that no decode calls are made during a reset:

// Note: No VideoDecoder calls should be made before |closure| is executed.

And as far as I can tell, DecoderStream correctly avoids making any Decode()
calls while a reset is happening.

[Pawel Osciak, 2014/08/28 12:09:12]

This class implements VideoDecodeAccelerator, not VideoDecoder :)
GpuVideoDecoder, the VideoDecoder implementation that is the HTML5 <video>
client of VDA impls, is a VD, but other VDA clients are not.

In fact, one significant example is Flash, the ppapi-based client of VDA, which
does call Decode immediately after Reset, before getting NotifyResetDone. We had
to address this once before, see crrev.com/179384 for example. But this is fully
intended behavior, so it was just a bug in one of the VDAs that we had to fix.

And, well, it's slightly more efficient to accept Decode while resetting and
there is no real reason not to. I think I'll file an exploratory bug why we
can't accumulate more buffers from VideoDecoder.

You could solve this perhaps by managing Reset like other VDAs, see VaapiVDA for
example.

   decoder_thread_.message_loop_proxy()->PostTask(FROM_HERE, base::Bind(
       &VTVideoDecodeAccelerator::DecodeTask, base::Unretained(this),
       bitstream));
 }
 
 // TODO(sandersd): Proper error reporting instead of CHECKs.
 void VTVideoDecodeAccelerator::DecodeTask(
     const media::BitstreamBuffer bitstream) {
   DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
 
@@ skipping 105 matching lines @@
   CHECK_EQ(CFGetTypeID(image_buffer), CVPixelBufferGetTypeID());
   CFRetain(image_buffer);
   gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
       &VTVideoDecodeAccelerator::OutputTask,
       weak_this_factory_.GetWeakPtr(),
       DecodedFrame(bitstream_id, image_buffer)));
 }
 
 void VTVideoDecodeAccelerator::OutputTask(DecodedFrame frame) {
   DCHECK(CalledOnValidThread());
-  decoded_frames_.push(frame);
-  SendPictures();
+  if (state_ == RESETTING || state_ == DESTROYING) {
+    // Drop all decoded frames when resetting or destroying.
+    client_->NotifyEndOfBitstreamBuffer(frame.bitstream_id);
+    frames_pending_decode_--;
+    if (!frames_pending_decode_)
+      FlushDone();
+  } else {
+    decoded_frames_.push(frame);
+    SendPictures();
+  }
 }
 
 void VTVideoDecodeAccelerator::SizeChangedTask(gfx::Size coded_size) {
   DCHECK(CalledOnValidThread());
   texture_size_ = coded_size;
   // TODO(sandersd): Dismiss existing picture buffers.
   client_->ProvidePictureBuffers(
       kNumPictureBuffers, texture_size_, GL_TEXTURE_RECTANGLE_ARB);
 }
 
 void VTVideoDecodeAccelerator::AssignPictureBuffers(
     const std::vector<media::PictureBuffer>& pictures) {
   DCHECK(CalledOnValidThread());
 
   for (size_t i = 0; i < pictures.size(); i++) {
     CHECK(!texture_ids_.count(pictures[i].id()));
     available_picture_ids_.push(pictures[i].id());
     texture_ids_[pictures[i].id()] = pictures[i].texture_id();
   }
 
   // Pictures are not marked as uncleared until this method returns. They will
-  // become broken if they are used before that happens.
+  // become broken if they are used before that happens. So, instead of calling
+  // SendPictures(), arrange for it to be called later.
   gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
       &VTVideoDecodeAccelerator::SendPictures,
       weak_this_factory_.GetWeakPtr()));
 }
 
 void VTVideoDecodeAccelerator::ReusePictureBuffer(int32_t picture_id) {
   DCHECK(CalledOnValidThread());
   DCHECK_EQ(CFGetRetainCount(picture_bindings_[picture_id]), 1);
   picture_bindings_.erase(picture_id);
   available_picture_ids_.push(picture_id);
   SendPictures();
 }
 
 // TODO(sandersd): Proper error reporting instead of CHECKs.
 void VTVideoDecodeAccelerator::SendPictures() {
   DCHECK(CalledOnValidThread());
   if (available_picture_ids_.empty() || decoded_frames_.empty())
     return;
 
   gfx::ScopedCGLSetCurrentContext scoped_set_current_context(cgl_context_);
   glEnable(GL_TEXTURE_RECTANGLE_ARB);
 
   while (!available_picture_ids_.empty() && !decoded_frames_.empty()) {
     int32_t picture_id = available_picture_ids_.front();
     available_picture_ids_.pop();
     DecodedFrame frame = decoded_frames_.front();
     decoded_frames_.pop();
     IOSurfaceRef surface = CVPixelBufferGetIOSurface(frame.image_buffer);
 
+    // TODO(sandersd): Find out why this fails due to no GL context sometimes.
     gfx::ScopedTextureBinder
         texture_binder(GL_TEXTURE_RECTANGLE_ARB, texture_ids_[picture_id]);
     CHECK(!CGLTexImageIOSurface2D(
         cgl_context_,                 // ctx
         GL_TEXTURE_RECTANGLE_ARB,     // target
         GL_RGB,                       // internal_format
         texture_size_.width(),        // width
         texture_size_.height(),       // height
         GL_YCBCR_422_APPLE,           // format
         GL_UNSIGNED_SHORT_8_8_APPLE,  // type
         surface,                      // io_surface
         0));                          // plane
 
     picture_bindings_[picture_id] = frame.image_buffer;
     client_->PictureReady(media::Picture(picture_id, frame.bitstream_id));
     client_->NotifyEndOfBitstreamBuffer(frame.bitstream_id);
+    frames_pending_decode_--;
   }
 
   glDisable(GL_TEXTURE_RECTANGLE_ARB);
+
+  if (state_ != NORMAL && !frames_pending_decode_)
+    FlushDone();
+}
+
+void VTVideoDecodeAccelerator::FlushStart(State state) {
+  DCHECK(CalledOnValidThread());
+  state_ = state;
+
+  // If resetting or destroying, drop all pending frames.
+  if (state_ == RESETTING || state_ == DESTROYING) {
+    while (!decoded_frames_.empty()) {
+      DecodedFrame frame = decoded_frames_.front();
+      decoded_frames_.pop();
+      client_->NotifyEndOfBitstreamBuffer(frame.bitstream_id);
+      frames_pending_decode_--;
+    }
+  }
+
+  if (frames_pending_decode_) {
+    decoder_thread_.message_loop_proxy()->PostTask(FROM_HERE, base::Bind(
+        &VTVideoDecodeAccelerator::FlushTask, base::Unretained(this)));
+  } else {
+    FlushDone();
+  }
+}
+
+void VTVideoDecodeAccelerator::FlushTask() {
+  DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
+  if (session_)
+    CHECK(!VTDecompressionSessionFinishDelayedFrames(session_));
+}
+
+void VTVideoDecodeAccelerator::FlushDone() {
+  DCHECK(CalledOnValidThread());
+  switch (state_) {
+    case NORMAL:
+      NOTREACHED();
+      break;
+    case FLUSHING:
+      client_->NotifyFlushDone();
+      state_ = NORMAL;
+      break;
+    case RESETTING:
+      client_->NotifyResetDone();
+      state_ = NORMAL;
+      break;
+    case DESTROYING:
+      delete this;
+      break;
+  }
 }
 
 void VTVideoDecodeAccelerator::Flush() {
   DCHECK(CalledOnValidThread());
-  // TODO(sandersd): Trigger flush, sending frames.
+  FlushStart(FLUSHING);
 }
 
 void VTVideoDecodeAccelerator::Reset() {
   DCHECK(CalledOnValidThread());
-  // TODO(sandersd): Trigger flush, discarding frames.
+  FlushStart(RESETTING);
 }
 
 void VTVideoDecodeAccelerator::Destroy() {
   DCHECK(CalledOnValidThread());
-  // TODO(sandersd): Trigger flush, discarding frames, and wait for them.
-  delete this;
+  FlushStart(DESTROYING);
 }
 
 bool VTVideoDecodeAccelerator::CanDecodeOnIOThread() {
   return false;
 }
 
 }  // namespace content