Add callback in VideoDecoder and AudioDecoder to return decoded frames.

media/filters/gpu_video_decoder.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/filters/gpu_video_decoder.h"
 
 #include <algorithm>
 
 #include "base/bind.h"
 #include "base/callback_helpers.h"
@@ skipping 23 matching lines @@
 // Size of shared-memory segments we allocate.  Since we reuse them we let them
 // be on the beefy side.
 static const size_t kSharedMemorySegmentBytes = 100 << 10;
 
 GpuVideoDecoder::SHMBuffer::SHMBuffer(base::SharedMemory* m, size_t s)
     : shm(m), size(s) {
 }
 
 GpuVideoDecoder::SHMBuffer::~SHMBuffer() {}
 
-GpuVideoDecoder::BufferPair::BufferPair(
-    SHMBuffer* s, const scoped_refptr<DecoderBuffer>& b)
-    : shm_buffer(s), buffer(b) {
+GpuVideoDecoder::PendingDecodeBuffer::PendingDecodeBuffer(
+    SHMBuffer* s,
+    const scoped_refptr<DecoderBuffer>& b,
+    const DecodeCB& done_cb)
+    : shm_buffer(s), buffer(b), done_cb(done_cb) {
 }
 
-GpuVideoDecoder::BufferPair::~BufferPair() {}
+GpuVideoDecoder::PendingDecodeBuffer::~PendingDecodeBuffer() {}
 
 GpuVideoDecoder::BufferData::BufferData(
     int32 bbid, base::TimeDelta ts, const gfx::Rect& vr, const gfx::Size& ns)
     : bitstream_buffer_id(bbid), timestamp(ts), visible_rect(vr),
       natural_size(ns) {
 }
 
 GpuVideoDecoder::BufferData::~BufferData() {}
 
 GpuVideoDecoder::GpuVideoDecoder(
@@ skipping 13 matching lines @@
 
 void GpuVideoDecoder::Reset(const base::Closure& closure)  {
   DVLOG(3) << "Reset()";
   DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
 
   if (state_ == kDrainingDecoder) {
     base::MessageLoop::current()->PostTask(
         FROM_HERE,
         base::Bind(
             &GpuVideoDecoder::Reset, weak_factory_.GetWeakPtr(), closure));
-    // NOTE: if we're deferring Reset() until a Flush() completes, return
-    // queued pictures to the VDA so they can be used to finish that Flush().
-    if (pending_decode_cb_.is_null())
-      ready_video_frames_.clear();
     return;
   }
 
-  // Throw away any already-decoded, not-yet-delivered frames.
-  ready_video_frames_.clear();
-
   if (!vda_) {
     base::MessageLoop::current()->PostTask(FROM_HERE, closure);
     return;
   }
 
-  if (!pending_decode_cb_.is_null())
-    EnqueueFrameAndTriggerFrameDelivery(VideoFrame::CreateEOSFrame());
-
   DCHECK(pending_reset_cb_.is_null());
   pending_reset_cb_ = BindToCurrentLoop(closure);
 
   vda_->Reset();
 }
 
 void GpuVideoDecoder::Stop() {
   DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
   if (vda_)
     DestroyVDA();
-  if (!pending_decode_cb_.is_null())
-    EnqueueFrameAndTriggerFrameDelivery(VideoFrame::CreateEOSFrame());
+  DCHECK(bitstream_buffers_in_decoder_.empty());
   if (!pending_reset_cb_.is_null())
     base::ResetAndReturn(&pending_reset_cb_).Run();
 }
 
 static bool IsCodedSizeSupported(const gfx::Size& coded_size) {
   // Only non-Windows, Ivy Bridge+ platforms can support more than 1920x1080.
   // We test against 1088 to account for 16x16 macroblocks.
   if (coded_size.width() <= 1920 && coded_size.height() <= 1088)
     return true;
 
@@ skipping 13 matching lines @@
 static void ReportGpuVideoDecoderInitializeStatusToUMAAndRunCB(
     const PipelineStatusCB& cb,
     PipelineStatus status) {
   UMA_HISTOGRAM_ENUMERATION(
       "Media.GpuVideoDecoderInitializeStatus", status, PIPELINE_STATUS_MAX + 1);
   cb.Run(status);
 }
 
 void GpuVideoDecoder::Initialize(const VideoDecoderConfig& config,
                                  bool live_mode,
-                                 const PipelineStatusCB& orig_status_cb) {
+                                 const PipelineStatusCB& orig_status_cb,
+                                 const OutputCB& output_cb) {
   DVLOG(3) << "Initialize()";
   DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
   DCHECK(config.IsValidConfig());
   DCHECK(!config.is_encrypted());
 
   PipelineStatusCB status_cb =
       base::Bind(&ReportGpuVideoDecoderInitializeStatusToUMAAndRunCB,
                  BindToCurrentLoop(orig_status_cb));
 
   bool previously_initialized = config_.IsValidConfig();
   DVLOG(1) << "(Re)initializing GVD with config: "
            << config.AsHumanReadableString();
 
   // TODO(posciak): destroy and create a new VDA on codec/profile change
   // (http://crbug.com/260224).
   if (previously_initialized && (config_.profile() != config.profile())) {
     DVLOG(1) << "Codec or profile changed, cannot reinitialize.";
     status_cb.Run(DECODER_ERROR_NOT_SUPPORTED);
     return;
   }
 
   if (!IsCodedSizeSupported(config.coded_size())) {
     status_cb.Run(DECODER_ERROR_NOT_SUPPORTED);
     return;
   }
 
   config_ = config;
   needs_bitstream_conversion_ = (config.codec() == kCodecH264);
+  output_cb_ = output_cb;
 
   if (previously_initialized) {
     // Reinitialization with a different config (but same codec and profile).
     // VDA should handle it by detecting this in-stream by itself,
     // no need to notify it.
     status_cb.Run(PIPELINE_OK);
     return;
   }
 
   vda_ = factories_->CreateVideoDecodeAccelerator().Pass();
@@ skipping 31 matching lines @@
        ++it) {
     assigned_picture_buffers_.erase(it->first);
   }
   DestroyPictureBuffers(&assigned_picture_buffers_);
 }
 
 void GpuVideoDecoder::Decode(const scoped_refptr<DecoderBuffer>& buffer,
                              const DecodeCB& decode_cb) {
   DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
   DCHECK(pending_reset_cb_.is_null());
-  DCHECK(pending_decode_cb_.is_null());
-
-  pending_decode_cb_ = BindToCurrentLoop(decode_cb);
 
   if (state_ == kError || !vda_) {
-    base::ResetAndReturn(&pending_decode_cb_).Run(kDecodeError, NULL);
+    decode_cb.Run(kDecodeError);

[xhwang, 2014/05/29 22:15:14]

VideoDecoder API says that |decode_cb| should not be fired in the same callstack
of Decode() call. Need a post here. 

[Sergey Ulanov, 2014/06/03 00:08:11]

Thanks for catching it. Added BindToCurrentLoop().

     return;
   }
 
   switch (state_) {
     case kDecoderDrained:
-      if (!ready_video_frames_.empty()) {
-        EnqueueFrameAndTriggerFrameDelivery(NULL);
-        return;
-      }
       state_ = kNormal;
       // Fall-through.
     case kNormal:
       break;
     case kDrainingDecoder:
-      DCHECK(buffer->end_of_stream());
-      // Do nothing.  Will be satisfied either by a PictureReady or
-      // NotifyFlushDone below.
-      return;
     case kError:
       NOTREACHED();
       return;
   }
 
+  DCHECK_EQ(state_, kNormal);
+
   if (buffer->end_of_stream()) {
-    if (state_ == kNormal) {
-      state_ = kDrainingDecoder;
-      vda_->Flush();
-      // If we have ready frames, go ahead and process them to ensure that the
-      // Flush operation does not block in the VDA due to lack of picture
-      // buffers.
-      if (!ready_video_frames_.empty())
-        EnqueueFrameAndTriggerFrameDelivery(NULL);
-    }
+    state_ = kDrainingDecoder;
+    flush_cb_ = decode_cb;

[xhwang, 2014/05/29 22:15:14]

hmm, now I see why it's called |flush_cb_|.... This is kinda a grey area. My
understanding is that "decode_cb" is a signal that the input has been
accepted/consumed by the real codec so that the caller can feed more. In this
case there's no real input buffer and the caller knows that. Can we just fire
|decode_cb| right away here? The caller (DecoderStream) should be able to easily
handle this case.

[Sergey Ulanov, 2014/06/03 00:08:11]

See my previous comments about flushing. When flushing |decode_cb| must be
called after flushing has finished. Otherwise DecodeStream wouldn't know when
the decoder can be reset after flushing.

+    vda_->Flush();
     return;
   }
 
   size_t size = buffer->data_size();
   SHMBuffer* shm_buffer = GetSHM(size);
   if (!shm_buffer) {
-    base::ResetAndReturn(&pending_decode_cb_).Run(kDecodeError, NULL);
+    decode_cb.Run(kDecodeError);

[xhwang, 2014/05/29 22:15:14]

ditto

[Sergey Ulanov, 2014/06/03 00:08:11]

Done.

     return;
   }
 
   memcpy(shm_buffer->shm->memory(), buffer->data(), size);
   BitstreamBuffer bitstream_buffer(
       next_bitstream_buffer_id_, shm_buffer->shm->handle(), size);
   // Mask against 30 bits, to avoid (undefined) wraparound on signed integer.
   next_bitstream_buffer_id_ = (next_bitstream_buffer_id_ + 1) & 0x3FFFFFFF;
-  bool inserted = bitstream_buffers_in_decoder_.insert(std::make_pair(
-      bitstream_buffer.id(), BufferPair(shm_buffer, buffer))).second;
+  bool inserted = bitstream_buffers_in_decoder_
+                      .insert(std::make_pair(
+                          bitstream_buffer.id(),
+                          PendingDecodeBuffer(shm_buffer, buffer, decode_cb)))
+                      .second;

[xhwang, 2014/05/29 22:15:14]

hmm... this can be improved :) We can do it later though... add a TODO?

[Sergey Ulanov, 2014/06/03 00:08:11]

reformatted this code to make it more readable.

   DCHECK(inserted);
+  DCHECK_LE(static_cast<int>(bitstream_buffers_in_decoder_.size()),
+            kMaxInFlightDecodes);
   RecordBufferData(bitstream_buffer, *buffer.get());
 
   vda_->Decode(bitstream_buffer);
-
-  if (!ready_video_frames_.empty()) {
-    EnqueueFrameAndTriggerFrameDelivery(NULL);
-    return;
-  }
-
-  if (CanMoreDecodeWorkBeDone())
-    base::ResetAndReturn(&pending_decode_cb_).Run(kNotEnoughData, NULL);
-}
-
-bool GpuVideoDecoder::CanMoreDecodeWorkBeDone() {
-  return bitstream_buffers_in_decoder_.size() < kMaxInFlightDecodes;
 }
 
 void GpuVideoDecoder::RecordBufferData(const BitstreamBuffer& bitstream_buffer,
                                        const DecoderBuffer& buffer) {
   input_buffer_data_.push_front(BufferData(bitstream_buffer.id(),
                                            buffer.timestamp(),
                                            config_.visible_rect(),
                                            config_.natural_size()));
   // Why this value?  Because why not.  avformat.h:MAX_REORDER_DELAY is 16, but
   // that's too small for some pathological B-frame test videos.  The cost of
@@ skipping 23 matching lines @@
 
 bool GpuVideoDecoder::NeedsBitstreamConversion() const {
   DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
   return needs_bitstream_conversion_;
 }
 
 bool GpuVideoDecoder::CanReadWithoutStalling() const {
   DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
   return
       next_picture_buffer_id_ == 0 ||  // Decode() will ProvidePictureBuffers().
-      available_pictures_ > 0 || !ready_video_frames_.empty();
+      available_pictures_ > 0;
+}
+
+int GpuVideoDecoder::GetMaxDecodeRequests() const {
+  return kMaxInFlightDecodes;
 }
 
 void GpuVideoDecoder::ProvidePictureBuffers(uint32 count,
                                             const gfx::Size& size,
                                             uint32 texture_target) {
   DVLOG(3) << "ProvidePictureBuffers(" << count << ", "
            << size.width() << "x" << size.height() << ")";
   DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
 
   std::vector<uint32> texture_ids;
@@ skipping 112 matching lines @@
       timestamp,
       base::Bind(&ReadPixelsSync, factories_, pb.texture_id(), visible_rect)));
   CHECK_GT(available_pictures_, 0);
   --available_pictures_;
   bool inserted =
       picture_buffers_at_display_.insert(std::make_pair(
                                              picture.picture_buffer_id(),
                                              pb.texture_id())).second;
   DCHECK(inserted);
 
-  EnqueueFrameAndTriggerFrameDelivery(frame);
+  DeliverFrame(frame);
 }
 
-void GpuVideoDecoder::EnqueueFrameAndTriggerFrameDelivery(
+void GpuVideoDecoder::DeliverFrame(
     const scoped_refptr<VideoFrame>& frame) {
   DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
 
   // During a pending vda->Reset(), we don't accumulate frames.  Drop it on the
   // floor and return.
   if (!pending_reset_cb_.is_null())
     return;
 
-  if (frame.get())
-    ready_video_frames_.push_back(frame);
-  else
-    DCHECK(!ready_video_frames_.empty());
-
-  if (pending_decode_cb_.is_null())
-    return;
-
-  base::ResetAndReturn(&pending_decode_cb_)
-      .Run(kOk, ready_video_frames_.front());
-  ready_video_frames_.pop_front();
+  output_cb_.Run(frame);
 }
 
 // static
 void GpuVideoDecoder::ReleaseMailbox(
     base::WeakPtr<GpuVideoDecoder> decoder,
     const scoped_refptr<media::GpuVideoAcceleratorFactories>& factories,
     int64 picture_buffer_id,
     uint32 texture_id,
     const std::vector<uint32>& release_sync_points) {
   DCHECK(factories->GetTaskRunner()->BelongsToCurrentThread());
@@ skipping 52 matching lines @@
 
 void GpuVideoDecoder::PutSHM(SHMBuffer* shm_buffer) {
   DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
   available_shm_segments_.push_back(shm_buffer);
 }
 
 void GpuVideoDecoder::NotifyEndOfBitstreamBuffer(int32 id) {
   DVLOG(3) << "NotifyEndOfBitstreamBuffer(" << id << ")";
   DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
 
-  std::map<int32, BufferPair>::iterator it =
+  std::map<int32, PendingDecodeBuffer>::iterator it =
       bitstream_buffers_in_decoder_.find(id);
   if (it == bitstream_buffers_in_decoder_.end()) {
     NotifyError(VideoDecodeAccelerator::PLATFORM_FAILURE);
     NOTREACHED() << "Missing bitstream buffer: " << id;
     return;
   }
 
   PutSHM(it->second.shm_buffer);
+  it->second.done_cb.Run(state_ == kError ? kDecodeError : kOk);
   bitstream_buffers_in_decoder_.erase(it);
-
-  if (pending_reset_cb_.is_null() && state_ != kDrainingDecoder &&
-      CanMoreDecodeWorkBeDone() && !pending_decode_cb_.is_null()) {
-    base::ResetAndReturn(&pending_decode_cb_).Run(kNotEnoughData, NULL);
-  }
 }
 
 GpuVideoDecoder::~GpuVideoDecoder() {
   DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
   // Stop should have been already called.
   DCHECK(!vda_.get() && assigned_picture_buffers_.empty());
-  DCHECK(pending_decode_cb_.is_null());
+  DCHECK(bitstream_buffers_in_decoder_.empty());
   for (size_t i = 0; i < available_shm_segments_.size(); ++i) {
     available_shm_segments_[i]->shm->Close();
     delete available_shm_segments_[i];
   }
   available_shm_segments_.clear();
-  for (std::map<int32, BufferPair>::iterator it =
+  for (std::map<int32, PendingDecodeBuffer>::iterator it =
            bitstream_buffers_in_decoder_.begin();
        it != bitstream_buffers_in_decoder_.end(); ++it) {
     it->second.shm_buffer->shm->Close();
   }
   bitstream_buffers_in_decoder_.clear();
 }
 
 void GpuVideoDecoder::NotifyFlushDone() {
   DVLOG(3) << "NotifyFlushDone()";
   DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
   DCHECK_EQ(state_, kDrainingDecoder);
   state_ = kDecoderDrained;
-  EnqueueFrameAndTriggerFrameDelivery(VideoFrame::CreateEOSFrame());
+  DeliverFrame(VideoFrame::CreateEOSFrame());
+  base::ResetAndReturn(&flush_cb_).Run(kOk);
 }
 
 void GpuVideoDecoder::NotifyResetDone() {
   DVLOG(3) << "NotifyResetDone()";
   DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
-  DCHECK(ready_video_frames_.empty());
+  DCHECK(bitstream_buffers_in_decoder_.empty());
 
   // This needs to happen after the Reset() on vda_ is done to ensure pictures
   // delivered during the reset can find their time data.
   input_buffer_data_.clear();
 
   if (!pending_reset_cb_.is_null())
     base::ResetAndReturn(&pending_reset_cb_).Run();
-
-  if (!pending_decode_cb_.is_null())
-    EnqueueFrameAndTriggerFrameDelivery(VideoFrame::CreateEOSFrame());
 }
 
 void GpuVideoDecoder::NotifyError(media::VideoDecodeAccelerator::Error error) {
   DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent();
   if (!vda_)
     return;
 
+  state_ = kError;
+
   DLOG(ERROR) << "VDA Error: " << error;
   DestroyVDA();
-
-  state_ = kError;
-
-  if (!pending_decode_cb_.is_null()) {
-    base::ResetAndReturn(&pending_decode_cb_).Run(kDecodeError, NULL);
-    return;
-  }
 }
 
 void GpuVideoDecoder::DCheckGpuVideoAcceleratorFactoriesTaskRunnerIsCurrent()
     const {
   DCHECK(factories_->GetTaskRunner()->BelongsToCurrentThread());
 }
 
 }  // namespace media