Implement a reorder queue in VTVideoDecodeAccelerator.

content/common/gpu/media/vt_video_decode_accelerator.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <CoreVideo/CoreVideo.h>
 #include <OpenGL/CGLIOSurface.h>
 #include <OpenGL/gl.h>
 
 #include "base/bind.h"
 #include "base/command_line.h"
@@ skipping 20 matching lines @@
 
 // Size to use for NALU length headers in AVC format (can be 1, 2, or 4).
 static const int kNALUHeaderLength = 4;
 
 // We request 5 picture buffers from the client, each of which has a texture ID
 // that we can bind decoded frames to. We need enough to satisfy preroll, and
 // enough to avoid unnecessary stalling, but no more than that. The resource
 // requirements are low, as we don't need the textures to be backed by storage.
 static const int kNumPictureBuffers = media::limits::kMaxVideoFrames + 1;
 
+// TODO(sandersd): Use the configured reorder window instead.
+static const int kMinReorderQueueSize = 4;
+static const int kMaxReorderQueueSize = 16;
+
 // Route decoded frame callbacks back into the VTVideoDecodeAccelerator.
 static void OutputThunk(
     void* decompression_output_refcon,
     void* source_frame_refcon,
     OSStatus status,
     VTDecodeInfoFlags info_flags,
     CVImageBufferRef image_buffer,
     CMTime presentation_time_stamp,
     CMTime presentation_duration) {
   VTVideoDecodeAccelerator* vda =
       reinterpret_cast<VTVideoDecodeAccelerator*>(decompression_output_refcon);
   vda->Output(source_frame_refcon, status, image_buffer);
 }
 
 VTVideoDecodeAccelerator::Task::Task(TaskType type) : type(type) {
 }
 
 VTVideoDecodeAccelerator::Task::~Task() {
 }
 
 VTVideoDecodeAccelerator::Frame::Frame(int32_t bitstream_id)
-    : bitstream_id(bitstream_id) {
+    : bitstream_id(bitstream_id), pic_order_cnt(0) {
 }
 
 VTVideoDecodeAccelerator::Frame::~Frame() {
 }
 
+bool VTVideoDecodeAccelerator::FrameOrder::operator()(
+    const linked_ptr<Frame>& lhs,
+    const linked_ptr<Frame>& rhs) const {
+  if (lhs->pic_order_cnt != rhs->pic_order_cnt)
+    return lhs->pic_order_cnt > rhs->pic_order_cnt;
+  // If the pic_order is the same, fallback on using the bitstream order.
+  // TODO(sandersd): Assign a sequence number in Decode().
+  return lhs->bitstream_id > rhs->bitstream_id;
+}
+
+
 VTVideoDecodeAccelerator::VTVideoDecodeAccelerator(
     CGLContextObj cgl_context,
     const base::Callback<bool(void)>& make_context_current)
     : cgl_context_(cgl_context),
       make_context_current_(make_context_current),
       client_(NULL),
       state_(STATE_DECODING),
       format_(NULL),
       session_(NULL),
+      last_sps_id_(-1),
+      last_pps_id_(-1),
       gpu_task_runner_(base::ThreadTaskRunnerHandle::Get()),
       weak_this_factory_(this),
       decoder_thread_("VTDecoderThread") {
   DCHECK(!make_context_current_.is_null());
   callback_.decompressionOutputCallback = OutputThunk;
   callback_.decompressionOutputRefCon = this;
 }
 
 VTVideoDecodeAccelerator::~VTVideoDecodeAccelerator() {
 }
@@ skipping 167 matching lines @@
   size_t data_size = 0;
   std::vector<media::H264NALU> nalus;
   parser_.SetStream(buf, size);
   media::H264NALU nalu;
   while (true) {
     media::H264Parser::Result result = parser_.AdvanceToNextNALU(&nalu);
     if (result == media::H264Parser::kEOStream)
       break;
     if (result != media::H264Parser::kOk) {
       DLOG(ERROR) << "Failed to find H.264 NALU";
-      NotifyError(PLATFORM_FAILURE);
+      NotifyError(UNREADABLE_INPUT);
       return;
     }
     switch (nalu.nal_unit_type) {
       case media::H264NALU::kSPS:
         last_sps_.assign(nalu.data, nalu.data + nalu.size);
         last_spsext_.clear();
         config_changed = true;
+        if (parser_.ParseSPS(&last_sps_id_) != media::H264Parser::kOk) {
+          DLOG(ERROR) << "Could not parse SPS";
+          NotifyError(UNREADABLE_INPUT);
+          return;
+        }
         break;
+
       case media::H264NALU::kSPSExt:
         // TODO(sandersd): Check that the previous NALU was an SPS.
         last_spsext_.assign(nalu.data, nalu.data + nalu.size);
         config_changed = true;
         break;
+
       case media::H264NALU::kPPS:
         last_pps_.assign(nalu.data, nalu.data + nalu.size);
         config_changed = true;
+        if (parser_.ParsePPS(&last_pps_id_) != media::H264Parser::kOk) {
+          DLOG(ERROR) << "Could not parse PPS";
+          NotifyError(UNREADABLE_INPUT);
+          return;
+        }
         break;
+
       case media::H264NALU::kSliceDataA:
       case media::H264NALU::kSliceDataB:
       case media::H264NALU::kSliceDataC:
         DLOG(ERROR) << "Coded slide data partitions not implemented.";
         NotifyError(PLATFORM_FAILURE);
         return;
+
+      case media::H264NALU::kNonIDRSlice:
+        // TODO(sandersd): Check that there has been an SPS or IDR slice since
+        // the last reset.
       case media::H264NALU::kIDRSlice:
-      case media::H264NALU::kNonIDRSlice:
-        // TODO(sandersd): Compute pic_order_count.
+        {
+          // TODO(sandersd): Make sure this only happens once per frame.
+          DCHECK_EQ(frame->pic_order_cnt, 0);
+
+          media::H264SliceHeader slice_hdr;
+          result = parser_.ParseSliceHeader(nalu, &slice_hdr);
+          if (result != media::H264Parser::kOk) {
+            DLOG(ERROR) << "Could not parse slice header";
+            NotifyError(UNREADABLE_INPUT);
+            return;
+          }
+
+          // TODO(sandersd): Keep a cache of recent SPS/PPS units instead of
+          // only the most recent ones.
+          DCHECK_EQ(slice_hdr.pic_parameter_set_id, last_pps_id_);
+          const media::H264PPS* pps =
+              parser_.GetPPS(slice_hdr.pic_parameter_set_id);
+          if (!pps) {
+            DLOG(ERROR) << "Mising PPS referenced by slice";
+            NotifyError(UNREADABLE_INPUT);
+            return false;
+          }
+
+          DCHECK_EQ(pps->seq_parameter_set_id, last_sps_id_);
+          const media::H264SPS* sps = parser_.GetSPS(pps->seq_parameter_set_id);
+          if (!sps) {
+            DLOG(ERROR) << "Mising SPS referenced by PPS";
+            NotifyError(UNREADABLE_INPUT);
+            return false;
+          }
+
+          // TODO(sandersd): Compute pic_order_cnt.
+          DCHECK(!slice_hdr.field_pic_flag);
+          frame->pic_order_cnt = 0;
+        }
       default:
         nalus.push_back(nalu);
         data_size += kNALUHeaderLength + nalu.size;
         break;
     }
   }
 
   // Initialize VideoToolbox.
   // TODO(sandersd): Instead of assuming that the last SPS and PPS units are
   // always the correct ones, maintain a cache of recent SPS and PPS units and
@@ skipping 19 matching lines @@
     return;
   }
 
   // If the session is not configured by this point, fail.
   if (!session_) {
     DLOG(ERROR) << "Image slice without configuration";
     NotifyError(INVALID_ARGUMENT);
     return;
   }
 
+  // Update the frame metadata with configuration data.
+  frame->coded_size = coded_size_;
+
   // Create a memory-backed CMBlockBuffer for the translated data.
   // TODO(sandersd): Pool of memory blocks.
   base::ScopedCFTypeRef<CMBlockBufferRef> data;
   OSStatus status = CMBlockBufferCreateWithMemoryBlock(
       kCFAllocatorDefault,
       NULL,                 // &memory_block
       data_size,            // block_length
       kCFAllocatorDefault,  // block_allocator
       NULL,                 // &custom_block_source
       0,                    // offset_to_data
@@ skipping 38 matching lines @@
       0,                    // num_sample_timing_entries
       NULL,                 // &sample_timing_array
       0,                    // num_sample_size_entries
       NULL,                 // &sample_size_array
       sample.InitializeInto());
   if (status) {
     NOTIFY_STATUS("CMSampleBufferCreate()", status);
     return;
   }
 
-  // Update the frame data.
-  frame->coded_size = coded_size_;
-
   // Send the frame for decoding.
   // Asynchronous Decompression allows for parallel submission of frames
   // (without it, DecodeFrame() does not return until the frame has been
   // decoded). We don't enable Temporal Processing so that frames are always
   // returned in decode order; this makes it easier to avoid deadlock.
   VTDecodeFrameFlags decode_flags =
       kVTDecodeFrame_EnableAsynchronousDecompression;
   status = VTDecompressionSessionDecodeFrame(
       session_,
       sample,                                 // sample_buffer
@@ skipping 24 matching lines @@
         weak_this_factory_.GetWeakPtr(), frame));
   }
 }
 
 void VTVideoDecodeAccelerator::DecodeDone(Frame* frame) {
   DCHECK(gpu_thread_checker_.CalledOnValidThread());
   DCHECK_EQ(frame->bitstream_id, pending_frames_.front()->bitstream_id);
   Task task(TASK_FRAME);
   task.frame = pending_frames_.front();
   pending_frames_.pop();
-  pending_tasks_.push(task);
-  ProcessTasks();
+  task_queue_.push(task);
+  ProcessWorkQueues();
 }
 
 void VTVideoDecodeAccelerator::FlushTask(TaskType type) {
   DCHECK(decoder_thread_.message_loop_proxy()->BelongsToCurrentThread());
   FinishDelayedFrames();
 
   // Always queue a task, even if FinishDelayedFrames() fails, so that
   // destruction always completes.
   gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
       &VTVideoDecodeAccelerator::FlushDone,
       weak_this_factory_.GetWeakPtr(), type));
 }
 
 void VTVideoDecodeAccelerator::FlushDone(TaskType type) {
   DCHECK(gpu_thread_checker_.CalledOnValidThread());
-  pending_tasks_.push(Task(type));
-  ProcessTasks();
+  task_queue_.push(Task(type));
+  ProcessWorkQueues();
 }
 
 void VTVideoDecodeAccelerator::Decode(const media::BitstreamBuffer& bitstream) {
   DCHECK(gpu_thread_checker_.CalledOnValidThread());
   DCHECK_EQ(assigned_bitstream_ids_.count(bitstream.id()), 0u);
   assigned_bitstream_ids_.insert(bitstream.id());
   Frame* frame = new Frame(bitstream.id());
   pending_frames_.push(make_linked_ptr(frame));
   decoder_thread_.message_loop_proxy()->PostTask(FROM_HERE, base::Bind(
       &VTVideoDecodeAccelerator::DecodeTask, base::Unretained(this),
       bitstream, frame));
 }
 
 void VTVideoDecodeAccelerator::AssignPictureBuffers(
     const std::vector<media::PictureBuffer>& pictures) {
   DCHECK(gpu_thread_checker_.CalledOnValidThread());
 
   for (const media::PictureBuffer& picture : pictures) {
     DCHECK(!texture_ids_.count(picture.id()));
     assigned_picture_ids_.insert(picture.id());
     available_picture_ids_.push_back(picture.id());
     texture_ids_[picture.id()] = picture.texture_id();
   }
 
   // Pictures are not marked as uncleared until after this method returns, and
   // they will be broken if they are used before that happens. So, schedule
   // future work after that happens.
   gpu_task_runner_->PostTask(FROM_HERE, base::Bind(
-      &VTVideoDecodeAccelerator::ProcessTasks,
+      &VTVideoDecodeAccelerator::ProcessWorkQueues,
       weak_this_factory_.GetWeakPtr()));
 }
 
 void VTVideoDecodeAccelerator::ReusePictureBuffer(int32_t picture_id) {
   DCHECK(gpu_thread_checker_.CalledOnValidThread());
   DCHECK_EQ(CFGetRetainCount(picture_bindings_[picture_id]), 1);
   picture_bindings_.erase(picture_id);
   if (assigned_picture_ids_.count(picture_id) != 0) {
     available_picture_ids_.push_back(picture_id);
-    ProcessTasks();
+    ProcessWorkQueues();
   } else {
     client_->DismissPictureBuffer(picture_id);
   }
 }
 
-void VTVideoDecodeAccelerator::ProcessTasks() {
+void VTVideoDecodeAccelerator::ProcessWorkQueues() {
   DCHECK(gpu_thread_checker_.CalledOnValidThread());
+  switch (state_) {
+    case STATE_DECODING:
+      // TODO(sandersd): Batch where possible.
+      while (ProcessReorderQueue() || ProcessTaskQueue());

[DaleCurtis, 2014/11/14 21:11:45]

Is this something that could block for a long time? Possibly better off with a
PostTask past some limit?

[sandersd (OOO until July 31), 2014/11/14 21:28:52]

This would be worth investigating. I do expect this processing to be relatively
short.

It is bounded by the size of the task queue, since that cannot grow during
processing, and by the number of available pictures, as the reorder queue will
fill up once they are gone.

In the common case, less than three items (reorder queue or task queue items)
would be processed. The extreme cases are:
  - Flush arrives, entire reorder queue will be sent and then possibly refilled.
(~10 items processed, if there were enough available pictures.)
  - Pictures are assigned, most of the reorder queue is sent and refilled. (~10
again, as pictures are requested 5 at a time.)

[DaleCurtis, 2014/11/14 21:41:54]

More important than the length is the time taken. Do you have estimates for how
long this takes? This is on the main gpu thread as far as I can tell, which
seems like it could impact compositing.

+      return;
 
-  while (!pending_tasks_.empty()) {
-    const Task& task = pending_tasks_.front();
+    case STATE_ERROR:
+      // Do nothing until Destroy() is called.
+      return;
 
-    switch (state_) {
-      case STATE_DECODING:
-        if (!ProcessTask(task))
-          return;
-        pending_tasks_.pop();
-        break;
-
-      case STATE_ERROR:
-        // Do nothing until Destroy() is called.
-        return;
-
-      case STATE_DESTROYING:
-        // Discard tasks until destruction is complete.
-        if (task.type == TASK_DESTROY) {
+    case STATE_DESTROYING:
+      // Drop tasks until we are ready to destruct.
+      while (!task_queue_.empty()) {
+        if (task_queue_.front().type == TASK_DESTROY) {
           delete this;
           return;
         }
-        pending_tasks_.pop();
-        break;
-    }
+        task_queue_.pop();
+      }
+      return;
   }
 }
 
-bool VTVideoDecodeAccelerator::ProcessTask(const Task& task) {
+bool VTVideoDecodeAccelerator::ProcessTaskQueue() {
   DCHECK(gpu_thread_checker_.CalledOnValidThread());
   DCHECK_EQ(state_, STATE_DECODING);
 
+  if (task_queue_.empty())
+    return false;
+
+  const Task& task = task_queue_.front();
   switch (task.type) {
     case TASK_FRAME:
-      return ProcessFrame(*task.frame);
+      // TODO(sandersd): Signal IDR explicitly (not using pic_order_cnt == 0).
+      if (reorder_queue_.size() < kMaxReorderQueueSize &&
+          (task.frame->pic_order_cnt != 0 || reorder_queue_.empty())) {
+        assigned_bitstream_ids_.erase(task.frame->bitstream_id);
+        client_->NotifyEndOfBitstreamBuffer(task.frame->bitstream_id);
+        reorder_queue_.push(task.frame);
+        task_queue_.pop();
+        return true;
+      }
+      return false;
 
     case TASK_FLUSH:
       DCHECK_EQ(task.type, pending_flush_tasks_.front());
-      pending_flush_tasks_.pop();
-      client_->NotifyFlushDone();
-      return true;
+      if (reorder_queue_.size() == 0) {
+        pending_flush_tasks_.pop();
+        client_->NotifyFlushDone();
+        task_queue_.pop();
+        return true;
+      }
+      return false;
 
     case TASK_RESET:
       DCHECK_EQ(task.type, pending_flush_tasks_.front());
-      pending_flush_tasks_.pop();
-      client_->NotifyResetDone();
-      return true;
+      if (reorder_queue_.size() == 0) {
+        pending_flush_tasks_.pop();
+        client_->NotifyResetDone();
+        task_queue_.pop();
+        return true;
+      }
+      return false;
 
     case TASK_DESTROY:
       NOTREACHED() << "Can't destroy while in STATE_DECODING.";
       NotifyError(ILLEGAL_STATE);
       return false;
   }
 }
 
+bool VTVideoDecodeAccelerator::ProcessReorderQueue() {
+  DCHECK(gpu_thread_checker_.CalledOnValidThread());
+  DCHECK_EQ(state_, STATE_DECODING);
+
+  if (reorder_queue_.empty())
+    return false;
+
+  // If the next task is a flush (because there is a pending flush or becuase
+  // the next frame is an IDR), then we don't need a full reorder buffer to send
+  // the next frame.
+  bool flushing = !task_queue_.empty() &&
+                  (task_queue_.front().type != TASK_FRAME ||
+                   task_queue_.front().frame->pic_order_cnt == 0);
+  if (flushing || reorder_queue_.size() >= kMinReorderQueueSize) {
+    if (ProcessFrame(*reorder_queue_.top())) {
+      reorder_queue_.pop();
+      return true;
+    }
+  }
+
+  return false;
+}
+
 bool VTVideoDecodeAccelerator::ProcessFrame(const Frame& frame) {
   DCHECK(gpu_thread_checker_.CalledOnValidThread());
   DCHECK_EQ(state_, STATE_DECODING);
+
   // If the next pending flush is for a reset, then the frame will be dropped.
   bool resetting = !pending_flush_tasks_.empty() &&
                    pending_flush_tasks_.front() == TASK_RESET;
+
   if (!resetting && frame.image.get()) {
     // If the |coded_size| has changed, request new picture buffers and then
     // wait for them.
     // TODO(sandersd): If GpuVideoDecoder didn't specifically check the size of
     // textures, this would be unnecessary, as the size is actually a property
     // of the texture binding, not the texture. We rebind every frame, so the
     // size passed to ProvidePictureBuffers() is meaningless.
     if (picture_size_ != frame.coded_size) {
       // Dismiss current pictures.
       for (int32_t picture_id : assigned_picture_ids_)
         client_->DismissPictureBuffer(picture_id);
       assigned_picture_ids_.clear();
       available_picture_ids_.clear();
 
       // Request new pictures.
       picture_size_ = frame.coded_size;
       client_->ProvidePictureBuffers(
           kNumPictureBuffers, coded_size_, GL_TEXTURE_RECTANGLE_ARB);
       return false;
     }
     if (!SendFrame(frame))
       return false;
   }
-  assigned_bitstream_ids_.erase(frame.bitstream_id);
-  client_->NotifyEndOfBitstreamBuffer(frame.bitstream_id);
+
   return true;
 }
 
 bool VTVideoDecodeAccelerator::SendFrame(const Frame& frame) {
   DCHECK(gpu_thread_checker_.CalledOnValidThread());
   DCHECK_EQ(state_, STATE_DECODING);
 
   if (available_picture_ids_.empty())
     return false;
 
@@ skipping 45 matching lines @@
 
 void VTVideoDecodeAccelerator::QueueFlush(TaskType type) {
   DCHECK(gpu_thread_checker_.CalledOnValidThread());
   pending_flush_tasks_.push(type);
   decoder_thread_.message_loop_proxy()->PostTask(FROM_HERE, base::Bind(
       &VTVideoDecodeAccelerator::FlushTask, base::Unretained(this),
       type));
 
   // If this is a new flush request, see if we can make progress.
   if (pending_flush_tasks_.size() == 1)
-    ProcessTasks();
+    ProcessWorkQueues();
 }
 
 void VTVideoDecodeAccelerator::Flush() {
   DCHECK(gpu_thread_checker_.CalledOnValidThread());
   QueueFlush(TASK_FLUSH);
 }
 
 void VTVideoDecodeAccelerator::Reset() {
   DCHECK(gpu_thread_checker_.CalledOnValidThread());
   QueueFlush(TASK_RESET);
 }
 
 void VTVideoDecodeAccelerator::Destroy() {
   DCHECK(gpu_thread_checker_.CalledOnValidThread());
+  // TODO(sandersd): Make sure the decoder won't try to read the buffers again
+  // before discarding them.
   for (int32_t bitstream_id : assigned_bitstream_ids_)
     client_->NotifyEndOfBitstreamBuffer(bitstream_id);
   assigned_bitstream_ids_.clear();
   state_ = STATE_DESTROYING;
   QueueFlush(TASK_DESTROY);
 }
 
 bool VTVideoDecodeAccelerator::CanDecodeOnIOThread() {
   return false;
 }
 
 }  // namespace content