Implement software fallback for PPB_VideoDecoder.

content/renderer/pepper/video_decoder_shim.cc

+// Copyright (c) 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <GLES2/gl2.h>
+#include <GLES2/gl2ext.h>
+#include <GLES2/gl2extchromium.h>
+
+#include "content/renderer/pepper/video_decoder_shim.h"
+
+#include "base/bind.h"
+#include "base/numerics/safe_conversions.h"
+#include "content/public/renderer/render_thread.h"
+#include "content/renderer/pepper/pepper_video_decoder_host.h"
+#include "content/renderer/render_thread_impl.h"
+#include "gpu/command_buffer/client/gles2_implementation.h"
+#include "media/base/decoder_buffer.h"
+#include "media/base/limits.h"
+#include "media/base/video_decoder.h"
+#include "media/filters/ffmpeg_video_decoder.h"
+#include "media/video/picture.h"
+#include "media/video/video_decode_accelerator.h"
+#include "ppapi/c/pp_errors.h"
+#include "third_party/libyuv/include/libyuv.h"
+#include "webkit/common/gpu/context_provider_web_context.h"
+
+namespace content {
+
+struct VideoDecoderShim::PendingDecode {
+  PendingDecode(uint32_t decode_id,
+                const scoped_refptr<media::DecoderBuffer>& buffer);
+  ~PendingDecode();
+
+  const uint32_t decode_id;
+  const scoped_refptr<media::DecoderBuffer> buffer;
+};
+
+VideoDecoderShim::PendingDecode::PendingDecode(
+    uint32_t decode_id,
+    const scoped_refptr<media::DecoderBuffer>& buffer)
+    : decode_id(decode_id), buffer(buffer) {
+}
+
+VideoDecoderShim::PendingDecode::~PendingDecode() {
+}
+
+struct VideoDecoderShim::PendingFrame {
+  explicit PendingFrame(uint32_t decode_id);
+  PendingFrame(uint32_t decode_id, const gfx::Size& size);
+  ~PendingFrame();
+
+  const uint32_t decode_id;
+  const gfx::Size size;
+  std::vector<uint8_t> argb_pixels;
+
+ private:
+  // This could be expensive to copy, so guard against that.
+  DISALLOW_COPY_AND_ASSIGN(PendingFrame);
+};
+
+VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id)
+    : decode_id(decode_id) {
+}
+
+VideoDecoderShim::PendingFrame::PendingFrame(uint32_t decode_id,
+                                             const gfx::Size& size)
+    : decode_id(decode_id),
+      size(size),
+      argb_pixels(size.width() * size.height() * 4) {
+}
+
+VideoDecoderShim::PendingFrame::~PendingFrame() {
+}
+
+// DecoderImpl runs the underlying VideoDecoder on the media thread, receiving
+// calls from the VideoDecodeShim on the main thread and sending results back.
+// This class is constructed on the main thread, but used and destructed on the
+// media thread.
+class VideoDecoderShim::DecoderImpl {
+ public:
+  explicit DecoderImpl(const base::WeakPtr<VideoDecoderShim>& proxy);
+  ~DecoderImpl();
+
+  void Initialize(media::VideoDecoderConfig config);
+  void Decode(uint32_t decode_id, scoped_refptr<media::DecoderBuffer> buffer);
+  void Reset();
+  void Stop();
+
+ private:
+  void OnPipelineStatus(media::PipelineStatus status);
+  void DoDecode();
+  void OnDecodeComplete(uint32_t decode_id,
+                        media::VideoDecoder::Status status,
+                        const scoped_refptr<media::VideoFrame>& frame);
+  void OnResetComplete();
+
+  // WeakPtr is bound to main_message_loop_. Use only in shim callbacks.
+  base::WeakPtr<VideoDecoderShim> shim_;
+  scoped_ptr<media::VideoDecoder> decoder_;
+  scoped_refptr<base::MessageLoopProxy> main_message_loop_;
+  // Queue of decodes waiting for the decoder.
+  typedef std::queue<PendingDecode> PendingDecodeQueue;
+  PendingDecodeQueue pending_decodes_;
+  int max_decodes_at_decoder_;
+  int num_decodes_at_decoder_;
+};
+
+VideoDecoderShim::DecoderImpl::DecoderImpl(
+    const base::WeakPtr<VideoDecoderShim>& proxy)
+    : shim_(proxy),
+      main_message_loop_(base::MessageLoopProxy::current()),
+      max_decodes_at_decoder_(0),
+      num_decodes_at_decoder_(0) {
+}
+
+VideoDecoderShim::DecoderImpl::~DecoderImpl() {
+  DCHECK(pending_decodes_.empty());
+}
+
+void VideoDecoderShim::DecoderImpl::Initialize(
+    media::VideoDecoderConfig config) {
+  DCHECK(!decoder_);
+  decoder_.reset(
+      new media::FFmpegVideoDecoder(base::MessageLoopProxy::current()));
+  max_decodes_at_decoder_ = decoder_->GetMaxDecodeRequests();
+  // We can use base::Unretained() safely in decoder callbacks because we call
+  // VideoDecoder::Stop() before deletion. Stop() guarantees there will be no
+  // outstanding callbacks after it returns.
+  decoder_->Initialize(
+      config,
+      true /* low_delay */,

[Ami GONE FROM CHROMIUM, 2014/06/07 18:26:40]

Igor: the commentary on your new API is that it disables low_delay.  Does that
mean this true should turn to false to avoid misleading readers?
Or should it stay true in case the future improves, but with a comment saying it
is ignored by the ffmpeg impl?  
Or something else?

[igorc, 2014/06/11 20:13:18]

It could be poor commenting and/or incomplete understanding on my side. See
https://codereview.chromium.org/301243008/.

Specifically, comments by dalecurtis@:
https://codereview.chromium.org/301243008/diff/1/media/filters/ffmpeg_video_d...

And later: "You could dovetail this with the existing low delay flag since that
already disables frame threading."

It's probably better to set low_delay to false. Will ask dalecurtis@ first
though.

[DaleCurtis, 2014/06/11 21:08:08]

The comment is incorrect / opposite and should probably be updated.  Setting the
thread type to FF_THREAD_SLICE is Chrome's definition of "low delay," despite
the fact that FFmpeg does expose a CODEC_FLAG_LOW_DELAY that we don't use.

Setting CODEC_FLAG2_CHUNKS as set_decode_nalus() does, transitively causes
FFmpeg to use FF_THREAD_SLICE instead of FF_THREAD_FRAME which makes it
equivalent to Chrome's "low delay" mode.

[igorc, 2014/06/11 22:30:08]

Just spoke to Dale, and we agreed to delete "Disables low-latency mode" phrase
from set_decode_nalus()'s comment, see
https://codereview.chromium.org/301243008/patch/40001/50002.

The main reasoning is that there're many other parameters that affect
"active_thread_type" selection.

+      base::Bind(&VideoDecoderShim::DecoderImpl::OnPipelineStatus,
+                 base::Unretained(this)));
+}
+
+void VideoDecoderShim::DecoderImpl::OnPipelineStatus(
+    media::PipelineStatus status) {
+  int32_t result;
+  switch (status) {
+    case media::PIPELINE_OK:
+      result = PP_OK;
+      break;
+    case media::DECODER_ERROR_NOT_SUPPORTED:
+      result = PP_ERROR_NOTSUPPORTED;
+      break;
+    default:
+      result = PP_ERROR_FAILED;
+      break;
+  }
+
+  // Calculate how many textures the shim should create.
+  uint32_t texture_pool_size =
+      max_decodes_at_decoder_ + media::limits::kMaxVideoFrames;
+  main_message_loop_->PostTask(
+      FROM_HERE,
+      base::Bind(&VideoDecoderShim::OnInitializeComplete,
+                 shim_,
+                 result,
+                 texture_pool_size));
+}
+
+void VideoDecoderShim::DecoderImpl::Decode(
+    uint32_t decode_id,
+    scoped_refptr<media::DecoderBuffer> buffer) {
+  DCHECK(decoder_);
+  pending_decodes_.push(PendingDecode(decode_id, buffer));
+  DoDecode();

[Ami GONE FROM CHROMIUM, 2014/06/07 18:26:40]

lol I start out thinking it was a bug but convinced myself it wasn't possible
that to arrive at a situation where one buffer is queued and a later buffer is
passed to the decoder ahead of the queued one.  Anyway I like this formulation
better.
ffmpeg is _very_ forgiving; it will try as hard as it can to keep chugging even
in the face of corrupt inputs (at the cost of visual corruption / stuttering /
jank).  It can be useful to test the returned frames for bit-exactness with some
expectation.
(see e.g. the hash & thumbnail-related code in content/common/gpu/media's video
decode tests)

[bbudge, 2014/06/07 21:28:33]

I am still searching for some way to test this more rigorously (i.e.
browser_tests, content_browsertests). What I need is some compact source of
bitstream data, like a test pattern. Any pointers greatly appreciated!

+}
+
+void VideoDecoderShim::DecoderImpl::Reset() {
+  DCHECK(decoder_);
+  // Abort all pending decodes.
+  while (!pending_decodes_.empty()) {
+    const PendingDecode& decode = pending_decodes_.front();
+    scoped_ptr<PendingFrame> pending_frame(new PendingFrame(decode.decode_id));
+    main_message_loop_->PostTask(FROM_HERE,
+                                 base::Bind(&VideoDecoderShim::OnDecodeComplete,
+                                            shim_,
+                                            media::VideoDecoder::kAborted,
+                                            base::Passed(&pending_frame)));
+    pending_decodes_.pop();
+  }
+  decoder_->Reset(base::Bind(&VideoDecoderShim::DecoderImpl::OnResetComplete,
+                             base::Unretained(this)));
+}
+
+void VideoDecoderShim::DecoderImpl::Stop() {
+  DCHECK(decoder_);
+  // Clear pending decodes now. We don't want OnDecodeComplete to call Decode
+  // again.
+  while (!pending_decodes_.empty())
+    pending_decodes_.pop();
+  decoder_->Stop();
+  // This instance is deleted once we exit this scope.
+}
+
+void VideoDecoderShim::DecoderImpl::DoDecode() {
+  while (!pending_decodes_.empty() &&
+         num_decodes_at_decoder_ < max_decodes_at_decoder_) {
+    num_decodes_at_decoder_++;
+    const PendingDecode& decode = pending_decodes_.front();
+    decoder_->Decode(
+        decode.buffer,
+        base::Bind(&VideoDecoderShim::DecoderImpl::OnDecodeComplete,
+                   base::Unretained(this),
+                   decode.decode_id));
+    pending_decodes_.pop();
+  }
+}
+
+void VideoDecoderShim::DecoderImpl::OnDecodeComplete(
+    uint32_t decode_id,
+    media::VideoDecoder::Status status,
+    const scoped_refptr<media::VideoFrame>& frame) {
+  num_decodes_at_decoder_--;
+  scoped_ptr<PendingFrame> pending_frame;
+  if (status == media::VideoDecoder::kOk && !frame->end_of_stream()) {
+    pending_frame.reset(new PendingFrame(decode_id, frame->coded_size()));
+    // Convert the VideoFrame pixels to ARGB.
+    libyuv::I420ToARGB(frame->data(media::VideoFrame::kYPlane),
+                       frame->stride(media::VideoFrame::kYPlane),
+                       frame->data(media::VideoFrame::kUPlane),
+                       frame->stride(media::VideoFrame::kUPlane),
+                       frame->data(media::VideoFrame::kVPlane),
+                       frame->stride(media::VideoFrame::kVPlane),
+                       &pending_frame->argb_pixels.front(),
+                       frame->coded_size().width() * 4,
+                       frame->coded_size().width(),
+                       frame->coded_size().height());
+  } else {
+    pending_frame.reset(new PendingFrame(decode_id));
+  }
+
+  int32_t result;
+  switch (status) {
+    case media::VideoDecoder::kOk:
+    // We allow partial frames, so this is OK.
+    case media::VideoDecoder::kNotEnoughData:
+      result = PP_OK;
+      break;
+    case media::VideoDecoder::kAborted:
+      result = PP_ERROR_ABORTED;
+      break;
+    case media::VideoDecoder::kDecodeError:
+      result = PP_ERROR_RESOURCE_FAILED;
+      break;
+    default:
+      NOTREACHED();
+      result = PP_ERROR_FAILED;
+      break;
+  }
+
+  main_message_loop_->PostTask(FROM_HERE,
+                               base::Bind(&VideoDecoderShim::OnDecodeComplete,
+                                          shim_,
+                                          result,
+                                          base::Passed(&pending_frame)));
+
+  DoDecode();
+}
+
+void VideoDecoderShim::DecoderImpl::OnResetComplete() {
+  main_message_loop_->PostTask(
+      FROM_HERE, base::Bind(&VideoDecoderShim::OnResetComplete, shim_));
+}
+
+VideoDecoderShim::VideoDecoderShim(PepperVideoDecoderHost* host)
+    : state_(UNINITIALIZED),
+      host_(host),
+      media_message_loop_(
+          RenderThreadImpl::current()->GetMediaThreadMessageLoopProxy()),
+      context_provider_(
+          RenderThreadImpl::current()->SharedMainThreadContextProvider()),
+      num_pending_decodes_(0),
+      texture_pool_size_(0),
+      weak_ptr_factory_(this) {
+  DCHECK(host_);
+  DCHECK(media_message_loop_);
+  DCHECK(context_provider_);
+  decoder_impl_.reset(new DecoderImpl(weak_ptr_factory_.GetWeakPtr()));
+}
+
+VideoDecoderShim::~VideoDecoderShim() {
+  DCHECK(RenderThreadImpl::current());
+  // Delete any remaining textures.
+  TextureIdMap::iterator it = texture_id_map_.begin();
+  for (; it != texture_id_map_.end(); ++it)
+    DeleteTexture(it->second);
+  texture_id_map_.clear();
+
+  FlushCommandBuffer();
+
+  host_ = NULL;
+  // No methods can post tasks to the delegate now.
+  weak_ptr_factory_.InvalidateWeakPtrs();
+  // No more callbacks from the delegate will be received now.
+
+  // The callback now holds the only reference to the DecoderImpl, which will be
+  // deleted when Stop completes.
+  media_message_loop_->PostTask(
+      FROM_HERE,
+      base::Bind(&VideoDecoderShim::DecoderImpl::Stop,
+                 base::Owned(decoder_impl_.release())));
+
+}
+
+bool VideoDecoderShim::Initialize(
+    media::VideoCodecProfile profile,
+    media::VideoDecodeAccelerator::Client* client) {
+  // Ignore the |client| parameter, which is the same as |host_|.

[Ami GONE FROM CHROMIUM, 2014/06/07 18:26:39]

DCHECK this instead of the comment?

[bbudge, 2014/06/07 21:28:33]

Done.

+  DCHECK(RenderThreadImpl::current());
+  DCHECK_EQ(state_, UNINITIALIZED);
+  media::VideoCodec codec = media::kUnknownVideoCodec;
+  if (profile <= media::H264PROFILE_MAX)
+    codec = media::kCodecH264;
+  else if (profile <= media::VP8PROFILE_MAX)
+    codec = media::kCodecVP8;
+  DCHECK_NE(codec, media::kUnknownVideoCodec);
+
+  media::VideoDecoderConfig config(
+      codec,
+      profile,
+      media::VideoFrame::YV12,
+      gfx::Size(32, 24),  // Small sizes that won't fail.
+      gfx::Rect(32, 24),
+      gfx::Size(32, 24),
+      NULL /* extra_data */,  // TODO(bbudge) Verify this isn't needed.
+      0 /* extra_data_size */,
+      false /* decryption */);
+
+  media_message_loop_->PostTask(
+      FROM_HERE,
+      base::Bind(&VideoDecoderShim::DecoderImpl::Initialize,
+                 base::Unretained(decoder_impl_.get()),
+                 config));
+  // Return success, even though we are asynchronous, to mimic
+  // media::VideoDecodeAccelerator.
+  return true;
+}
+
+void VideoDecoderShim::Decode(const media::BitstreamBuffer& bitstream_buffer) {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK_EQ(state_, DECODING);
+
+  num_pending_decodes_++;
+
+  // We need the address of the shared memory, so we can copy into our buffer.
+  const uint8_t* buffer = host_->ShmHandleToAddress(bitstream_buffer.handle());
+  DCHECK(buffer);
+
+  media_message_loop_->PostTask(
+      FROM_HERE,
+      base::Bind(
+          &VideoDecoderShim::DecoderImpl::Decode,
+          base::Unretained(decoder_impl_.get()),
+          bitstream_buffer.id(),
+          media::DecoderBuffer::CopyFrom(buffer, bitstream_buffer.size())));
+}
+
+void VideoDecoderShim::AssignPictureBuffers(
+    const std::vector<media::PictureBuffer>& buffers) {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK_EQ(state_, DECODING);
+  if (buffers.empty()) {
+    NOTREACHED();
+    return;
+  }
+  DCHECK_EQ(buffers.size(), pending_texture_mailboxes_.size());
+  GLuint num_textures = base::checked_cast<GLuint>(buffers.size());
+  std::vector<uint32_t> local_texture_ids(num_textures);
+  gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
+  gles2->GenTextures(num_textures, &local_texture_ids.front());
+  for (uint32_t i = 0; i < num_textures; i++) {
+    gles2->ActiveTexture(GL_TEXTURE0);
+    gles2->BindTexture(GL_TEXTURE_2D, local_texture_ids[i]);
+    gles2->ConsumeTextureCHROMIUM(GL_TEXTURE_2D,
+                                  pending_texture_mailboxes_[i].name);
+    // Map the plugin texture id to the local texture id.
+    uint32_t plugin_texture_id = buffers[i].texture_id();
+    texture_id_map_[plugin_texture_id] = local_texture_ids[i];
+    available_textures_.push_back(plugin_texture_id);
+  }
+  pending_texture_mailboxes_.clear();
+  SendPictures();
+}
+
+void VideoDecoderShim::ReusePictureBuffer(int32 picture_buffer_id) {
+  DCHECK(RenderThreadImpl::current());
+  uint32_t texture_id = static_cast<uint32_t>(picture_buffer_id);
+  if (textures_to_dismiss_.find(texture_id) != textures_to_dismiss_.end()) {
+    DismissTexture(texture_id);
+  } else if (texture_id_map_.find(texture_id) != texture_id_map_.end()) {
+    available_textures_.push_back(texture_id);
+    SendPictures();
+  } else {
+    NOTREACHED();
+  }
+}
+
+void VideoDecoderShim::Flush() {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK_EQ(state_, DECODING);
+  state_ = FLUSHING;
+}
+
+void VideoDecoderShim::Reset() {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK_EQ(state_, DECODING);
+  state_ = RESETTING;
+  media_message_loop_->PostTask(
+      FROM_HERE,
+      base::Bind(&VideoDecoderShim::DecoderImpl::Reset,
+                 base::Unretained(decoder_impl_.get())));
+}
+
+void VideoDecoderShim::Destroy() {
+  // This will be called, but our destructor can do the actual work.
+}
+
+void VideoDecoderShim::OnInitializeComplete(int32_t result,
+                                            uint32_t texture_pool_size) {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK(host_);
+
+  if (result == PP_OK) {
+    state_ = DECODING;
+    texture_pool_size_ = texture_pool_size;
+  }
+
+  host_->OnInitializeComplete(result);
+}
+
+void VideoDecoderShim::OnDecodeComplete(int32_t result,
+                                        scoped_ptr<PendingFrame> frame) {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK(host_);
+
+  num_pending_decodes_--;
+
+  if (result == PP_OK && !frame->argb_pixels.empty()) {
+    if (texture_size_ != frame->size) {
+      // If the size has changed, all current textures must be dismissed. Add
+      // all textures to |textures_to_dismiss_| and dismiss any that aren't in
+      // use by the plugin. We dismiss the rest as they are recycled.
+      for (TextureIdMap::const_iterator it = texture_id_map_.begin();
+           it != texture_id_map_.end();
+           ++it) {
+        textures_to_dismiss_.insert(it->second);
+      }
+      for (std::vector<uint32_t>::const_iterator it =
+               available_textures_.begin();
+           it != available_textures_.end();
+           ++it) {
+        DismissTexture(*it);
+      }
+      available_textures_.clear();
+      FlushCommandBuffer();
+
+      DCHECK(pending_texture_mailboxes_.empty());
+      for (uint32_t i = 0; i < texture_pool_size_; i++)
+        pending_texture_mailboxes_.push_back(gpu::Mailbox::Generate());
+
+      host_->RequestTextures(texture_pool_size_,
+                             frame->size,
+                             GL_TEXTURE_2D,
+                             pending_texture_mailboxes_);
+      texture_size_ = frame->size;
+    }
+
+    pending_frames_.push(linked_ptr<PendingFrame>(frame.release()));
+    SendPictures();
+  } else {
+    host_->NotifyEndOfBitstreamBuffer(frame->decode_id);
+  }
+
+  // Report any VDA type errors.
+  if (result == PP_ERROR_RESOURCE_FAILED)
+    host_->NotifyError(media::VideoDecodeAccelerator::PLATFORM_FAILURE);
+}
+
+void VideoDecoderShim::SendPictures() {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK(host_);
+
+  while (!pending_frames_.empty() && !available_textures_.empty()) {
+    const linked_ptr<PendingFrame>& frame = pending_frames_.front();
+
+    uint32_t texture_id = available_textures_.back();
+    available_textures_.pop_back();
+
+    uint32_t local_texture_id = texture_id_map_[texture_id];
+    gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
+    gles2->ActiveTexture(GL_TEXTURE0);
+    gles2->BindTexture(GL_TEXTURE_2D, local_texture_id);
+    gles2->TexImage2D(GL_TEXTURE_2D,
+                      0,
+                      GL_RGBA,
+                      texture_size_.width(),
+                      texture_size_.height(),
+                      0,
+                      GL_RGBA,
+                      GL_UNSIGNED_BYTE,
+                      &frame->argb_pixels.front());
+
+    host_->NotifyEndOfBitstreamBuffer(frame->decode_id);
+    host_->PictureReady(media::Picture(texture_id, frame->decode_id));
+    pending_frames_.pop();
+  }
+
+  FlushCommandBuffer();
+
+  if (state_ == FLUSHING && !num_pending_decodes_ && pending_frames_.empty()) {
+    state_ = DECODING;
+    host_->NotifyFlushDone();
+  }
+}
+
+void VideoDecoderShim::OnResetComplete() {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK(host_);
+
+  while (!pending_frames_.empty()) {
+    const linked_ptr<PendingFrame>& frame = pending_frames_.front();
+    host_->NotifyEndOfBitstreamBuffer(frame->decode_id);
+    pending_frames_.pop();
+  }
+
+  state_ = DECODING;
+  host_->NotifyResetDone();
+}
+
+void VideoDecoderShim::DismissTexture(uint32_t texture_id) {
+  DCHECK(host_);
+  textures_to_dismiss_.erase(texture_id);
+  DCHECK(texture_id_map_.find(texture_id) != texture_id_map_.end());
+  DeleteTexture(texture_id_map_[texture_id]);
+  texture_id_map_.erase(texture_id);
+  host_->DismissPictureBuffer(texture_id);
+}
+
+void VideoDecoderShim::DeleteTexture(uint32_t texture_id) {
+  gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
+  gles2->DeleteTextures(1, &texture_id);
+}
+
+void VideoDecoderShim::FlushCommandBuffer() {
+  context_provider_->ContextGL()->Flush();
+}
+
+}  // namespace content