Implement software fallback for PPB_VideoDecoder.

content/renderer/pepper/video_decoder_adapter.cc

+// Copyright (c) 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <GLES2/gl2.h>
+#include <GLES2/gl2ext.h>
+#include <GLES2/gl2extchromium.h>
+
+#include "content/renderer/pepper/video_decoder_adapter.h"
+
+#include "base/bind.h"
+#include "base/numerics/safe_conversions.h"
+#include "content/public/renderer/render_thread.h"
+#include "content/renderer/pepper/pepper_video_decoder_host.h"
+#include "content/renderer/render_thread_impl.h"
+#include "gpu/command_buffer/client/gles2_implementation.h"
+#include "media/base/decoder_buffer.h"
+#include "media/filters/ffmpeg_video_decoder.h"
+#include "media/video/picture.h"
+#include "media/video/video_decode_accelerator.h"
+#include "ppapi/c/pp_errors.h"
+#include "third_party/libyuv/include/libyuv.h"
+#include "webkit/common/gpu/context_provider_web_context.h"
+
+namespace content {
+
+struct VideoDecoderAdapter::PendingDecode {
+  PendingDecode(uint32_t decode_id,
+                const scoped_refptr<media::DecoderBuffer>& buffer);
+  ~PendingDecode();
+
+  const uint32_t decode_id;
+  const scoped_refptr<media::DecoderBuffer> buffer;
+};
+
+VideoDecoderAdapter::PendingDecode::PendingDecode(
+    uint32_t decode_id,
+    const scoped_refptr<media::DecoderBuffer>& buffer)
+    : decode_id(decode_id), buffer(buffer) {
+}
+
+VideoDecoderAdapter::PendingDecode::~PendingDecode() {
+}
+
+struct VideoDecoderAdapter::PendingFrame {
+  explicit PendingFrame(uint32_t decode_id);
+  PendingFrame(uint32_t decode_id, const gfx::Size& size);
+  ~PendingFrame();
+
+  const uint32_t decode_id;
+  const gfx::Size size;
+  std::vector<uint8_t> argb_pixels;

[Ami GONE FROM CHROMIUM, 2014/06/06 17:14:33]

Experimentation is the only authority :(

[dmichael (off chromium), 2014/06/06 17:24:00]

private:
 DISALLOW_COPY_AND_ASSIGN(PendingFrame);

?

It's copyable, but that would be slightly expensive, so I think you want to
protect yourself from doing it accidentally.

[bbudge, 2014/06/07 00:31:09]

Done.

[bbudge, 2014/06/07 00:31:09]

Looks like ARGB. I guess as long as there are an even number of endian flips...

+};
+
+VideoDecoderAdapter::PendingFrame::PendingFrame(uint32_t decode_id)
+    : decode_id(decode_id) {
+}
+
+VideoDecoderAdapter::PendingFrame::PendingFrame(uint32_t decode_id,
+                                                const gfx::Size& size)
+    : decode_id(decode_id),
+      size(size),
+      argb_pixels(size.width() * size.height() * 4) {
+}
+
+VideoDecoderAdapter::PendingFrame::~PendingFrame() {
+}
+
+VideoDecoderAdapter::Delegate::Delegate(
+    const base::WeakPtr<VideoDecoderAdapter>& proxy)
+    : adapter_(proxy),
+      main_message_loop_(base::MessageLoopProxy::current()),
+      max_pending_decodes_(0),
+      num_pending_decodes_(0) {
+}
+
+VideoDecoderAdapter::Delegate::~Delegate() {
+  DCHECK(pending_decodes_.empty());
+}
+
+void VideoDecoderAdapter::Delegate::Initialize(
+    media::VideoDecoderConfig config) {
+  DCHECK(!decoder_);
+  decoder_.reset(
+      new media::FFmpegVideoDecoder(base::MessageLoopProxy::current()));
+  max_pending_decodes_ = decoder_->GetMaxDecodeRequests();
+  // We can use base::Unretained() safely in decoder callbacks because we call
+  // VideoDecoder::Stop() before deletion. Stop() guarantees there will be no
+  // outstanding callbacks after it returns.
+  decoder_->Initialize(
+      config,
+      true /* low_delay */,
+      base::Bind(&VideoDecoderAdapter::Delegate::OnPipelineStatus,
+                 base::Unretained(this)));
+}
+
+void VideoDecoderAdapter::Delegate::OnPipelineStatus(
+    media::PipelineStatus status) {
+  main_message_loop_->PostTask(
+      FROM_HERE,
+      base::Bind(&VideoDecoderAdapter::OnPipelineStatus, adapter_, status));
+}
+
+void VideoDecoderAdapter::Delegate::Decode(
+    uint32_t decode_id,
+    scoped_refptr<media::DecoderBuffer> buffer) {
+  DCHECK(decoder_);
+  if (num_pending_decodes_ < max_pending_decodes_) {

[Ami GONE FROM CHROMIUM, 2014/06/06 17:14:33]

num_pending_decodes_ holding a value other than
pending_decodes_.size() is an unfortunate naming situation.

(elsewhere in the VDA world we use the terminology "at decoder" for what you've
named {num,max}_pending_decodes_)

[bbudge, 2014/06/07 00:31:09]

Done.

+    num_pending_decodes_++;
+    decoder_->Decode(

[Ami GONE FROM CHROMIUM, 2014/06/06 17:14:33]

DCHECK(pending_decodes_.empty());

[bbudge, 2014/06/07 00:31:09]

It might not be true. I think you identified a real problem though.

I changed back to having a separate DoDecode function, and having Decode just
push the latest buffer and then call DoDecode, to ensure that buffers never get
sent in the wrong order.

This might have been a bug. Is it possible that the example video can tolerate
some buffers out of order (since it only has 1 seek point)?

+        buffer,
+        base::Bind(&VideoDecoderAdapter::Delegate::OnDecodeComplete,
+                   base::Unretained(this),
+                   decode_id));
+  } else {
+    pending_decodes_.push(PendingDecode(decode_id, buffer));
+  }
+}
+
+void VideoDecoderAdapter::Delegate::Reset() {
+  DCHECK(decoder_);
+  // Abort all pending decodes.
+  while (!pending_decodes_.empty()) {
+    PendingDecode& decode = pending_decodes_.front();

[dmichael (off chromium), 2014/06/06 17:24:00]

nit: const& ?

[bbudge, 2014/06/07 00:31:09]

Done.

+    scoped_ptr<PendingFrame> pending_frame(new PendingFrame(decode.decode_id));
+    main_message_loop_->PostTask(
+        FROM_HERE,
+        base::Bind(&VideoDecoderAdapter::OnDecodeComplete,
+                   adapter_,
+                   media::VideoDecoder::kAborted,
+                   base::Passed(&pending_frame)));
+    pending_decodes_.pop();
+  }
+  decoder_->Reset(base::Bind(&VideoDecoderAdapter::Delegate::OnResetComplete,
+                             base::Unretained(this)));
+}
+
+void VideoDecoderAdapter::Delegate::Stop() {
+  DCHECK(decoder_);
+  // Clear pending decodes now. We don't want OnDecodeComplete to call Decode
+  // again.
+  while (!pending_decodes_.empty())
+    pending_decodes_.pop();

[Ami GONE FROM CHROMIUM, 2014/06/06 17:14:33]

pending_decodes_.clear() instead of the previous two lines?

[bbudge, 2014/06/07 00:31:09]

Can't, it's std::queue.

+  decoder_->Stop();
+  // This instance is deleted once we exit this scope.
+}
+
+void VideoDecoderAdapter::Delegate::OnDecodeComplete(
+    uint32_t decode_id,
+    media::VideoDecoder::Status status,
+    const scoped_refptr<media::VideoFrame>& frame) {
+  num_pending_decodes_--;
+  scoped_ptr<PendingFrame> pending_frame;
+  if (status == media::VideoDecoder::kOk && frame && !frame->end_of_stream()) {

[Ami GONE FROM CHROMIUM, 2014/06/06 17:14:33]

&& frame is unnecessary here.

[bbudge, 2014/06/07 00:31:09]

Done.

+    pending_frame.reset(new PendingFrame(decode_id, frame->coded_size()));
+    // Convert the VideoFrame pixels to ARGB.
+    libyuv::I420ToARGB(frame->data(media::VideoFrame::kYPlane),
+                       frame->stride(media::VideoFrame::kYPlane),
+                       frame->data(media::VideoFrame::kUPlane),
+                       frame->stride(media::VideoFrame::kUPlane),
+                       frame->data(media::VideoFrame::kVPlane),
+                       frame->stride(media::VideoFrame::kVPlane),
+                       &pending_frame->argb_pixels.front(),
+                       frame->coded_size().width() * 4,
+                       frame->coded_size().width(),
+                       frame->coded_size().height());
+  } else {
+    pending_frame.reset(new PendingFrame(decode_id));
+  }
+
+  main_message_loop_->PostTask(
+      FROM_HERE,
+      base::Bind(&VideoDecoderAdapter::OnDecodeComplete,
+                 adapter_,
+                 status,
+                 base::Passed(&pending_frame)));
+
+  if (!pending_decodes_.empty()) {
+    const PendingDecode& decode = pending_decodes_.front();
+    Decode(decode.decode_id, decode.buffer);
+    pending_decodes_.pop();
+  }
+}
+
+void VideoDecoderAdapter::Delegate::OnResetComplete() {
+  main_message_loop_->PostTask(
+      FROM_HERE, base::Bind(&VideoDecoderAdapter::OnResetComplete, adapter_));
+}
+
+VideoDecoderAdapter::VideoDecoderAdapter(PepperVideoDecoderHost* host)
+    : state_(UNINITIALIZED),
+      host_(host),
+      media_message_loop_(
+          RenderThreadImpl::current()->GetMediaThreadMessageLoopProxy()),
+      context_provider_(
+          RenderThreadImpl::current()->SharedMainThreadContextProvider()),
+      num_pending_decodes_(0),
+      weak_ptr_factory_(this) {
+  DCHECK(host_);
+  DCHECK(media_message_loop_);
+  DCHECK(context_provider_);
+  delegate_.reset(new Delegate(weak_ptr_factory_.GetWeakPtr()));
+}
+
+VideoDecoderAdapter::~VideoDecoderAdapter() {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK(!host_);
+  // Delete any remaining textures.
+  TextureIdMap::iterator it = texture_id_map_.begin();
+  for (; it != texture_id_map_.end(); ++it)
+    DeleteTexture(it->second);

[Ami GONE FROM CHROMIUM, 2014/06/06 17:14:33]

Yeah, I realize; still that's what I'd do.
(SEGV's at 0x0 are always more pleasant to debug than UAFs IMO)

[bbudge, 2014/06/07 00:31:09]

Done.

+
+  FlushCommandBuffer();
+}
+
+void VideoDecoderAdapter::Initialize(media::VideoCodecProfile profile) {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK_EQ(state_, UNINITIALIZED);
+  media::VideoCodec codec = media::kUnknownVideoCodec;
+  if (profile <= media::H264PROFILE_MAX)
+    codec = media::kCodecH264;
+  else if (profile <= media::VP8PROFILE_MAX)
+    codec = media::kCodecVP8;
+  DCHECK_NE(codec, media::kUnknownVideoCodec);
+
+  media::VideoDecoderConfig config(
+      codec,
+      profile,
+      media::VideoFrame::YV12,
+      gfx::Size(32, 24),  // Small sizes that won't fail.
+      gfx::Rect(32, 24),
+      gfx::Size(32, 24),
+      NULL /* extra_data */,

[Ami GONE FROM CHROMIUM, 2014/06/06 17:14:33]

Please add a TODO.
The demuxer gives you the extradata.

[bbudge, 2014/06/07 00:31:09]

Done.

+      0 /* extra_data_size */,
+      false /* decryption */);
+
+  media_message_loop_->PostTask(
+      FROM_HERE,
+      base::Bind(&VideoDecoderAdapter::Delegate::Initialize,
+                 base::Unretained(delegate_.get()),
+                 config));
+}
+
+void VideoDecoderAdapter::Decode(uint32_t decode_id,
+                                 const uint8_t* buffer,
+                                 uint32_t size) {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK_EQ(state_, DECODING);
+
+  num_pending_decodes_++;
+
+  media_message_loop_->PostTask(
+      FROM_HERE,
+      base::Bind(&VideoDecoderAdapter::Delegate::Decode,
+                 base::Unretained(delegate_.get()),
+                 decode_id,
+                 media::DecoderBuffer::CopyFrom(buffer, size)));
+}
+
+void VideoDecoderAdapter::AssignTextures(
+    const std::vector<uint32_t>& texture_ids) {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK_EQ(state_, DECODING);
+  DCHECK(texture_ids.size());

[Ami GONE FROM CHROMIUM, 2014/06/06 17:14:33]

Wouldn't such a texture be in textures_to_dismiss_ as opposed to t_i_m_?

[dmichael (off chromium), 2014/06/06 17:24:00]

I think you should maybe reformat this to something like:
if (textured_ids.empty()) {
  NOTREACHED();
  return;
}

So that you avoid calling front() on an empty vector below if somehow the
condition doesn't hold.

[bbudge, 2014/06/07 00:31:09]

Done.

[bbudge, 2014/06/07 00:31:09]

It would be in both. textures_to_dismiss_ may contain textures that are still in
use by the plugin.

+  DCHECK_EQ(texture_ids.size(), pending_texture_mailboxes_.size());
+  GLuint num_textures = base::checked_cast<GLuint>(texture_ids.size());
+  std::vector<uint32_t> local_texture_ids(num_textures);
+  gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
+  gles2->GenTextures(num_textures, &local_texture_ids.front());
+  for (uint32_t i = 0; i < num_textures; i++) {
+    gles2->ActiveTexture(GL_TEXTURE0);
+    gles2->BindTexture(GL_TEXTURE_2D, local_texture_ids[i]);
+    gles2->ConsumeTextureCHROMIUM(GL_TEXTURE_2D,
+                                  pending_texture_mailboxes_[i].name);
+    // Map the plugin texture id to the local texture id.
+    texture_id_map_[texture_ids[i]] = local_texture_ids[i];
+  }
+  pending_texture_mailboxes_.clear();
+  available_textures_.insert(
+      available_textures_.end(), texture_ids.begin(), texture_ids.end());
+  SendPictures();
+}
+
+void VideoDecoderAdapter::RecycleTexture(uint32_t texture_id) {
+  DCHECK(RenderThreadImpl::current());
+  if (textures_to_dismiss_.find(texture_id) != textures_to_dismiss_.end()) {
+    DismissTexture(texture_id);
+  } else if (texture_id_map_.find(texture_id) != texture_id_map_.end()) {
+    available_textures_.push_back(texture_id);
+    SendPictures();
+  } else {
+    NOTREACHED();
+  }
+}
+
+void VideoDecoderAdapter::Flush() {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK_EQ(state_, DECODING);
+  state_ = FLUSHING;
+}
+
+void VideoDecoderAdapter::Reset() {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK_EQ(state_, DECODING);
+  state_ = RESETTING;
+  media_message_loop_->PostTask(
+      FROM_HERE,
+      base::Bind(&VideoDecoderAdapter::Delegate::Reset,
+                 base::Unretained(delegate_.get())));
+}
+
+void VideoDecoderAdapter::Destroy() {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK(host_);
+  host_ = NULL;
+  state_ = DESTROYED;
+  // No methods can post tasks to the delegate now.
+  weak_ptr_factory_.InvalidateWeakPtrs();
+  // No more callbacks from the delegate will be received now.
+
+  // The callback now holds the only reference to the delegate, which will be
+  // deleted when Stop completes.
+  media_message_loop_->PostTask(FROM_HERE,
+                                base::Bind(&VideoDecoderAdapter::Delegate::Stop,
+                                           base::Owned(delegate_.release())));

[dmichael (off chromium), 2014/06/06 17:24:00]

You convinced me that VideoDecoderAdapter::Delegate should have Stop and not
just do everything in Delegate's destructor. But...  this above still looks like
the above could all happen in VideoDecoderAdapter's destructor, and there's no
need to do anything special for scoped_ptr<> for VideoDecoderAdapter.

[bbudge, 2014/06/07 00:31:09]

We're posting a task to the media loop to Stop the decoder. But there may be
tasks on the main loop with base::Unretained's to this adapter, or tasks on the
media thread ahead of the Stop which will post such tasks. We can't delete this
class until we're sure the Delegate has stopped.

[bbudge, 2014/06/07 01:01:23]

Ugh, there you go being right again. Yeah, since I'm invoking the dtor below,
this could all be in the regular dtor and we don't need the Deleter.

Done.

+
+  delete this;
+}
+
+void VideoDecoderAdapter::OnPipelineStatus(media::PipelineStatus status) {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK(host_);
+
+  int32_t result;
+  switch (status) {
+    case media::PIPELINE_OK:
+      result = PP_OK;
+      state_ = DECODING;
+      break;
+    case media::DECODER_ERROR_NOT_SUPPORTED:
+      result = PP_ERROR_NOTSUPPORTED;
+      break;
+    default:
+      result = PP_ERROR_FAILED;
+      break;
+  }
+  host_->OnInitializeComplete(result);
+}
+
+void VideoDecoderAdapter::OnDecodeComplete(media::VideoDecoder::Status status,
+                                           scoped_ptr<PendingFrame> frame) {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK(host_);
+
+  num_pending_decodes_--;
+
+  if (!frame->argb_pixels.empty()) {
+    if (texture_size_ != frame->size) {
+      // If the size has changed, all current textures must be dismissed. Add
+      // all textures to |textures_to_dismiss_| and dismiss any that aren't in
+      // use by the plugin. We dismiss the rest as they are recycled.
+      for (TextureIdMap::const_iterator it = texture_id_map_.begin();
+           it != texture_id_map_.end();
+           ++it) {
+        textures_to_dismiss_.insert(it->second);
+      }
+      for (std::vector<uint32_t>::const_iterator it =
+               available_textures_.begin();
+           it != available_textures_.end();
+           ++it) {
+        DismissTexture(*it);
+      }
+      available_textures_.clear();
+      FlushCommandBuffer();
+
+      DCHECK(pending_texture_mailboxes_.empty());
+      const uint32_t num_textures = 8;

[Ami GONE FROM CHROMIUM, 2014/06/06 17:14:33]

In order to saturate both the decoder and the renderer you'll want
GetMaxDecodeRequests() + media::limits::kMaxVideoFrames

[bbudge, 2014/06/07 00:31:09]

Done.

+      for (uint32_t i = 0; i < num_textures; i++)
+        pending_texture_mailboxes_.push_back(gpu::Mailbox::Generate());
+
+      host_->RequestTextures(
+          num_textures, frame->size, GL_TEXTURE_2D, pending_texture_mailboxes_);
+      texture_size_ = frame->size;
+    }
+
+    pending_frames_.push(linked_ptr<PendingFrame>(frame.release()));
+    SendPictures();
+  } else {
+    host_->NotifyEndOfBitstreamBuffer(frame->decode_id);
+  }
+
+  switch (status) {
+    case media::VideoDecoder::kOk:
+    case media::VideoDecoder::kAborted:
+    // This is not necessarily an error.
+    case media::VideoDecoder::kNotEnoughData:
+      break;
+    case media::VideoDecoder::kDecodeError:
+      host_->NotifyError(PP_ERROR_RESOURCE_FAILED);
+      break;
+    case media::VideoDecoder::kDecryptError:
+      NOTREACHED();
+      break;
+      // No default case, to catch unhandled status values.
+  }
+}
+
+void VideoDecoderAdapter::SendPictures() {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK(host_);
+
+  while (!pending_frames_.empty() && !available_textures_.empty()) {
+    const linked_ptr<PendingFrame>& frame = pending_frames_.front();
+
+    uint32_t texture_id = available_textures_.back();
+    available_textures_.pop_back();
+
+    uint32_t local_texture_id = texture_id_map_[texture_id];
+    gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
+    gles2->ActiveTexture(GL_TEXTURE0);
+    gles2->BindTexture(GL_TEXTURE_2D, local_texture_id);
+    gles2->TexImage2D(GL_TEXTURE_2D,
+                      0,
+                      GL_RGBA,
+                      texture_size_.width(),
+                      texture_size_.height(),
+                      0,
+                      GL_RGBA,
+                      GL_UNSIGNED_BYTE,
+                      &frame->argb_pixels.front());
+
+    host_->NotifyEndOfBitstreamBuffer(frame->decode_id);
+    host_->PictureReady(media::Picture(texture_id, frame->decode_id));
+    pending_frames_.pop();
+  }
+
+  FlushCommandBuffer();
+
+  if (state_ == FLUSHING && !num_pending_decodes_ && pending_frames_.empty()) {
+    state_ = DECODING;
+    host_->NotifyFlushDone();
+  }
+}
+
+void VideoDecoderAdapter::OnResetComplete() {
+  DCHECK(RenderThreadImpl::current());
+  DCHECK(host_);
+
+  while (!pending_frames_.empty()) {
+    const linked_ptr<PendingFrame>& frame = pending_frames_.front();
+    host_->NotifyEndOfBitstreamBuffer(frame->decode_id);
+    pending_frames_.pop();
+  }
+
+  state_ = DECODING;
+  host_->NotifyResetDone();
+}
+
+void VideoDecoderAdapter::DismissTexture(uint32_t texture_id) {
+  DCHECK(host_);
+  textures_to_dismiss_.erase(texture_id);
+  DCHECK(texture_id_map_.find(texture_id) != texture_id_map_.end());
+  DeleteTexture(texture_id_map_[texture_id]);
+  texture_id_map_.erase(texture_id);
+  host_->DismissPictureBuffer(texture_id);
+}
+
+void VideoDecoderAdapter::DeleteTexture(uint32_t texture_id) {
+  gpu::gles2::GLES2Interface* gles2 = context_provider_->ContextGL();
+  gles2->DeleteTextures(1, &texture_id);
+}
+
+void VideoDecoderAdapter::FlushCommandBuffer() {
+  context_provider_->ContextGL()->Flush();
+}
+
+}  // namespace content