Integrate VDA with WebRTC.

content/renderer/media/rtc_video_decoder_bridge.cc

Index: content/renderer/media/rtc_video_decoder_bridge.cc
diff --git a/content/renderer/media/rtc_video_decoder_bridge.cc b/content/renderer/media/rtc_video_decoder_bridge.cc
new file mode 100644
index 0000000000000000000000000000000000000000..7bc555c5a9f5a4339d25663e36c52573570cb6a2
--- /dev/null
+++ b/content/renderer/media/rtc_video_decoder_bridge.cc
@@ -0,0 +1,325 @@
+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/renderer/media/rtc_video_decoder_bridge.h"
+
+#include "base/bind.h"
+#include "base/callback_helpers.h"
+#include "base/location.h"
+#include "base/logging.h"
+#include "base/memory/ref_counted.h"
+#include "base/message_loop_proxy.h"
+#include "base/time.h"
+#include "content/renderer/media/native_handle_impl.h"
+#include "media/base/decoder_buffer.h"
+#include "media/base/yuv_convert.h"
+#include "third_party/libjingle/source/talk/base/ratetracker.h" // XXX remove
+#include "third_party/webrtc/system_wrappers/interface/ref_count.h"
+
+
+namespace content {
+
+#define LOG_LINE() VLOG(0) << __FILE__ << ":" << __FUNCTION__

[Ami GONE FROM CHROMIUM, 2013/04/26 00:42:05]

Planning to remove before landing?

[wuchengli, 2013/05/08 15:58:56]

Removed.
On 2013/04/26 00:42:05, Ami Fischman wrote:

+
+// static
+const base::TimeDelta RTCVideoDecoderBridge::kDecoderTimeOut =

[Ami GONE FROM CHROMIUM, 2013/04/26 00:42:05]

Why is this a good idea?

[wuchengli, 2013/05/08 15:58:56]

Removed. Now InitDecode waits for GpuVideoDecoder::Initialize to finish.
On 2013/04/26 00:42:05, Ami Fischman wrote:

+     base::TimeDelta::FromMilliseconds(3000);
+
+// Helper function that makes sure |read_cb| runs on |message_loop_proxy|.
+static void RunOnMessageLoop(

[Ami GONE FROM CHROMIUM, 2013/04/26 00:42:05]

Any reason this can't be replaced by BindToLoop?

[wuchengli, 2013/05/08 15:58:56]

Removed.

+    const media::DemuxerStream::ReadCB& read_cb,
+    const scoped_refptr<base::MessageLoopProxy>& message_loop_proxy,
+    media::DemuxerStream::Status status,
+    const scoped_refptr<media::DecoderBuffer>& buffer) {
+  if (!message_loop_proxy->BelongsToCurrentThread()) {
+    message_loop_proxy->PostTask(FROM_HERE, base::Bind(
+        &RunOnMessageLoop, read_cb, message_loop_proxy, status, buffer));
+    return;
+  }
+
+  read_cb.Run(status, buffer);
+}
+
+void RTCDemuxerStream::Read(const ReadCB& read_cb) {
+  CHECK(read_cb_.is_null());
+  read_cb_ = base::Bind(&RunOnMessageLoop, read_cb,
+                        base::MessageLoopProxy::current());
+
+  if(!buffer_queue_.IsEmpty()) {
+    base::ResetAndReturn(&read_cb_).Run(
+        DemuxerStream::kOk, buffer_queue_.Pop());
+  }
+}
+
+const media::AudioDecoderConfig& RTCDemuxerStream::audio_decoder_config() {
+  CHECK(false) << "Does not support audio.";

[Ami GONE FROM CHROMIUM, 2013/04/26 00:42:05]

FWIW 
LOG(FATAL)
reads slightly more clearly to me than
CHECK(false)

[wuchengli, 2013/05/08 15:58:56]

Done.

+  return dummy_audio_decoder_config_;

[Ami GONE FROM CHROMIUM, 2013/04/26 00:42:05]

this is unreachable, and therefore the member is unnecessary.

[wuchengli, 2013/05/08 15:58:56]

Removed.

+}
+
+const media::VideoDecoderConfig& RTCDemuxerStream::video_decoder_config() {
+  return video_decoder_config_;
+}
+
+media::DemuxerStream::Type RTCDemuxerStream::type() {
+  return media::DemuxerStream::VIDEO;
+}
+
+void RTCDemuxerStream::EnableBitstreamConverter() {
+}

[Ami GONE FROM CHROMIUM, 2013/04/26 00:42:05]

LOG(FATAL) << "Unreachable!";

[wuchengli, 2013/05/08 15:58:56]

Changed to NOTREACHED();

+
+void RTCDemuxerStream::UpdateSize(gfx::Size size) {
+  gfx::Rect rect(size);
+  video_decoder_config_.Initialize(
+      media::kCodecVP8,
+      media::VP8PROFILE_MAIN,
+      media::VideoFrame::NATIVE_TEXTURE,
+      size, rect, size, NULL, 0, false, false);

[Ami GONE FROM CHROMIUM, 2013/04/26 00:42:05]

Coded==visible==natural sizes/rects?
I suspect there's an impedance mismatch between media::VF and webrtc that at
least warrants a comment.

[wuchengli, 2013/05/08 15:58:56]

Can you explain more? I'm not sure what to put in the comment. I cannot find
information about visible rect or pixel aspect ratio in webrtc.

+}
+
+void RTCDemuxerStream::QueueBuffer(
+    scoped_refptr<media::DecoderBuffer> buffer) {
+  LOG_LINE();
+  LOG_LINE() << " end";
+  buffer_queue_.Push(buffer);
+
+  if (!read_cb_.is_null()) {
+    base::ResetAndReturn(&read_cb_).Run(
+        DemuxerStream::kOk, buffer_queue_.Pop());
+  }
+}
+
+RTCVideoDecoderBridge::RTCVideoDecoderBridge(
+    scoped_refptr<media::VideoDecoder> video_decoder,
+    const scoped_refptr<base::MessageLoopProxy>& message_loop)
+  : video_decoder_(video_decoder),
+    loop_proxy_(message_loop),
+    decode_complete_callback_(NULL),
+    pipeline_status_(media::PIPELINE_OK),
+    decoder_init_waiter_(false, false),
+    decoder_reset_waiter_(false, false),
+    stream_(new RTCDemuxerStream()),
+    status_(kNoInit),
+    decoding_error_occured_(false) {
+  LOG_LINE();
+}
+
+RTCVideoDecoderBridge::~RTCVideoDecoderBridge() {
+  LOG_LINE();
+}
+
+WebRtc_Word32 RTCVideoDecoderBridge::InitDecode(
+    const webrtc::VideoCodec* codecSettings,
+    WebRtc_Word32 numberOfCores) {
+  LOG_LINE();
+  if (status_ != kNoInit) {
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  if (codecSettings->codecType != webrtc::kVideoCodecVP8) {

[Pawel Osciak, 2013/04/26 07:08:01]

Can this happen if the check in RTCVideoDecoderFactory::CreateVideoDecoder()
doesn't fail? Perhaps we could drop it and keep this one?

Or actually, ideally, let VideoDecoder::Initialize decide.

[wuchengli, 2013/05/08 15:58:56]

All possible codecType are kVideoCodecVP8, kVideoCodecI420, kVideoCodecRED,
kVideoCodecULPFEC, kVideoCodecGeneric, and kVideoCodecUnknown. In
RTCVideoDecoderFactory::CreateVideoDecoder, we don't have
video_codec_config.size. So we cannot call VideoDecoder::Initialize there.

We can let VideoDecoder::Initialize decide. A mapping table will be needed.
Since only VP8 is the only option that can be mapped to
media::VideoCodecProfile. We don't need to add the code not used now. Thought?

+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  if (codecSettings->codecSpecific.VP8.feedbackModeOn) {
+    // We don't support feedback mode.
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
+  loop_proxy_->PostTask(
+      FROM_HERE,
+      base::Bind(&RTCDemuxerStream::UpdateSize, stream_,
+                 gfx::Size(codecSettings->width, codecSettings->height)));
+  loop_proxy_->PostTask(
+      FROM_HERE,
+      base::Bind(&media::VideoDecoder::Initialize,
+                 video_decoder_,
+                 stream_,
+                 base::Bind(&RTCVideoDecoderBridge::OnUpdatePipelineStatus,
+                            base::Unretained(this)),
+                 base::Bind(&RTCVideoDecoderBridge::OnUpdateStatistics,
+                            base::Unretained(this))));

[Ami GONE FROM CHROMIUM, 2013/04/26 00:42:05]

why is this Unretained safe?

+  decoder_init_waiter_.TimedWait(kDecoderTimeOut);
+  if (pipeline_status_ != media::PIPELINE_OK) {
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }

[Pawel Osciak, 2013/04/26 07:08:01]

No need for braces.

[wuchengli, 2013/05/08 15:58:56]

I added one more line.

+  status_ = kInitialized;
+  LOG_LINE() << "OK";
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+void RTCVideoDecoderBridge::OnUpdateStatistics(
+    const media::PipelineStatistics& stats) {
+  // We don't use statistics for now.
+}
+
+void RTCVideoDecoderBridge::OnUpdatePipelineStatus(
+    const media::PipelineStatus status) {
+  VLOG(0) << "RTCVideoDecoderBridge::OnUpdatePipelineStatus. status=" << status;
+  pipeline_status_ = status;
+  decoder_init_waiter_.Signal();
+}
+
+void RTCVideoDecoderBridge::ResetComplete() {
+  LOG_LINE();
+  decoder_reset_waiter_.Signal();
+}
+
+
+WebRtc_Word32 RTCVideoDecoderBridge::Decode(
+    const webrtc::EncodedImage& inputImage,
+    bool missingFrames,
+    const webrtc::RTPFragmentationHeader* fragmentation,
+    const webrtc::CodecSpecificInfo* codecSpecificInfo,
+    WebRtc_Word64 renderTimeMs) {

[Ami GONE FROM CHROMIUM, 2013/04/26 00:42:05]

unused parameters can have their name omitted, which makes it more obvious that
they are unused.  Please do that for unused params.

[wuchengli, 2013/05/08 15:58:56]

Done.

+  VLOG(0) << "RTCVideoDecoderBridge::Decode. status=" << status_;
+  {
+    base::AutoLock auto_lock(lock_);

[Pawel Osciak, 2013/04/26 07:08:01]

Lock usage is mysterious here, especially since there is a return after this and
no change to decoding_error_occured_.

[wuchengli, 2013/05/08 15:58:56]

decoding_error_occurred_ is read by WebRTC DecodingThread and set by
decoder_message_loop_. I added comments in the header file and used volatile
instead.

+    if (decoding_error_occured_) {
+      // TODO(dwkang): consider creating a SW decoder and falling back to it.

[Ami GONE FROM CHROMIUM, 2013/04/26 00:42:05]

s/dwkang/wuchengli/g ? :)

[wuchengli, 2013/05/08 15:58:56]

Removed the todo. This doesn't look like the right place for SW decoder.

+      return WEBRTC_VIDEO_CODEC_ERROR;
+    }
+  }
+  if (status_ == kNoInit || decode_complete_callback_ == NULL) {
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  }

[Pawel Osciak, 2013/04/26 07:08:01]

Braces not needed and below too.

[wuchengli, 2013/05/08 15:58:56]

Done.

+  if (!(status_ == kInitialized || status_ == kDecoding)) {

[Pawel Osciak, 2013/04/26 07:08:01]

if (status_ != kInitialized && status_ != kDecoding)

[wuchengli, 2013/05/08 15:58:56]

Done.

+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  if (missingFrames || !inputImage._completeFrame) {
+    // Unlike the SW decoder in libvpx, hw decoder can not handle broken frames.
+    // Here, we return an error in order to request a key frame.
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+
+  if (inputImage._frameType == webrtc::kKeyFrame &&
+      inputImage._encodedWidth != 0 && inputImage._encodedHeight != 0) {
+    // Only key frame has the size.
+    gfx::Size new_size(inputImage._encodedWidth, inputImage._encodedHeight);
+    if (size_ != new_size) {
+      size_ = new_size;

[Ami GONE FROM CHROMIUM, 2013/04/26 00:42:05]

member never used; drop?

[wuchengli, 2013/05/08 15:58:56]

This is used so that RTCDemuxerStream::UpdateSize is not called when the size is
unchanged.

+      loop_proxy_->PostTask(
+          FROM_HERE,
+          base::Bind(&RTCDemuxerStream::UpdateSize, stream_, new_size));

[Ami GONE FROM CHROMIUM, 2013/04/26 00:42:05]

In this case, you need to signal DemuxerStream::kConfigChanged via read_cb_.

+    }
+  }
+  scoped_refptr<media::DecoderBuffer> buffer;

[Ami GONE FROM CHROMIUM, 2013/04/26 00:42:05]

I only got down to here but I think I see your problem now; nothing is calling
GpuVideoDecoder::Decode()!  

WebRTC's model is that the "decoder" is given a callback to call when pictures
are ready and is then called repeatedly to decode encoded images (bitstream
buffers).

media/'s model is that the decoder is given a demuxer stream (the source of
bitstream buffers) and is repeatedly called asking for available images.

In other words, WebRTC uses a "push" model, and media/ uses a "pull" model.

Ironically, the VDA decode API is also a push model, but GVD converts it to pull
for the benefit of the media pipeline.

I suspect if you dropped GVD from the picture and instead used the
GpuVideoDecodeAcceleratorHost directly (as a VDA; it's not even important that
it happens to be a GVDAH), your code would be a lot simpler and at the same time
would start working :)

[wuchengli, 2013/04/26 05:08:08]

GpuVideoDecoder::Read is called in RTCVideoDecoderBridge::RequestFrame. I think
it triggers GpuVideoDecoder decoding. From the log, the decode did start.
https://p.prom.corp.google.com/2594062  It was just sluggish intermittently. 
On 2013/04/26 00:42:05, Ami Fischman wrote:

[Ami GONE FROM CHROMIUM, 2013/04/26 15:13:34]

I see.  IIUC the code will wait for each frame to round-trip from the GPU
process before requesting the next frame (and thus sending the bitstream to the
GPU process) so sluggishness is not at all surprising (instead, you want to
pipeline the decodes).  I still think GVD is doing you a disservice here and
that you should be using VDA directly (via the concrete impl of GVDAH, which is
what GVD uses).

[wuchengli, 2013/05/09 15:43:14]

The latest patchset does not wait from the GPU process before requesting the
next frame. RTCVideoDecoder::Decode is run by webrtc DecodingThread and keeps
queuing buffers. FrameReady is run by decoder_message_loop_ (same as
gvd_loop_proxy_ in GpuVideoDecoder and is a new thread created in
MediaStreamDependencyFactory).

One question is whether it is safe for webrtc to call decode complete callback
from the other thread. video_codec_interface.h doesn't mention this.

+  buffer = media::DecoderBuffer::CopyFrom(

[Ami GONE FROM CHROMIUM, 2013/04/26 00:42:05]

assign on prev line?

[wuchengli, 2013/05/08 15:58:56]

Done.

+      inputImage._buffer, inputImage._length);
+  buffer->SetTimestamp(base::TimeDelta::FromInternalValue(
+      inputImage._timeStamp));

[Ami GONE FROM CHROMIUM, 2013/04/26 00:42:05]

This looks strange; internalvalue is in microseconds but _timeStamp is an int32
so likely not.

[wuchengli, 2013/05/09 15:43:14]

inputImage._timeStamp is in 90KHz. We need to divide it by 90 here and multiply
it by 90 when getting the frame from decoder in FrameReady(). It may lose the
precision. However, webrtc uses the timestamp as key of a map. I cannot think of
a good solution. Adding another map here is not good, either. Multiple 90Khz
timestamps can be converted to the same timestamp in microsecond. Any
suggestion?

Use timestamp as key to get other data from the map:
https://code.google.com/p/chromium/codesearch#chromium/src/third_party/webrtc...

Other webrtc::VideoDecoder::Decoder implementation are blocking. They return the
decoded frame and can easily use the same timestamp.
https://code.google.com/p/chromium/codesearch#chromium/src/third_party/webrtc...

+
+  loop_proxy_->PostTask(
+      FROM_HERE,
+      base::Bind(&RTCDemuxerStream::QueueBuffer, stream_, buffer));
+
+  {
+    base::AutoLock auto_lock(lock_);
+    for (VideoFrameList::iterator it = ready_video_frames_.begin();
+        it != ready_video_frames_.end(); ++it) {
+      gfx::Rect rect = (*it)->visible_rect();
+      decoded_image_.CreateEmptyFrame(
+          rect.width(), rect.height(),
+          rect.width(), rect.width() / 2, rect.width() / 2);
+      webrtc::RefCountImpl<media::NativeHandleImpl>* handle =
+         new webrtc::RefCountImpl<media::NativeHandleImpl>();
+      handle->SetHandle((*it).get());
+      decoded_image_.set_native_handle(handle);
+      decoded_image_.set_timestamp(
+          static_cast<uint32_t>((*it)->GetTimestamp().InMicroseconds()));
+      decode_complete_callback_->Decoded(decoded_image_);
+      decoded_image_.set_native_handle(NULL);
+    }
+    ready_video_frames_.clear();
+  }
+
+  if (status_ == kInitialized) {
+    status_ = kDecoding;
+    RequestFrame();
+  }
+
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+void RTCVideoDecoderBridge::RequestFrame() {
+  loop_proxy_->PostTask(
+      FROM_HERE,
+      base::Bind(&media::VideoDecoder::Read,
+                 video_decoder_,
+                 base::Bind(&RTCVideoDecoderBridge::FrameReady,
+                            base::AsWeakPtr(this))));
+}
+
+WebRtc_Word32 RTCVideoDecoderBridge::RegisterDecodeCompleteCallback(
+    webrtc::DecodedImageCallback* callback) {
+  decode_complete_callback_ = callback;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+WebRtc_Word32 RTCVideoDecoderBridge::Release() {
+  LOG_LINE();
+  /*if (status_ == kNoInit) {
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  }
+  base::WaitableEvent event(false, false);
+  loop_proxy_->PostTask(
+      FROM_HERE,
+      base::Bind(&media::VideoDecoder::Stop,
+                 video_decoder_,
+                 base::Bind(&base::WaitableEvent::Signal,
+                            base::Unretained(&event))));
+  decoder_init_waiter_.Wait();
+  status_ = kReleased;*/
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+WebRtc_Word32 RTCVideoDecoderBridge::Reset() {
+  LOG_LINE();
+  if (status_ == kNoInit) {
+    LOG(ERROR) << "RTCVideoDecoderBridge::Reset. status==kNoInit";
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  }
+  status_ = kReseting;
+  base::WaitableEvent event(false, false);

[Pawel Osciak, 2013/04/26 07:08:01]

unused?

[wuchengli, 2013/05/08 15:58:56]

Removed.

+  loop_proxy_->PostTask(
+      FROM_HERE,
+      base::Bind(&media::VideoDecoder::Reset,
+                 video_decoder_,
+                 base::Bind(&RTCVideoDecoderBridge::ResetComplete,
+                            base::Unretained(this))));
+  decoder_reset_waiter_.Wait();
+  status_ = kInitialized;
+  LOG_LINE() << " OK";
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+void RTCVideoDecoderBridge::FrameReady(
+    media::VideoDecoder::Status status,
+    const scoped_refptr<media::VideoFrame>& frame) {
+  VLOG(0) << "RTCVideoDecoderBridge::FrameReady. status=" << status
+          << ". status_=" << status_;
+  /*{
+    static talk_base::RateTracker frame_rate_tracker_;
+    static base::Time start_time = base::Time::Now();
+    base::Time diff = base::Time::Now() - start_time;
+    VLOG(1) << "RTCVideoDecoderBridge::FrameReady DeliverFrame framerate "
+        << frame_rate_tracker_.units_second()
+        << " avg framerate "
+        << frame_rate_tracker_.total_units() / diff.InSecondsF();
+    frame_rate_tracker_.Update(1);
+  }*/
+
+  {
+    base::AutoLock auto_lock(lock_);
+    if (status != media::VideoDecoder::kOk) {
+      LOG(ERROR) << "RTCVideoDecoderBridge::FrameReady. status!=kOk";
+      decoding_error_occured_ = true;
+      return;
+    }
+
+    ready_video_frames_.push_back(frame);

[wuchengli, 2013/04/25 12:38:54]

How can I call decode_complete_callback_ using webrtc DecodingThread here? I
tried to store DecodingThread message loop proxy in InitDecode. But
MessageLoopProxy::current() was null.

+  }
+  if (status_ == kDecoding) {
+    RequestFrame();
+  }
+}
+
+}  // namespace content
+