Implement WebRTC in Chrome for TV

content/renderer/media/rtc_video_decoder_bridge_tv.cc

+// Copyright (c) 2013 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/renderer/media/rtc_video_decoder_bridge_tv.h"
+
+#include <queue>
+
+#include "base/bind.h"
+#include "base/callback_helpers.h"
+#include "base/location.h"
+#include "base/logging.h"
+#include "base/memory/ref_counted.h"
+#include "base/memory/singleton.h"
+#include "base/message_loop_proxy.h"
+#include "base/synchronization/lock.h"
+#include "base/time.h"
+#include "media/base/bind_to_loop.h"
+#include "media/base/decoder_buffer.h"
+#include "third_party/libjingle/source/talk/base/ratetracker.h"
+
+namespace content {
+
+// Helper function that makes sure |read_cb| runs on |message_loop_proxy|.
+static void RunOnMessageLoop(
+    media::DemuxerStream::ReadCB read_cb,
+    scoped_refptr<base::MessageLoopProxy> message_loop_proxy,
+    media::DemuxerStream::Status status,
+    const scoped_refptr<media::DecoderBuffer>& buffer) {
+  if (!message_loop_proxy->BelongsToCurrentThread()) {
+    message_loop_proxy->PostTask(FROM_HERE, base::Bind(
+        &RunOnMessageLoop, read_cb, message_loop_proxy, status, buffer));
+    return;
+  }
+  read_cb.Run(status, buffer);
+}
+
+// RTCDemuxerStream ------------------------------------------------------------
+
+namespace {
+
+class RTCDemuxerStream : public media::DemuxerStream {
+ public:
+  explicit RTCDemuxerStream(const gfx::Size& size);
+  virtual ~RTCDemuxerStream();
+  // media::DemuxerStream implementation.
+  virtual void Read(const ReadCB& read_cb) OVERRIDE;
+  virtual const media::AudioDecoderConfig& audio_decoder_config() OVERRIDE;
+  virtual const media::VideoDecoderConfig& video_decoder_config() OVERRIDE;
+  virtual Type type() OVERRIDE;
+  virtual void EnableBitstreamConverter() OVERRIDE;
+
+  void QueueBuffer(scoped_refptr<media::DecoderBuffer> buffer,
+                   const base::Closure& done_cb,
+                   const gfx::Size& size);
+
+ private:
+  struct DecoderBuffer {

[ycheo (away), 2013/05/06 05:45:34]

This name is a little confused with media::DecoderBuffer.

[wonsik, 2013/05/06 12:28:43]

Done --- Used BufferEntry

+    DecoderBuffer(const scoped_refptr<media::DecoderBuffer>& decoder_buffer,
+                  const base::Closure& done_cb,
+                  const gfx::Size& size)
+        : decoder_buffer_(decoder_buffer), done_cb_(done_cb), size_(size) {}
+
+    scoped_refptr<media::DecoderBuffer> decoder_buffer_;

[ycheo (away), 2013/05/06 05:45:34]

We don't add '_' suffix at the data members in structs.

[wonsik, 2013/05/06 12:28:43]

Done.

+    base::Closure done_cb_;
+    gfx::Size size_;

[dwkang1, 2013/05/06 09:43:38]

Could you add a doc for this variable? (when this is empty.)

[wonsik, 2013/05/06 12:28:43]

Done.

+  };
+
+  void RunReadCallback();
+
+  std::queue<DecoderBuffer> buffer_queue_;

[dwkang1, 2013/05/06 09:43:38]

Member variables need to be protected because QueueBuffer() will be called in
the webrtc thread.

[wonsik, 2013/05/06 12:28:43]

Done.

+  ReadCB read_cb_;
+  base::Closure pending_done_cb_;
+
+  media::AudioDecoderConfig dummy_audio_decoder_config_;
+  media::VideoDecoderConfig video_decoder_config_;
+  talk_base::RateTracker frame_rate_tracker_;
+};
+
+RTCDemuxerStream::RTCDemuxerStream(const gfx::Size& size)
+  : video_decoder_config_(
+      media::kCodecVP8,
+      media::VP8PROFILE_MAIN,
+      media::VideoFrame::NATIVE_TEXTURE,
+      size, gfx::Rect(size), size, NULL, 0, false) {
+}
+
+RTCDemuxerStream::~RTCDemuxerStream() {}
+
+const media::AudioDecoderConfig& RTCDemuxerStream::audio_decoder_config() {
+  LOG(FATAL) << "Does not support audio.";
+  return dummy_audio_decoder_config_;
+}
+
+const media::VideoDecoderConfig& RTCDemuxerStream::video_decoder_config() {
+  return video_decoder_config_;
+}
+
+media::DemuxerStream::Type RTCDemuxerStream::type() {
+  return media::DemuxerStream::VIDEO;
+}
+
+void RTCDemuxerStream::EnableBitstreamConverter() {
+  LOG(FATAL) << "Not reachable.";
+}
+
+void RTCDemuxerStream::QueueBuffer(scoped_refptr<media::DecoderBuffer> buffer,
+                                   const base::Closure& done_cb,
+                                   const gfx::Size& size) {
+  buffer_queue_.push(DecoderBuffer(buffer, done_cb, size));
+  frame_rate_tracker_.Update(1);
+  DLOG(INFO) << "frame rate received : " << frame_rate_tracker_.units_second();
+  RunReadCallback();
+}
+
+void RTCDemuxerStream::Read(const ReadCB& read_cb) {
+  CHECK(read_cb_.is_null());
+  // A call to |Read| operation means that |MediaSourceDelegate| is done with
+  // the previous buffer.
+  if (!pending_done_cb_.is_null())
+    base::ResetAndReturn(&pending_done_cb_).Run();
+  read_cb_ = media::BindToLoop(base::MessageLoopProxy::current(), read_cb);
+  RunReadCallback();
+}
+
+void RTCDemuxerStream::RunReadCallback() {
+  if (!read_cb_.is_null() && !buffer_queue_.empty()) {
+    const DecoderBuffer& front = buffer_queue_.front();
+    pending_done_cb_ = front.done_cb_;
+    DemuxerStream::Status status = DemuxerStream::kOk;
+    if (!front.size_.IsEmpty()) {
+      // No VideoFrame actually reaches cc in Google TV case. We just make
+      // coded_size == visible_rect == natural_size here.
+      video_decoder_config_.Initialize(media::kCodecVP8,
+                                       media::VP8PROFILE_MAIN,
+                                       media::VideoFrame::NATIVE_TEXTURE,
+                                       front.size_,
+                                       gfx::Rect(front.size_),
+                                       front.size_,
+                                       NULL, 0, false, false);
+      status = DemuxerStream::kConfigChanged;
+    }
+    base::ResetAndReturn(&read_cb_).Run(status, front.decoder_buffer_);
+    buffer_queue_.pop();
+  }
+}
+
+// RTCDemuxerProxy -------------------------------------------------------------
+
+class RTCDemuxerProxy : public base::RefCountedThreadSafe<RTCDemuxerProxy> {
+ public:
+  RTCDemuxerProxy(const scoped_refptr<base::MessageLoopProxy>& message_loop);
+
+  void Initialize(media::DemuxerHost* host, const media::PipelineStatusCB& cb);
+  media::DemuxerStream* GetStream(media::DemuxerStream::Type type);
+  void UpdateSize(const gfx::Size& size);
+  void QueueBuffer(scoped_refptr<media::DecoderBuffer> buffer,
+                   const base::Closure& done_cb,
+                   const gfx::Size& size);
+
+ protected:
+  friend class base::RefCountedThreadSafe<RTCDemuxerProxy>;
+
+  virtual ~RTCDemuxerProxy();
+
+ private:
+  scoped_refptr<base::MessageLoopProxy> loop_proxy_;
+  scoped_ptr<RTCDemuxerStream> stream_;
+  media::DemuxerHost* host_;
+  media::PipelineStatusCB init_cb_;
+};
+
+RTCDemuxerProxy::RTCDemuxerProxy(
+    const scoped_refptr<base::MessageLoopProxy>& message_loop)
+  : loop_proxy_(message_loop),
+    host_(NULL) {
+}
+
+RTCDemuxerProxy::~RTCDemuxerProxy() {}
+
+void RTCDemuxerProxy::Initialize(media::DemuxerHost* host,
+                                 const media::PipelineStatusCB& cb) {
+  host_ = host;
+  init_cb_ = cb;
+}
+
+media::DemuxerStream* RTCDemuxerProxy::GetStream(
+    media::DemuxerStream::Type type) {
+  if (type == media::DemuxerStream::VIDEO)
+    return stream_.get();
+  return NULL;
+}
+
+void RTCDemuxerProxy::UpdateSize(const gfx::Size& size) {
+  if (!loop_proxy_->BelongsToCurrentThread()) {
+    loop_proxy_->PostTask(FROM_HERE, base::Bind(
+        &RTCDemuxerProxy::UpdateSize, this, size));
+    return;
+  }
+  DCHECK(!stream_);
+  stream_.reset(new RTCDemuxerStream(size));
+  if (!init_cb_.is_null())
+    base::ResetAndReturn(&init_cb_).Run(media::PIPELINE_OK);
+}
+
+void RTCDemuxerProxy::QueueBuffer(scoped_refptr<media::DecoderBuffer> buffer,
+                                  const base::Closure& done_cb,
+                                  const gfx::Size& size) {
+  if (!loop_proxy_->BelongsToCurrentThread()) {
+    loop_proxy_->PostTask(FROM_HERE, base::Bind(
+        &RTCDemuxerProxy::QueueBuffer, this, buffer, done_cb, size));
+    return;
+  }
+  if (stream_)
+    stream_->QueueBuffer(buffer, done_cb, size);
+  else
+    done_cb.Run();
+}
+
+// RTCDemuxer ------------------------------------------------------------------
+
+class RTCDemuxer : public media::Demuxer {
+ public:
+  RTCDemuxer(const scoped_refptr<base::MessageLoopProxy>& message_loop);
+  virtual ~RTCDemuxer();
+
+  // media::Demuxer implementation.
+  virtual void Initialize(media::DemuxerHost* host,
+                          const media::PipelineStatusCB& cb) OVERRIDE;
+  virtual media::DemuxerStream* GetStream(
+      media::DemuxerStream::Type type) OVERRIDE;
+  virtual base::TimeDelta GetStartTime() const OVERRIDE;
+
+  void UpdateSize(const gfx::Size& size);
+  void QueueBuffer(scoped_refptr<media::DecoderBuffer> buffer,
+                   const base::Closure& done_cb,
+                   const gfx::Size& size);
+
+ private:
+  friend class RTCVideoDecoderBridgeTv;
+
+  scoped_refptr<RTCDemuxerProxy> proxy_;
+};
+
+RTCDemuxer::RTCDemuxer(
+    const scoped_refptr<base::MessageLoopProxy>& message_loop)
+  : proxy_(new RTCDemuxerProxy(message_loop)) {}
+
+RTCDemuxer::~RTCDemuxer() {}
+
+void RTCDemuxer::Initialize(media::DemuxerHost* host,
+                            const media::PipelineStatusCB& cb) {
+  proxy_->Initialize(host, cb);
+}
+
+media::DemuxerStream* RTCDemuxer::GetStream(media::DemuxerStream::Type type) {
+  return proxy_->GetStream(type);
+}
+
+base::TimeDelta RTCDemuxer::GetStartTime() const {
+  return base::TimeDelta();
+}
+
+void RTCDemuxer::UpdateSize(const gfx::Size& size) {
+  proxy_->UpdateSize(size);
+}
+
+void RTCDemuxer::QueueBuffer(scoped_refptr<media::DecoderBuffer> buffer,
+                             const base::Closure& done_cb,
+                             const gfx::Size& size) {
+  proxy_->QueueBuffer(buffer, done_cb, size);
+}
+
+// RTCVideoDecoderBridgeTvImpl -------------------------------------------------
+
+class RTCVideoDecoderBridgeTvImpl : public RTCVideoDecoderBridgeTv {
+ public:
+  static RTCVideoDecoderBridgeTvImpl* GetInstance() {
+    return Singleton<RTCVideoDecoderBridgeTvImpl>::get();
+  }
+  RTCVideoDecoderBridgeTvImpl();
+  virtual ~RTCVideoDecoderBridgeTvImpl();
+
+  // webrtc::VideoDecoder implementation.
+  virtual WebRtc_Word32 InitDecode(
+      const webrtc::VideoCodec* codecSettings,
+      WebRtc_Word32 numberOfCores) OVERRIDE;
+  virtual WebRtc_Word32 Decode(
+      const webrtc::EncodedImage& inputImage,
+      bool missingFrames,
+      const webrtc::RTPFragmentationHeader* /* fragmentation */,

[ycheo (away), 2013/05/06 05:45:34]

Why do you wrap this /* */?

[wonsik, 2013/05/06 12:28:43]

I tried to indicate that this argument is not used, but after second thought, it
seems useless. Unwrapped.

+      const webrtc::CodecSpecificInfo* codecSpecificInfo = NULL,
+      WebRtc_Word64 renderTimeMs = -1) OVERRIDE;
+  virtual WebRtc_Word32 RegisterDecodeCompleteCallback(
+      webrtc::DecodedImageCallback* callback) OVERRIDE;
+  virtual WebRtc_Word32 Release() OVERRIDE;
+  virtual WebRtc_Word32 Reset() OVERRIDE;
+
+  virtual media::Demuxer* CreateDemuxer(
+      const MediaStreamDependencyFactory* media_stream_dependency_factory,
+      const scoped_refptr<base::MessageLoopProxy>& message_loop) OVERRIDE;
+  virtual void DestroyDemuxer(const media::Demuxer* demuxer) OVERRIDE;
+  virtual bool AcquireOwnership(
+      const MediaStreamDependencyFactory* media_stream_dependency_factory)
+      OVERRIDE;
+  virtual void ReleaseOwnership(
+      const MediaStreamDependencyFactory* media_stream_dependency_factory)
+      OVERRIDE;
+
+  static void RunDecodeCompleteCallback(webrtc::DecodedImageCallback* callback,
+                                        WebRtc_Word64 timestamp);
+
+ private:
+  friend struct DefaultSingletonTraits<RTCVideoDecoderBridgeTv>;
+
+  static const base::TimeDelta kDecoderTimeOut;
+  enum Status {
+    kNoInit,
+    kInitialized,
+  };
+
+  // Lock protected, since these can be accessed in CreateDemuxer,
+  // DestroyDemuxer, AcquireOwnership, and ReleaseOwnership. No guarantee of
+  // calling thread for these methods.
+  base::Lock lock_;
+  const MediaStreamDependencyFactory* ownership_tag_;
+  const MediaStreamDependencyFactory* demuxer_tag_;
+  scoped_ptr<RTCDemuxer> demuxer_;
+  gfx::Size size_;
+  Status status_;
+
+  // Only used by decoder thread.
+  bool first_frame_;
+  webrtc::DecodedImageCallback* decode_complete_callback_;
+  WebRtc_Word64 timestamp_offset_;
+
+  DISALLOW_COPY_AND_ASSIGN(RTCVideoDecoderBridgeTvImpl);
+};
+
+RTCVideoDecoderBridgeTvImpl::RTCVideoDecoderBridgeTvImpl()
+    : ownership_tag_(NULL),
+      demuxer_tag_(NULL),
+      status_(kNoInit),
+      first_frame_(true),
+      decode_complete_callback_(NULL) {}
+
+RTCVideoDecoderBridgeTvImpl::~RTCVideoDecoderBridgeTvImpl() {}
+
+media::Demuxer* RTCVideoDecoderBridgeTvImpl::CreateDemuxer(
+    const MediaStreamDependencyFactory* media_stream_dependency_factory,
+    const scoped_refptr<base::MessageLoopProxy>& message_loop) {
+  base::AutoLock lock(lock_);
+  if (demuxer_tag_ != NULL ||
+      (ownership_tag_ != NULL &&
+       ownership_tag_ != media_stream_dependency_factory))
+    return NULL;
+  DCHECK(!demuxer_);
+  demuxer_tag_ = media_stream_dependency_factory;
+  demuxer_.reset(new RTCDemuxer(message_loop));
+  // If this is initialized before demuxer is set, update the demuxer with size
+  // information.
+  if (status_ == kInitialized)
+    demuxer_->UpdateSize(size_);
+  return demuxer_.get();
+}
+
+void RTCVideoDecoderBridgeTvImpl::DestroyDemuxer(
+    const media::Demuxer* demuxer) {
+  base::AutoLock lock(lock_);
+  DCHECK(demuxer_.get() == demuxer);
+  demuxer_.reset();
+  demuxer_tag_ = NULL;
+}
+
+bool RTCVideoDecoderBridgeTvImpl::AcquireOwnership(
+    const MediaStreamDependencyFactory* media_stream_dependency_factory) {
+  base::AutoLock lock(lock_);
+  if (ownership_tag_ != NULL ||
+      (demuxer_tag_ != NULL && demuxer_tag_ != media_stream_dependency_factory))
+    return false;
+
+  ownership_tag_ = media_stream_dependency_factory;
+  return true;
+}
+
+void RTCVideoDecoderBridgeTvImpl::ReleaseOwnership(
+    const MediaStreamDependencyFactory* media_stream_dependency_factory) {
+  base::AutoLock lock(lock_);
+  DCHECK(ownership_tag_ == media_stream_dependency_factory);
+  ownership_tag_ = NULL;
+}
+
+WebRtc_Word32 RTCVideoDecoderBridgeTvImpl::InitDecode(
+    const webrtc::VideoCodec* codecSettings,
+    WebRtc_Word32 numberOfCores) {
+  if (codecSettings->codecType != webrtc::kVideoCodecVP8)
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  // We don't support feedback mode.
+  if (codecSettings->codecSpecific.VP8.feedbackModeOn)
+    return WEBRTC_VIDEO_CODEC_ERROR;
+
+  base::AutoLock lock(lock_);
+  if (status_ != kNoInit)
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  size_ = gfx::Size(codecSettings->width, codecSettings->height);
+  status_ = kInitialized;
+  first_frame_ = true;
+  if (demuxer_)
+    demuxer_->UpdateSize(size_);
+
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+WebRtc_Word32 RTCVideoDecoderBridgeTvImpl::Decode(
+    const webrtc::EncodedImage& inputImage,
+    bool missingFrames,
+    const webrtc::RTPFragmentationHeader* /* fragmentation */,
+    const webrtc::CodecSpecificInfo* /* codecSpecificInfo */,
+    WebRtc_Word64 renderTimeMs) {
+  // Unlike the SW decoder in libvpx, hw decoder can not handle broken frames.
+  // Here, we return an error in order to request a key frame.
+  if (missingFrames || !inputImage._completeFrame)
+    return WEBRTC_VIDEO_CODEC_ERROR;
+
+  base::AutoLock lock(lock_);
+  if (status_ == kNoInit || decode_complete_callback_ == NULL)
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
+  if (status_ != kInitialized)
+    return WEBRTC_VIDEO_CODEC_ERROR;
+  // Drop frames until demuxer is up and running. We'll request key frame
+  // once demuxer is ready.
+  if (!demuxer_)
+    return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
+
+  if (first_frame_) {
+    // If the first frame is not the key frame, return an error to request a key
+    // frame.
+    if (inputImage._frameType != webrtc::kKeyFrame)
+      return WEBRTC_VIDEO_CODEC_ERROR;
+
+    // Google TV expects timestamp from 0.
+    timestamp_offset_ = renderTimeMs;
+  }
+  first_frame_ = false;
+  gfx::Size new_size;
+  if (inputImage._frameType == webrtc::kKeyFrame &&
+      inputImage._encodedWidth != 0 && inputImage._encodedHeight != 0) {
+    // Only key frame has the size.
+    new_size.SetSize(inputImage._encodedWidth, inputImage._encodedHeight);
+    size_ = new_size;
+  }
+  scoped_refptr<media::DecoderBuffer> buffer;
+  buffer = media::DecoderBuffer::CopyFrom(
+      inputImage._buffer, inputImage._length);
+  if (renderTimeMs != -1) {
+    buffer->SetTimestamp(
+        base::TimeDelta::FromMilliseconds(renderTimeMs - timestamp_offset_));
+  }
+
+  demuxer_->QueueBuffer(
+      buffer,
+      base::Bind(&RTCVideoDecoderBridgeTvImpl::RunDecodeCompleteCallback,
+                 decode_complete_callback_,
+                 renderTimeMs),
+      new_size);
+
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+WebRtc_Word32 RTCVideoDecoderBridgeTvImpl::RegisterDecodeCompleteCallback(
+    webrtc::DecodedImageCallback* callback) {
+  decode_complete_callback_ = callback;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+WebRtc_Word32 RTCVideoDecoderBridgeTvImpl::Release() {
+  base::AutoLock lock(lock_);
+  status_ = kNoInit;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+WebRtc_Word32 RTCVideoDecoderBridgeTvImpl::Reset() {
+  first_frame_ = true;
+  return WEBRTC_VIDEO_CODEC_OK;
+}
+
+// static
+void RTCVideoDecoderBridgeTvImpl::RunDecodeCompleteCallback(
+    webrtc::DecodedImageCallback* callback, WebRtc_Word64 timestamp) {
+  webrtc::I420VideoFrame dummy_video_frame;
+  dummy_video_frame.CreateEmptyFrame(2, 1, 2, 1, 1);
+  dummy_video_frame.set_timestamp(timestamp);
+  callback->Decoded(dummy_video_frame);

[dwkang1, 2013/05/06 09:43:38]

Could you add some comments about the background of this?

[wonsik, 2013/05/06 12:28:43]

Done.

+}
+
+}  // anonymous namespace
+
+// RTCVideoDecoderBridgeTv -----------------------------------------------------
+
+// static
+RTCVideoDecoderBridgeTv* RTCVideoDecoderBridgeTv::Get() {
+  return RTCVideoDecoderBridgeTvImpl::GetInstance();
+}
+
+// static
+void RTCVideoDecoderBridgeTv::RunDestroyDemuxer(const media::Demuxer* demuxer) {
+  Get()->DestroyDemuxer(demuxer);
+}
+
+}  // namespace content