Integrate VDA with WebRTC.

content/renderer/media/rtc_video_decoder.h

Index: content/renderer/media/rtc_video_decoder.h
diff --git a/content/renderer/media/rtc_video_decoder.h b/content/renderer/media/rtc_video_decoder.h
new file mode 100644
index 0000000000000000000000000000000000000000..6e1b47e53a7a6e6dd8abf43d382485dcb626ea69
--- /dev/null
+++ b/content/renderer/media/rtc_video_decoder.h
@@ -0,0 +1,237 @@
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CONTENT_RENDERER_MEDIA_RTC_VIDEO_DECODER_H_
+#define CONTENT_RENDERER_MEDIA_RTC_VIDEO_DECODER_H_
+
+#include <deque>
+#include <set>
+#include <utility>
+
+#include "base/basictypes.h"
+#include "base/memory/weak_ptr.h"
+#include "base/message_loop.h"
+#include "base/synchronization/lock.h"
+#include "base/synchronization/waitable_event.h"
+#include "base/threading/thread.h"
+#include "content/common/content_export.h"
+#include "media/base/bitstream_buffer.h"
+#include "media/base/video_decoder.h"
+#include "media/filters/gpu_video_decoder.h"
+#include "media/video/picture.h"
+#include "media/video/video_decode_accelerator.h"
+#include "third_party/webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
+
+namespace base {
+class MessageLoopProxy;
+};
+
+namespace media {
+class DecoderBuffer;
+}
+
+namespace content {
+
+// This class uses hardware accelerated video decoder to decode video for
+// WebRTC. The message loop of RendererGpuVideoDecoderFactories is stored as
+// |vda_message_loop_|. It is the compositor thread, or the renderer thread if
+// threaded compositing is disabled. VDA::Client methods run on
+// |vda_message_loop_|. webrtc::VideoDecoder methods run on WebRTC
+// DecodingThread or Chrome_libJingle_WorkerThread, which are trampolined to
+// |vda_message_loop_|. Decode() is non-blocking and queues the buffers. Decoded
+// frames are delivered on |vda_message_loop_|.
+class CONTENT_EXPORT RTCVideoDecoder
+    : NON_EXPORTED_BASE(public webrtc::VideoDecoder),
+      public media::VideoDecodeAccelerator::Client,
+      public base::MessageLoop::DestructionObserver {
+ public:
+  virtual ~RTCVideoDecoder();
+
+  // Creates a RTCVideoDecoder. Returns NULL if failed.
+  static scoped_ptr<RTCVideoDecoder> Create(
+      const scoped_refptr<media::GpuVideoDecoder::Factories>& factories);
+
+  // webrtc::VideoDecoder implementation.
+  // Called on WebRTC DecodingThread.
+  virtual int32_t InitDecode(const webrtc::VideoCodec* codecSettings,
+                             int32_t numberOfCores) OVERRIDE;
+  // Called on WebRTC DecodingThread.
+  virtual int32_t Decode(
+      const webrtc::EncodedImage& inputImage,
+      bool missingFrames,
+      const webrtc::RTPFragmentationHeader* fragmentation,
+      const webrtc::CodecSpecificInfo* codecSpecificInfo = NULL,
+      int64_t renderTimeMs = -1) OVERRIDE;
+  // Called on WebRTC DecodingThread.
+  virtual int32_t RegisterDecodeCompleteCallback(
+      webrtc::DecodedImageCallback* callback) OVERRIDE;
+  // Called on Chrome_libJingle_WorkerThread. The child thread is blocked while
+  // this runs.
+  virtual int32_t Release() OVERRIDE;
+  // Called on Chrome_libJingle_WorkerThread. The child thread is blocked while
+  // this runs.
+  virtual int32_t Reset() OVERRIDE;
+
+  // VideoDecodeAccelerator::Client implementation.
+  virtual void NotifyInitializeDone() OVERRIDE;
+  virtual void ProvidePictureBuffers(uint32 count,
+                                     const gfx::Size& size,
+                                     uint32 texture_target) OVERRIDE;
+  virtual void DismissPictureBuffer(int32 id) OVERRIDE;
+  virtual void PictureReady(const media::Picture& picture) OVERRIDE;
+  virtual void NotifyEndOfBitstreamBuffer(int32 id) OVERRIDE;
+  virtual void NotifyFlushDone() OVERRIDE;
+  virtual void NotifyResetDone() OVERRIDE;
+  virtual void NotifyError(media::VideoDecodeAccelerator::Error error) OVERRIDE;
+
+  // base::DestructionObserver implementation. Called when |vda_message_loop_|
+  // is stopped.
+  virtual void WillDestroyCurrentMessageLoop();
+
+ private:
+  struct SHMBuffer;
+  struct BufferData;
+
+  RTCVideoDecoder(
+      const scoped_refptr<media::GpuVideoDecoder::Factories>& factories);
+
+  void Initialize(base::WaitableEvent* waiter);
+
+  // Requests a buffer to be decoded by VDA.
+  void RequestBufferDecode();
+
+  bool CanMoreDecodeWorkBeDone();
+
+  // Sends the WebRTC buffer for decode on |vda_loop_proxy_|.
+  void SendBufferForDecode(const webrtc::EncodedImage& inputImage,

[Ami GONE FROM CHROMIUM, 2013/06/26 00:11:58]

s/inputImage/input_image/ since this isn't an override for a webrtc method.

[wuchengli, 2013/06/28 15:08:44]

Done.

+                           SHMBuffer* shm_buffer,

[Ami GONE FROM CHROMIUM, 2013/06/26 00:11:58]

indent is off here and elsewhere

[Ami GONE FROM CHROMIUM, 2013/06/26 00:11:58]

style: output params follow input params
But this isn't really an output param; you're passing shm_buffer so it can be
used by SBFD but you don't care about the return value.  Maybe I'm missing
something but instead of
SendBufferFD
SendPendingBuffersFD
SaveToPendingBuffers
why not have:
AppendBufferToDecodeQueue
DrainDecodeQueue
where the former unconditionally adds the input_image to a queue, and the latter
drains as many images as it has SHMs for?  I think that will simplify both the
calling code and the impl.
In particular because running out of SHMs because of in-flight decodes is one
thing, but having SHMs that are too small is another.  (the former should just
wait till a SHM can be recycled, the latter should create new SHMs)

[wuchengli, 2013/06/28 15:08:44]

Indent looks correct. I've run git cl format. Where are you referring to?

[wuchengli, 2013/06/28 15:08:44]

They are all output params. It's a pointer because I need to save it to stl
deque. It's not const because make_pair requires it to be non-const.

I didn't use AppendBufferToDecodeQueue and DrainDecodeQueue because
SaveToPendingBuffers needs an extra memcpy. We own the buffers of
|webrtc_buffers_|. But we don't own |inputImage._buffer| and we cannot swap the
buffer. So I don't want to put them in the same queue.

On 2013/06/26 00:11:58, Ami Fischman wrote:

+                           const BufferData& buffer_data);
+
+  // Sends all WebRTC buffers for decode on |vda_loop_proxy_|.
+  void SendPendingBuffersForDecode();
+
+  // Saves a WebRTC buffer for decode later.
+  void SaveToPendingBuffers(const webrtc::EncodedImage& inputImage,
+                            const BufferData& buffer_data);
+
+  scoped_refptr<media::VideoFrame> CreateVideoFrame(
+      const media::Picture& picture,
+      const media::PictureBuffer& pb,
+      uint32_t timestamp,
+      uint32_t width,
+      uint32_t height,
+      size_t size);
+
+  // Resets VDA.
+  void ResetInternal();
+
+  // Tells VDA that a picture buffer can be recycled.
+  void ReusePictureBuffer(int64 picture_buffer_id);
+
+  void DestroyTextures();
+  void DestroyVDA();
+
+  // Requests a shared-memory segment of at least |min_size| bytes. Caller does
+  // not own returned pointer.

[Ami GONE FROM CHROMIUM, 2013/06/26 00:11:58]

Doco meaning of NULL return

[wuchengli, 2013/06/28 15:08:44]

Done.

+  SHMBuffer* GetSHM(size_t min_size);
+
+  // Allocates shared memory of at least |min_size| bytes.
+  void CreateSHM(size_t min_size);
+
+  // Returns a shared-memory segment to the available pool.
+  void PutSHM(SHMBuffer* shm_buffer);
+
+  // Stores the buffer metadata to |input_buffer_data_|.
+  void RecordBufferData(const BufferData& buffer_data);
+  // Gets the buffer metadata from |input_buffer_data_|.
+  void GetBufferData(int32 bitstream_buffer_id,
+                     uint32_t* timestamp,
+                     uint32_t* width,
+                     uint32_t* height,
+                     size_t* size);
+
+  enum State {
+    UNINITIALIZED,  // The decoder has not initialized.
+    INITIALIZED,    // The decoder has initialized.
+    RESETTING,      // The decoder is being reset.
+    DECODE_ERROR,   // Decoding error happened.
+  };
+
+  // The hardware video decoder.
+  scoped_ptr<media::VideoDecodeAccelerator> vda_;
+
+  // The size of the incoming video frames.
+  gfx::Size frame_size_;
+
+  // Protects |state_|, |decode_complete_callback_| , |available_shm_segments_|,
+  // |buffers_to_be_decoded_|, |next_bitstream_buffer_id_| and
+  // |reset_bitstream_buffer_id_|.
+  base::Lock lock_;
+
+  // The state of RTCVideoDecoder. Guarded by |lock_|.
+  State state_;
+
+  // Guarded by |lock_|.
+  webrtc::DecodedImageCallback* decode_complete_callback_;
+
+  // Shared-memory buffer pool.  Since allocating SHM segments requires a
+  // round-trip to the browser process, we keep allocation out of the
+  // steady-state of the decoder. Guarded by |lock_|.
+  std::vector<SHMBuffer*> available_shm_segments_;
+
+  // The weak pointer should live and die on the |vda_loop_proxy_|;
+  base::WeakPtrFactory<RTCVideoDecoder> weak_factory_;
+  base::WeakPtr<RTCVideoDecoder> weak_this_;
+
+  scoped_refptr<media::GpuVideoDecoder::Factories> factories_;
+
+  // The message loop to run callbacks on. This is should be the same as the one
+  // of |factories_|.
+  scoped_refptr<base::MessageLoopProxy> vda_loop_proxy_;
+
+  // The texture target used for decoded pictures.
+  uint32 decoder_texture_target_;
+
+  // A queue storing WebRTC encoding images (and their metadata) that are
+  // waiting for the allocation of shared memory. Only accessed by WebRTC
+  // DecodingThread.

[Ami GONE FROM CHROMIUM, 2013/06/26 00:11:58]

Generally vars not protected by locks go above the lock in class layout.

[wuchengli, 2013/06/28 15:08:44]

Done.

+  std::deque<std::pair<webrtc::EncodedImage, BufferData> > webrtc_buffers_;
+
+  // Metadata of the buffers that have been sent for decode.
+  std::list<BufferData> input_buffer_data_;
+
+  // A queue storing buffers (and their metadata) that will be sent to VDA for
+  // decode. Guarded by |lock_|.
+  std::deque<std::pair<SHMBuffer*, BufferData> > buffers_to_be_decoded_;
+
+  // A map from bitstream buffer IDs to bitstream buffers that are being
+  // processed by VDA.
+  std::map<int32, SHMBuffer*> bitstream_buffers_in_decoder_;
+
+  // A map from picture buffer IDs to texture-backed picture buffers.
+  std::map<int32, media::PictureBuffer> assigned_picture_buffers_;
+
+  // Picture buffers that are dismissed but not deleted yet.
+  std::map<int32, media::PictureBuffer> dismissed_picture_buffers_;
+
+  // PictureBuffers given to us by VDA via PictureReady, which we sent forward
+  // as VideoFrames to be rendered via read_cb_, and which will be returned
+  // to us via ReusePictureBuffer.
+  std::set<int32> picture_buffers_at_display_;
+
+  // The id that will be given to the next picture buffer.
+  int32 next_picture_buffer_id_;
+
+  // The id that will be given to the next bitstream buffer. Guarded by |lock_|.
+  int32 next_bitstream_buffer_id_;
+
+  // A buffer that has an id less than this should be dropped because Reset or
+  // Release has been called. Guarded by |lock_|.
+  int32 reset_bitstream_buffer_id_;
+
+  DISALLOW_COPY_AND_ASSIGN(RTCVideoDecoder);
+};
+
+}  // namespace content
+
+#endif  // CONTENT_RENDERER_MEDIA_RTC_VIDEO_DECODER_H_