MediaStream: Adding VideoTrackRecorder class and unittests

content/renderer/media/video_track_recorder.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/renderer/media/video_track_recorder.h"
+
+#include "base/bind.h"
+#include "base/logging.h"
+#include "base/threading/thread.h"
+#include "base/time/time.h"
+#include "base/trace_event/trace_event.h"
+#include "content/child/child_process.h"
+#include "media/base/bind_to_current_loop.h"
+#include "media/base/video_frame.h"
+
+extern "C" {
+// VPX_CODEC_DISABLE_COMPAT excludes parts of the libvpx API that provide
+// backwards compatibility for legacy applications using the library.
+#define VPX_CODEC_DISABLE_COMPAT 1
+#include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
+#include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h"
+}
+
+using media::VideoFrame;
+using media::VideoFrameMetadata;
+
+namespace content {
+
+namespace {
+
+const vpx_codec_flags_t kNoFlags = 0;
+
+static double GetFrameRate(const scoped_refptr<VideoFrame>& video_frame) {
+  double frame_rate = 0.0f;
+  base::IgnoreResult(video_frame->metadata()->GetDouble(
+      VideoFrameMetadata::FRAME_RATE, &frame_rate));
+  return frame_rate;
+}
+
+}  // anonymous namespace
+
+// Inner class holding all libvpx configuration and encoding received frames.
+// This class is created and destroyed on its parent's thread: the main render
+// thread. It receives VideoFrames and must Run() the callbacks on IO thread,
+// and finally uses an internal |encoding_thread_| for libvpx interactions,
+// notably for encoding (which might take some time). All this forces a bit of
+// hopping back and forth between threads.
+// Only VP8 is supported for the time being.
+// TODO(mcasas): Consider not getting |io_task_runner| in ctor and instead
+// caching it on first StartFrameEncode().
+class VideoTrackRecorder::VpxEncoder
+    : public base::RefCountedThreadSafe<VpxEncoder> {
+ public:
+  VpxEncoder(const scoped_refptr<base::SingleThreadTaskRunner>& io_task_runner,
+             const OnFirstFrameCB& on_first_frame_callback,
+             const OnEncodedVideoCB& on_encoded_video_callback);
+
+  void StartFrameEncode(const scoped_refptr<VideoFrame>& frame,
+                        const base::TimeTicks& capture_timestamp);
+
+ private:
+  friend class base::RefCountedThreadSafe<VpxEncoder>;
+  virtual ~VpxEncoder();
+
+  void EncodeOnEncodingThread(const scoped_refptr<VideoFrame>& frame,
+                              const base::TimeTicks& capture_timestamp);
+
+  void OnFrameEncodeCompleted(scoped_ptr<std::string> data,
+                          base::TimeDelta timestamp,
+                          bool keyframe);
+
+  void ConfigureVp8Encoding(const gfx::Size& size);
+  // Returns true if |codec_config_| has been filled in at least once.
+  bool IsInitialized() const;
+
+  base::TimeDelta CalculateFrameDuration(
+      const scoped_refptr<VideoFrame>& frame);
+
+  // Used to check that we are destroyed on the same thread we were created.
+  base::ThreadChecker main_render_thread_checker_;
+
+  // Render IO thread task runner for posting tasks and DCHECK()s.
+  const scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_;
+
+  // Used to ping |on_first_frame_callback_|. Used on IO thread only.
+  bool first_frame_received_;
+  // Written once on IO thread and used (read) from capture thread.
+  uint64_t track_index_;
+  // Callbacks that should be exercised on IO thread.
+  const OnFirstFrameCB on_first_frame_callback_;
+  const OnEncodedVideoCB on_encoded_video_callback_;
+
+  // Thread for encoding. Active as long as VpxEncoder exists. All variables
+  // below this are used in this thread.
+  base::Thread encoding_thread_;
+  // VP8 internal objects: configuration, encoder and Vpx Image wrapper.
+  vpx_codec_enc_cfg_t codec_config_;
+  vpx_codec_ctx_t encoder_;
+  // Origin of times for frame timestamps.
+  base::TimeTicks first_frame_timestamp_;
+  // The |VideoFrame::timestamp()| of the last encoded frame.  This is used to
+  // predict the duration of the next frame.
+  base::TimeDelta last_frame_timestamp_;
+};
+
+VideoTrackRecorder::VpxEncoder::VpxEncoder(
+    const scoped_refptr<base::SingleThreadTaskRunner>& io_task_runner,
+    const OnFirstFrameCB& on_first_frame_callback,
+    const OnEncodedVideoCB& on_encoded_video_callback)
+    : io_task_runner_(io_task_runner),
+      first_frame_received_(false),
+      track_index_(0),
+      on_first_frame_callback_(on_first_frame_callback),
+      on_encoded_video_callback_(on_encoded_video_callback),
+      encoding_thread_("EncodingThread") {
+  DCHECK(!on_first_frame_callback_.is_null());
+  DCHECK(!on_encoded_video_callback_.is_null());
+
+  codec_config_.g_timebase.den = 0;  // Not initialized.
+
+  DCHECK(!encoding_thread_.IsRunning());
+  encoding_thread_.Start();
+}
+
+void VideoTrackRecorder::VpxEncoder::StartFrameEncode(
+    const scoped_refptr<VideoFrame>& frame,
+    const base::TimeTicks& capture_timestamp) {
+  DCHECK(io_task_runner_->BelongsToCurrentThread());
+
+  if (!first_frame_received_) {
+    track_index_ = on_first_frame_callback_.Run(frame->visible_rect().size(),
+                                                GetFrameRate(frame));
+    first_frame_timestamp_ = capture_timestamp;
+    first_frame_received_ = true;
+  }
+  encoding_thread_.task_runner()->PostTask(FROM_HERE,
+      base::Bind(&VpxEncoder::EncodeOnEncodingThread,
+                 this, frame, capture_timestamp));
+}
+
+VideoTrackRecorder::VpxEncoder::~VpxEncoder() {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  DCHECK(encoding_thread_.IsRunning());
+  encoding_thread_.Stop();
+}
+
+void VideoTrackRecorder::VpxEncoder::EncodeOnEncodingThread(
+    const scoped_refptr<VideoFrame>& frame,
+    const base::TimeTicks& capture_timestamp) {
+  TRACE_EVENT0("video",
+               "VideoTrackRecorder::VpxEncoder::EncodeOnEncodingThread");
+  DCHECK(encoding_thread_.task_runner()->BelongsToCurrentThread());
+
+  const gfx::Size frame_size = frame->visible_rect().size();
+  if (!IsInitialized() ||
+      gfx::Size(codec_config_.g_w, codec_config_.g_h) != frame_size) {
+    ConfigureVp8Encoding(frame_size);
+  }
+
+  vpx_image_t vpx_image;
+  vpx_image_t* const result = vpx_img_wrap(&vpx_image,
+                                           VPX_IMG_FMT_I420,
+                                           frame_size.width(),
+                                           frame_size.height(),
+                                           1  /* align */,
+                                           frame->data(VideoFrame::kYPlane));
+  DCHECK_EQ(result, &vpx_image);
+  vpx_image.planes[VPX_PLANE_Y] = frame->visible_data(VideoFrame::kYPlane);
+  vpx_image.planes[VPX_PLANE_U] = frame->visible_data(VideoFrame::kUPlane);
+  vpx_image.planes[VPX_PLANE_V] = frame->visible_data(VideoFrame::kVPlane);
+  vpx_image.stride[VPX_PLANE_Y] = frame->stride(VideoFrame::kYPlane);
+  vpx_image.stride[VPX_PLANE_U] = frame->stride(VideoFrame::kUPlane);
+  vpx_image.stride[VPX_PLANE_V] = frame->stride(VideoFrame::kVPlane);
+
+  const base::TimeDelta duration = CalculateFrameDuration(frame);
+  DVLOG(1) << duration.InMicroseconds();
+  // Encode the frame.  The presentation time stamp argument here is fixed to
+  // zero to force the encoder to base its single-frame bandwidth calculations
+  // entirely on |predicted_frame_duration|.
+  const vpx_codec_err_t ret = vpx_codec_encode(&encoder_,
+                                               &vpx_image,
+                                               0  /* pts */,
+                                               duration.InMicroseconds(),
+                                               kNoFlags,
+                                               VPX_DL_REALTIME);
+  DCHECK_EQ(ret, VPX_CODEC_OK) << vpx_codec_err_to_string(ret) << ", #"
+                               << vpx_codec_error(&encoder_) << " -"
+                               << vpx_codec_error_detail(&encoder_);
+
+  scoped_ptr<std::string> data(new std::string);
+  bool keyframe = false;
+  vpx_codec_iter_t iter = NULL;
+  const vpx_codec_cx_pkt_t* pkt = NULL;
+  while ((pkt = vpx_codec_get_cx_data(&encoder_, &iter)) != NULL) {
+    if (pkt->kind != VPX_CODEC_CX_FRAME_PKT)
+      continue;
+    data->assign(static_cast<char*>(pkt->data.frame.buf), pkt->data.frame.sz);
+    keyframe = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0;
+    break;
+  }
+  const int timestamp =
+      (capture_timestamp - first_frame_timestamp_).InMilliseconds();
+  io_task_runner_->PostTask(FROM_HERE,
+      base::Bind(&VpxEncoder::OnFrameEncodeCompleted,
+                 this,
+                 base::Passed(&data),
+                 base::TimeDelta::FromMilliseconds(timestamp),
+                 keyframe ));
+}
+
+void VideoTrackRecorder::VpxEncoder::OnFrameEncodeCompleted(
+    scoped_ptr<std::string> data,
+    base::TimeDelta timestamp,
+    bool keyframe) {
+  DVLOG(1) << (keyframe ? "" : "non ") << "keyframe "
+           << timestamp.InMilliseconds() << " ms - " << data->length() << "B ";
+  DCHECK(io_task_runner_->BelongsToCurrentThread());
+  on_encoded_video_callback_.Run(track_index_, base::StringPiece(*data),
+                                 timestamp, keyframe);
+}
+
+void VideoTrackRecorder::VpxEncoder::ConfigureVp8Encoding(
+    const gfx::Size& size) {
+  if (IsInitialized()) {
+    // TODO(mcasas): Workaround for certain bug.
+    DVLOG(1) << "Destroying/Re-Creating encoder for larger frame size: "
+             << gfx::Size(codec_config_.g_w, codec_config_.g_h).ToString()
+             << " --> " << size.ToString();
+    //vpx_codec_destroy(&encoder_);
+  }
+  const vpx_codec_iface_t* interface = vpx_codec_vp8_cx();
+  vpx_codec_enc_config_default(interface, &codec_config_, 0 /* reserved */);
+
+  // Adjust default bit rate to account for the actual size.
+  codec_config_.rc_target_bitrate = size.width() * size.height() *
+                                    codec_config_.rc_target_bitrate /
+                                    codec_config_.g_w / codec_config_.g_h;
+  DCHECK(size.width());
+  DCHECK(size.height());
+  codec_config_.g_w = size.width();
+  codec_config_.g_h = size.height();
+  codec_config_.g_pass = VPX_RC_ONE_PASS;
+
+  // Timebase is the smallest interval used by the stream, can be set to the
+  // frame rate or just to milliseconds.
+  codec_config_.g_timebase.num = 1;
+  codec_config_.g_timebase.den = base::Time::kMillisecondsPerSecond;
+
+  // Let the encoder decide where to place the Keyframes, between min and max.
+  // In VPX_KF_AUTO mode libvpx will sometimes emit keyframes regardless of min/
+  // max distance out of necessity. Due to http://crbug.com/440223, decoding
+  // fails after 30,000 non-key frames, so force an "unnecessary" key-frame
+  // every 10,000 frames.
+  codec_config_.kf_mode = VPX_KF_AUTO;
+  codec_config_.kf_min_dist = 10000;
+  codec_config_.kf_max_dist = 10000;
+
+  // Number of frames to consume before producing output.
+  codec_config_.g_lag_in_frames = 0;
+
+  const vpx_codec_err_t ret = vpx_codec_enc_init(&encoder_, interface,
+                                                 &codec_config_, kNoFlags);
+  DCHECK_EQ(VPX_CODEC_OK, ret);
+}
+
+bool VideoTrackRecorder::VpxEncoder::IsInitialized() const {
+  DCHECK(encoding_thread_.task_runner()->BelongsToCurrentThread());
+  return codec_config_.g_timebase.den != 0;
+}
+
+base::TimeDelta VideoTrackRecorder::VpxEncoder::CalculateFrameDuration(
+    const scoped_refptr<VideoFrame>& frame) {
+  DCHECK(encoding_thread_.task_runner()->BelongsToCurrentThread());
+
+  base::TimeDelta predicted_frame_duration;
+  if (!frame->metadata()->GetTimeDelta(VideoFrameMetadata::FRAME_DURATION,
+                                       &predicted_frame_duration) ||
+      predicted_frame_duration <= base::TimeDelta()) {
+    // The source of the video frame did not provide the frame duration.  Use
+    // the actual amount of time between the current and previous frame as a
+    // prediction for the next frame's duration.

[Niklas Enbom, 2015/07/30 19:46:47]

Drive by shooting. This (using last frames duration as estimate of current frame
duration) could create some weird artifacts if we have a camera freeze, pause,
or large packet loss for remote streams. Add a TODO to look into that later?

[mcasas, 2015/07/31 11:26:27]

Done.

+    predicted_frame_duration = frame->timestamp() - last_frame_timestamp_;
+  }
+  last_frame_timestamp_ = frame->timestamp();
+  const base::TimeDelta kMinFrameDuration =
+      base::TimeDelta::FromMilliseconds(1);
+  return std::max(predicted_frame_duration, kMinFrameDuration);
+}
+
+VideoTrackRecorder::VideoTrackRecorder(
+    const scoped_refptr<base::SingleThreadTaskRunner>& io_task_runner,
+    const blink::WebMediaStreamTrack& track,
+    const OnFirstFrameCB& on_first_frame_cb,
+    const OnEncodedVideoCB& on_encoded_video_cb)
+    : track_(track),
+      encoder_(new VpxEncoder(io_task_runner,
+                              on_first_frame_cb,
+                              on_encoded_video_cb)) {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  DCHECK(track.extraData());
+  AddToVideoTrack(this,
+      base::Bind(&VpxEncoder::StartFrameEncode, encoder_), track_);
+}
+
+VideoTrackRecorder::~VideoTrackRecorder() {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  RemoveFromVideoTrack(this, track_);
+}
+
+void VideoTrackRecorder::StartFrameEncodeForTesting(
+    const scoped_refptr<VideoFrame>& frame,
+    const base::TimeTicks& capture_timestamp) {
+  encoder_->StartFrameEncode(frame, capture_timestamp);
+}
+
+}  // namespace content