MediaStream: Adding VideoTrackRecorder class and unittests

content/renderer/media/video_track_recorder.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/renderer/media/video_track_recorder.h"
+
+#include "base/bind.h"
+#include "base/logging.h"
+#include "base/sys_info.h"
+#include "base/threading/thread.h"
+#include "base/time/time.h"
+#include "base/trace_event/trace_event.h"
+#include "content/child/child_process.h"
+#include "media/base/bind_to_current_loop.h"
+#include "media/base/video_frame.h"
+
+extern "C" {
+// VPX_CODEC_DISABLE_COMPAT excludes parts of the libvpx API that provide
+// backwards compatibility for legacy applications using the library.
+#define VPX_CODEC_DISABLE_COMPAT 1
+#include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
+#include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h"
+}
+
+using media::VideoFrame;
+using media::VideoFrameMetadata;
+
+namespace content {
+
+namespace {
+const vpx_codec_flags_t kNoFlags = 0;
+}  // anonymous namespace
+
+// Inner class encapsulating all libvpx interactions and the encoding+delivery
+// of received frames. This class is:
+// - created and destroyed on its parent's thread (usually the main render
+// thread),
+// - receives VideoFrames and Run()s the callbacks on another thread (supposedly
+// the render IO thread), which is cached on first frame arrival,
+// - uses an internal |encoding_thread_| for libvpx interactions, notably for
+// encoding (which might take some time).
+// Only VP8 is supported for the time being.
+class VideoTrackRecorder::VpxEncoder
+    : public base::RefCountedThreadSafe<VpxEncoder> {
+ public:
+  explicit VpxEncoder(const OnEncodedVideoCB& on_encoded_video_callback);
+
+  void StartFrameEncode(const scoped_refptr<VideoFrame>& frame,
+                        const base::TimeTicks& capture_timestamp);
+
+ private:
+  friend class base::RefCountedThreadSafe<VpxEncoder>;
+  virtual ~VpxEncoder();
+
+  void EncodeOnEncodingThread(const scoped_refptr<VideoFrame>& frame,
+                              const base::TimeTicks& capture_timestamp);
+
+  void OnFrameEncodeCompleted(const scoped_refptr<VideoFrame>& frame,
+                              scoped_ptr<std::string> data,
+                              const base::TimeTicks& capture_timestamp,
+                              bool keyframe);
+
+  void ConfigureVp8Encoding(const gfx::Size& size);
+
+  // Returns true if |codec_config_| has been filled in at least once.
+  bool IsInitialized() const;
+
+  // Estimate the frame duration from |frame| and |last_frame_timestamp_|.
+  base::TimeDelta CalculateFrameDuration(
+      const scoped_refptr<VideoFrame>& frame);
+
+  // Used to check that we are destroyed on the same thread we were created.
+  base::ThreadChecker main_render_thread_checker_;
+
+  // Task runner where frames to encode and reply callbacks must happen.
+  scoped_refptr<base::SingleThreadTaskRunner> origin_task_runner_;
+
+  // This callback should be exercised on IO thread.
+  const OnEncodedVideoCB on_encoded_video_callback_;
+
+  // Thread for encoding. Active as long as VpxEncoder exists. All variables
+  // below this are used in this thread.
+  base::Thread encoding_thread_;
+  // VP8 internal objects: configuration, encoder and Vpx Image wrapper.
+  vpx_codec_enc_cfg_t codec_config_;
+  vpx_codec_ctx_t encoder_;
+
+  // The |VideoFrame::timestamp()| of the last encoded frame.  This is used to
+  // predict the duration of the next frame.
+  base::TimeDelta last_frame_timestamp_;
+};
+
+VideoTrackRecorder::VpxEncoder::VpxEncoder(
+    const OnEncodedVideoCB& on_encoded_video_callback)
+    : on_encoded_video_callback_(on_encoded_video_callback),
+      encoding_thread_("EncodingThread") {
+  DCHECK(!on_encoded_video_callback_.is_null());
+
+  codec_config_.g_timebase.den = 0;  // Not initialized.
+
+  DCHECK(!encoding_thread_.IsRunning());
+  encoding_thread_.Start();
+}
+
+void VideoTrackRecorder::VpxEncoder::StartFrameEncode(
+    const scoped_refptr<VideoFrame>& frame,
+    const base::TimeTicks& capture_timestamp) {
+  // Cache the thread sending frames on first frame arrival.
+  if (!origin_task_runner_.get())
+    origin_task_runner_ = base::MessageLoop::current()->task_runner();
+  DCHECK(origin_task_runner_->BelongsToCurrentThread());
+
+  encoding_thread_.task_runner()->PostTask(FROM_HERE,
+      base::Bind(&VpxEncoder::EncodeOnEncodingThread,
+                 this, frame, capture_timestamp));
+}
+
+VideoTrackRecorder::VpxEncoder::~VpxEncoder() {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  DCHECK(encoding_thread_.IsRunning());
+  encoding_thread_.Stop();
+  vpx_codec_destroy(&encoder_);
+}
+
+void VideoTrackRecorder::VpxEncoder::EncodeOnEncodingThread(
+    const scoped_refptr<VideoFrame>& frame,
+    const base::TimeTicks& capture_timestamp) {
+  TRACE_EVENT0("video",
+               "VideoTrackRecorder::VpxEncoder::EncodeOnEncodingThread");
+  DCHECK(encoding_thread_.task_runner()->BelongsToCurrentThread());
+
+  const gfx::Size frame_size = frame->visible_rect().size();
+  if (!IsInitialized() ||
+      gfx::Size(codec_config_.g_w, codec_config_.g_h) != frame_size) {
+    ConfigureVp8Encoding(frame_size);
+  }
+
+  vpx_image_t vpx_image;
+  vpx_image_t* const result = vpx_img_wrap(&vpx_image,
+                                           VPX_IMG_FMT_I420,
+                                           frame_size.width(),
+                                           frame_size.height(),
+                                           1  /* align */,
+                                           frame->data(VideoFrame::kYPlane));
+  DCHECK_EQ(result, &vpx_image);
+  vpx_image.planes[VPX_PLANE_Y] = frame->visible_data(VideoFrame::kYPlane);
+  vpx_image.planes[VPX_PLANE_U] = frame->visible_data(VideoFrame::kUPlane);
+  vpx_image.planes[VPX_PLANE_V] = frame->visible_data(VideoFrame::kVPlane);
+  vpx_image.stride[VPX_PLANE_Y] = frame->stride(VideoFrame::kYPlane);
+  vpx_image.stride[VPX_PLANE_U] = frame->stride(VideoFrame::kUPlane);
+  vpx_image.stride[VPX_PLANE_V] = frame->stride(VideoFrame::kVPlane);
+
+  const base::TimeDelta duration = CalculateFrameDuration(frame);
+  // Encode the frame.  The presentation time stamp argument here is fixed to
+  // zero to force the encoder to base its single-frame bandwidth calculations
+  // entirely on |predicted_frame_duration|.
+  const vpx_codec_err_t ret = vpx_codec_encode(&encoder_,
+                                               &vpx_image,
+                                               0  /* pts */,
+                                               duration.InMicroseconds(),
+                                               kNoFlags,
+                                               VPX_DL_REALTIME);
+  DCHECK_EQ(ret, VPX_CODEC_OK) << vpx_codec_err_to_string(ret) << ", #"
+                               << vpx_codec_error(&encoder_) << " -"
+                               << vpx_codec_error_detail(&encoder_);
+
+  scoped_ptr<std::string> data(new std::string);
+  bool keyframe = false;
+  vpx_codec_iter_t iter = NULL;
+  const vpx_codec_cx_pkt_t* pkt = NULL;
+  while ((pkt = vpx_codec_get_cx_data(&encoder_, &iter)) != NULL) {
+    if (pkt->kind != VPX_CODEC_CX_FRAME_PKT)
+      continue;
+    data->assign(static_cast<char*>(pkt->data.frame.buf), pkt->data.frame.sz);
+    keyframe = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0;
+    break;
+  }
+  origin_task_runner_->PostTask(FROM_HERE,
+      base::Bind(&VpxEncoder::OnFrameEncodeCompleted,
+                 this,

[miu, 2015/08/19 19:43:45]

You can get rid of ref-counting if OnFrameEncodeCompleted() is made a static
method.  All you need to do is also pass on_encoded_video_callback_ as an
additional arg.

[mcasas, 2015/08/20 00:37:04]

Done.

+                 frame,
+                 base::Passed(&data),
+                 capture_timestamp,
+                 keyframe));
+}
+
+void VideoTrackRecorder::VpxEncoder::OnFrameEncodeCompleted(
+    const scoped_refptr<VideoFrame>& frame,
+    scoped_ptr<std::string> data,
+    const base::TimeTicks& capture_timestamp,
+    bool keyframe) {
+  DVLOG(1) << (keyframe ? "" : "non ") << "keyframe "
+           << capture_timestamp << " ms - " << data->length() << "B ";
+  DCHECK(origin_task_runner_->BelongsToCurrentThread());
+  on_encoded_video_callback_.Run(frame,
+                                 base::StringPiece(*data),
+                                 capture_timestamp,
+                                 keyframe);
+}
+
+void VideoTrackRecorder::VpxEncoder::ConfigureVp8Encoding(
+    const gfx::Size& size) {
+  if (IsInitialized()) {
+    // VP8 quirk: If the new |size| is strictly less-than-or-equal than the old
+    // size, in terms of area, the existing encoder instance _could_ continue.

[miu, 2015/08/19 19:43:45]

I don't understand.  This comment is explaining something the code doesn't do
here.  Did you mean to grab more of the code from
media/cast/sender/vp8_encoder.cc?

Otherwise, this sounds more like a TODO comment.

[mcasas, 2015/08/20 00:37:04]

Done.

+    DVLOG(1) << "Destroying/Re-Creating encoder for new frame size: "
+             << gfx::Size(codec_config_.g_w, codec_config_.g_h).ToString()
+             << " --> " << size.ToString();
+    vpx_codec_destroy(&encoder_);
+  }
+
+  const vpx_codec_iface_t* interface = vpx_codec_vp8_cx();
+  const vpx_codec_err_t result = vpx_codec_enc_config_default(interface,
+                                                              &codec_config_,
+                                                              0 /* reserved */);
+  DCHECK_EQ(VPX_CODEC_OK, result);
+
+  // Adjust default bit rate to account for the actual size.
+  DCHECK_EQ(320u, codec_config_.g_w);
+  DCHECK_EQ(240u, codec_config_.g_h);
+  DCHECK_EQ(256u, codec_config_.rc_target_bitrate);
+  codec_config_.rc_target_bitrate = size.GetArea() *
+     codec_config_.rc_target_bitrate / codec_config_.g_w / codec_config_.g_h;
+
+  DCHECK(size.width());
+  DCHECK(size.height());
+  codec_config_.g_w = size.width();
+  codec_config_.g_h = size.height();
+  codec_config_.g_pass = VPX_RC_ONE_PASS;
+
+  // Timebase is the smallest interval used by the stream, can be set to the
+  // frame rate or to e.g. microseconds.
+  codec_config_.g_timebase.num = 1;
+  codec_config_.g_timebase.den = base::Time::kMicrosecondsPerSecond;
+
+  // Let the encoder decide where to place the Keyframes, between min and max.
+  // In VPX_KF_AUTO mode libvpx will sometimes emit keyframes regardless of min/
+  // max distance out of necessity.
+  // Note that due to http://crbug.com/440223, it might be necessary to force a
+  // key frame after 10,000frames since decoding fails after 30,000 non-key
+  // frames.
+  codec_config_.kf_mode = VPX_KF_AUTO;
+  codec_config_.kf_min_dist = 0;
+  codec_config_.kf_max_dist = 30000;
+
+  // Do not saturate CPU utilization just for encoding. On a lower-end system
+  // with only 1 or 2 cores, use only one thread for encoding. On systems with
+  // more cores, allow half of the cores to be used for encoding.
+  codec_config_.g_threads =
+      std::min(8, (base::SysInfo::NumberOfProcessors() + 1) / 2);
+
+  // Number of frames to consume before producing output.
+  codec_config_.g_lag_in_frames = 0;
+
+  const vpx_codec_err_t ret = vpx_codec_enc_init(&encoder_, interface,
+                                                 &codec_config_, kNoFlags);
+  DCHECK_EQ(VPX_CODEC_OK, ret);
+}
+
+bool VideoTrackRecorder::VpxEncoder::IsInitialized() const {
+  DCHECK(encoding_thread_.task_runner()->BelongsToCurrentThread());
+  return codec_config_.g_timebase.den != 0;
+}
+
+base::TimeDelta VideoTrackRecorder::VpxEncoder::CalculateFrameDuration(
+    const scoped_refptr<VideoFrame>& frame) {
+  DCHECK(encoding_thread_.task_runner()->BelongsToCurrentThread());
+
+  using base::TimeDelta;
+  TimeDelta predicted_frame_duration;
+  if (!frame->metadata()->GetTimeDelta(VideoFrameMetadata::FRAME_DURATION,
+                                       &predicted_frame_duration) ||
+      predicted_frame_duration <= TimeDelta()) {
+    // The source of the video frame did not provide the frame duration.  Use
+    // the actual amount of time between the current and previous frame as a
+    // prediction for the next frame's duration.
+    // TODO(mcasas): This duration estimation could lead to artifacts if the
+    // cadence of the received stream is compromised (e.g. camera freeze, pause,
+    // remote packet loss).  Investigate using GetFrameRate() in this case.
+    predicted_frame_duration = frame->timestamp() - last_frame_timestamp_;
+  }
+  last_frame_timestamp_ = frame->timestamp();
+  // Make sure |predicted_frame_duration| is in a safe range of values.
+  const TimeDelta kMaxFrameDuration = TimeDelta::FromSecondsD(1.0 / 8);
+  const TimeDelta kMinFrameDuration = TimeDelta::FromMilliseconds(1);
+  return std::min(kMaxFrameDuration, std::max(predicted_frame_duration,
+                                              kMinFrameDuration));
+}
+
+VideoTrackRecorder::VideoTrackRecorder(
+    const blink::WebMediaStreamTrack& track,
+    const OnEncodedVideoCB& on_encoded_video_callback)
+    : track_(track),
+      encoder_(new VpxEncoder(on_encoded_video_callback)),
+      weak_factory_(this) {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  DCHECK(track.extraData());
+  AddToVideoTrack(this,
+                  media::BindToCurrentLoop(
+                      base::Bind(&VideoTrackRecorder::OnVideoFrame,
+                                 weak_factory_.GetWeakPtr())),
+                  track_);
+}
+
+VideoTrackRecorder::~VideoTrackRecorder() {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  RemoveFromVideoTrack(this, track_);
+  weak_factory_.InvalidateWeakPtrs();
+  track_.reset();
+}
+
+void VideoTrackRecorder::OnVideoFrame(const scoped_refptr<VideoFrame>& frame,
+                                      const base::TimeTicks& timestamp) {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  if (!track_.isNull())

[miu, 2015/08/19 19:43:45]

This should be DCHECK(!track_.isNull()).  Or, better yet, put this DCHECK in the
ctor.

[mcasas, 2015/08/20 00:37:05]

Done.

+    encoder_->StartFrameEncode(frame, timestamp);
+}
+
+}  // namespace content