MediaStream: Adding VideoTrackRecorder class and unittests

content/renderer/media/video_track_recorder.cc

+// Copyright 2015 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/renderer/media/video_track_recorder.h"
+
+#include "base/bind.h"
+#include "base/logging.h"
+#include "base/time/time.h"
+#include "content/child/child_process.h"
+#include "media/base/bind_to_current_loop.h"
+#include "media/base/video_frame.h"
+
+extern "C" {
+#define VPX_CODEC_DISABLE_COMPAT 1
+#include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
+#include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h"
+}
+
+using media::VideoFrame;
+using media::VideoFrameMetadata;
+
+namespace content {
+
+namespace {
+
+// Originally from remoting/codec/scoped_vpx_codec.h
+struct VpxCodecDeleter {
+  void operator()(vpx_codec_ctx_t* codec) {
+    if (!codec)
+      return;
+    const vpx_codec_err_t ret = vpx_codec_destroy(codec);
+    CHECK_EQ(VPX_CODEC_OK, ret) << "Failed to destroy codec";

[Tom Finegan, 2015/07/23 22:18:39]

Won't this kill a tab? Killing a tab for a weird vpx failure seems like
overkill.

[mcasas, 2015/07/24 15:00:09]

Yes, changed.

Btw this comes from [1], eternal glory to the one that fills in
a bug about it :)

[1]
https://code.google.com/p/chromium/codesearch#chromium/src/remoting/codec/sco...

+    delete codec;
+  }
+};
+
+const vpx_codec_flags_t kNoFlags = 0;
+const uint32_t kDefaultFrameDurationInMilliseconds = 30u;
+
+static scoped_ptr<vpx_image_t> InitVp8Encoding(const gfx::Size& size,
+                                               vpx_codec_ctx_t* codec_context) {

[Tom Finegan, 2015/07/23 22:18:39]

A static method in an anon namespace seems unnecessary. One or the other should
suffice, no?

[mcasas, 2015/07/24 15:00:10]

It would but content rules "encourage" putting static functions
and ad hoc classes in an anonymous namespace _inside_
content namespace (see for instance video_track_adapter.cc
or media_stream_video_capturer_source.cc). Note that this is
different from media conventions, where everything is piled up
in media namespace, not in an anonymous namespace either
inside or outside of the former.

[Tom Finegan, 2015/07/24 20:56:16]

Ah, ok. I was just going on my admittedly foggy memories from the last time I
was working within media/filters... Kind of missed that the parent was in
content... Oops. :)

[mcasas, 2015/07/27 11:41:00]

Acknowledged.

+  const vpx_codec_iface_t* interface = vpx_codec_vp8_cx();
+  scoped_ptr<vpx_codec_enc_cfg_t> codec_config(new vpx_codec_enc_cfg_t);
+  vpx_codec_enc_config_default(interface, codec_config.get(), 0 /* reserved */);
+
+  // Adjust default bit rate to account for the actual size.
+  codec_config->rc_target_bitrate = size.width() * size.height() *
+                                    codec_config->rc_target_bitrate /
+                                    codec_config->g_w / codec_config->g_h;
+  DCHECK(size.width());
+  DCHECK(size.height());
+  codec_config->g_w = size.width();
+  codec_config->g_h = size.height();
+  codec_config->g_pass = VPX_RC_ONE_PASS;
+
+  // Millisecond granurality.
+  codec_config->g_timebase.num = 1;
+  codec_config->g_timebase.den = base::Time::kMillisecondsPerSecond * 90;

[Tom Finegan, 2015/07/23 22:18:38]

90000? RTP?

[mcasas, 2015/07/24 15:00:09]

Might have taken it inadvertently, removed :)

+
+  // Let the encoder decide where to place the Keyframes, between min and max.
+  // Due to http://crbug.com/440223, decoding fails after 30,000 non-key
+  // frames, so force an "unnecessary" key-frame every 10,000 frames.
+  codec_config->kf_mode = VPX_KF_AUTO;
+  codec_config->kf_min_dist = 10000;
+  codec_config->kf_max_dist = 10000;

[Tom Finegan, 2015/07/23 22:18:39]

note that you can end up with more keyframes than just the one (I assume you
know this; just being pedantic)

[mcasas, 2015/07/24 15:00:09]

Hmmkay, should I add a note?

[Tom Finegan, 2015/07/24 20:56:16]

Doesn't hurt to add something like, "When kf_mode is VPX_KF_AUTO libvpx will
sometimes emit keyframes regardless of min/max distance out of necessity." It
might prevent some confusion in the far future during a debug run. :)

+

[Tom Finegan, 2015/07/23 22:18:39]

One more config flag that should always be set by real time encoders:
g_lag_in_frames. Set it to 0. 

[mcasas, 2015/07/24 15:00:10]

Done.

+  const vpx_codec_err_t ret = vpx_codec_enc_init(codec_context, interface,
+                                                 codec_config.get(), kNoFlags);
+  DCHECK_EQ(VPX_CODEC_OK, ret);
+
+  scoped_ptr<vpx_image_t> image(new vpx_image_t());
+  vpx_image_t* const returned_image_ptr =
+      vpx_img_wrap(image.get(), VPX_IMG_FMT_I420, size.width(), size.height(),
+                   1 /* align */, nullptr /* img_data */);
+  DCHECK_EQ(image.get(), returned_image_ptr);
+  return image.Pass();
+}
+
+}  // anonymous namespace
+
+// Inner class holding all libvpx configuration and encoding received frames
+class VideoTrackRecorder::VpxEncoder
+    : public base::RefCountedThreadSafe<VpxEncoder> {
+ public:
+  VpxEncoder(scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
+             const OnFirstFrameCB& on_first_frame_callback,
+             const OnEncodedVideoCB& on_encoded_video_callback);
+
+  void EncodeOnIo(const scoped_refptr<VideoFrame>& frame,
+                  const base::TimeTicks& estimated_capture_time);
+
+  bool configuration_done() const { return configuration_done_; }
+
+ private:
+  friend class base::RefCountedThreadSafe<VpxEncoder>;
+  virtual ~VpxEncoder();
+
+  // Used to DCHECK that we are destructed where we were constructed.
+  base::ThreadChecker main_render_thread_checker_;
+  const scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_;
+
+  const OnFirstFrameCB on_first_frame_callback_;
+  const OnEncodedVideoCB on_encoded_video_callback_;
+  uint64_t track_index_;
+
+  // VP8 internal objects: codec configuration and Vpx Image wrapper.
+  scoped_ptr<vpx_codec_ctx_t, VpxCodecDeleter> codec_;
+  scoped_ptr<vpx_image_t> vpx_image_;
+
+  // Configuration has to happen on first frame arrival.
+  bool configuration_done_;
+
+  // Origin of times for frame timestamps.
+  base::TimeTicks timestamp_base_;
+};
+
+VideoTrackRecorder::VpxEncoder::VpxEncoder(
+    scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
+    const OnFirstFrameCB& on_first_frame_callback,
+    const OnEncodedVideoCB& on_encoded_video_callback)
+    : io_task_runner_(io_task_runner),
+      on_first_frame_callback_(on_first_frame_callback),
+      on_encoded_video_callback_(on_encoded_video_callback),
+      track_index_(0),
+      codec_(new vpx_codec_ctx_t),
+      configuration_done_(false) {
+  DCHECK(!on_encoded_video_callback_.is_null());
+}
+
+VideoTrackRecorder::VpxEncoder::~VpxEncoder() {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+}
+
+void VideoTrackRecorder::VpxEncoder::EncodeOnIo(
+    const scoped_refptr<VideoFrame>& frame,
+    const base::TimeTicks& estimated_capture_time) {
+  DVLOG(3) << __FUNCTION__ << " " << frame->coded_size().ToString();
+  DCHECK(io_task_runner_->BelongsToCurrentThread());
+
+  vpx_enc_frame_flags_t flags = kNoFlags;
+  double frame_rate = 0.0f;
+  if (!configuration_done_) {
+    vpx_image_ = InitVp8Encoding(frame->coded_size(), codec_.get());
+    DCHECK(vpx_image_.get());
+    timestamp_base_ = estimated_capture_time;
+    base::IgnoreResult(frame->metadata()->GetDouble(
+        VideoFrameMetadata::FRAME_RATE, &frame_rate));
+    track_index_ =
+        on_first_frame_callback_.Run(frame->coded_size(), frame_rate);
+
+    flags |= VPX_EFLAG_FORCE_KF;
+    configuration_done_ = true;
+  }
+
+  DCHECK(frame->visible_data(VideoFrame::kYPlane));
+  vpx_image_->planes[VPX_PLANE_Y] = frame->visible_data(VideoFrame::kYPlane);
+  vpx_image_->planes[VPX_PLANE_U] = frame->visible_data(VideoFrame::kUPlane);
+  vpx_image_->planes[VPX_PLANE_V] = frame->visible_data(VideoFrame::kVPlane);
+  vpx_image_->stride[VPX_PLANE_Y] = frame->stride(VideoFrame::kYPlane);
+  vpx_image_->stride[VPX_PLANE_U] = frame->stride(VideoFrame::kUPlane);
+  vpx_image_->stride[VPX_PLANE_V] = frame->stride(VideoFrame::kVPlane);
+
+  // TODO(mcasas): Use VideoFrameMetadata::FRAME_DURATION if available.
+  uint32_t duration = kDefaultFrameDurationInMilliseconds;
+  base::IgnoreResult(frame->metadata()->GetDouble(
+      VideoFrameMetadata::FRAME_RATE, &frame_rate));
+  if (frame_rate)
+    duration = base::Time::kMillisecondsPerSecond / frame_rate;
+  const int timestamp =
+      (estimated_capture_time - timestamp_base_).InMilliseconds();
+  const vpx_codec_err_t ret =
+      vpx_codec_encode(codec_.get(), vpx_image_.get(), timestamp, duration,
+                       flags, VPX_DL_REALTIME);
+  DCHECK_EQ(ret, VPX_CODEC_OK) << vpx_codec_err_to_string(ret) << ", #"
+                               << vpx_codec_error(codec_.get()) << " -"
+                               << vpx_codec_error_detail(codec_.get());
+
+  std::string data;
+  bool keyframe = false;
+  vpx_codec_iter_t iter = NULL;
+  const vpx_codec_cx_pkt_t* pkt = NULL;
+  while ((pkt = vpx_codec_get_cx_data(codec_.get(), &iter)) != NULL) {
+    if (pkt->kind != VPX_CODEC_CX_FRAME_PKT)
+      continue;
+    data.append(static_cast<char*>(pkt->data.frame.buf), pkt->data.frame.sz);
+    keyframe = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0;
+    break;
+  }
+  DVLOG(1) << timestamp << " - " << data.length() << "B "
+           << (keyframe ? "" : "non") << " keyframe";
+  on_encoded_video_callback_.Run(track_index_,
+                                 base::StringPiece(data),
+                                 base::TimeDelta::FromMilliseconds(timestamp),
+                                 keyframe);
+}
+
+VideoTrackRecorder::VideoTrackRecorder(
+    const blink::WebMediaStreamTrack& track,
+    const OnFirstFrameCB& on_first_frame_cb,
+    const OnEncodedVideoCB& on_encoded_video_cb)
+    : track_(track),
+      encoder_(new VpxEncoder(ChildProcess::current()->io_task_runner(),
+                              on_first_frame_cb,
+                              on_encoded_video_cb)) {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  DCHECK(track.extraData());
+  AddToVideoTrack(
+      this,
+      base::Bind(&VideoTrackRecorder::VpxEncoder::EncodeOnIo, encoder_),
+      track_);
+}
+
+VideoTrackRecorder::~VideoTrackRecorder() {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  RemoveFromVideoTrack(this, track_);
+}
+
+void VideoTrackRecorder::EncodeOnIoForTesting(
+    const scoped_refptr<VideoFrame>& frame,
+    const base::TimeTicks& estimated_capture_time) {
+  encoder_->EncodeOnIo(frame, estimated_capture_time);
+}
+
+bool VideoTrackRecorder::IsEncoderConfigurationDoneForTesting() const {
+  return encoder_->configuration_done();
+}
+
+}  // namespace content