MediaRecorderHandler (video part) and unittests

content/renderer/media/video_track_recorder.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/video_track_recorder.h"
 
 #include "base/bind.h"
 #include "base/logging.h"
 #include "base/sys_info.h"
 #include "base/threading/thread.h"
 #include "base/time/time.h"
 #include "base/trace_event/trace_event.h"
-#include "content/child/child_process.h"
-#include "media/base/bind_to_current_loop.h"
 #include "media/base/video_frame.h"
 
 extern "C" {
 // VPX_CODEC_DISABLE_COMPAT excludes parts of the libvpx API that provide
 // backwards compatibility for legacy applications using the library.
 #define VPX_CODEC_DISABLE_COMPAT 1
 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
 #include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h"
 }
 
 using media::VideoFrame;
 using media::VideoFrameMetadata;
 
 namespace content {
 
 namespace {
+
 const vpx_codec_flags_t kNoFlags = 0;
 
+// Originally from remoting/codec/scoped_vpx_codec.h.
+// TODO(mcasas): Refactor into a common location.
+struct VpxCodecDeleter {
+  void operator()(vpx_codec_ctx_t* codec) {
+    if (!codec)
+      return;
+    vpx_codec_err_t ret = vpx_codec_destroy(codec);
+    CHECK_EQ(ret, VPX_CODEC_OK) << "Failed to destroy codec";

[miu, 2015/09/02 21:28:13]

nit: No need for the logging string.

[mcasas, 2015/09/04 02:16:30]

Done.

+    delete codec;
+  }
+};
+
+typedef scoped_ptr<vpx_codec_ctx_t, VpxCodecDeleter> ScopedVpxCodec;
+
 void OnFrameEncodeCompleted(
-    const content::VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_cb,
+    const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_cb,
     const scoped_refptr<VideoFrame>& frame,
     scoped_ptr<std::string> data,
     base::TimeTicks capture_timestamp,
     bool keyframe) {
-  DVLOG(1) << (keyframe ? "" : "non ") << "keyframe "
-           << capture_timestamp << " ms - " << data->length() << "B ";
+  DVLOG(1) << (keyframe ? "" : "non ") << "keyframe "<< data->length() << "B, "
+           << capture_timestamp << " ms";
   on_encoded_video_cb.Run(frame, base::StringPiece(*data), capture_timestamp,
                           keyframe);
 }
 
 }  // anonymous namespace
 
 // Inner class encapsulating all libvpx interactions and the encoding+delivery
-// of received frames. This class is:
-// - created and destroyed on its parent's thread (usually the main render
-// thread),
-// - receives VideoFrames and Run()s the callbacks on another thread (supposedly
-// the render IO thread), which is cached on first frame arrival,
+// of received frames. Limitation: Only VP8 is supported for the time being.
+// This class needs to be ref-counted to avoid destruction (of |encoder_| in
+// particular) while there's still a frame being processed in either IO or
+// encode threads. This class:
+// - is created and destroyed on its parent's thread, usually the main Render
+// thread;
+// - receives VideoFrames and Run()s the callbacks on |origin_task_runner_|,
+// which is cached on first frame arrival, usually this thread is to be the
+// render IO thread, but this is not enforced;
 // - uses an internal |encoding_thread_| for libvpx interactions, notably for
 // encoding (which might take some time).
-// Only VP8 is supported for the time being.
-class VideoTrackRecorder::VpxEncoder final {
+class VideoTrackRecorder::VpxEncoder final
+    : public base::RefCountedThreadSafe<VpxEncoder> {
  public:
   explicit VpxEncoder(const OnEncodedVideoCB& on_encoded_video_callback);
-  ~VpxEncoder();
 
   void StartFrameEncode(const scoped_refptr<VideoFrame>& frame,
                         base::TimeTicks capture_timestamp);
 
  private:
+  friend class base::RefCountedThreadSafe<VpxEncoder>;
+  ~VpxEncoder();
+
   void EncodeOnEncodingThread(const scoped_refptr<VideoFrame>& frame,
                               base::TimeTicks capture_timestamp);
 
   void ConfigureVp8Encoding(const gfx::Size& size);
 
   // Returns true if |codec_config_| has been filled in at least once.
   bool IsInitialized() const;
 
   // Estimate the frame duration from |frame| and |last_frame_timestamp_|.
   base::TimeDelta CalculateFrameDuration(
       const scoped_refptr<VideoFrame>& frame);
 
   // Used to check that we are destroyed on the same thread we were created.
   base::ThreadChecker main_render_thread_checker_;
 
   // Task runner where frames to encode and reply callbacks must happen.
   scoped_refptr<base::SingleThreadTaskRunner> origin_task_runner_;
 
   // This callback should be exercised on IO thread.
   const OnEncodedVideoCB on_encoded_video_callback_;
 
   // Thread for encoding. Active as long as VpxEncoder exists. All variables
   // below this are used in this thread.
   base::Thread encoding_thread_;
-  // VP8 internal objects: configuration, encoder and Vpx Image wrapper.
+  // VP8 internal objects: configuration and encoder.
   vpx_codec_enc_cfg_t codec_config_;
-  vpx_codec_ctx_t encoder_;
+  // |encoder_| is a special scoped pointer to guarantee proper destruction.
+  ScopedVpxCodec encoder_;
 
   // The |VideoFrame::timestamp()| of the last encoded frame.  This is used to
   // predict the duration of the next frame.
   base::TimeDelta last_frame_timestamp_;
 
   DISALLOW_COPY_AND_ASSIGN(VpxEncoder);
 };
 
 VideoTrackRecorder::VpxEncoder::VpxEncoder(
     const OnEncodedVideoCB& on_encoded_video_callback)
     : on_encoded_video_callback_(on_encoded_video_callback),
       encoding_thread_("EncodingThread") {
   DCHECK(!on_encoded_video_callback_.is_null());
 
   codec_config_.g_timebase.den = 0;  // Not initialized.
 
   DCHECK(!encoding_thread_.IsRunning());
   encoding_thread_.Start();
 }
 
 VideoTrackRecorder::VpxEncoder::~VpxEncoder() {
   DCHECK(main_render_thread_checker_.CalledOnValidThread());

[miu, 2015/09/02 21:28:13]

Because this is now ref-counted, there is no guarantee the dtor will be called
on the main render thread.

[mcasas, 2015/09/04 02:16:30]

You're right.

I've checked other MediaStreamVideoSinks and they don't 
seem to address this issue in a clean way.
MediaStreamVideoRendererSink avoids it by using 
BindToCurrentLoop() with its OnVideoFrame(), then it
goes on to using a WeakPtrFactory.

VideoTrackToPepperAdapter::PpFrameReceiver also uses
a BindToCurrentLoop() and a WeakPtrFactory.

I could reuse that pattern here and make sure VpxEncoder
is created and lives in Main Render Thread, and that would
simplify WebmMuxer, which could stay single-threaded, 
but that would not work for the back-and-forth hop to the
encoding thread, unless I also use BindToCurrentLoop. 

WDYT?

[miu, 2015/09/04 21:48:52]

Seem to me that you can write a dtor that is able to be run on any thread:

VideoTrackRecorder::VpxEncoder::~VpxEncoder() {
  main_task_runner_->PostTask(FROM_HERE,
                              base::Bind(&VpxEncoder::ShutdownEncoder,
                                         base::Passed(&encoding_thread_),
                                         base::Passed(&encoder_)));
}

// static
void VideoTrackRecorder::VpxEncoder::ShutdownEncoder(
  scoped_ptr<base::Thread> encoding_thread,
  ScopedVpxCodecCtxPtr encoder) {
  encoding_thread->Stop();
  // Both |encoding_thread| and |encoder| will be destroyed at end-of-scope.
}

...and now I will stop harping on you for writing the custom scoped_ptr<> for
the VP8 encoder context, since it is actually needed now.  ;-)

[mcasas, 2015/09/05 02:43:01]

Done.

   DCHECK(encoding_thread_.IsRunning());
   encoding_thread_.Stop();
-  vpx_codec_destroy(&encoder_);
 }
 
 void VideoTrackRecorder::VpxEncoder::StartFrameEncode(
     const scoped_refptr<VideoFrame>& frame,
     base::TimeTicks capture_timestamp) {
   // Cache the thread sending frames on first frame arrival.
   if (!origin_task_runner_.get())
     origin_task_runner_ = base::MessageLoop::current()->task_runner();
   DCHECK(origin_task_runner_->BelongsToCurrentThread());
 
   encoding_thread_.task_runner()->PostTask(
       FROM_HERE, base::Bind(&VpxEncoder::EncodeOnEncodingThread,
-                            base::Unretained(this), frame, capture_timestamp));
+                            this, frame, capture_timestamp));
 }
 
 void VideoTrackRecorder::VpxEncoder::EncodeOnEncodingThread(
     const scoped_refptr<VideoFrame>& frame,
     base::TimeTicks capture_timestamp) {
   TRACE_EVENT0("video",
                "VideoTrackRecorder::VpxEncoder::EncodeOnEncodingThread");
   DCHECK(encoding_thread_.task_runner()->BelongsToCurrentThread());
 
   const gfx::Size frame_size = frame->visible_rect().size();
@@ skipping 14 matching lines @@
   vpx_image.planes[VPX_PLANE_U] = frame->visible_data(VideoFrame::kUPlane);
   vpx_image.planes[VPX_PLANE_V] = frame->visible_data(VideoFrame::kVPlane);
   vpx_image.stride[VPX_PLANE_Y] = frame->stride(VideoFrame::kYPlane);
   vpx_image.stride[VPX_PLANE_U] = frame->stride(VideoFrame::kUPlane);
   vpx_image.stride[VPX_PLANE_V] = frame->stride(VideoFrame::kVPlane);
 
   const base::TimeDelta duration = CalculateFrameDuration(frame);
   // Encode the frame.  The presentation time stamp argument here is fixed to
   // zero to force the encoder to base its single-frame bandwidth calculations
   // entirely on |predicted_frame_duration|.
-  const vpx_codec_err_t ret = vpx_codec_encode(&encoder_,
+  const vpx_codec_err_t ret = vpx_codec_encode(encoder_.get(),
                                                &vpx_image,
                                                0  /* pts */,
                                                duration.InMicroseconds(),
                                                kNoFlags,
                                                VPX_DL_REALTIME);
   DCHECK_EQ(ret, VPX_CODEC_OK) << vpx_codec_err_to_string(ret) << ", #"
-                               << vpx_codec_error(&encoder_) << " -"
-                               << vpx_codec_error_detail(&encoder_);
+                               << vpx_codec_error(encoder_.get()) << " -"
+                               << vpx_codec_error_detail(encoder_.get());
 
   scoped_ptr<std::string> data(new std::string);
   bool keyframe = false;
   vpx_codec_iter_t iter = NULL;
   const vpx_codec_cx_pkt_t* pkt = NULL;
-  while ((pkt = vpx_codec_get_cx_data(&encoder_, &iter)) != NULL) {
+  while ((pkt = vpx_codec_get_cx_data(encoder_.get(), &iter)) != NULL) {
     if (pkt->kind != VPX_CODEC_CX_FRAME_PKT)
       continue;
     data->assign(static_cast<char*>(pkt->data.frame.buf), pkt->data.frame.sz);
     keyframe = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0;
     break;
   }
   origin_task_runner_->PostTask(FROM_HERE,
-      base::Bind(&OnFrameEncodeCompleted,
+      base::Bind(OnFrameEncodeCompleted,
                  on_encoded_video_callback_,
                  frame,
                  base::Passed(&data),
                  capture_timestamp,
                  keyframe));
 }
 
 void VideoTrackRecorder::VpxEncoder::ConfigureVp8Encoding(
     const gfx::Size& size) {
   if (IsInitialized()) {
     // TODO(mcasas) VP8 quirk/optimisation: If the new |size| is strictly less-
     // than-or-equal than the old size, in terms of area, the existing encoder
     // instance could be reused after changing |codec_config_.{g_w,g_h}|.
     DVLOG(1) << "Destroying/Re-Creating encoder for new frame size: "
              << gfx::Size(codec_config_.g_w, codec_config_.g_h).ToString()
              << " --> " << size.ToString();
-    vpx_codec_destroy(&encoder_);
+    encoder_.reset();
   }
 
   const vpx_codec_iface_t* interface = vpx_codec_vp8_cx();
   const vpx_codec_err_t result = vpx_codec_enc_config_default(interface,
                                                               &codec_config_,
                                                               0 /* reserved */);
   DCHECK_EQ(VPX_CODEC_OK, result);
 
   // Adjust default bit rate to account for the actual size.
   DCHECK_EQ(320u, codec_config_.g_w);
@@ skipping 25 matching lines @@
 
   // Do not saturate CPU utilization just for encoding. On a lower-end system
   // with only 1 or 2 cores, use only one thread for encoding. On systems with
   // more cores, allow half of the cores to be used for encoding.
   codec_config_.g_threads =
       std::min(8, (base::SysInfo::NumberOfProcessors() + 1) / 2);
 
   // Number of frames to consume before producing output.
   codec_config_.g_lag_in_frames = 0;
 
-  const vpx_codec_err_t ret = vpx_codec_enc_init(&encoder_, interface,
+  DCHECK(!encoder_);
+  encoder_.reset(new vpx_codec_ctx_t);

[miu, 2015/09/02 21:28:13]

Why did you change this to a custom scoped ptr?  It doesn't seem to affect
anything except to allocate the encoder as a separate heap object.

[mcasas, 2015/09/04 02:16:30]

Is due to the possible case of ctor() + dtor() without
ConfigureVp8Encoding() (i.e. create-destroy with no
frames received. Using this specific scoped_ptr<> 
does clean up automagically, which means less
troubles down the road. Note that there is a unit
test for this, InitializeStartStop.

[miu, 2015/09/04 21:48:52]

But you have IsInitialized() to tell you whether you need to call
vpx_codec_destroy().  In any case, if you follow my advice for the dtor, this is
now a non-issue.

[mcasas, 2015/09/05 02:43:01]

Acknowledged.

+  const vpx_codec_err_t ret = vpx_codec_enc_init(encoder_.get(), interface,
                                                  &codec_config_, kNoFlags);
   DCHECK_EQ(VPX_CODEC_OK, ret);
 }
 
 bool VideoTrackRecorder::VpxEncoder::IsInitialized() const {
   DCHECK(encoding_thread_.task_runner()->BelongsToCurrentThread());
   return codec_config_.g_timebase.den != 0;
 }
 
 base::TimeDelta VideoTrackRecorder::VpxEncoder::CalculateFrameDuration(
@@ skipping 18 matching lines @@
   const TimeDelta kMaxFrameDuration = TimeDelta::FromSecondsD(1.0 / 8);
   const TimeDelta kMinFrameDuration = TimeDelta::FromMilliseconds(1);
   return std::min(kMaxFrameDuration, std::max(predicted_frame_duration,
                                               kMinFrameDuration));
 }
 
 VideoTrackRecorder::VideoTrackRecorder(
     const blink::WebMediaStreamTrack& track,
     const OnEncodedVideoCB& on_encoded_video_callback)
     : track_(track),
-      encoder_(new VpxEncoder(on_encoded_video_callback)),
-      weak_factory_(this) {
+      encoder_(new VpxEncoder(on_encoded_video_callback)) {
   DCHECK(main_render_thread_checker_.CalledOnValidThread());
   DCHECK(!track_.isNull());
   DCHECK(track.extraData());
+
+  // StartFrameEncode() will be called on Render IO thread.
   AddToVideoTrack(this,
-                  media::BindToCurrentLoop(
-                      base::Bind(&VideoTrackRecorder::OnVideoFrame,
-                                 weak_factory_.GetWeakPtr())),
+                  base::Bind(&VideoTrackRecorder::VpxEncoder::StartFrameEncode,
+                             encoder_),
                   track_);
 }
 
 VideoTrackRecorder::~VideoTrackRecorder() {
   DCHECK(main_render_thread_checker_.CalledOnValidThread());
   RemoveFromVideoTrack(this, track_);
-  weak_factory_.InvalidateWeakPtrs();
   track_.reset();
 }
 
-void VideoTrackRecorder::OnVideoFrame(const scoped_refptr<VideoFrame>& frame,
-                                      base::TimeTicks timestamp) {
-  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+void VideoTrackRecorder::OnVideoFrameForTesting(
+    const scoped_refptr<media::VideoFrame>& frame,
+    base::TimeTicks timestamp) {
   encoder_->StartFrameEncode(frame, timestamp);
 }
 
 }  // namespace content