Release video frames earlier in MediaRecorder

content/renderer/media/video_track_recorder.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/video_track_recorder.h"
 
 #include <utility>
 
 #include "base/bind.h"
 #include "base/logging.h"
@@ skipping 148 matching lines @@
 
   void SetPaused(bool paused);
 
  protected:
   friend class base::RefCountedThreadSafe<Encoder>;
   virtual ~Encoder() {
     main_task_runner_->DeleteSoon(FROM_HERE, video_renderer_.release());
   }
 
   virtual void EncodeOnEncodingTaskRunner(
-      const scoped_refptr<VideoFrame>& frame,
+      scoped_refptr<VideoFrame> frame,
       base::TimeTicks capture_timestamp) = 0;
   virtual void ConfigureEncoderOnEncodingTaskRunner(const gfx::Size& size) = 0;
 
   // Used to shutdown properly on the same thread we were created.
   const scoped_refptr<base::SingleThreadTaskRunner> main_task_runner_;
 
   // Task runner where frames to encode and reply callbacks must happen.
   scoped_refptr<base::SingleThreadTaskRunner> origin_task_runner_;
 
   // Task runner where encoding interactions happen.
@@ skipping 151 matching lines @@
       return;
     vpx_codec_err_t ret = vpx_codec_destroy(codec);
     CHECK_EQ(ret, VPX_CODEC_OK);
     delete codec;
   }
 };
 typedef std::unique_ptr<vpx_codec_ctx_t, VpxCodecDeleter> ScopedVpxCodecCtxPtr;
 
 static void OnFrameEncodeCompleted(
     const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_cb,
-    const scoped_refptr<VideoFrame>& frame,
+    const media::WebmMuxer::VideoParameters& params,
     std::unique_ptr<std::string> data,
     base::TimeTicks capture_timestamp,
     bool keyframe) {
   DVLOG(1) << (keyframe ? "" : "non ") << "keyframe "<< data->length() << "B, "
            << capture_timestamp << " ms";
-  on_encoded_video_cb.Run(frame, std::move(data), capture_timestamp, keyframe);
+  on_encoded_video_cb.Run(params, std::move(data), capture_timestamp, keyframe);
 }
 
 static int GetNumberOfThreadsForEncoding() {
   // Do not saturate CPU utilization just for encoding. On a lower-end system
   // with only 1 or 2 cores, use only one thread for encoding. On systems with
   // more cores, allow half of the cores to be used for encoding.
   return std::min(8, (base::SysInfo::NumberOfProcessors() + 1) / 2);
 }
 
 // Class encapsulating VideoEncodeAccelerator interactions.
@@ skipping 13 matching lines @@
                                const gfx::Size& input_coded_size,
                                size_t output_buffer_size) override;
   void BitstreamBufferReady(int32_t bitstream_buffer_id,
                             size_t payload_size,
                             bool key_frame,
                             base::TimeDelta timestamp) override;
   void NotifyError(media::VideoEncodeAccelerator::Error error) override;
 
  private:
   using VideoFrameAndTimestamp =
-      std::pair<scoped_refptr<VideoFrame>, base::TimeTicks>;
+      std::pair<scoped_refptr<media::VideoFrame>, base::TimeTicks>;
+  using VideoParamsAndTimestamp =
+      std::pair<media::WebmMuxer::VideoParameters, base::TimeTicks>;
 
   void UseOutputBitstreamBufferId(int32_t bitstream_buffer_id);
   void FrameFinished(std::unique_ptr<base::SharedMemory> shm);
 
   // VideoTrackRecorder::Encoder implementation.
   ~VEAEncoder() override;
-  void EncodeOnEncodingTaskRunner(const scoped_refptr<VideoFrame>& frame,
+  void EncodeOnEncodingTaskRunner(scoped_refptr<VideoFrame> frame,
                                   base::TimeTicks capture_timestamp) override;
   void ConfigureEncoderOnEncodingTaskRunner(const gfx::Size& size) override;
 
   media::GpuVideoAcceleratorFactories* const gpu_factories_;
 
   const media::VideoCodecProfile codec_;
 
   // The underlying VEA to perform encoding on.
   std::unique_ptr<media::VideoEncodeAccelerator> video_encoder_;
 
   // Shared memory buffers for output with the VEA.
   std::vector<std::unique_ptr<base::SharedMemory>> output_buffers_;
 
   // Shared memory buffers for output with the VEA as FIFO.
   std::queue<std::unique_ptr<base::SharedMemory>> input_buffers_;
 
   // Tracks error status.
   bool error_notified_;
 
   // Tracks the last frame that we delay the encode.
   std::unique_ptr<VideoFrameAndTimestamp> last_frame_;
 
   // Size used to initialize encoder.
   gfx::Size input_size_;
 
   // Coded size that encoder requests as input.
   gfx::Size vea_requested_input_size_;
 
   // Frames and corresponding timestamps in encode as FIFO.
-  std::queue<VideoFrameAndTimestamp> frames_in_encode_;
+  std::queue<VideoParamsAndTimestamp> frames_in_encode_;
 };
 
 // Class encapsulating all libvpx interactions for VP8/VP9 encoding.
 class VpxEncoder final : public VideoTrackRecorder::Encoder {
  public:
   static void ShutdownEncoder(std::unique_ptr<base::Thread> encoding_thread,
                               ScopedVpxCodecCtxPtr encoder);
 
   VpxEncoder(
       bool use_vp9,
       const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_callback,
       int32_t bits_per_second);
 
  private:
   // VideoTrackRecorder::Encoder implementation.
   ~VpxEncoder() override;
-  void EncodeOnEncodingTaskRunner(const scoped_refptr<VideoFrame>& frame,
+  void EncodeOnEncodingTaskRunner(scoped_refptr<VideoFrame> frame,
                                   base::TimeTicks capture_timestamp) override;
   void ConfigureEncoderOnEncodingTaskRunner(const gfx::Size& size) override;
 
   // Returns true if |codec_config_| has been filled in at least once.
   bool IsInitialized() const;
 
   // Estimate the frame duration from |frame| and |last_frame_timestamp_|.
   base::TimeDelta EstimateFrameDuration(const scoped_refptr<VideoFrame>& frame);
 
   // Force usage of VP9 for encoding, instead of VP8 which is the default.
@@ skipping 31 matching lines @@
   static void ShutdownEncoder(std::unique_ptr<base::Thread> encoding_thread,
                               ScopedISVCEncoderPtr encoder);
 
   H264Encoder(
       const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_callback,
       int32_t bits_per_second);
 
  private:
   // VideoTrackRecorder::Encoder implementation.
   ~H264Encoder() override;
-  void EncodeOnEncodingTaskRunner(const scoped_refptr<VideoFrame>& frame,
+  void EncodeOnEncodingTaskRunner(scoped_refptr<VideoFrame> frame,
                               base::TimeTicks capture_timestamp) override;
   void ConfigureEncoderOnEncodingTaskRunner(const gfx::Size& size) override;
 
   // |openh264_encoder_| is a special scoped pointer to guarantee proper
   // destruction, also when reconfiguring due to parameters change. Only used on
   // |encoding_thread_|.
   gfx::Size configured_size_;
   ScopedISVCEncoderPtr openh264_encoder_;
 
   // The |VideoFrame::timestamp()| of the first received frame. Only used on
@@ skipping 91 matching lines @@
       output_buffers_[bitstream_buffer_id]->mapped_size()));
 }
 
 void VEAEncoder::FrameFinished(std::unique_ptr<base::SharedMemory> shm) {
   DVLOG(3) << __func__;
   DCHECK(encoding_task_runner_->BelongsToCurrentThread());
   input_buffers_.push(std::move(shm));
 }
 
 void VEAEncoder::EncodeOnEncodingTaskRunner(
-    const scoped_refptr<VideoFrame>& frame,
+   scoped_refptr<VideoFrame> frame,
     base::TimeTicks capture_timestamp) {
   DVLOG(3) << __func__;
   DCHECK(encoding_task_runner_->BelongsToCurrentThread());
 
   if (input_size_ != frame->visible_rect().size() && video_encoder_) {
     video_encoder_->Destroy();
     video_encoder_.reset();
   }
 
   if (!video_encoder_)
@@ skipping 63 matching lines @@
                      frame->visible_data(media::VideoFrame::kVPlane),
                      frame->stride(media::VideoFrame::kVPlane),
                      video_frame->visible_data(media::VideoFrame::kYPlane),
                      video_frame->stride(media::VideoFrame::kYPlane),
                      video_frame->visible_data(media::VideoFrame::kUPlane),
                      video_frame->stride(media::VideoFrame::kUPlane),
                      video_frame->visible_data(media::VideoFrame::kVPlane),
                      video_frame->stride(media::VideoFrame::kVPlane),
                      input_size_.width(), input_size_.height());
   }
-  frames_in_encode_.push(std::make_pair(video_frame, capture_timestamp));
+  frames_in_encode_.push(std::make_pair(
+      media::WebmMuxer::VideoParameters(frame), capture_timestamp));
 
   encoding_task_runner_->PostTask(
       FROM_HERE,
       base::Bind(&media::VideoEncodeAccelerator::Encode,
                  base::Unretained(video_encoder_.get()), video_frame, false));
 }
 
 void VEAEncoder::ConfigureEncoderOnEncodingTaskRunner(const gfx::Size& size) {
   DVLOG(3) << __func__;
   DCHECK(encoding_task_runner_->BelongsToCurrentThread());
@@ skipping 27 matching lines @@
 }
 
 VpxEncoder::~VpxEncoder() {
   main_task_runner_->PostTask(FROM_HERE,
                               base::Bind(&VpxEncoder::ShutdownEncoder,
                                          base::Passed(&encoding_thread_),
                                          base::Passed(&encoder_)));
 }
 
 void VpxEncoder::EncodeOnEncodingTaskRunner(
-    const scoped_refptr<VideoFrame>& frame,
+    scoped_refptr<VideoFrame> frame,
     base::TimeTicks capture_timestamp) {
   TRACE_EVENT0("video", "VpxEncoder::EncodeOnEncodingTaskRunner");
   DCHECK(encoding_task_runner_->BelongsToCurrentThread());
 
   const gfx::Size frame_size = frame->visible_rect().size();
   if (!IsInitialized() ||
       gfx::Size(codec_config_.g_w, codec_config_.g_h) != frame_size) {
     ConfigureEncoderOnEncodingTaskRunner(frame_size);
   }
 
@@ skipping 19 matching lines @@
   const vpx_codec_err_t ret = vpx_codec_encode(encoder_.get(),
                                                &vpx_image,
                                                0  /* pts */,
                                                duration.InMicroseconds(),
                                                0 /* flags */,
                                                VPX_DL_REALTIME);
   DCHECK_EQ(ret, VPX_CODEC_OK) << vpx_codec_err_to_string(ret) << ", #"
                                << vpx_codec_error(encoder_.get()) << " -"
                                << vpx_codec_error_detail(encoder_.get());
 
+  const media::WebmMuxer::VideoParameters video_params(frame);
+  frame = nullptr;
+
   std::unique_ptr<std::string> data(new std::string);
   bool keyframe = false;
   vpx_codec_iter_t iter = NULL;
   const vpx_codec_cx_pkt_t* pkt = NULL;
   while ((pkt = vpx_codec_get_cx_data(encoder_.get(), &iter)) != NULL) {
     if (pkt->kind != VPX_CODEC_CX_FRAME_PKT)
       continue;
     data->assign(static_cast<char*>(pkt->data.frame.buf), pkt->data.frame.sz);
     keyframe = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0;
     break;
   }
   origin_task_runner_->PostTask(FROM_HERE,
       base::Bind(OnFrameEncodeCompleted,
                  on_encoded_video_callback_,
-                 frame,
+                 video_params,
                  base::Passed(&data),
                  capture_timestamp,
                  keyframe));
 }
 
 void VpxEncoder::ConfigureEncoderOnEncodingTaskRunner(const gfx::Size& size) {
   DCHECK(encoding_task_runner_->BelongsToCurrentThread());
   if (IsInitialized()) {
     // TODO(mcasas) VP8 quirk/optimisation: If the new |size| is strictly less-
     // than-or-equal than the old size, in terms of area, the existing encoder
@@ skipping 129 matching lines @@
 }
 
 H264Encoder::~H264Encoder() {
   main_task_runner_->PostTask(FROM_HERE,
                               base::Bind(&H264Encoder::ShutdownEncoder,
                                          base::Passed(&encoding_thread_),
                                          base::Passed(&openh264_encoder_)));
 }
 
 void H264Encoder::EncodeOnEncodingTaskRunner(
-    const scoped_refptr<VideoFrame>& frame,
+    scoped_refptr<VideoFrame> frame,
     base::TimeTicks capture_timestamp) {
   TRACE_EVENT0("video", "H264Encoder::EncodeOnEncodingTaskRunner");
   DCHECK(encoding_task_runner_->BelongsToCurrentThread());
 
   const gfx::Size frame_size = frame->visible_rect().size();
   if (!openh264_encoder_ || configured_size_ != frame_size) {
     ConfigureEncoderOnEncodingTaskRunner(frame_size);
     first_frame_timestamp_ = capture_timestamp;
   }
 
   SSourcePicture picture = {};
   picture.iPicWidth = frame_size.width();
   picture.iPicHeight = frame_size.height();
   picture.iColorFormat = EVideoFormatType::videoFormatI420;
   picture.uiTimeStamp =
       (capture_timestamp - first_frame_timestamp_).InMilliseconds();
   picture.iStride[0] = frame->stride(VideoFrame::kYPlane);
   picture.iStride[1] = frame->stride(VideoFrame::kUPlane);
   picture.iStride[2] = frame->stride(VideoFrame::kVPlane);
   picture.pData[0] = frame->visible_data(VideoFrame::kYPlane);
   picture.pData[1] = frame->visible_data(VideoFrame::kUPlane);
   picture.pData[2] = frame->visible_data(VideoFrame::kVPlane);
 
   SFrameBSInfo info = {};
   if (openh264_encoder_->EncodeFrame(&picture, &info) != cmResultSuccess) {
     NOTREACHED() << "OpenH264 encoding failed";
     return;
   }
+  const media::WebmMuxer::VideoParameters video_params(frame);
+  frame = nullptr;
 
   std::unique_ptr<std::string> data(new std::string);
   const uint8_t kNALStartCode[4] = {0, 0, 0, 1};
   for (int layer = 0; layer < info.iLayerNum; ++layer) {
     const SLayerBSInfo& layerInfo = info.sLayerInfo[layer];
     // Iterate NAL units making up this layer, noting fragments.
     size_t layer_len = 0;
     for (int nal = 0; nal < layerInfo.iNalCount; ++nal) {
       // The following DCHECKs make sure that the header of each NAL unit is OK.
       DCHECK_GE(layerInfo.pNalLengthInByte[nal], 4);
       DCHECK_EQ(kNALStartCode[0], layerInfo.pBsBuf[layer_len+0]);
       DCHECK_EQ(kNALStartCode[1], layerInfo.pBsBuf[layer_len+1]);
       DCHECK_EQ(kNALStartCode[2], layerInfo.pBsBuf[layer_len+2]);
       DCHECK_EQ(kNALStartCode[3], layerInfo.pBsBuf[layer_len+3]);
 
       layer_len += layerInfo.pNalLengthInByte[nal];
     }
     // Copy the entire layer's data (including NAL start codes).
     data->append(reinterpret_cast<char*>(layerInfo.pBsBuf), layer_len);
   }
 
   const bool is_key_frame = info.eFrameType == videoFrameTypeIDR;
   origin_task_runner_->PostTask(
-      FROM_HERE,
-      base::Bind(OnFrameEncodeCompleted, on_encoded_video_callback_, frame,
-                 base::Passed(&data), capture_timestamp, is_key_frame));
+      FROM_HERE, base::Bind(OnFrameEncodeCompleted, on_encoded_video_callback_,
+                            video_params, base::Passed(&data),
+                            capture_timestamp, is_key_frame));
 }
 
 void H264Encoder::ConfigureEncoderOnEncodingTaskRunner(const gfx::Size& size) {
   DCHECK(encoding_task_runner_->BelongsToCurrentThread());
   ISVCEncoder* temp_encoder = nullptr;
   if (WelsCreateSVCEncoder(&temp_encoder) != 0) {
     NOTREACHED() << "Failed to create OpenH264 encoder";
     return;
   }
   openh264_encoder_.reset(temp_encoder);
@@ skipping 156 matching lines @@
     encoder_->SetPaused(paused_before_init_);
 
   // StartFrameEncode() will be called on Render IO thread.
   MediaStreamVideoSink::ConnectToTrack(
       track_,
       base::Bind(&VideoTrackRecorder::Encoder::StartFrameEncode, encoder_),
       false);
 }
 
 }  // namespace content