Initialize based on frame sizes in VideoTrackRecorder

content/renderer/media/video_track_recorder.cc

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/video_track_recorder.h"
 
 #include <utility>
 
 #include "base/bind.h"
 #include "base/logging.h"
@@ skipping 21 matching lines @@
 extern "C" {
 // VPX_CODEC_DISABLE_COMPAT excludes parts of the libvpx API that provide
 // backwards compatibility for legacy applications using the library.
 #define VPX_CODEC_DISABLE_COMPAT 1
 #include "third_party/libvpx/source/libvpx/vpx/vp8cx.h"
 #include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h"
 }
 
 using media::VideoFrame;
 using media::VideoFrameMetadata;
+using video_track_recorder::kVEAEncoderMinResolutionWidth;
+using video_track_recorder::kVEAEncoderMinResolutionHeight;
 
 namespace content {
 
 namespace {
 
-const int kVEAEncoderMinResolutionWidth = 640;
-const int kVEAEncoderMinResolutionHeight = 480;
 const int kVEAEncoderOutputBufferCount = 4;
 
 static struct {
   VideoTrackRecorder::CodecId codec_id;
   media::VideoCodecProfile min_profile;
   media::VideoCodecProfile max_profile;
 } const kSupportedVideoCodecIdToProfile[] = {
     {VideoTrackRecorder::CodecId::VP8,
      media::VP8PROFILE_MIN,
      media::VP8PROFILE_MAX},
     {VideoTrackRecorder::CodecId::VP9,
      media::VP9PROFILE_MIN,
      media::VP9PROFILE_MAX},
     {VideoTrackRecorder::CodecId::H264,
      media::H264PROFILE_MIN,
      media::H264PROFILE_MAX}};
 
 // Returns the corresponding codec profile from VEA supported codecs. If no
 // profile is found, returns VIDEO_CODEC_PROFILE_UNKNOWN.
 media::VideoCodecProfile CodecIdToVEAProfile(
     content::VideoTrackRecorder::CodecId codec) {
   // See https://crbug.com/616659.
 #if defined(OS_CHROMEOS)
   return media::VIDEO_CODEC_PROFILE_UNKNOWN;
 #endif  // defined(OS_CHROMEOS)
-  content::RenderThreadImpl* render_thread_impl =
+  content::RenderThreadImpl* const render_thread_impl =
       content::RenderThreadImpl::current();
-  if (!render_thread_impl)
+  if (!render_thread_impl) {
+    DVLOG(3) << "Couldn't access the render thread";
     return media::VIDEO_CODEC_PROFILE_UNKNOWN;
+  }
 
-  media::GpuVideoAcceleratorFactories* gpu_factories =
-      content::RenderThreadImpl::current()->GetGpuFactories();
+  media::GpuVideoAcceleratorFactories* const gpu_factories =
+      render_thread_impl->GetGpuFactories();
   if (!gpu_factories || !gpu_factories->IsGpuVideoAcceleratorEnabled()) {
     DVLOG(3) << "Couldn't initialize GpuVideoAcceleratorFactories";
     return media::VIDEO_CODEC_PROFILE_UNKNOWN;
   }
 
   const media::VideoEncodeAccelerator::SupportedProfiles& vea_profiles =
       gpu_factories->GetVideoEncodeAcceleratorSupportedProfiles();
   for (const auto& vea_profile : vea_profiles) {
     for (const auto& supported_profile : kSupportedVideoCodecIdToProfile) {
       if (codec == supported_profile.codec_id &&
@@ skipping 152 matching lines @@
 
 // Class encapsulating VideoEncodeAccelerator interactions.
 // This class is created and destroyed in its owner thread. All other methods
 // operate on the task runner pointed by GpuFactories.
 class VEAEncoder final : public VideoTrackRecorder::Encoder,
                          public media::VideoEncodeAccelerator::Client {
  public:
   VEAEncoder(
       const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_callback,
       int32_t bits_per_second,
-      media::VideoCodecProfile codec);
+      media::VideoCodecProfile codec,
+      const gfx::Size& size);
 
   // media::VideoEncodeAccelerator::Client implementation.
   void RequireBitstreamBuffers(unsigned int input_count,
                                const gfx::Size& input_coded_size,
                                size_t output_buffer_size) override;
   void BitstreamBufferReady(int32_t bitstream_buffer_id,
                             size_t payload_size,
                             bool key_frame,
                             base::TimeDelta timestamp) override;
   void NotifyError(media::VideoEncodeAccelerator::Error error) override;
@@ skipping 20 matching lines @@
 
   // Shared memory buffers for output with the VEA.
   std::vector<std::unique_ptr<base::SharedMemory>> output_buffers_;
 
   // Shared memory buffers for output with the VEA as FIFO.
   std::queue<std::unique_ptr<base::SharedMemory>> input_buffers_;
 
   // Tracks error status.
   bool error_notified_;
 
-  // Tracks the first frame to encode.
-  std::unique_ptr<VideoFrameAndTimestamp> first_frame_;
+  // Tracks the last frame that we delay the encode.
+  std::unique_ptr<VideoFrameAndTimestamp> last_frame_;
 
   // Size used to initialize encoder.
   gfx::Size input_size_;
 
   // Coded size that encoder requests as input.
   gfx::Size vea_requested_input_size_;
 
   // Frames and corresponding timestamps in encode as FIFO.
   std::queue<VideoFrameAndTimestamp> frames_in_encode_;
 };
@@ skipping 79 matching lines @@
   base::TimeTicks first_frame_timestamp_;
 
   DISALLOW_COPY_AND_ASSIGN(H264Encoder);
 };
 
 #endif  // #if BUILDFLAG(RTC_USE_H264)
 
 VEAEncoder::VEAEncoder(
     const VideoTrackRecorder::OnEncodedVideoCB& on_encoded_video_callback,
     int32_t bits_per_second,
-    media::VideoCodecProfile codec)
+    media::VideoCodecProfile codec,
+    const gfx::Size& size)
     : Encoder(on_encoded_video_callback,
               bits_per_second,
               RenderThreadImpl::current()->GetGpuFactories()->GetTaskRunner()),
       gpu_factories_(RenderThreadImpl::current()->GetGpuFactories()),
       codec_(codec),
       error_notified_(false) {
   DCHECK(gpu_factories_);
+  DCHECK_GE(size.width(), kVEAEncoderMinResolutionWidth);
+  DCHECK_GE(size.height(), kVEAEncoderMinResolutionHeight);
+
+  encoding_task_runner_->PostTask(
+      FROM_HERE, base::Bind(&VEAEncoder::ConfigureEncoderOnEncodingTaskRunner,
+                            this, size));
 }
 
 VEAEncoder::~VEAEncoder() {
   encoding_task_runner_->PostTask(
       FROM_HERE, base::Bind(&media::VideoEncodeAccelerator::Destroy,
                             base::Unretained(video_encoder_.release())));
 }
 
 void VEAEncoder::RequireBitstreamBuffers(unsigned int /*input_count*/,
                                          const gfx::Size& input_coded_size,
@@ skipping 65 matching lines @@
     const scoped_refptr<VideoFrame>& frame,
     base::TimeTicks capture_timestamp) {
   DVLOG(3) << __FUNCTION__;
   DCHECK(encoding_task_runner_->BelongsToCurrentThread());
 
   if (input_size_ != frame->visible_rect().size() && video_encoder_) {
     video_encoder_->Destroy();
     video_encoder_.reset();
   }
 
-  if (!video_encoder_) {
+  if (!video_encoder_)
     ConfigureEncoderOnEncodingTaskRunner(frame->visible_rect().size());
-    first_frame_.reset(
-        new std::pair<scoped_refptr<VideoFrame>, base::TimeTicks>(
-            frame, capture_timestamp));
-  }
 
   if (error_notified_) {
     DVLOG(3) << "An error occurred in VEA encoder";
     return;
   }
 
   // Drop frames if there is no output buffers available.
   if (output_buffers_.empty()) {
     // TODO(emircan): Investigate if resetting encoder would help.
-    DVLOG(3) << "Dropped frame.";
+    DVLOG(3) << "Might drop frame.";
+    last_frame_.reset(
+        new std::pair<scoped_refptr<VideoFrame>, base::TimeTicks>(
+            frame, capture_timestamp));
     return;
   }
 
   // If first frame hasn't been encoded, do it first.
-  if (first_frame_) {
-    std::unique_ptr<VideoFrameAndTimestamp> first_frame(first_frame_.release());
-    EncodeOnEncodingTaskRunner(first_frame->first, first_frame->second);
+  if (last_frame_) {
+    std::unique_ptr<VideoFrameAndTimestamp> last_frame(last_frame_.release());
+    EncodeOnEncodingTaskRunner(last_frame->first, last_frame->second);
   }
 
   // Lower resolutions may fall back to SW encoder in some platforms, i.e. Mac.
   // In that case, the encoder expects more frames before returning result.
   // Therefore, a copy is necessary to release the current frame.
   scoped_refptr<media::VideoFrame> video_frame = frame;
   if (vea_requested_input_size_ != input_size_ ||
       input_size_.width() < kVEAEncoderMinResolutionWidth ||
       input_size_.height() < kVEAEncoderMinResolutionHeight) {
     // Create SharedMemory backed input buffers as necessary. These SharedMemory
@@ skipping 408 matching lines @@
 }
 #endif //#if BUILDFLAG(RTC_USE_H264)
 
 }  // anonymous namespace
 
 VideoTrackRecorder::VideoTrackRecorder(
     CodecId codec,
     const blink::WebMediaStreamTrack& track,
     const OnEncodedVideoCB& on_encoded_video_callback,
     int32_t bits_per_second)
-    : track_(track) {
+    : track_(track),
+      paused_before_init_(false),
+      weak_ptr_factory_(this) {
   DCHECK(main_render_thread_checker_.CalledOnValidThread());
   DCHECK(!track_.isNull());
   DCHECK(track_.getTrackData());
 
+  initialize_encoder_callback_ = base::Bind(
+      &VideoTrackRecorder::InitializeEncoder, weak_ptr_factory_.GetWeakPtr(),
+      codec, on_encoded_video_callback, bits_per_second);
+
+  // InitializeEncoder() will be called on Render Main thread.
+  MediaStreamVideoSink::ConnectToTrack(
+      track_, media::BindToCurrentLoop(initialize_encoder_callback_), false);
+}
+
+VideoTrackRecorder::~VideoTrackRecorder() {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  MediaStreamVideoSink::DisconnectFromTrack();
+  track_.reset();
+}
+
+void VideoTrackRecorder::Pause() {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  if (encoder_)
+    encoder_->SetPaused(true);
+  else
+    paused_before_init_ = true;
+}
+
+void VideoTrackRecorder::Resume() {
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+  if (encoder_)
+    encoder_->SetPaused(false);
+  else
+    paused_before_init_ = false;
+}
+
+void VideoTrackRecorder::OnVideoFrameForTesting(
+    const scoped_refptr<media::VideoFrame>& frame,
+    base::TimeTicks timestamp) {
+  DVLOG(3) << __FUNCTION__;
+
+  if (!encoder_) {
+    DCHECK(!initialize_encoder_callback_.is_null());
+    initialize_encoder_callback_.Run(frame, timestamp);
+  }
+
+  encoder_->StartFrameEncode(frame, timestamp);
+}
+
+void VideoTrackRecorder::InitializeEncoder(
+    CodecId codec,
+    const OnEncodedVideoCB& on_encoded_video_callback,
+    int32_t bits_per_second,
+    const scoped_refptr<media::VideoFrame>& frame,
+    base::TimeTicks capture_time) {
+  DVLOG(3) << __FUNCTION__ << frame->visible_rect().size().ToString();
+  DCHECK(main_render_thread_checker_.CalledOnValidThread());
+
+  MediaStreamVideoSink::DisconnectFromTrack();
+
+  const gfx::Size& input_size = frame->visible_rect().size();
   const auto& vea_supported_profile = CodecIdToVEAProfile(codec);
-  // TODO(emircan): Prioritize software based encoders in lower resolutions.
-  if (vea_supported_profile != media::VIDEO_CODEC_PROFILE_UNKNOWN) {
+  if (vea_supported_profile != media::VIDEO_CODEC_PROFILE_UNKNOWN &&
+      input_size.width() >= kVEAEncoderMinResolutionWidth &&
+      input_size.height() >= kVEAEncoderMinResolutionHeight) {
     encoder_ = new VEAEncoder(on_encoded_video_callback, bits_per_second,
-                              vea_supported_profile);
+                              vea_supported_profile, input_size);
   } else {
     switch (codec) {
 #if BUILDFLAG(RTC_USE_H264)
       case CodecId::H264:
-        encoder_ = new H264Encoder(on_encoded_video_callback, bits_per_second);
+        encoder_ =
+            new H264Encoder(on_encoded_video_callback, bits_per_second);
         break;
 #endif
       case CodecId::VP8:
       case CodecId::VP9:
         encoder_ = new VpxEncoder(codec == CodecId::VP9,
                                   on_encoded_video_callback, bits_per_second);
         break;
       default:
         NOTREACHED() << "Unsupported codec";
     }
   }
 
+  if (paused_before_init_)
+    encoder_->SetPaused(paused_before_init_);
+
   // StartFrameEncode() will be called on Render IO thread.
   MediaStreamVideoSink::ConnectToTrack(
       track_,
       base::Bind(&VideoTrackRecorder::Encoder::StartFrameEncode, encoder_),
       false);
 }
 
-VideoTrackRecorder::~VideoTrackRecorder() {
-  DCHECK(main_render_thread_checker_.CalledOnValidThread());
-  MediaStreamVideoSink::DisconnectFromTrack();
-  track_.reset();
-}
-
-void VideoTrackRecorder::Pause() {
-  DCHECK(main_render_thread_checker_.CalledOnValidThread());
-  DCHECK(encoder_);
-  encoder_->SetPaused(true);
-}
-
-void VideoTrackRecorder::Resume() {
-  DCHECK(main_render_thread_checker_.CalledOnValidThread());
-  DCHECK(encoder_);
-  encoder_->SetPaused(false);
-}
-
-void VideoTrackRecorder::OnVideoFrameForTesting(
-    const scoped_refptr<media::VideoFrame>& frame,
-    base::TimeTicks timestamp) {
-  encoder_->StartFrameEncode(frame, timestamp);
-}
-
 }  // namespace content