RELAND: [Cast] Software encoder support for varying video frame sizes.

media/cast/sender/vp8_encoder.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "media/cast/sender/vp8_encoder.h"
 
 #include "base/logging.h"
 #include "media/base/video_frame.h"
 #include "media/cast/cast_defines.h"
 #include "media/cast/net/cast_transport_config.h"
@@ skipping 10 matching lines @@
 // pause in the video stream.
 const int kRestartFramePeriods = 3;
 
 }  // namespace
 
 Vp8Encoder::Vp8Encoder(const VideoSenderConfig& video_config)
     : cast_config_(video_config),
       use_multiple_video_buffers_(
           cast_config_.max_number_of_video_buffers_used ==
           kNumberOfVp8VideoBuffers),
-      raw_image_(nullptr),
       key_frame_requested_(true),
+      bitrate_kbit_(cast_config_.start_bitrate / 1000),
       last_encoded_frame_id_(kStartFrameId),
       last_acked_frame_id_(kStartFrameId),
       undroppable_frames_(0) {
   config_.g_timebase.den = 0;  // Not initialized.
 
+  for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
+    buffer_state_[i].frame_id = last_encoded_frame_id_;
+    buffer_state_[i].state = kBufferStartState;
+  }
+
   // VP8 have 3 buffers available for prediction, with
   // max_number_of_video_buffers_used set to 1 we maximize the coding efficiency
   // however in this mode we can not skip frames in the receiver to catch up
   // after a temporary network outage; with max_number_of_video_buffers_used
   // set to 3 we allow 2 frames to be skipped by the receiver without error
   // propagation.
   DCHECK(cast_config_.max_number_of_video_buffers_used == 1 ||
          cast_config_.max_number_of_video_buffers_used ==
              kNumberOfVp8VideoBuffers)
       << "Invalid argument";
 
   thread_checker_.DetachFromThread();
 }
 
 Vp8Encoder::~Vp8Encoder() {
   DCHECK(thread_checker_.CalledOnValidThread());
   if (is_initialized())
     vpx_codec_destroy(&encoder_);
-  vpx_img_free(raw_image_);
 }
 
 void Vp8Encoder::Initialize() {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(!is_initialized());
+  // The encoder will be created/configured when the first frame encode is
+  // requested.
+}
 
-  // Creating a wrapper to the image - setting image data to NULL. Actual
-  // pointer will be set during encode. Setting align to 1, as it is
-  // meaningless (actual memory is not allocated).
-  raw_image_ = vpx_img_wrap(
-      NULL, VPX_IMG_FMT_I420, cast_config_.width, cast_config_.height, 1, NULL);
+void Vp8Encoder::ConfigureForNewFrameSize(const gfx::Size& frame_size) {
+  if (is_initialized()) {
+    // Workaround for VP8 bug: If the new size is strictly less-than-or-equal to
+    // the old size, in terms of area, the existing encoder instance can
+    // continue.  Otherwise, completely tear-down and re-create a new encoder to
+    // avoid a shutdown crash.
+    if (frame_size.GetArea() <= gfx::Size(config_.g_w, config_.g_h).GetArea() &&
+        !use_multiple_video_buffers_) {
+      DVLOG(1) << "Continuing to use existing encoder at smaller frame size: "
+               << gfx::Size(config_.g_w, config_.g_h).ToString() << " --> "
+               << frame_size.ToString();
+      config_.g_w = frame_size.width();
+      config_.g_h = frame_size.height();
+      CHECK_EQ(vpx_codec_enc_config_set(&encoder_, &config_), VPX_CODEC_OK)
+          << "Failed to update frame size in encoder config.";
+      return;
+    }
 
+    DVLOG(1) << "Destroying/Re-Creating encoder for larger frame size: "
+             << gfx::Size(config_.g_w, config_.g_h).ToString() << " --> "
+             << frame_size.ToString();
+    vpx_codec_destroy(&encoder_);
+  } else {
+    DVLOG(1) << "Creating encoder for the first frame; size: "
+             << frame_size.ToString();
+  }
+
+  // Reset multi-buffer mode state.
+  last_acked_frame_id_ = last_encoded_frame_id_;
+  undroppable_frames_ = 0;
   for (int i = 0; i < kNumberOfVp8VideoBuffers; ++i) {
-    buffer_state_[i].frame_id = kStartFrameId;
+    buffer_state_[i].frame_id = last_encoded_frame_id_;
     buffer_state_[i].state = kBufferStartState;
   }
 
   // Populate encoder configuration with default values.
-  if (vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &config_, 0)) {
-    NOTREACHED() << "Invalid return value";
-    config_.g_timebase.den = 0;  // Do not call vpx_codec_destroy() in dtor.
-    return;
-  }
+  CHECK_EQ(vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &config_, 0),
+           VPX_CODEC_OK);
 
   config_.g_threads = cast_config_.number_of_encode_threads;
-  config_.g_w = cast_config_.width;
-  config_.g_h = cast_config_.height;
+  config_.g_w = frame_size.width();
+  config_.g_h = frame_size.height();
   // Set the timebase to match that of base::TimeDelta.
   config_.g_timebase.num = 1;
   config_.g_timebase.den = base::Time::kMicrosecondsPerSecond;
   if (use_multiple_video_buffers_) {
     // We must enable error resilience when we use multiple buffers, due to
     // codec requirements.
     config_.g_error_resilient = 1;
   }
+  // |g_pass| and |g_lag_in_frames| must be "one pass" and zero, respectively,
+  // in order for VP8 to support changing frame sizes during encoding:
   config_.g_pass = VPX_RC_ONE_PASS;
   config_.g_lag_in_frames = 0;  // Immediate data output for each frame.
 
   // Rate control settings.
   config_.rc_dropframe_thresh = 0;  // The encoder may not drop any frames.
   config_.rc_resize_allowed = 0;  // TODO(miu): Why not?  Investigate this.
   config_.rc_end_usage = VPX_CBR;
-  config_.rc_target_bitrate = cast_config_.start_bitrate / 1000;  // In kbit/s.
+  config_.rc_target_bitrate = bitrate_kbit_;
   config_.rc_min_quantizer = cast_config_.min_qp;
   config_.rc_max_quantizer = cast_config_.max_qp;
   // TODO(miu): Revisit these now that the encoder is being successfully
   // micro-managed.
   config_.rc_undershoot_pct = 100;
   config_.rc_overshoot_pct = 15;
   // TODO(miu): Document why these rc_buf_*_sz values were chosen and/or
   // research for better values.  Should they be computed from the target
   // playout delay?
   config_.rc_buf_initial_sz = 500;
   config_.rc_buf_optimal_sz = 600;
   config_.rc_buf_sz = 1000;
 
   config_.kf_mode = VPX_KF_DISABLED;
 
   vpx_codec_flags_t flags = 0;
-  if (vpx_codec_enc_init(&encoder_, vpx_codec_vp8_cx(), &config_, flags)) {
-    NOTREACHED() << "vpx_codec_enc_init() failed.";
-    config_.g_timebase.den = 0;  // Do not call vpx_codec_destroy() in dtor.
-    return;
-  }
+  CHECK_EQ(vpx_codec_enc_init(&encoder_, vpx_codec_vp8_cx(), &config_, flags),
+           VPX_CODEC_OK);
 
   // Raise the threshold for considering macroblocks as static.  The default is
   // zero, so this setting makes the encoder less sensitive to motion.  This
   // lowers the probability of needing to utilize more CPU to search for motion
   // vectors.
-  vpx_codec_control(&encoder_, VP8E_SET_STATIC_THRESHOLD, 1);
+  CHECK_EQ(vpx_codec_control(&encoder_, VP8E_SET_STATIC_THRESHOLD, 1),
+           VPX_CODEC_OK);
 
   // Improve quality by enabling sets of codec features that utilize more CPU.
   // The default is zero, with increasingly more CPU to be used as the value is
   // more negative.
   // TODO(miu): Document why this value was chosen and expected behaviors.
   // Should this be dynamic w.r.t. hardware performance?
-  vpx_codec_control(&encoder_, VP8E_SET_CPUUSED, -6);
+  CHECK_EQ(vpx_codec_control(&encoder_, VP8E_SET_CPUUSED, -6), VPX_CODEC_OK);
 }
 
 void Vp8Encoder::Encode(const scoped_refptr<media::VideoFrame>& video_frame,
                         const base::TimeTicks& reference_time,
                         EncodedFrame* encoded_frame) {
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK(encoded_frame);
 
-  CHECK(is_initialized());  // No illegal reference to |config_| or |encoder_|.
-
-  // Image in vpx_image_t format.
-  // Input image is const. VP8's raw image is not defined as const.
-  raw_image_->planes[VPX_PLANE_Y] =
-      const_cast<uint8*>(video_frame->data(VideoFrame::kYPlane));
-  raw_image_->planes[VPX_PLANE_U] =
-      const_cast<uint8*>(video_frame->data(VideoFrame::kUPlane));
-  raw_image_->planes[VPX_PLANE_V] =
-      const_cast<uint8*>(video_frame->data(VideoFrame::kVPlane));
-
-  raw_image_->stride[VPX_PLANE_Y] = video_frame->stride(VideoFrame::kYPlane);
-  raw_image_->stride[VPX_PLANE_U] = video_frame->stride(VideoFrame::kUPlane);
-  raw_image_->stride[VPX_PLANE_V] = video_frame->stride(VideoFrame::kVPlane);
+  // Initialize on-demand.  Later, if the video frame size has changed, update
+  // the encoder configuration.
+  const gfx::Size frame_size = video_frame->visible_rect().size();
+  if (!is_initialized() || gfx::Size(config_.g_w, config_.g_h) != frame_size)
+    ConfigureForNewFrameSize(frame_size);
 
   uint32 latest_frame_id_to_reference;
   Vp8Buffers buffer_to_update;
   vpx_codec_flags_t flags = 0;
   if (key_frame_requested_) {
     flags = VPX_EFLAG_FORCE_KF;
     // Self reference.
     latest_frame_id_to_reference = last_encoded_frame_id_ + 1;
     // We can pick any buffer as buffer_to_update since we update
     // them all.
     buffer_to_update = kLastBuffer;
   } else {
     // Reference all acked frames (buffers).
     latest_frame_id_to_reference = GetCodecReferenceFlags(&flags);
     buffer_to_update = GetNextBufferToUpdate();
     GetCodecUpdateFlags(buffer_to_update, &flags);
   }
 
+  // Wrapper for vpx_codec_encode() to access the YUV data in the |video_frame|.
+  // Only the VISIBLE rectangle within |video_frame| is exposed to the codec.
+  vpx_image_t vpx_image;
+  vpx_image_t* const result = vpx_img_wrap(
+      &vpx_image,
+      VPX_IMG_FMT_I420,
+      frame_size.width(),
+      frame_size.height(),
+      1,
+      video_frame->data(VideoFrame::kYPlane));
+  DCHECK_EQ(result, &vpx_image);
+  vpx_image.planes[VPX_PLANE_Y] =
+      video_frame->visible_data(VideoFrame::kYPlane);
+  vpx_image.planes[VPX_PLANE_U] =
+      video_frame->visible_data(VideoFrame::kUPlane);
+  vpx_image.planes[VPX_PLANE_V] =
+      video_frame->visible_data(VideoFrame::kVPlane);
+  vpx_image.stride[VPX_PLANE_Y] = video_frame->stride(VideoFrame::kYPlane);
+  vpx_image.stride[VPX_PLANE_U] = video_frame->stride(VideoFrame::kUPlane);
+  vpx_image.stride[VPX_PLANE_V] = video_frame->stride(VideoFrame::kVPlane);
+
   // The frame duration given to the VP8 codec affects a number of important
   // behaviors, including: per-frame bandwidth, CPU time spent encoding,
   // temporal quality trade-offs, and key/golden/alt-ref frame generation
   // intervals.  Use the actual amount of time between the current and previous
   // frames as a prediction for the next frame's duration, but bound the
   // prediction to account for the fact that the frame rate can be highly
   // variable, including long pauses in the video stream.
   const base::TimeDelta minimum_frame_duration =
       base::TimeDelta::FromSecondsD(1.0 / cast_config_.max_frame_rate);
   const base::TimeDelta maximum_frame_duration =
       base::TimeDelta::FromSecondsD(static_cast<double>(kRestartFramePeriods) /
                                         cast_config_.max_frame_rate);
   const base::TimeDelta last_frame_duration =
       video_frame->timestamp() - last_frame_timestamp_;
   const base::TimeDelta predicted_frame_duration =
       std::max(minimum_frame_duration,
                std::min(maximum_frame_duration, last_frame_duration));
   last_frame_timestamp_ = video_frame->timestamp();
 
   // Encode the frame.  The presentation time stamp argument here is fixed to
   // zero to force the encoder to base its single-frame bandwidth calculations
   // entirely on |predicted_frame_duration| and the target bitrate setting being
   // micro-managed via calls to UpdateRates().
   CHECK_EQ(vpx_codec_encode(&encoder_,
-                            raw_image_,
+                            &vpx_image,
                             0,
                             predicted_frame_duration.InMicroseconds(),
                             flags,
                             VPX_DL_REALTIME),
            VPX_CODEC_OK)
       << "BUG: Invalid arguments passed to vpx_codec_encode().";
 
   // Pull data from the encoder, populating a new EncodedFrame.
   encoded_frame->frame_id = ++last_encoded_frame_id_;
   const vpx_codec_cx_pkt_t* pkt = NULL;
@@ skipping 186 matching lines @@
 void Vp8Encoder::UpdateRates(uint32 new_bitrate) {
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (!is_initialized())
     return;
 
   uint32 new_bitrate_kbit = new_bitrate / 1000;
   if (config_.rc_target_bitrate == new_bitrate_kbit)
     return;
 
-  config_.rc_target_bitrate = new_bitrate_kbit;
+  config_.rc_target_bitrate = bitrate_kbit_ = new_bitrate_kbit;
 
   // Update encoder context.
   if (vpx_codec_enc_config_set(&encoder_, &config_)) {
     NOTREACHED() << "Invalid return value";
   }
 
   VLOG(1) << "VP8 new rc_target_bitrate: " << new_bitrate_kbit << " kbps";
 }
 
 void Vp8Encoder::LatestFrameIdToReference(uint32 frame_id) {
@@ skipping 13 matching lines @@
   }
 }
 
 void Vp8Encoder::GenerateKeyFrame() {
   DCHECK(thread_checker_.CalledOnValidThread());
   key_frame_requested_ = true;
 }
 
 }  // namespace cast
 }  // namespace media