Extract bitrate allocation of spatial/temporal layers out of codec impl.

webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 94 matching lines @@
 VP8Encoder* VP8Encoder::Create() {
   return new VP8EncoderImpl();
 }
 
 VP8Decoder* VP8Decoder::Create() {
   return new VP8DecoderImpl();
 }
 
 VP8EncoderImpl::VP8EncoderImpl()
     : encoded_complete_callback_(nullptr),
-      rate_allocator_(new SimulcastRateAllocator(codec_)),
+      fallback_rate_allocator_(new SimulcastRateAllocator(codec_)),
       inited_(false),
       timestamp_(0),
       feedback_mode_(false),
       qp_max_(56),  // Setting for max quantizer.
       cpu_speed_default_(-6),
       rc_max_intra_target_(0),
       token_partitions_(VP8_ONE_TOKENPARTITION),
       down_scale_requested_(false),
       down_scale_bitrate_(0),
       key_frame_request_(kMaxSimulcastStreams, false),
@@ skipping 36 matching lines @@
   send_stream_.clear();
   cpu_speed_.clear();
   while (!raw_images_.empty()) {
     vpx_img_free(&raw_images_.back());
     raw_images_.pop_back();
   }
   while (!temporal_layers_.empty()) {
     delete temporal_layers_.back();
     temporal_layers_.pop_back();
   }
+  fallback_rate_allocator_.reset();
   inited_ = false;
   return ret_val;
 }
 
 int VP8EncoderImpl::SetRates(uint32_t new_bitrate_kbit,
                              uint32_t new_framerate) {
-  if (!inited_) {
+  if (!fallback_rate_allocator_.get())
+    return WEBRTC_VIDEO_CODEC_UNINITIALIZED;  // Indicate !initiated_.
+
+  // TODO(sprang): Remove this fallback path.
+  return SetRateAllocation(fallback_rate_allocator_->GetAllocation(
+                               new_bitrate_kbit * 1000, new_framerate),
+                           new_framerate);
+}
+
+int VP8EncoderImpl::SetRateAllocation(const BitrateAllocation& bitrate,
+                                      uint32_t new_framerate) {
+  if (!inited_)
     return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
-  }
-  if (encoders_[0].err) {
+
+  if (encoders_[0].err)
     return WEBRTC_VIDEO_CODEC_ERROR;
-  }
-  if (new_framerate < 1) {
+
+  if (new_framerate < 1)
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+
+  // At this point, bitrate allocation should already match codec settings.
+  if (codec_.maxBitrate > 0 && bitrate.get_sum_kbps() > codec_.maxBitrate)
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+
+  if (bitrate.get_sum_kbps() < codec_.minBitrate)
+    return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
+
+  if (codec_.numberOfSimulcastStreams > 0 &&

[perkj_webrtc, 2016/10/21 08:24:29]

dont' we always set this to at least 1.

[sprang_webrtc, 2016/10/25 10:44:25]

I don't think so. At least there seem to be many test cases which don't. Maybe
that can be a followup cl, this is really quite large as it is?

+      bitrate.get_sum_kbps() < codec_.simulcastStream[0].minBitrate) {
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
-  if (codec_.maxBitrate > 0 && new_bitrate_kbit > codec_.maxBitrate) {
-    new_bitrate_kbit = codec_.maxBitrate;
-  }
-  if (new_bitrate_kbit < codec_.minBitrate) {
-    new_bitrate_kbit = codec_.minBitrate;
-  }
-  if (codec_.numberOfSimulcastStreams > 0 &&
-      new_bitrate_kbit < codec_.simulcastStream[0].minBitrate) {
-    new_bitrate_kbit = codec_.simulcastStream[0].minBitrate;
-  }
+
   codec_.maxFramerate = new_framerate;
 
   if (encoders_.size() == 1) {
     // 1:1.
     // Calculate a rough limit for when to trigger a potental down scale.
     uint32_t k_pixels_per_frame = codec_.width * codec_.height / 1000;
     // TODO(pwestin): we currently lack CAMA, this is a temporary fix to work
     // around the current limitations.
     // Only trigger keyframes if we are allowed to scale down.
     if (configurations_[0].rc_resize_allowed) {
       if (!down_scale_requested_) {
-        if (k_pixels_per_frame > new_bitrate_kbit) {
+        if (k_pixels_per_frame > bitrate.get_sum_kbps()) {
           down_scale_requested_ = true;
-          down_scale_bitrate_ = new_bitrate_kbit;
+          down_scale_bitrate_ = bitrate.get_sum_kbps();
           key_frame_request_[0] = true;
         }
       } else {
-        if (new_bitrate_kbit > (2 * down_scale_bitrate_) ||
-            new_bitrate_kbit < (down_scale_bitrate_ / 2)) {
+        if (bitrate.get_sum_kbps() > (2 * down_scale_bitrate_) ||
+            bitrate.get_sum_kbps() < (down_scale_bitrate_ / 2)) {
           down_scale_requested_ = false;
         }
       }
     }
   } else {
     // If we have more than 1 stream, reduce the qp_max for the low resolution
     // stream if frame rate is not too low. The trade-off with lower qp_max is
     // possibly more dropped frames, so we only do this if the frame rate is
     // above some threshold (base temporal layer is down to 1/4 for 3 layers).
     // We may want to condition this on bitrate later.
     if (new_framerate > 20) {
       configurations_[encoders_.size() - 1].rc_max_quantizer = 45;
     } else {
       // Go back to default value set in InitEncode.
       configurations_[encoders_.size() - 1].rc_max_quantizer = qp_max_;
     }
   }
 
-  std::vector<uint32_t> stream_bitrates =
-      rate_allocator_->GetAllocation(new_bitrate_kbit);
   size_t stream_idx = encoders_.size() - 1;
   for (size_t i = 0; i < encoders_.size(); ++i, --stream_idx) {
+    unsigned int target_bitrate_kbps =
+        bitrate.get_spatial_layer_sum(stream_idx) / 1000;
+
     if (encoders_.size() > 1)
-      SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx);
+      SetStreamState(target_bitrate_kbps > 0, stream_idx);
 
-    unsigned int target_bitrate = stream_bitrates[stream_idx];
-    unsigned int max_bitrate = codec_.maxBitrate;
-    int framerate = new_framerate;
-    // TODO(holmer): This is a temporary hack for screensharing, where we
-    // interpret the startBitrate as the encoder target bitrate. This is
-    // to allow for a different max bitrate, so if the codec can't meet
-    // the target we still allow it to overshoot up to the max before dropping
-    // frames. This hack should be improved.
-    if (codec_.targetBitrate > 0 &&
-        (codec_.codecSpecific.VP8.numberOfTemporalLayers == 2 ||
-         codec_.simulcastStream[0].numberOfTemporalLayers == 2)) {
-      int tl0_bitrate = std::min(codec_.targetBitrate, target_bitrate);
-      max_bitrate = std::min(codec_.maxBitrate, target_bitrate);
-      target_bitrate = tl0_bitrate;
-    }
-    configurations_[i].rc_target_bitrate = target_bitrate;
-    temporal_layers_[stream_idx]->ConfigureBitrates(
-        target_bitrate, max_bitrate, framerate, &configurations_[i]);
+    configurations_[i].rc_target_bitrate = target_bitrate_kbps;
+    temporal_layers_[stream_idx]->UpdateConfiguration(&configurations_[i]);
+
     if (vpx_codec_enc_config_set(&encoders_[i], &configurations_[i])) {
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
   }
   quality_scaler_.ReportFramerate(new_framerate);
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
 void VP8EncoderImpl::OnDroppedFrame() {
   if (quality_scaler_enabled_)
     quality_scaler_.ReportDroppedFrame();
 }
 
 const char* VP8EncoderImpl::ImplementationName() const {
   return "libvpx";
 }
 
 void VP8EncoderImpl::SetStreamState(bool send_stream,
                                             int stream_idx) {
   if (send_stream && !send_stream_[stream_idx]) {
     // Need a key frame if we have not sent this stream before.
     key_frame_request_[stream_idx] = true;
   }
   send_stream_[stream_idx] = send_stream;
 }
 
 void VP8EncoderImpl::SetupTemporalLayers(int num_streams,
                                          int num_temporal_layers,
-                                         const VideoCodec& codec) {
-  TemporalLayersFactory default_factory;
+                                         const VideoCodec& codec,
+                                         SimulcastRateAllocator* allocator) {
+  RTC_DCHECK(codec.codecSpecific.VP8.tl_factory != nullptr);
   const TemporalLayersFactory* tl_factory = codec.codecSpecific.VP8.tl_factory;
-  if (!tl_factory)
-    tl_factory = &default_factory;
   if (num_streams == 1) {
-    if (codec.mode == kScreensharing) {
-      // Special mode when screensharing on a single stream.
-      temporal_layers_.push_back(new ScreenshareLayers(
-          num_temporal_layers, rand(), webrtc::Clock::GetRealTimeClock()));
-    } else {
-      temporal_layers_.push_back(
-          tl_factory->Create(num_temporal_layers, rand()));
-    }
+    temporal_layers_.push_back(
+        tl_factory->Create(0, num_temporal_layers, rand()));
+    allocator->OnTemporalLayersCreated(0, temporal_layers_.back());
   } else {
     for (int i = 0; i < num_streams; ++i) {
       // TODO(andresp): crash if layers is invalid.

[perkj_webrtc, 2016/10/21 08:24:28]

fix or remove this todo?

[sprang_webrtc, 2016/10/25 10:44:25]

Done.

-      int layers = codec.simulcastStream[i].numberOfTemporalLayers;
-      if (layers < 1)
-        layers = 1;
-      temporal_layers_.push_back(tl_factory->Create(layers, rand()));
+      int layers = std::max(static_cast<uint8_t>(1),
+                            codec.simulcastStream[i].numberOfTemporalLayers);
+      temporal_layers_.push_back(tl_factory->Create(i, layers, rand()));
+      allocator->OnTemporalLayersCreated(i, temporal_layers_.back());
     }
   }
 }
 
 int VP8EncoderImpl::InitEncode(const VideoCodec* inst,
                                int number_of_cores,
                                size_t /*maxPayloadSize */) {
   if (inst == NULL) {
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
@@ skipping 30 matching lines @@
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
 
   int num_temporal_layers =
       doing_simulcast ? inst->simulcastStream[0].numberOfTemporalLayers
                       : inst->codecSpecific.VP8.numberOfTemporalLayers;
 
   // TODO(andresp): crash if num temporal layers is bananas.
   if (num_temporal_layers < 1)
     num_temporal_layers = 1;
-  SetupTemporalLayers(number_of_streams, num_temporal_layers, *inst);
+
+  fallback_rate_allocator_.reset(new SimulcastRateAllocator(*inst));
+  SetupTemporalLayers(number_of_streams, num_temporal_layers, *inst,
+                      fallback_rate_allocator_.get());
 
   feedback_mode_ = inst->codecSpecific.VP8.feedbackModeOn;
-
   timestamp_ = 0;
   codec_ = *inst;
-  rate_allocator_.reset(new SimulcastRateAllocator(codec_));
 
   // Code expects simulcastStream resolutions to be correct, make sure they are
   // filled even when there are no simulcast layers.
   if (codec_.numberOfSimulcastStreams == 0) {
     codec_.simulcastStream[0].width = codec_.width;
     codec_.simulcastStream[0].height = codec_.height;
   }
 
   picture_id_.resize(number_of_streams);
   last_key_frame_picture_id_.resize(number_of_streams);
@@ skipping 132 matching lines @@
   // TODO(fbarchard): Consider number of Simulcast layers.
   configurations_[0].g_threads = NumberOfThreads(
       configurations_[0].g_w, configurations_[0].g_h, number_of_cores);
 
   // Creating a wrapper to the image - setting image data to NULL.
   // Actual pointer will be set in encode. Setting align to 1, as it
   // is meaningless (no memory allocation is done here).
   vpx_img_wrap(&raw_images_[0], VPX_IMG_FMT_I420, inst->width, inst->height, 1,
                NULL);
 
-  if (encoders_.size() == 1) {
-    configurations_[0].rc_target_bitrate = inst->startBitrate;
-    temporal_layers_[0]->ConfigureBitrates(inst->startBitrate, inst->maxBitrate,
-                                           inst->maxFramerate,
-                                           &configurations_[0]);
-  } else {
-    // Note the order we use is different from webm, we have lowest resolution
-    // at position 0 and they have highest resolution at position 0.
-    int stream_idx = encoders_.size() - 1;
-    std::vector<uint32_t> stream_bitrates =
-        rate_allocator_->GetAllocation(inst->startBitrate);
+  // Note the order we use is different from webm, we have lowest resolution
+  // at position 0 and they have highest resolution at position 0.
+  int stream_idx = encoders_.size() - 1;
+  std::vector<uint32_t> stream_bitrates;
+  BitrateAllocation allocation = fallback_rate_allocator_->GetAllocation(
+      inst->startBitrate * 1000, inst->maxFramerate);
+  for (int i = 0; i == 0 || i < inst->numberOfSimulcastStreams; ++i) {
+    uint32_t bitrate = allocation.get_spatial_layer_sum(i) / 1000;
+    stream_bitrates.push_back(bitrate);
+  }
+
+  configurations_[0].rc_target_bitrate = stream_bitrates[stream_idx];
+  temporal_layers_[stream_idx]->OnRatesUpdated(
+      stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate);
+  temporal_layers_[stream_idx]->UpdateConfiguration(&configurations_[0]);
+  --stream_idx;
+  for (size_t i = 1; i < encoders_.size(); ++i, --stream_idx) {
+    memcpy(&configurations_[i], &configurations_[0],
+           sizeof(configurations_[0]));
+
+    configurations_[i].g_w = inst->simulcastStream[stream_idx].width;
+    configurations_[i].g_h = inst->simulcastStream[stream_idx].height;
+
+    // Use 1 thread for lower resolutions.
+    configurations_[i].g_threads = 1;
+
+    // Setting alignment to 32 - as that ensures at least 16 for all
+    // planes (32 for Y, 16 for U,V). Libvpx sets the requested stride for
+    // the y plane, but only half of it to the u and v planes.
+    vpx_img_alloc(&raw_images_[i], VPX_IMG_FMT_I420,
+                  inst->simulcastStream[stream_idx].width,
+                  inst->simulcastStream[stream_idx].height, kVp832ByteAlign);
     SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx);
-    configurations_[0].rc_target_bitrate = stream_bitrates[stream_idx];
-    temporal_layers_[stream_idx]->ConfigureBitrates(
-        stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate,
-        &configurations_[0]);
-    --stream_idx;
-    for (size_t i = 1; i < encoders_.size(); ++i, --stream_idx) {
-      memcpy(&configurations_[i], &configurations_[0],
-             sizeof(configurations_[0]));
-
-      configurations_[i].g_w = inst->simulcastStream[stream_idx].width;
-      configurations_[i].g_h = inst->simulcastStream[stream_idx].height;
-
-      // Use 1 thread for lower resolutions.
-      configurations_[i].g_threads = 1;
-
-      // Setting alignment to 32 - as that ensures at least 16 for all
-      // planes (32 for Y, 16 for U,V). Libvpx sets the requested stride for
-      // the y plane, but only half of it to the u and v planes.
-      vpx_img_alloc(&raw_images_[i], VPX_IMG_FMT_I420,
-                    inst->simulcastStream[stream_idx].width,
-                    inst->simulcastStream[stream_idx].height, kVp832ByteAlign);
-      SetStreamState(stream_bitrates[stream_idx] > 0, stream_idx);
-      configurations_[i].rc_target_bitrate = stream_bitrates[stream_idx];
-      temporal_layers_[stream_idx]->ConfigureBitrates(
-          stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate,
-          &configurations_[i]);
-    }
+    configurations_[i].rc_target_bitrate = stream_bitrates[stream_idx];
+    temporal_layers_[stream_idx]->OnRatesUpdated(
+        stream_bitrates[stream_idx], inst->maxBitrate, inst->maxFramerate);
+    temporal_layers_[stream_idx]->UpdateConfiguration(&configurations_[i]);
   }
 
   rps_.Init();
   quality_scaler_.Init(codec_.codecType, codec_.startBitrate, codec_.width,
                        codec_.height, codec_.maxFramerate);
 
   // Only apply scaling to improve for single-layer streams. The scaling metrics
   // use frame drops as a signal and is only applicable when we drop frames.
   quality_scaler_enabled_ = encoders_.size() == 1 &&
                             configurations_[0].rc_dropframe_thresh > 0 &&
@@ skipping 753 matching lines @@
     return -1;
   }
   if (vpx_codec_control(copy->decoder_, VP8_SET_REFERENCE, ref_frame_) !=
       VPX_CODEC_OK) {
     return -1;
   }
   return 0;
 }
 
 }  // namespace webrtc