Run VideoProcessor on task queue in VideoProcessorIntegrationTest.

webrtc/modules/video_coding/codecs/test/videoprocessor.cc

 /*
  *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
  *
  *  Use of this source code is governed by a BSD-style license
  *  that can be found in the LICENSE file in the root of the source
  *  tree. An additional intellectual property rights grant can be found
  *  in the file PATENTS.  All contributing project authors may
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
@@ skipping 41 matching lines @@
          CodecTypeToPayloadName(codec_settings.codecType).value_or("Unknown"));
   printf("  Start bitrate     : %d kbps\n", codec_settings.startBitrate);
   printf("  Max bitrate       : %d kbps\n", codec_settings.maxBitrate);
   printf("  Min bitrate       : %d kbps\n", codec_settings.minBitrate);
   printf("  Width             : %d\n", codec_settings.width);
   printf("  Height            : %d\n", codec_settings.height);
   printf("  Max frame rate    : %d\n", codec_settings.maxFramerate);
   printf("  QPmax             : %d\n", codec_settings.qpMax);
   if (codec_settings.codecType == kVideoCodecVP8) {
     printf("  Complexity        : %d\n", codec_settings.VP8().complexity);
+    printf("  Resilience        : %d\n", codec_settings.VP8().resilience);
+    printf("  # temporal layers : %d\n",
+           codec_settings.VP8().numberOfTemporalLayers);
     printf("  Denoising         : %d\n", codec_settings.VP8().denoisingOn);
     printf("  Error concealment : %d\n",
            codec_settings.VP8().errorConcealmentOn);
+    printf("  Automatic resize  : %d\n",
+           codec_settings.VP8().automaticResizeOn);
     printf("  Frame dropping    : %d\n", codec_settings.VP8().frameDroppingOn);
-    printf("  Resilience        : %d\n", codec_settings.VP8().resilience);
     printf("  Key frame interval: %d\n", codec_settings.VP8().keyFrameInterval);
   } else if (codec_settings.codecType == kVideoCodecVP9) {
     printf("  Complexity        : %d\n", codec_settings.VP9().complexity);
+    printf("  Resilience        : %d\n", codec_settings.VP9().resilienceOn);
+    printf("  # temporal layers : %d\n",
+           codec_settings.VP9().numberOfTemporalLayers);
     printf("  Denoising         : %d\n", codec_settings.VP9().denoisingOn);
     printf("  Frame dropping    : %d\n", codec_settings.VP9().frameDroppingOn);
-    printf("  Resilience        : %d\n", codec_settings.VP9().resilienceOn);
     printf("  Key frame interval: %d\n", codec_settings.VP9().keyFrameInterval);
     printf("  Adaptive QP mode  : %d\n", codec_settings.VP9().adaptiveQpMode);
+    printf("  Automatic resize  : %d\n",
+           codec_settings.VP9().automaticResizeOn);
+    printf("  # spatial layers  : %d\n",
+           codec_settings.VP9().numberOfSpatialLayers);
+    printf("  Flexible mode     : %d\n", codec_settings.VP9().flexibleMode);
   } else if (codec_settings.codecType == kVideoCodecH264) {
     printf("  Frame dropping    : %d\n", codec_settings.H264().frameDroppingOn);
     printf("  Key frame interval: %d\n",
            codec_settings.H264().keyFrameInterval);
     printf("  Profile           : %d\n", codec_settings.H264().profile);
   }
 }
 
 int GetElapsedTimeMicroseconds(int64_t start_ns, int64_t stop_ns) {
   int64_t diff_us = (stop_ns - start_ns) / rtc::kNumNanosecsPerMicrosec;
@@ skipping 18 matching lines @@
 
 VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder,
                                webrtc::VideoDecoder* decoder,
                                FrameReader* analysis_frame_reader,
                                FrameWriter* analysis_frame_writer,
                                PacketManipulator* packet_manipulator,
                                const TestConfig& config,
                                Stats* stats,
                                IvfFileWriter* encoded_frame_writer,
                                FrameWriter* decoded_frame_writer)
-    : config_(config),
+    : initialized_(false),
+      config_(config),
       encoder_(encoder),
       decoder_(decoder),
       bitrate_allocator_(CreateBitrateAllocator(&config_)),
-      encode_callback_(new VideoProcessorEncodeCompleteCallback(this)),
-      decode_callback_(new VideoProcessorDecodeCompleteCallback(this)),
+      encode_callback_(this),
+      decode_callback_(this),
+      num_frames_decoded_(0),
       packet_manipulator_(packet_manipulator),
       analysis_frame_reader_(analysis_frame_reader),
       analysis_frame_writer_(analysis_frame_writer),
       encoded_frame_writer_(encoded_frame_writer),
       decoded_frame_writer_(decoded_frame_writer),
-      initialized_(false),
       last_encoded_frame_num_(-1),
       last_decoded_frame_num_(-1),
       first_key_frame_has_been_excluded_(false),
-      last_decoded_frame_buffer_(0, analysis_frame_reader->FrameLength()),
+      last_decoded_frame_buffer_(analysis_frame_reader->FrameLength()),
       stats_(stats),
-      num_dropped_frames_(0),
-      num_spatial_resizes_(0) {
+      rate_update_index_(-1) {
   RTC_DCHECK(encoder);
   RTC_DCHECK(decoder);
   RTC_DCHECK(packet_manipulator);
   RTC_DCHECK(analysis_frame_reader);
   RTC_DCHECK(analysis_frame_writer);
   RTC_DCHECK(stats);
   frame_infos_.reserve(analysis_frame_reader->NumberOfFrames());
 }
 
 VideoProcessor::~VideoProcessor() = default;
 
 void VideoProcessor::Init() {
+  RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
   RTC_DCHECK(!initialized_) << "VideoProcessor already initialized.";
   initialized_ = true;
 
   // Setup required callbacks for the encoder and decoder.
-  RTC_CHECK_EQ(encoder_->RegisterEncodeCompleteCallback(encode_callback_.get()),
+  RTC_CHECK_EQ(encoder_->RegisterEncodeCompleteCallback(&encode_callback_),
                WEBRTC_VIDEO_CODEC_OK)
       << "Failed to register encode complete callback";
-  RTC_CHECK_EQ(decoder_->RegisterDecodeCompleteCallback(decode_callback_.get()),
+  RTC_CHECK_EQ(decoder_->RegisterDecodeCompleteCallback(&decode_callback_),
                WEBRTC_VIDEO_CODEC_OK)
       << "Failed to register decode complete callback";
 
   // Initialize the encoder and decoder.
   uint32_t num_cores =
       config_.use_single_core ? 1 : CpuInfo::DetectNumberOfCores();
   RTC_CHECK_EQ(
       encoder_->InitEncode(&config_.codec_settings, num_cores,
                            config_.networking_config.max_payload_size_in_bytes),
       WEBRTC_VIDEO_CODEC_OK)
@@ skipping 18 matching lines @@
              CodecTypeToPayloadName(config_.codec_settings.codecType)
                  .value_or("Unknown"),
              encoder_->ImplementationName());
     }
     PrintCodecSettings(config_.codec_settings);
     printf("\n");
   }
 }
 
 void VideoProcessor::Release() {
-  encoder_->RegisterEncodeCompleteCallback(nullptr);
-  decoder_->RegisterDecodeCompleteCallback(nullptr);
+  RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
 
   RTC_CHECK_EQ(encoder_->Release(), WEBRTC_VIDEO_CODEC_OK);
   RTC_CHECK_EQ(decoder_->Release(), WEBRTC_VIDEO_CODEC_OK);
 
+  encoder_->RegisterEncodeCompleteCallback(nullptr);
+  decoder_->RegisterDecodeCompleteCallback(nullptr);
+
   initialized_ = false;
 }
 
-bool VideoProcessor::ProcessFrame(int frame_number) {
-  RTC_DCHECK_GE(frame_number, 0);
-  RTC_DCHECK_LE(frame_number, frame_infos_.size())
-      << "Must process frames without gaps.";
+void VideoProcessor::ProcessFrame(int frame_number) {
+  RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
+  RTC_DCHECK_EQ(frame_number, frame_infos_.size())
+      << "Must process frames in sequence.";
   RTC_DCHECK(initialized_) << "VideoProcessor not initialized.";
 
+  // Get frame from file.
   rtc::scoped_refptr<I420BufferInterface> buffer(
       analysis_frame_reader_->ReadFrame());
-
-  if (!buffer) {
-    // Last frame has been reached.
-    return false;
-  }
-
-  uint32_t timestamp = FrameNumberToTimestamp(frame_number);
-  VideoFrame source_frame(buffer, timestamp, 0, webrtc::kVideoRotation_0);
-
-  // Store frame information during the different stages of encode and decode.
-  frame_infos_.emplace_back();
-  FrameInfo* frame_info = &frame_infos_.back();
-  frame_info->timestamp = timestamp;
+  RTC_CHECK(buffer) << "Tried to read too many frames from the file.";
+  const int64_t kNoRenderTime = 0;
+  VideoFrame source_frame(buffer, FrameNumberToTimestamp(frame_number),
+                          kNoRenderTime, webrtc::kVideoRotation_0);
 
   // Decide if we are going to force a keyframe.
   std::vector<FrameType> frame_types(1, kVideoFrameDelta);
   if (config_.keyframe_interval > 0 &&
       frame_number % config_.keyframe_interval == 0) {
     frame_types[0] = kVideoFrameKey;
   }
 
+  // Store frame information during the different stages of encode and decode.
+  frame_infos_.emplace_back();
+  FrameInfo* frame_info = &frame_infos_.back();
+
   // Create frame statistics object used for aggregation at end of test run.
   FrameStatistic* frame_stat = &stats_->NewFrame(frame_number);
 
   // For the highest measurement accuracy of the encode time, the start/stop
   // time recordings should wrap the Encode call as tightly as possible.
   frame_info->encode_start_ns = rtc::TimeNanos();
   frame_stat->encode_return_code =
       encoder_->Encode(source_frame, nullptr, &frame_types);
 
   if (frame_stat->encode_return_code != WEBRTC_VIDEO_CODEC_OK) {
     LOG(LS_WARNING) << "Failed to encode frame " << frame_number
                     << ", return code: " << frame_stat->encode_return_code
                     << ".";
   }
-
-  return true;
 }
 
-void VideoProcessor::SetRates(int bit_rate, int frame_rate) {
-  config_.codec_settings.maxFramerate = frame_rate;
+void VideoProcessor::SetRates(int bitrate_kbps, int framerate_fps) {
+  RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
+
+  config_.codec_settings.maxFramerate = framerate_fps;
   int set_rates_result = encoder_->SetRateAllocation(
-      bitrate_allocator_->GetAllocation(bit_rate * 1000, frame_rate),
-      frame_rate);
+      bitrate_allocator_->GetAllocation(bitrate_kbps * 1000, framerate_fps),
+      framerate_fps);
   RTC_DCHECK_GE(set_rates_result, 0)
-      << "Failed to update encoder with new rate " << bit_rate;
-  num_dropped_frames_ = 0;
-  num_spatial_resizes_ = 0;
+      << "Failed to update encoder with new rate " << bitrate_kbps << ".";
+  ++rate_update_index_;
+  num_dropped_frames_.push_back(0);
+  num_spatial_resizes_.push_back(0);
 }
 
-size_t VideoProcessor::EncodedFrameSize(int frame_number) {
-  RTC_DCHECK_LT(frame_number, frame_infos_.size());
-  return frame_infos_[frame_number].encoded_frame_size;
+int VideoProcessor::NumFramesDecoded() const {
+  RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
+  return num_frames_decoded_;
 }
 
-FrameType VideoProcessor::EncodedFrameType(int frame_number) {
-  RTC_DCHECK_LT(frame_number, frame_infos_.size());
-  return frame_infos_[frame_number].encoded_frame_type;
-}
-
-int VideoProcessor::GetQpFromEncoder(int frame_number) {
-  RTC_DCHECK_LT(frame_number, frame_infos_.size());
+int VideoProcessor::GetQpFromEncoder(int frame_number) const {
+  RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
+  RTC_CHECK_LT(frame_number, frame_infos_.size());
   return frame_infos_[frame_number].qp_encoder;
 }
 
-int VideoProcessor::GetQpFromBitstream(int frame_number) {
-  RTC_DCHECK_LT(frame_number, frame_infos_.size());
+int VideoProcessor::GetQpFromBitstream(int frame_number) const {
+  RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
+  RTC_CHECK_LT(frame_number, frame_infos_.size());
   return frame_infos_[frame_number].qp_bitstream;
 }
 
-int VideoProcessor::NumberDroppedFrames() {
+std::vector<int> VideoProcessor::NumberDroppedFramesPerRateUpdate() const {
+  RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
   return num_dropped_frames_;
 }
 
-int VideoProcessor::NumberSpatialResizes() {
+std::vector<int> VideoProcessor::NumberSpatialResizesPerRateUpdate() const {
+  RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
   return num_spatial_resizes_;
 }
 
 void VideoProcessor::FrameEncoded(
     webrtc::VideoCodecType codec,
     const EncodedImage& encoded_image,
     const webrtc::RTPFragmentationHeader* fragmentation) {
+  RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
+
   // For the highest measurement accuracy of the encode time, the start/stop
   // time recordings should wrap the Encode call as tightly as possible.
   int64_t encode_stop_ns = rtc::TimeNanos();
 
   if (encoded_frame_writer_) {
     RTC_CHECK(encoded_frame_writer_->WriteFrame(encoded_image, codec));
   }
 
   // Timestamp is proportional to frame number, so this gives us number of
   // dropped frames.
   int frame_number = TimestampToFrameNumber(encoded_image._timeStamp);
   bool last_frame_missing = false;
   if (frame_number > 0) {
     RTC_DCHECK_GE(last_encoded_frame_num_, 0);
     int num_dropped_from_last_encode =
         frame_number - last_encoded_frame_num_ - 1;
     RTC_DCHECK_GE(num_dropped_from_last_encode, 0);
-    num_dropped_frames_ += num_dropped_from_last_encode;
+    RTC_CHECK_GE(rate_update_index_, 0);
+    num_dropped_frames_[rate_update_index_] += num_dropped_from_last_encode;
     if (num_dropped_from_last_encode > 0) {
       // For dropped frames, we write out the last decoded frame to avoid
       // getting out of sync for the computation of PSNR and SSIM.
       for (int i = 0; i < num_dropped_from_last_encode; i++) {
         RTC_DCHECK_EQ(last_decoded_frame_buffer_.size(),
                       analysis_frame_writer_->FrameLength());
         RTC_CHECK(analysis_frame_writer_->WriteFrame(
             last_decoded_frame_buffer_.data()));
         if (decoded_frame_writer_) {
           RTC_DCHECK_EQ(last_decoded_frame_buffer_.size(),
                         decoded_frame_writer_->FrameLength());
           RTC_CHECK(decoded_frame_writer_->WriteFrame(
               last_decoded_frame_buffer_.data()));
         }
       }
     }
 
     last_frame_missing =
         (frame_infos_[last_encoded_frame_num_].manipulated_length == 0);
   }
   // Ensure strict monotonicity.
   RTC_CHECK_GT(frame_number, last_encoded_frame_num_);
   last_encoded_frame_num_ = frame_number;
 
   // Frame is not dropped, so update frame information and statistics.
-  RTC_DCHECK_LT(frame_number, frame_infos_.size());
+  RTC_CHECK_LT(frame_number, frame_infos_.size());
   FrameInfo* frame_info = &frame_infos_[frame_number];
-  frame_info->encoded_frame_size = encoded_image._length;
-  frame_info->encoded_frame_type = encoded_image._frameType;
   frame_info->qp_encoder = encoded_image.qp_;
   if (codec == kVideoCodecVP8) {
     vp8::GetQp(encoded_image._buffer, encoded_image._length,
       &frame_info->qp_bitstream);
   } else if (codec == kVideoCodecVP9) {
     vp9::GetQp(encoded_image._buffer, encoded_image._length,
       &frame_info->qp_bitstream);
   }
   FrameStatistic* frame_stat = &stats_->stats_[frame_number];
   frame_stat->encode_time_in_us =
@@ skipping 66 matching lines @@
     if (decoded_frame_writer_) {
       RTC_DCHECK_EQ(last_decoded_frame_buffer_.size(),
                     decoded_frame_writer_->FrameLength());
       RTC_CHECK(
           decoded_frame_writer_->WriteFrame(last_decoded_frame_buffer_.data()));
     }
   }
 }
 
 void VideoProcessor::FrameDecoded(const VideoFrame& image) {
+  RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
+
   // For the highest measurement accuracy of the decode time, the start/stop
   // time recordings should wrap the Decode call as tightly as possible.
   int64_t decode_stop_ns = rtc::TimeNanos();
 
   // Update frame information and statistics.
   int frame_number = TimestampToFrameNumber(image.timestamp());
   RTC_DCHECK_LT(frame_number, frame_infos_.size());
   FrameInfo* frame_info = &frame_infos_[frame_number];
   frame_info->decoded_width = image.width();
   frame_info->decoded_height = image.height();
   FrameStatistic* frame_stat = &stats_->stats_[frame_number];
   frame_stat->decode_time_in_us =
       GetElapsedTimeMicroseconds(frame_info->decode_start_ns, decode_stop_ns);
   frame_stat->decoding_successful = true;
 
   // Check if the codecs have resized the frame since previously decoded frame.
   if (frame_number > 0) {
     RTC_DCHECK_GE(last_decoded_frame_num_, 0);
     const FrameInfo& last_decoded_frame_info =
         frame_infos_[last_decoded_frame_num_];
     if (static_cast<int>(image.width()) !=
             last_decoded_frame_info.decoded_width ||
         static_cast<int>(image.height()) !=
             last_decoded_frame_info.decoded_height) {
-      ++num_spatial_resizes_;
+      RTC_CHECK_GE(rate_update_index_, 0);
+      ++num_spatial_resizes_[rate_update_index_];
     }
   }
   // Ensure strict monotonicity.
   RTC_CHECK_GT(frame_number, last_decoded_frame_num_);
   last_decoded_frame_num_ = frame_number;
 
   // Check if codec size is different from the original size, and if so,
   // scale back to original size. This is needed for the PSNR and SSIM
   // calculations.
   size_t extracted_length;
@@ skipping 20 matching lines @@
   }
 
   RTC_DCHECK_EQ(extracted_length, analysis_frame_writer_->FrameLength());
   RTC_CHECK(analysis_frame_writer_->WriteFrame(extracted_buffer.data()));
   if (decoded_frame_writer_) {
     RTC_DCHECK_EQ(extracted_length, decoded_frame_writer_->FrameLength());
     RTC_CHECK(decoded_frame_writer_->WriteFrame(extracted_buffer.data()));
   }
 
   last_decoded_frame_buffer_ = std::move(extracted_buffer);
+
+  ++num_frames_decoded_;
 }
 
-uint32_t VideoProcessor::FrameNumberToTimestamp(int frame_number) {
+uint32_t VideoProcessor::FrameNumberToTimestamp(int frame_number) const {
+  RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
+
   RTC_DCHECK_GE(frame_number, 0);
   const int ticks_per_frame =
       kRtpClockRateHz / config_.codec_settings.maxFramerate;
   return (frame_number + 1) * ticks_per_frame;
 }
 
-int VideoProcessor::TimestampToFrameNumber(uint32_t timestamp) {
+int VideoProcessor::TimestampToFrameNumber(uint32_t timestamp) const {
+  RTC_DCHECK_CALLED_SEQUENTIALLY(&sequence_checker_);
+
   RTC_DCHECK_GT(timestamp, 0);
   const int ticks_per_frame =
       kRtpClockRateHz / config_.codec_settings.maxFramerate;
   RTC_DCHECK_EQ(timestamp % ticks_per_frame, 0);
   return (timestamp / ticks_per_frame) - 1;
 }
 
 }  // namespace test
 }  // namespace webrtc