OLD | NEW |
1 /* | 1 /* |
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. | 2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license | 4 * Use of this source code is governed by a BSD-style license |
5 * that can be found in the LICENSE file in the root of the source | 5 * that can be found in the LICENSE file in the root of the source |
6 * tree. An additional intellectual property rights grant can be found | 6 * tree. An additional intellectual property rights grant can be found |
7 * in the file PATENTS. All contributing project authors may | 7 * in the file PATENTS. All contributing project authors may |
8 * be found in the AUTHORS file in the root of the source tree. | 8 * be found in the AUTHORS file in the root of the source tree. |
9 */ | 9 */ |
10 | 10 |
11 #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_ | 11 #ifndef WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_ |
12 #define WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_ | 12 #define WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_ |
13 | 13 |
14 #include <memory> | 14 #include <memory> |
15 #include <string> | 15 #include <string> |
16 #include <vector> | 16 #include <vector> |
17 | 17 |
18 #include "webrtc/api/video/video_frame.h" | 18 #include "webrtc/api/video/video_frame.h" |
19 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" | 19 #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" |
20 #include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h" | 20 #include "webrtc/modules/video_coding/codecs/test/packet_manipulator.h" |
21 #include "webrtc/modules/video_coding/codecs/test/stats.h" | 21 #include "webrtc/modules/video_coding/codecs/test/stats.h" |
22 #include "webrtc/modules/video_coding/include/video_codec_interface.h" | 22 #include "webrtc/modules/video_coding/include/video_codec_interface.h" |
23 #include "webrtc/modules/video_coding/utility/ivf_file_writer.h" | 23 #include "webrtc/modules/video_coding/utility/ivf_file_writer.h" |
24 #include "webrtc/modules/video_coding/utility/vp8_header_parser.h" | 24 #include "webrtc/modules/video_coding/utility/vp8_header_parser.h" |
25 #include "webrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.h" | 25 #include "webrtc/modules/video_coding/utility/vp9_uncompressed_header_parser.h" |
26 #include "webrtc/rtc_base/buffer.h" | 26 #include "webrtc/rtc_base/buffer.h" |
27 #include "webrtc/rtc_base/checks.h" | 27 #include "webrtc/rtc_base/checks.h" |
| 28 #include "webrtc/rtc_base/constructormagic.h" |
| 29 #include "webrtc/rtc_base/sequenced_task_checker.h" |
| 30 #include "webrtc/rtc_base/task_queue.h" |
28 #include "webrtc/test/testsupport/frame_reader.h" | 31 #include "webrtc/test/testsupport/frame_reader.h" |
29 #include "webrtc/test/testsupport/frame_writer.h" | 32 #include "webrtc/test/testsupport/frame_writer.h" |
30 | 33 |
31 namespace webrtc { | 34 namespace webrtc { |
32 | 35 |
33 class VideoBitrateAllocator; | 36 class VideoBitrateAllocator; |
34 | 37 |
35 namespace test { | 38 namespace test { |
36 | 39 |
37 // Defines which frame types shall be excluded from packet loss and when. | 40 // Defines which frame types shall be excluded from packet loss and when. |
(...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
103 // The codec settings to use for the test (target bitrate, video size, | 106 // The codec settings to use for the test (target bitrate, video size, |
104 // framerate and so on). This struct should be filled in using the | 107 // framerate and so on). This struct should be filled in using the |
105 // VideoCodingModule::Codec() method. | 108 // VideoCodingModule::Codec() method. |
106 webrtc::VideoCodec codec_settings; | 109 webrtc::VideoCodec codec_settings; |
107 | 110 |
108 // If printing of information to stdout shall be performed during processing. | 111 // If printing of information to stdout shall be performed during processing. |
109 bool verbose = true; | 112 bool verbose = true; |
110 | 113 |
111 // If HW or SW codec should be used. | 114 // If HW or SW codec should be used. |
112 bool hw_codec = false; | 115 bool hw_codec = false; |
113 | |
114 // In batch mode, the VideoProcessor is fed all the frames for processing | |
115 // before any metrics are calculated. This is useful for pipelining HW codecs, | |
116 // for which some calculated metrics otherwise would be incorrect. The | |
117 // downside with batch mode is that mid-test rate allocation is not supported. | |
118 bool batch_mode = false; | |
119 }; | 116 }; |
120 | 117 |
121 // Handles encoding/decoding of video using the VideoEncoder/VideoDecoder | 118 // Handles encoding/decoding of video using the VideoEncoder/VideoDecoder |
122 // interfaces. This is done in a sequential manner in order to be able to | 119 // interfaces. This is done in a sequential manner in order to be able to |
123 // measure times properly. | 120 // measure times properly. |
124 // The class processes a frame at the time for the configured input file. | 121 // The class processes a frame at the time for the configured input file. |
125 // It maintains state of where in the source input file the processing is at. | 122 // It maintains state of where in the source input file the processing is at. |
126 // | 123 // |
127 // Regarding packet loss: Note that keyframes are excluded (first or all | 124 // Regarding packet loss: Note that keyframes are excluded (first or all |
128 // depending on the ExcludeFrameTypes setting). This is because if key frames | 125 // depending on the ExcludeFrameTypes setting). This is because if key frames |
(...skipping 19 matching lines...) Expand all Loading... |
148 IvfFileWriter* encoded_frame_writer, | 145 IvfFileWriter* encoded_frame_writer, |
149 FrameWriter* decoded_frame_writer); | 146 FrameWriter* decoded_frame_writer); |
150 ~VideoProcessor(); | 147 ~VideoProcessor(); |
151 | 148 |
152 // Sets up callbacks and initializes the encoder and decoder. | 149 // Sets up callbacks and initializes the encoder and decoder. |
153 void Init(); | 150 void Init(); |
154 | 151 |
155 // Tears down callbacks and releases the encoder and decoder. | 152 // Tears down callbacks and releases the encoder and decoder. |
156 void Release(); | 153 void Release(); |
157 | 154 |
158 // Processes a single frame. Returns true as long as there's more frames | 155 // Processes a single frame. The frames must be processed in order, and the |
159 // available in the source clip. | 156 // VideoProcessor must be initialized first. |
160 // |frame_number| must be an integer >= 0. | 157 void ProcessFrame(int frame_number); |
161 bool ProcessFrame(int frame_number); | |
162 | 158 |
163 // Updates the encoder with the target |bit_rate| and the |frame_rate|. | 159 // Updates the encoder with target rates. Must be called at least once. |
164 void SetRates(int bit_rate, int frame_rate); | 160 void SetRates(int bitrate_kbps, int framerate_fps); |
165 | 161 |
166 // Return the size of the encoded frame in bytes. Dropped frames by the | 162 // Returns the number of frames that have been decoded. |
167 // encoder are regarded as zero size. | 163 int NumFramesDecoded() const; |
168 size_t EncodedFrameSize(int frame_number); | |
169 | 164 |
170 // Return the encoded frame type (key or delta). | 165 // TODO(brandtr): Get rid of these functions by moving the corresponding QP |
171 FrameType EncodedFrameType(int frame_number); | 166 // fields to the Stats object. |
| 167 int GetQpFromEncoder(int frame_number) const; |
| 168 int GetQpFromBitstream(int frame_number) const; |
172 | 169 |
173 // Return the qp used by encoder. | 170 // Returns the number of dropped frames. |
174 int GetQpFromEncoder(int frame_number); | 171 std::vector<int> NumberDroppedFramesPerRateUpdate() const; |
175 | 172 |
176 // Return the qp from the qp parser. | 173 // Returns the number of spatial resizes. |
177 int GetQpFromBitstream(int frame_number); | 174 std::vector<int> NumberSpatialResizesPerRateUpdate() const; |
178 | |
179 // Return the number of dropped frames. | |
180 int NumberDroppedFrames(); | |
181 | |
182 // Return the number of spatial resizes. | |
183 int NumberSpatialResizes(); | |
184 | 175 |
185 private: | 176 private: |
186 // Container that holds per-frame information that needs to be stored between | 177 // Container that holds per-frame information that needs to be stored between |
187 // calls to Encode and Decode, as well as the corresponding callbacks. It is | 178 // calls to Encode and Decode, as well as the corresponding callbacks. It is |
188 // not directly used for statistics -- for that, test::FrameStatistic is used. | 179 // not directly used for statistics -- for that, test::FrameStatistic is used. |
| 180 // TODO(brandtr): Get rid of this struct and use the Stats class instead. |
189 struct FrameInfo { | 181 struct FrameInfo { |
190 FrameInfo() | 182 int64_t encode_start_ns = 0; |
191 : timestamp(0), | 183 int64_t decode_start_ns = 0; |
192 encode_start_ns(0), | 184 int qp_encoder = 0; |
193 decode_start_ns(0), | 185 int qp_bitstream = 0; |
194 encoded_frame_size(0), | 186 int decoded_width = 0; |
195 encoded_frame_type(kVideoFrameDelta), | 187 int decoded_height = 0; |
196 decoded_width(0), | 188 size_t manipulated_length = 0; |
197 decoded_height(0), | |
198 manipulated_length(0), | |
199 qp_encoder(0), | |
200 qp_bitstream(0) {} | |
201 | |
202 uint32_t timestamp; | |
203 int64_t encode_start_ns; | |
204 int64_t decode_start_ns; | |
205 size_t encoded_frame_size; | |
206 FrameType encoded_frame_type; | |
207 int decoded_width; | |
208 int decoded_height; | |
209 size_t manipulated_length; | |
210 int qp_encoder; | |
211 int qp_bitstream; | |
212 }; | 189 }; |
213 | 190 |
214 // Callback class required to implement according to the VideoEncoder API. | |
215 class VideoProcessorEncodeCompleteCallback | 191 class VideoProcessorEncodeCompleteCallback |
216 : public webrtc::EncodedImageCallback { | 192 : public webrtc::EncodedImageCallback { |
217 public: | 193 public: |
218 explicit VideoProcessorEncodeCompleteCallback( | 194 explicit VideoProcessorEncodeCompleteCallback( |
219 VideoProcessor* video_processor) | 195 VideoProcessor* video_processor) |
220 : video_processor_(video_processor) {} | 196 : video_processor_(video_processor), |
| 197 task_queue_(rtc::TaskQueue::Current()) {} |
| 198 |
221 Result OnEncodedImage( | 199 Result OnEncodedImage( |
222 const webrtc::EncodedImage& encoded_image, | 200 const webrtc::EncodedImage& encoded_image, |
223 const webrtc::CodecSpecificInfo* codec_specific_info, | 201 const webrtc::CodecSpecificInfo* codec_specific_info, |
224 const webrtc::RTPFragmentationHeader* fragmentation) override { | 202 const webrtc::RTPFragmentationHeader* fragmentation) override { |
225 // Forward to parent class. | |
226 RTC_CHECK(codec_specific_info); | 203 RTC_CHECK(codec_specific_info); |
| 204 |
| 205 if (task_queue_ && !task_queue_->IsCurrent()) { |
| 206 task_queue_->PostTask(std::unique_ptr<rtc::QueuedTask>( |
| 207 new EncodeCallbackTask(video_processor_, encoded_image, |
| 208 codec_specific_info, fragmentation))); |
| 209 return Result(Result::OK, 0); |
| 210 } |
| 211 |
227 video_processor_->FrameEncoded(codec_specific_info->codecType, | 212 video_processor_->FrameEncoded(codec_specific_info->codecType, |
228 encoded_image, fragmentation); | 213 encoded_image, fragmentation); |
229 return Result(Result::OK, 0); | 214 return Result(Result::OK, 0); |
230 } | 215 } |
231 | 216 |
232 private: | 217 private: |
| 218 class EncodeCallbackTask : public rtc::QueuedTask { |
| 219 public: |
| 220 EncodeCallbackTask(VideoProcessor* video_processor, |
| 221 const webrtc::EncodedImage& encoded_image, |
| 222 const webrtc::CodecSpecificInfo* codec_specific_info, |
| 223 const webrtc::RTPFragmentationHeader* fragmentation) |
| 224 : video_processor_(video_processor), |
| 225 buffer_(encoded_image._buffer, encoded_image._length), |
| 226 encoded_image_(encoded_image), |
| 227 codec_specific_info_(*codec_specific_info) { |
| 228 encoded_image_._buffer = buffer_.data(); |
| 229 RTC_CHECK(fragmentation); |
| 230 fragmentation_.CopyFrom(*fragmentation); |
| 231 } |
| 232 |
| 233 bool Run() override { |
| 234 video_processor_->FrameEncoded(codec_specific_info_.codecType, |
| 235 encoded_image_, &fragmentation_); |
| 236 return true; |
| 237 } |
| 238 |
| 239 private: |
| 240 VideoProcessor* const video_processor_; |
| 241 rtc::Buffer buffer_; |
| 242 webrtc::EncodedImage encoded_image_; |
| 243 const webrtc::CodecSpecificInfo codec_specific_info_; |
| 244 webrtc::RTPFragmentationHeader fragmentation_; |
| 245 }; |
| 246 |
233 VideoProcessor* const video_processor_; | 247 VideoProcessor* const video_processor_; |
| 248 rtc::TaskQueue* const task_queue_; |
234 }; | 249 }; |
235 | 250 |
236 // Callback class required to implement according to the VideoDecoder API. | |
237 class VideoProcessorDecodeCompleteCallback | 251 class VideoProcessorDecodeCompleteCallback |
238 : public webrtc::DecodedImageCallback { | 252 : public webrtc::DecodedImageCallback { |
239 public: | 253 public: |
240 explicit VideoProcessorDecodeCompleteCallback( | 254 explicit VideoProcessorDecodeCompleteCallback( |
241 VideoProcessor* video_processor) | 255 VideoProcessor* video_processor) |
242 : video_processor_(video_processor) {} | 256 : video_processor_(video_processor), |
| 257 task_queue_(rtc::TaskQueue::Current()) {} |
| 258 |
243 int32_t Decoded(webrtc::VideoFrame& image) override { | 259 int32_t Decoded(webrtc::VideoFrame& image) override { |
244 // Forward to parent class. | 260 if (task_queue_ && !task_queue_->IsCurrent()) { |
| 261 task_queue_->PostTask( |
| 262 [this, image]() { video_processor_->FrameDecoded(image); }); |
| 263 return 0; |
| 264 } |
| 265 |
245 video_processor_->FrameDecoded(image); | 266 video_processor_->FrameDecoded(image); |
246 return 0; | 267 return 0; |
247 } | 268 } |
| 269 |
248 int32_t Decoded(webrtc::VideoFrame& image, | 270 int32_t Decoded(webrtc::VideoFrame& image, |
249 int64_t decode_time_ms) override { | 271 int64_t decode_time_ms) override { |
250 return Decoded(image); | 272 return Decoded(image); |
251 } | 273 } |
| 274 |
252 void Decoded(webrtc::VideoFrame& image, | 275 void Decoded(webrtc::VideoFrame& image, |
253 rtc::Optional<int32_t> decode_time_ms, | 276 rtc::Optional<int32_t> decode_time_ms, |
254 rtc::Optional<uint8_t> qp) override { | 277 rtc::Optional<uint8_t> qp) override { |
255 Decoded(image); | 278 Decoded(image); |
256 } | 279 } |
257 | 280 |
258 private: | 281 private: |
259 VideoProcessor* const video_processor_; | 282 VideoProcessor* const video_processor_; |
| 283 rtc::TaskQueue* const task_queue_; |
260 }; | 284 }; |
261 | 285 |
262 // Invoked by the callback when a frame has completed encoding. | 286 // Invoked by the callback adapter when a frame has completed encoding. |
263 void FrameEncoded(webrtc::VideoCodecType codec, | 287 void FrameEncoded(webrtc::VideoCodecType codec, |
264 const webrtc::EncodedImage& encodedImage, | 288 const webrtc::EncodedImage& encodedImage, |
265 const webrtc::RTPFragmentationHeader* fragmentation); | 289 const webrtc::RTPFragmentationHeader* fragmentation); |
266 | 290 |
267 // Invoked by the callback when a frame has completed decoding. | 291 // Invoked by the callback adapter when a frame has completed decoding. |
268 void FrameDecoded(const webrtc::VideoFrame& image); | 292 void FrameDecoded(const webrtc::VideoFrame& image); |
269 | 293 |
270 // Use the frame number as the basis for timestamp to identify frames. Let the | 294 // Use the frame number as the basis for timestamp to identify frames. Let the |
271 // first timestamp be non-zero, to not make the IvfFileWriter believe that we | 295 // first timestamp be non-zero, to not make the IvfFileWriter believe that we |
272 // want to use capture timestamps in the IVF files. | 296 // want to use capture timestamps in the IVF files. |
273 uint32_t FrameNumberToTimestamp(int frame_number); | 297 uint32_t FrameNumberToTimestamp(int frame_number) const; |
274 int TimestampToFrameNumber(uint32_t timestamp); | 298 int TimestampToFrameNumber(uint32_t timestamp) const; |
275 | 299 |
276 TestConfig config_; | 300 bool initialized_ GUARDED_BY(sequence_checker_); |
| 301 |
| 302 TestConfig config_ GUARDED_BY(sequence_checker_); |
277 | 303 |
278 webrtc::VideoEncoder* const encoder_; | 304 webrtc::VideoEncoder* const encoder_; |
279 webrtc::VideoDecoder* const decoder_; | 305 webrtc::VideoDecoder* const decoder_; |
280 const std::unique_ptr<VideoBitrateAllocator> bitrate_allocator_; | 306 const std::unique_ptr<VideoBitrateAllocator> bitrate_allocator_; |
281 | 307 |
282 // Adapters for the codec callbacks. | 308 // Adapters for the codec callbacks. |
283 const std::unique_ptr<EncodedImageCallback> encode_callback_; | 309 VideoProcessorEncodeCompleteCallback encode_callback_; |
284 const std::unique_ptr<DecodedImageCallback> decode_callback_; | 310 VideoProcessorDecodeCompleteCallback decode_callback_; |
| 311 int num_frames_decoded_ GUARDED_BY(sequence_checker_); |
285 | 312 |
286 // Fake network. | 313 // Fake network. |
287 PacketManipulator* const packet_manipulator_; | 314 PacketManipulator* const packet_manipulator_; |
288 | 315 |
289 // These (mandatory) file manipulators are used for, e.g., objective PSNR and | 316 // These (mandatory) file manipulators are used for, e.g., objective PSNR and |
290 // SSIM calculations at the end of a test run. | 317 // SSIM calculations at the end of a test run. |
291 FrameReader* const analysis_frame_reader_; | 318 FrameReader* const analysis_frame_reader_; |
292 FrameWriter* const analysis_frame_writer_; | 319 FrameWriter* const analysis_frame_writer_; |
293 | 320 |
294 // These (optional) file writers are used to persistently store the encoded | 321 // These (optional) file writers are used to persistently store the encoded |
295 // and decoded bitstreams. The purpose is to give the experimenter an option | 322 // and decoded bitstreams. The purpose is to give the experimenter an option |
296 // to subjectively evaluate the quality of the processing. Each frame writer | 323 // to subjectively evaluate the quality of the processing. Each frame writer |
297 // is enabled by being non-null. | 324 // is enabled by being non-null. |
298 IvfFileWriter* const encoded_frame_writer_; | 325 IvfFileWriter* const encoded_frame_writer_; |
299 FrameWriter* const decoded_frame_writer_; | 326 FrameWriter* const decoded_frame_writer_; |
300 | 327 |
301 bool initialized_; | |
302 | |
303 // Frame metadata for all frames that have been added through a call to | 328 // Frame metadata for all frames that have been added through a call to |
304 // ProcessFrames(). We need to store this metadata over the course of the | 329 // ProcessFrames(). We need to store this metadata over the course of the |
305 // test run, to support pipelining HW codecs. | 330 // test run, to support pipelining HW codecs. |
306 std::vector<FrameInfo> frame_infos_; | 331 std::vector<FrameInfo> frame_infos_ GUARDED_BY(sequence_checker_); |
307 int last_encoded_frame_num_; | 332 int last_encoded_frame_num_ GUARDED_BY(sequence_checker_); |
308 int last_decoded_frame_num_; | 333 int last_decoded_frame_num_ GUARDED_BY(sequence_checker_); |
309 | 334 |
310 // Keep track of if we have excluded the first key frame from packet loss. | 335 // Keep track of if we have excluded the first key frame from packet loss. |
311 bool first_key_frame_has_been_excluded_; | 336 bool first_key_frame_has_been_excluded_ GUARDED_BY(sequence_checker_); |
312 | 337 |
313 // Keep track of the last successfully decoded frame, since we write that | 338 // Keep track of the last successfully decoded frame, since we write that |
314 // frame to disk when decoding fails. | 339 // frame to disk when decoding fails. |
315 rtc::Buffer last_decoded_frame_buffer_; | 340 rtc::Buffer last_decoded_frame_buffer_ GUARDED_BY(sequence_checker_); |
316 | 341 |
317 // Statistics. | 342 // Statistics. |
318 Stats* stats_; | 343 Stats* stats_; |
319 int num_dropped_frames_; | 344 std::vector<int> num_dropped_frames_ GUARDED_BY(sequence_checker_); |
320 int num_spatial_resizes_; | 345 std::vector<int> num_spatial_resizes_ GUARDED_BY(sequence_checker_); |
| 346 int rate_update_index_ GUARDED_BY(sequence_checker_); |
| 347 |
| 348 rtc::SequencedTaskChecker sequence_checker_; |
| 349 |
| 350 RTC_DISALLOW_COPY_AND_ASSIGN(VideoProcessor); |
321 }; | 351 }; |
322 | 352 |
323 } // namespace test | 353 } // namespace test |
324 } // namespace webrtc | 354 } // namespace webrtc |
325 | 355 |
326 #endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_ | 356 #endif // WEBRTC_MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_ |
OLD | NEW |