VEA unit test: Add timestamp test

media/gpu/video_encode_accelerator_unittest.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <inttypes.h>
 #include <stddef.h>
 #include <stdint.h>
 
 #include <algorithm>
 #include <memory>
@@ skipping 774 matching lines @@
 class VEAClient : public VideoEncodeAccelerator::Client {
  public:
   VEAClient(TestStream* test_stream,
             ClientStateNotification<ClientState>* note,
             bool save_to_file,
             unsigned int keyframe_period,
             bool force_bitrate,
             bool test_perf,
             bool mid_stream_bitrate_switch,
             bool mid_stream_framerate_switch,
-            bool verify_output);
+            bool verify_output,
+            bool verify_output_timestamp);
   ~VEAClient() override;
   void CreateEncoder();
   void DestroyEncoder();
 
   // VideoDecodeAccelerator::Client implementation.
   void RequireBitstreamBuffers(unsigned int input_count,
                                const gfx::Size& input_coded_size,
                                size_t output_buffer_size) override;
   void BitstreamBufferReady(int32_t bitstream_buffer_id,
                             size_t payload_size,
@@ skipping 66 matching lines @@
 
   // Callback function of the |input_timer_|.
   void OnInputTimer();
 
   // Called when the quality validator has decoded all the frames.
   void DecodeCompleted();
 
   // Called when the quality validator fails to decode a frame.
   void DecodeFailed();
 
+  // Verify that the output timestamp matches input timestamp.
+  void VerifyOutputTimestamp(base::TimeDelta timestamp);
+
   ClientState state_;
   std::unique_ptr<VideoEncodeAccelerator> encoder_;
 
   TestStream* test_stream_;
 
   // Used to notify another thread about the state. VEAClient does not own this.
   ClientStateNotification<ClientState>* note_;
 
   // Ids assigned to VideoFrames.
   std::set<int32_t> inputs_at_client_;
@@ skipping 57 matching lines @@
   // Byte size of the encoded stream (for bitrate calculation) since last
   // time we checked bitrate.
   size_t encoded_stream_size_since_last_check_;
 
   // If true, verify performance at the end of the test.
   bool test_perf_;
 
   // Check the output frame quality of the encoder.
   bool verify_output_;
 
+  // Check whether the output timestamps match input timestamps.
+  bool verify_output_timestamp_;
+
   // Used to perform codec-specific sanity checks on the stream.
   std::unique_ptr<StreamValidator> stream_validator_;
 
   // Used to validate the encoded frame quality.
   std::unique_ptr<VideoFrameQualityValidator> quality_validator_;
 
   // The time when the first frame is submitted for encode.
   base::TimeTicks first_frame_start_time_;
 
   // The time when the last encoded frame is ready.
   base::TimeTicks last_frame_ready_time_;
 
   // All methods of this class should be run on the same thread.
   base::ThreadChecker thread_checker_;
 
   // Requested bitrate in bits per second.
   unsigned int requested_bitrate_;
 
   // Requested initial framerate.
   unsigned int requested_framerate_;
 
   // Bitrate to switch to in the middle of the stream.
   unsigned int requested_subsequent_bitrate_;
 
   // Framerate to switch to in the middle of the stream.
   unsigned int requested_subsequent_framerate_;
 
   // The timer used to feed the encoder with the input frames.
   std::unique_ptr<base::RepeatingTimer> input_timer_;
+
+  // The timestamps for each frame in the order of CreateFrame() invocation.
+  std::queue<base::TimeDelta> frame_timestamps_;
+
+  // The last timestamp popped from |frame_timestamps_|.
+  base::TimeDelta previous_timestamp_;
 };
 
 VEAClient::VEAClient(TestStream* test_stream,
                      ClientStateNotification<ClientState>* note,
                      bool save_to_file,
                      unsigned int keyframe_period,
                      bool force_bitrate,
                      bool test_perf,
                      bool mid_stream_bitrate_switch,
                      bool mid_stream_framerate_switch,
-                     bool verify_output)
+                     bool verify_output,
+                     bool verify_output_timestamp)
     : state_(CS_CREATED),
       test_stream_(test_stream),
       note_(note),
       next_input_id_(0),
       next_output_buffer_id_(0),
       pos_in_input_stream_(0),
       num_required_input_buffers_(0),
       output_buffer_size_(0),
       num_frames_to_encode_(0),
       num_encoded_frames_(0),
       num_frames_since_last_check_(0),
       seen_keyframe_in_this_buffer_(false),
       save_to_file_(save_to_file),
       keyframe_period_(keyframe_period),
       num_keyframes_requested_(0),
       next_keyframe_at_(0),
       force_bitrate_(force_bitrate),
       current_requested_bitrate_(0),
       current_framerate_(0),
       encoded_stream_size_since_last_check_(0),
       test_perf_(test_perf),
       verify_output_(verify_output),
+      verify_output_timestamp_(verify_output_timestamp),
       requested_bitrate_(0),
       requested_framerate_(0),
       requested_subsequent_bitrate_(0),
       requested_subsequent_framerate_(0) {
   if (keyframe_period_)
     LOG_ASSERT(kMaxKeyframeDelay < keyframe_period_);
 
   // Fake encoder produces an invalid stream, so skip validating it.
   if (!g_fake_encoder) {
     stream_validator_ = StreamValidator::Create(
@@ skipping 206 matching lines @@
     input_timer_->Start(
         FROM_HERE, base::TimeDelta::FromSeconds(1) / current_framerate_,
         base::Bind(&VEAClient::OnInputTimer, base::Unretained(this)));
   } else {
     while (inputs_at_client_.size() <
            num_required_input_buffers_ + kNumExtraInputFrames)
       FeedEncoderWithOneInput();
   }
 }
 
+void VEAClient::VerifyOutputTimestamp(base::TimeDelta timestamp) {
+  // One input frame may be mapped to multiple output frames, so the current
+  // timestamp should be equal to previous timestamp or the top of
+  // frame_timestamps_.
+  if (num_encoded_frames_ == 0 || timestamp != previous_timestamp_) {

[wuchengli, 2016/07/12 09:28:06]

This failed on oak for H264. Did you test any device with H264? The first input
buffer produces two outputs buffers. First output buffer is SPS, so
ProcessStreamBuffer does not call HandleEncodedFrame and num_encoded_frames_ is
not increased. The timestamps are like this:
num_encoded_frames_=0, timetsamp=0
num_encoded_frames_=0, timetsamp=0
num_encoded_frames_=1, timetsamp=33000

You can start the timestamp of first buffer at 33000. So you can do this here:
if (timestamp != previous_timestamp_) {
}

[shenghao1, 2016/07/12 11:05:51]

Done.

+    ASSERT_TRUE(!frame_timestamps_.empty());
+    EXPECT_EQ(frame_timestamps_.front(), timestamp);
+    previous_timestamp_ = frame_timestamps_.front();
+    frame_timestamps_.pop();
+  }
+}
+
 void VEAClient::BitstreamBufferReady(int32_t bitstream_buffer_id,
                                      size_t payload_size,
                                      bool key_frame,
                                      base::TimeDelta timestamp) {
   DCHECK(thread_checker_.CalledOnValidThread());
   ASSERT_LE(payload_size, output_buffer_size_);
 
   IdToSHM::iterator it = output_buffers_at_client_.find(bitstream_buffer_id);
   ASSERT_NE(it, output_buffers_at_client_.end());
   base::SharedMemory* shm = it->second;
   output_buffers_at_client_.erase(it);
 
   if (state_ == CS_FINISHED || state_ == CS_VALIDATED)
     return;
 
+  if (verify_output_timestamp_) {
+    VerifyOutputTimestamp(timestamp);
+  }
+
   encoded_stream_size_since_last_check_ += payload_size;
 
   const uint8_t* stream_ptr = static_cast<const uint8_t*>(shm->memory());
   if (payload_size > 0) {
     if (stream_validator_) {
       stream_validator_->ProcessStreamBuffer(stream_ptr, payload_size);
     } else {
       HandleEncodedFrame(key_frame);
     }
 
@@ skipping 61 matching lines @@
       position;
   uint8_t* frame_data_u = frame_data_y + test_stream_->aligned_plane_size[0];
   uint8_t* frame_data_v = frame_data_u + test_stream_->aligned_plane_size[1];
   CHECK_GT(current_framerate_, 0U);
 
   scoped_refptr<VideoFrame> video_frame = VideoFrame::WrapExternalYuvData(
       kInputFormat, input_coded_size_, gfx::Rect(test_stream_->visible_size),
       test_stream_->visible_size, input_coded_size_.width(),
       input_coded_size_.width() / 2, input_coded_size_.width() / 2,
       frame_data_y, frame_data_u, frame_data_v,
       base::TimeDelta().FromMilliseconds(next_input_id_ *

[wuchengli, 2016/07/12 09:28:07]

(next_input_id_ + 1)
See the another comment.

[shenghao1, 2016/07/12 11:05:51]

Done.

                                          base::Time::kMillisecondsPerSecond /
                                          current_framerate_));
   EXPECT_NE(nullptr, video_frame.get());
   return video_frame;
 }
 
 scoped_refptr<VideoFrame> VEAClient::PrepareInputFrame(off_t position,
                                                        int32_t* input_id) {
   CHECK_LE(position + test_stream_->aligned_buffer_size,
            test_stream_->aligned_in_file_data.size());
@@ skipping 34 matching lines @@
     // we require for bitrate tests.
     pos_in_input_stream_ = 0;
   }
 
   if (quality_validator_)
     quality_validator_->AddOriginalFrame(CreateFrame(pos_in_input_stream_));
 
   int32_t input_id;
   scoped_refptr<VideoFrame> video_frame =
       PrepareInputFrame(pos_in_input_stream_, &input_id);
+  frame_timestamps_.push(video_frame->timestamp());
   pos_in_input_stream_ += test_stream_->aligned_buffer_size;
 
   bool force_keyframe = false;
   if (keyframe_period_ && input_id % keyframe_period_ == 0) {
     force_keyframe = true;
     ++num_keyframes_requested_;
   }
 
   if (input_id == 0) {
     first_frame_start_time_ = base::TimeTicks::Now();
@@ skipping 73 matching lines @@
             FROM_HERE, base::TimeDelta::FromSeconds(1) / current_framerate_,
             base::Bind(&VEAClient::OnInputTimer, base::Unretained(this)));
     }
   } else if (num_encoded_frames_ == num_frames_to_encode_) {
     LogPerf();
     VerifyMinFPS();
     VerifyStreamProperties();
     SetState(CS_FINISHED);
     if (!quality_validator_)
       SetState(CS_VALIDATED);
+    if (verify_output_timestamp_) {
+      // The last timestamp in |frame_timestamps_| won't be popped out.

[wuchengli, 2016/07/12 09:28:07]

Update the comment.

[shenghao1, 2016/07/12 11:05:51]

Done.

+      EXPECT_LE(frame_timestamps_.size(),
+                static_cast<size_t>(next_input_id_ - num_frames_to_encode_));
+    }
     return false;
   }
 
   return true;
 }
 
 void VEAClient::LogPerf() {
   g_env->LogToFile("Measured encoder FPS",
                    base::StringPrintf("%.3f", frames_per_second()));
 
@@ skipping 79 matching lines @@
 //   one input stream; otherwise, one encoder per input stream will be
 //   instantiated.
 // - If true, save output to file (provided an output filename was supplied).
 // - Force a keyframe every n frames.
 // - Force bitrate; the actual required value is provided as a property
 //   of the input stream, because it depends on stream type/resolution/etc.
 // - If true, measure performance.
 // - If true, switch bitrate mid-stream.
 // - If true, switch framerate mid-stream.
 // - If true, verify the output frames of encoder.
+// - If true, verify the timestamps of output frames.
 class VideoEncodeAcceleratorTest
     : public ::testing::TestWithParam<
-          std::tuple<int, bool, int, bool, bool, bool, bool, bool>> {};
+          std::tuple<int, bool, int, bool, bool, bool, bool, bool, bool>> {};
 
 TEST_P(VideoEncodeAcceleratorTest, TestSimpleEncode) {
   size_t num_concurrent_encoders = std::get<0>(GetParam());
   const bool save_to_file = std::get<1>(GetParam());
   const unsigned int keyframe_period = std::get<2>(GetParam());
   const bool force_bitrate = std::get<3>(GetParam());
   const bool test_perf = std::get<4>(GetParam());
   const bool mid_stream_bitrate_switch = std::get<5>(GetParam());
   const bool mid_stream_framerate_switch = std::get<6>(GetParam());
   const bool verify_output =
       std::get<7>(GetParam()) || g_env->verify_all_output();
+  const bool verify_output_timestamp = std::get<8>(GetParam());
 
   ScopedVector<ClientStateNotification<ClientState>> notes;
   ScopedVector<VEAClient> clients;
   base::Thread encoder_thread("EncoderThread");
   ASSERT_TRUE(encoder_thread.Start());
 
   if (g_env->test_streams_.size() > 1)
     num_concurrent_encoders = g_env->test_streams_.size();
 
   // Create all encoders.
   for (size_t i = 0; i < num_concurrent_encoders; i++) {
     size_t test_stream_index = i % g_env->test_streams_.size();
     // Disregard save_to_file if we didn't get an output filename.
     bool encoder_save_to_file =
         (save_to_file &&
          !g_env->test_streams_[test_stream_index]->out_filename.empty());
 
     notes.push_back(new ClientStateNotification<ClientState>());
     clients.push_back(new VEAClient(
         g_env->test_streams_[test_stream_index], notes.back(),
         encoder_save_to_file, keyframe_period, force_bitrate, test_perf,
-        mid_stream_bitrate_switch, mid_stream_framerate_switch, verify_output));
+        mid_stream_bitrate_switch, mid_stream_framerate_switch, verify_output,
+        verify_output_timestamp));
 
     encoder_thread.task_runner()->PostTask(
         FROM_HERE, base::Bind(&VEAClient::CreateEncoder,
                               base::Unretained(clients.back())));
   }
 
   // All encoders must pass through states in this order.
   enum ClientState state_transitions[] = {
       CS_ENCODER_SET, CS_INITIALIZED, CS_ENCODING, CS_FINISHED, CS_VALIDATED};
 
@@ skipping 18 matching lines @@
 
   // This ensures all tasks have finished.
   encoder_thread.Stop();
 }
 
 #if !defined(OS_MACOSX)
 INSTANTIATE_TEST_CASE_P(
     SimpleEncode,
     VideoEncodeAcceleratorTest,
     ::testing::Values(
-        std::make_tuple(1, true, 0, false, false, false, false, false),
-        std::make_tuple(1, true, 0, false, false, false, false, true)));
+        std::make_tuple(1, true, 0, false, false, false, false, false, false),
+        std::make_tuple(1, true, 0, false, false, false, false, true, false)));
 
 INSTANTIATE_TEST_CASE_P(
     EncoderPerf,
     VideoEncodeAcceleratorTest,
     ::testing::Values(
-        std::make_tuple(1, false, 0, false, true, false, false, false)));
+        std::make_tuple(1, false, 0, false, true, false, false, false, false)));
 
-INSTANTIATE_TEST_CASE_P(
-    ForceKeyframes,
-    VideoEncodeAcceleratorTest,
-    ::testing::Values(
-        std::make_tuple(1, false, 10, false, false, false, false, false)));
+INSTANTIATE_TEST_CASE_P(ForceKeyframes,
+                        VideoEncodeAcceleratorTest,
+                        ::testing::Values(std::make_tuple(1,
+                                                          false,
+                                                          10,
+                                                          false,
+                                                          false,
+                                                          false,
+                                                          false,
+                                                          false,
+                                                          false)));
 
 INSTANTIATE_TEST_CASE_P(
     ForceBitrate,
     VideoEncodeAcceleratorTest,
     ::testing::Values(
-        std::make_tuple(1, false, 0, true, false, false, false, false)));
+        std::make_tuple(1, false, 0, true, false, false, false, false, false)));
 
 INSTANTIATE_TEST_CASE_P(
     MidStreamParamSwitchBitrate,
     VideoEncodeAcceleratorTest,
     ::testing::Values(
-        std::make_tuple(1, false, 0, true, false, true, false, false)));
+        std::make_tuple(1, false, 0, true, false, true, false, false, false)));
 
 INSTANTIATE_TEST_CASE_P(
     MidStreamParamSwitchFPS,
     VideoEncodeAcceleratorTest,
     ::testing::Values(
-        std::make_tuple(1, false, 0, true, false, false, true, false)));
+        std::make_tuple(1, false, 0, true, false, false, true, false, false)));
 
 INSTANTIATE_TEST_CASE_P(
     MultipleEncoders,
     VideoEncodeAcceleratorTest,
     ::testing::Values(
-        std::make_tuple(3, false, 0, false, false, false, false, false),
-        std::make_tuple(3, false, 0, true, false, false, true, false),
-        std::make_tuple(3, false, 0, true, false, true, false, false)));
+        std::make_tuple(3, false, 0, false, false, false, false, false, false),
+        std::make_tuple(3, false, 0, true, false, false, true, false, false),
+        std::make_tuple(3, false, 0, true, false, true, false, false, false)));
+
+INSTANTIATE_TEST_CASE_P(
+    VerifyTimestamp,
+    VideoEncodeAcceleratorTest,
+    ::testing::Values(
+        std::make_tuple(1, true, 0, false, false, false, false, false, true)));
+
 #else
 INSTANTIATE_TEST_CASE_P(
     SimpleEncode,
     VideoEncodeAcceleratorTest,
     ::testing::Values(
-        std::make_tuple(1, true, 0, false, false, false, false, false),
-        std::make_tuple(1, true, 0, false, false, false, false, true)));
+        std::make_tuple(1, true, 0, false, false, false, false, false, false),
+        std::make_tuple(1, true, 0, false, false, false, false, true, false)));
 
 INSTANTIATE_TEST_CASE_P(
     EncoderPerf,
     VideoEncodeAcceleratorTest,
     ::testing::Values(
-        std::make_tuple(1, false, 0, false, true, false, false, false)));
+        std::make_tuple(1, false, 0, false, true, false, false, false, false)));
 
-INSTANTIATE_TEST_CASE_P(
-    MultipleEncoders,
-    VideoEncodeAcceleratorTest,
-    ::testing::Values(
-        std::make_tuple(3, false, 0, false, false, false, false, false)));
+INSTANTIATE_TEST_CASE_P(MultipleEncoders,
+                        VideoEncodeAcceleratorTest,
+                        ::testing::Values(std::make_tuple(3,
+                                                          false,
+                                                          0,
+                                                          false,
+                                                          false,
+                                                          false,
+                                                          false,
+                                                          false,
+                                                          false)));

[wuchengli, 2016/07/12 09:28:07]

Try testing with you CL with verify_output_timestamp=true for all test cases. So
we can be sure there's no dependency between parameters.

[shenghao1, 2016/07/12 11:05:51]

Verified that they all pass.

 #endif
 
 // TODO(posciak): more tests:
 // - async FeedEncoderWithOutput
 // - out-of-order return of outputs to encoder
 // - multiple encoders + decoders
 // - mid-stream encoder_->Destroy()
 
 }  // namespace
 }  // namespace media
@@ skipping 77 matching lines @@
 
   media::g_env =
       reinterpret_cast<media::VideoEncodeAcceleratorTestEnvironment*>(
           testing::AddGlobalTestEnvironment(
               new media::VideoEncodeAcceleratorTestEnvironment(
                   std::move(test_stream_data), log_path, run_at_fps,
                   needs_encode_latency, verify_all_output)));
 
   return RUN_ALL_TESTS();
 }