Reland: Use webrtc::VideoFrame timestamp in RTCVideoEncoder

content/renderer/media/gpu/rtc_video_encoder.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/gpu/rtc_video_encoder.h"
 
 #include <string.h>
 
+#include <deque>
+
 #include "base/bind.h"
 #include "base/location.h"
 #include "base/logging.h"
 #include "base/macros.h"
 #include "base/memory/scoped_vector.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/numerics/safe_conversions.h"
 #include "base/rand_util.h"
 #include "base/single_thread_task_runner.h"
 #include "base/synchronization/lock.h"
 #include "base/synchronization/waitable_event.h"
 #include "base/threading/thread_task_runner_handle.h"
+#include "base/time/time.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/bitstream_buffer.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_util.h"
 #include "media/filters/h264_parser.h"
 #include "media/renderers/gpu_video_accelerator_factories.h"
 #include "media/video/video_encode_accelerator.h"
 #include "third_party/libyuv/include/libyuv.h"
 #include "third_party/webrtc/base/timeutils.h"
 
 namespace content {
 
 namespace {
 
+struct RTCTimestamps {
+  RTCTimestamps(base::TimeDelta media_timestamp, int32_t rtp_timestamp)
+      : media_timestamp_in_microseconds(media_timestamp.InMicroseconds()),
+        rtp_timestamp(rtp_timestamp) {}
+  const int64_t media_timestamp_in_microseconds;
+  const int32_t rtp_timestamp;
+
+ private:
+  DISALLOW_IMPLICIT_CONSTRUCTORS(RTCTimestamps);
+};
+
 // Translate from webrtc::VideoCodecType and webrtc::VideoCodec to
 // media::VideoCodecProfile.
 media::VideoCodecProfile WebRTCVideoCodecToVideoCodecProfile(
     webrtc::VideoCodecType type,
     const webrtc::VideoCodec* codec_settings) {
   DCHECK_EQ(type, codec_settings->codecType);
   switch (type) {
     case webrtc::kVideoCodecVP8:
       return media::VP8PROFILE_ANY;
     case webrtc::kVideoCodecH264:
@@ skipping 156 matching lines @@
   // webrtc::VideoEncoder expects InitEncode() and Encode() to be synchronous.
   // Do this by waiting on the |async_waiter_| and returning the return value in
   // |async_retval_| when initialization completes, encoding completes, or
   // an error occurs.
   base::WaitableEvent* async_waiter_;
   int32_t* async_retval_;
 
   // The underlying VEA to perform encoding on.
   std::unique_ptr<media::VideoEncodeAccelerator> video_encoder_;
 
+  // Used to match the encoded frame timestamp with WebRTC's given RTP
+  // timestamp.
+  std::deque<RTCTimestamps> pending_timestamps_;
+
   // Next input frame.  Since there is at most one next frame, a single-element
   // queue is sufficient.
   const webrtc::VideoFrame* input_next_frame_;
 
   // Whether to encode a keyframe next.
   bool input_next_frame_keyframe_;
 
   // Frame sizes.
   gfx::Size input_frame_coded_size_;
   gfx::Size input_visible_size_;
@@ skipping 243 matching lines @@
   base::SharedMemory* output_buffer = output_buffers_[bitstream_buffer_id];
   if (payload_size > output_buffer->mapped_size()) {
     LogAndNotifyError(FROM_HERE, "invalid payload_size",
                       media::VideoEncodeAccelerator::kPlatformFailureError);
     return;
   }
   output_buffers_free_count_--;
 
   // Derive the capture time (in ms) and RTP timestamp (in 90KHz ticks).
   int64_t capture_time_us, capture_time_ms;
-  uint32_t rtp_timestamp;
+  uint32_t rtp_timestamp = 0;

[pbos, 2017/02/10 00:05:08]

Can you make this optional?

[emircan, 2017/02/10 17:14:55]

Done.

 
   if (!timestamp.is_zero()) {
-    capture_time_us = timestamp.InMicroseconds();;
+    capture_time_us = timestamp.InMicroseconds();
     capture_time_ms = timestamp.InMilliseconds();
+    // Pop timestamps until we have a match.
+    while (!pending_timestamps_.empty()) {

[pbos, 2017/02/10 00:05:08]

This should never be empty, right? If so there's a bug in the VEA?

[emircan, 2017/02/10 17:14:55]

Yes. However I want to loop until all the dropped frames' timestamps are cleared
and I just use this as the condition to parse until everything is finished.
Suppose HW VEA is somewhat broken and returns a different timestamp then given,
then we would empty all timestamps and not let this queue grow.

+      const auto& front_timestamps = pending_timestamps_.front();
+      if (front_timestamps.media_timestamp_in_microseconds ==
+          timestamp.InMicroseconds()) {
+        rtp_timestamp = front_timestamps.rtp_timestamp;
+        pending_timestamps_.pop_front();
+        break;
+      }
+      pending_timestamps_.pop_front();
+    }

[pbos, 2017/02/10 00:05:07]

Can you DCHECK after this brace that rtp_timestamp actually got set?

[emircan, 2017/02/10 17:14:55]

Again, suppose HW VEA is somewhat broken and returns a different timestamp then
given,
then we would empty all timestamps and not let this queue grow. rtp_timestamp()
wouldn't be set then and we would fall to l.515.

   } else {
     // Fallback to the current time if encoder does not provide timestamp.
     capture_time_us = rtc::TimeMicros();
     capture_time_ms = capture_time_us / base::Time::kMicrosecondsPerMillisecond;
+    pending_timestamps_.clear();
   }
-  // RTP timestamp can wrap around. Get the lower 32 bits.
-  rtp_timestamp = static_cast<uint32_t>(
-      capture_time_us * 90 / base::Time::kMicrosecondsPerMillisecond);
+
+  if (rtp_timestamp == 0) {

[pbos, 2017/02/10 00:05:08]

0 is a valid RTP timestamp. Can you gate this on whether rtp_timestamp is set or
not instead?

[emircan, 2017/02/10 17:14:55]

Done.

+    // RTP timestamp can wrap around. Get the lower 32 bits.
+    rtp_timestamp = static_cast<uint32_t>(
+        capture_time_us * 90 / base::Time::kMicrosecondsPerMillisecond);
+  }
 
   webrtc::EncodedImage image(
       reinterpret_cast<uint8_t*>(output_buffer->memory()), payload_size,
       output_buffer->mapped_size());
   image._encodedWidth = input_visible_size_.width();
   image._encodedHeight = input_visible_size_.height();
   image._timeStamp = rtp_timestamp;
   image.capture_time_ms_ = capture_time_ms;
   image._frameType =
       (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta);
@@ skipping 106 matching lines @@
                           frame->visible_rect().width(),
                           frame->visible_rect().height(),
                           libyuv::kFilterBox)) {
       LogAndNotifyError(FROM_HERE, "Failed to copy buffer",
                         media::VideoEncodeAccelerator::kPlatformFailureError);
       return;
     }
   }
   frame->AddDestructionObserver(media::BindToCurrentLoop(
       base::Bind(&RTCVideoEncoder::Impl::EncodeFrameFinished, this, index)));
+  DCHECK(std::find_if(pending_timestamps_.begin(), pending_timestamps_.end(),
+                      [&frame](const RTCTimestamps& entry) {
+           return entry.media_timestamp_in_microseconds ==
+                  frame->timestamp().InMicroseconds();
+         }) == pending_timestamps_.end());
+  pending_timestamps_.emplace_back(frame->timestamp(), next_frame->timestamp());
+
   video_encoder_->Encode(frame, next_frame_keyframe);
   input_buffers_free_.pop_back();
   SignalAsyncWaiter(WEBRTC_VIDEO_CODEC_OK);
 }
 
 void RTCVideoEncoder::Impl::EncodeFrameFinished(int index) {
   DVLOG(3) << "Impl::EncodeFrameFinished(): index=" << index;
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK_GE(index, 0);
   DCHECK_LT(index, static_cast<int>(input_buffers_.size()));
@@ skipping 257 matching lines @@
   UMA_HISTOGRAM_BOOLEAN("Media.RTCVideoEncoderInitEncodeSuccess",
                         init_retval == WEBRTC_VIDEO_CODEC_OK);
   if (init_retval == WEBRTC_VIDEO_CODEC_OK) {
     UMA_HISTOGRAM_ENUMERATION("Media.RTCVideoEncoderProfile",
                               profile,
                               media::VIDEO_CODEC_PROFILE_MAX + 1);
   }
 }
 
 }  // namespace content