Reland: Use webrtc::VideoFrame timestamp in RTCVideoEncoder

content/renderer/media/gpu/rtc_video_encoder.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/gpu/rtc_video_encoder.h"
 
 #include <string.h>
 
+#include <deque>
+
 #include "base/bind.h"
 #include "base/location.h"
 #include "base/logging.h"
 #include "base/macros.h"
 #include "base/memory/scoped_vector.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/numerics/safe_conversions.h"
 #include "base/rand_util.h"
 #include "base/single_thread_task_runner.h"
 #include "base/synchronization/lock.h"
 #include "base/synchronization/waitable_event.h"
 #include "base/threading/thread_task_runner_handle.h"
+#include "base/time/time.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/bitstream_buffer.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_util.h"
 #include "media/filters/h264_parser.h"
 #include "media/renderers/gpu_video_accelerator_factories.h"
 #include "media/video/video_encode_accelerator.h"
 #include "third_party/libyuv/include/libyuv.h"
 #include "third_party/webrtc/base/timeutils.h"
 
 namespace content {
 
 namespace {
 
+struct RTCTimestamps {
+  RTCTimestamps(const base::TimeDelta& media_timestamp, int32_t rtp_timestamp)
+      : media_timestamp_in_microseconds(media_timestamp.InMicroseconds()),
+        rtp_timestamp(rtp_timestamp) {}
+  const int64_t media_timestamp_in_microseconds;
+  const int32_t rtp_timestamp;
+
+ private:
+  DISALLOW_IMPLICIT_CONSTRUCTORS(RTCTimestamps);
+};
+
 // Translate from webrtc::VideoCodecType and webrtc::VideoCodec to
 // media::VideoCodecProfile.
 media::VideoCodecProfile WebRTCVideoCodecToVideoCodecProfile(
     webrtc::VideoCodecType type,
     const webrtc::VideoCodec* codec_settings) {
   DCHECK_EQ(type, codec_settings->codecType);
   switch (type) {
     case webrtc::kVideoCodecVP8:
       return media::VP8PROFILE_ANY;
     case webrtc::kVideoCodecH264:
@@ skipping 156 matching lines @@
   // webrtc::VideoEncoder expects InitEncode() and Encode() to be synchronous.
   // Do this by waiting on the |async_waiter_| and returning the return value in
   // |async_retval_| when initialization completes, encoding completes, or
   // an error occurs.
   base::WaitableEvent* async_waiter_;
   int32_t* async_retval_;
 
   // The underlying VEA to perform encoding on.
   std::unique_ptr<media::VideoEncodeAccelerator> video_encoder_;
 
+  // Used to match the encoded frame timestamp with WebRTC's given RTP
+  // timestamp.
+  std::deque<RTCTimestamps> pending_timestamps_;
+
   // Next input frame.  Since there is at most one next frame, a single-element
   // queue is sufficient.
   const webrtc::VideoFrame* input_next_frame_;
 
   // Whether to encode a keyframe next.
   bool input_next_frame_keyframe_;
 
   // Frame sizes.
   gfx::Size input_frame_coded_size_;
   gfx::Size input_visible_size_;
@@ skipping 218 matching lines @@
     video_encoder_->UseOutputBitstreamBuffer(media::BitstreamBuffer(
         i, output_buffers_[i]->handle(), output_buffers_[i]->mapped_size()));
     output_buffers_free_count_++;
   }
   DCHECK_EQ(GetStatus(), WEBRTC_VIDEO_CODEC_UNINITIALIZED);
   SetStatus(WEBRTC_VIDEO_CODEC_OK);
   SignalAsyncWaiter(WEBRTC_VIDEO_CODEC_OK);
 }
 
 void RTCVideoEncoder::Impl::BitstreamBufferReady(int32_t bitstream_buffer_id,
-    size_t payload_size,
-    bool key_frame,
-    base::TimeDelta timestamp) {
+                                                 size_t payload_size,
+                                                 bool key_frame,
+                                                 base::TimeDelta timestamp) {
   DVLOG(3) << "Impl::BitstreamBufferReady(): bitstream_buffer_id="
            << bitstream_buffer_id << ", payload_size=" << payload_size
            << ", key_frame=" << key_frame
            << ", timestamp ms=" << timestamp.InMilliseconds();
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (bitstream_buffer_id < 0 ||
       bitstream_buffer_id >= static_cast<int>(output_buffers_.size())) {
     LogAndNotifyError(FROM_HERE, "invalid bitstream_buffer_id",
                       media::VideoEncodeAccelerator::kPlatformFailureError);
     return;
   }
   base::SharedMemory* output_buffer = output_buffers_[bitstream_buffer_id];
   if (payload_size > output_buffer->mapped_size()) {
     LogAndNotifyError(FROM_HERE, "invalid payload_size",
                       media::VideoEncodeAccelerator::kPlatformFailureError);
     return;
   }
   output_buffers_free_count_--;
 
   // Derive the capture time (in ms) and RTP timestamp (in 90KHz ticks).
   int64_t capture_time_us, capture_time_ms;
-  uint32_t rtp_timestamp;
+  base::Optional<uint32_t> rtp_timestamp;
 
   if (!timestamp.is_zero()) {
-    capture_time_us = timestamp.InMicroseconds();;
+    capture_time_us = timestamp.InMicroseconds();
     capture_time_ms = timestamp.InMilliseconds();
+    // Pop timestamps until we have a match.
+    while (!pending_timestamps_.empty()) {
+      const auto& front_timestamps = pending_timestamps_.front();
+      if (front_timestamps.media_timestamp_in_microseconds ==
+          timestamp.InMicroseconds()) {
+        rtp_timestamp = front_timestamps.rtp_timestamp;
+        pending_timestamps_.pop_front();
+        break;
+      }
+      pending_timestamps_.pop_front();
+    }

[pbos-webrtc, 2017/02/15 20:44:50]

Can we say that if pending_timestamps_ ever does not find a match we don't try
again in a release build? E.g. bool failed_timestamp_search_ = true, and don't
use the pending_timestamps_ loop again?

[emircan, 2017/02/15 21:03:16]

Done.

[nisse-chromium (ooo August 14), 2017/02/16 07:49:50]

That sounds like a failure event we might want some stats for.

+    DCHECK(rtp_timestamp.has_value());
   } else {
     // Fallback to the current time if encoder does not provide timestamp.
     capture_time_us = rtc::TimeMicros();
     capture_time_ms = capture_time_us / base::Time::kMicrosecondsPerMillisecond;
+    pending_timestamps_.clear();
   }
-  // RTP timestamp can wrap around. Get the lower 32 bits.
-  rtp_timestamp = static_cast<uint32_t>(
-      capture_time_us * 90 / base::Time::kMicrosecondsPerMillisecond);
+
+  if (!rtp_timestamp.has_value()) {
+    // RTP timestamp can wrap around. Get the lower 32 bits.
+    rtp_timestamp = static_cast<uint32_t>(
+        capture_time_us * 90 / base::Time::kMicrosecondsPerMillisecond);
+  }
 
   webrtc::EncodedImage image(
       reinterpret_cast<uint8_t*>(output_buffer->memory()), payload_size,
       output_buffer->mapped_size());
   image._encodedWidth = input_visible_size_.width();
   image._encodedHeight = input_visible_size_.height();
-  image._timeStamp = rtp_timestamp;
+  image._timeStamp = rtp_timestamp.value();
   image.capture_time_ms_ = capture_time_ms;
   image._frameType =
       (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta);
   image._completeFrame = true;
 
   ReturnEncodedImage(image, bitstream_buffer_id, picture_id_);
   // Picture ID must wrap after reaching the maximum.
   picture_id_ = (picture_id_ + 1) & 0x7FFF;
 }
 
@@ skipping 99 matching lines @@
                           frame->visible_rect().width(),
                           frame->visible_rect().height(),
                           libyuv::kFilterBox)) {
       LogAndNotifyError(FROM_HERE, "Failed to copy buffer",
                         media::VideoEncodeAccelerator::kPlatformFailureError);
       return;
     }
   }
   frame->AddDestructionObserver(media::BindToCurrentLoop(
       base::Bind(&RTCVideoEncoder::Impl::EncodeFrameFinished, this, index)));
+  DCHECK(std::find_if(pending_timestamps_.begin(), pending_timestamps_.end(),
+                      [&frame](const RTCTimestamps& entry) {
+           return entry.media_timestamp_in_microseconds ==
+                  frame->timestamp().InMicroseconds();
+         }) == pending_timestamps_.end());
+  pending_timestamps_.emplace_back(frame->timestamp(), next_frame->timestamp());
+
   video_encoder_->Encode(frame, next_frame_keyframe);
   input_buffers_free_.pop_back();
   SignalAsyncWaiter(WEBRTC_VIDEO_CODEC_OK);
 }
 
 void RTCVideoEncoder::Impl::EncodeFrameFinished(int index) {
   DVLOG(3) << "Impl::EncodeFrameFinished(): index=" << index;
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK_GE(index, 0);
   DCHECK_LT(index, static_cast<int>(input_buffers_.size()));
@@ skipping 257 matching lines @@
   UMA_HISTOGRAM_BOOLEAN("Media.RTCVideoEncoderInitEncodeSuccess",
                         init_retval == WEBRTC_VIDEO_CODEC_OK);
   if (init_retval == WEBRTC_VIDEO_CODEC_OK) {
     UMA_HISTOGRAM_ENUMERATION("Media.RTCVideoEncoderProfile",
                               profile,
                               media::VIDEO_CODEC_PROFILE_MAX + 1);
   }
 }
 
 }  // namespace content