Reland: Use webrtc::VideoFrame timestamp in RTCVideoEncoder

content/renderer/media/gpu/rtc_video_encoder.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/gpu/rtc_video_encoder.h"
 
 #include <string.h>
 
+#include <deque>
 #include <memory>
 #include <vector>
 
 #include "base/bind.h"
 #include "base/location.h"
 #include "base/logging.h"
 #include "base/macros.h"
 #include "base/metrics/histogram_macros.h"
 #include "base/numerics/safe_conversions.h"
 #include "base/rand_util.h"
 #include "base/single_thread_task_runner.h"
 #include "base/synchronization/lock.h"
 #include "base/synchronization/waitable_event.h"
 #include "base/threading/thread_task_runner_handle.h"
+#include "base/time/time.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/bitstream_buffer.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_util.h"
 #include "media/filters/h264_parser.h"
 #include "media/renderers/gpu_video_accelerator_factories.h"
 #include "media/video/video_encode_accelerator.h"
 #include "third_party/libyuv/include/libyuv.h"
 #include "third_party/webrtc/base/timeutils.h"
 
 namespace content {
 
 namespace {
 
+struct RTCTimestamps {
+  RTCTimestamps(const base::TimeDelta& media_timestamp, int32_t rtp_timestamp)
+      : media_timestamp_in_microseconds(media_timestamp.InMicroseconds()),
+        rtp_timestamp(rtp_timestamp) {}
+  const int64_t media_timestamp_in_microseconds;

[liberato (no reviews please), 2017/03/27 19:31:49]

why not use base::TimeDelta for this?  seems like almost every reference to it
includes InMicroseconds() on some other TimeDelta.

[emircan, 2017/03/27 20:20:32]

Done.

+  const int32_t rtp_timestamp;
+
+ private:
+  DISALLOW_IMPLICIT_CONSTRUCTORS(RTCTimestamps);
+};
+
 // Translate from webrtc::VideoCodecType and webrtc::VideoCodec to
 // media::VideoCodecProfile.
 media::VideoCodecProfile WebRTCVideoCodecToVideoCodecProfile(
     webrtc::VideoCodecType type,
     const webrtc::VideoCodec* codec_settings) {
   DCHECK_EQ(type, codec_settings->codecType);
   switch (type) {
     case webrtc::kVideoCodecVP8:
       return media::VP8PROFILE_ANY;
     case webrtc::kVideoCodecVP9:
@@ skipping 158 matching lines @@
   // webrtc::VideoEncoder expects InitEncode() and Encode() to be synchronous.
   // Do this by waiting on the |async_waiter_| and returning the return value in
   // |async_retval_| when initialization completes, encoding completes, or
   // an error occurs.
   base::WaitableEvent* async_waiter_;
   int32_t* async_retval_;
 
   // The underlying VEA to perform encoding on.
   std::unique_ptr<media::VideoEncodeAccelerator> video_encoder_;
 
+  // Used to match the encoded frame timestamp with WebRTC's given RTP
+  // timestamp.
+  std::deque<RTCTimestamps> pending_timestamps_;
+
+  // Indicates that timestamp match failed and we should no longer attempt
+  // matching.
+  bool failed_timestamp_match_;
+
   // Next input frame.  Since there is at most one next frame, a single-element
   // queue is sufficient.
   const webrtc::VideoFrame* input_next_frame_;
 
   // Whether to encode a keyframe next.
   bool input_next_frame_keyframe_;
 
   // Frame sizes.
   gfx::Size input_frame_coded_size_;
   gfx::Size input_visible_size_;
@@ skipping 32 matching lines @@
   // Instead, we cache an error status here and return it the next time an
   // interface entry point is called. This is protected by |status_lock_|.
   int32_t status_;
 
   DISALLOW_COPY_AND_ASSIGN(Impl);
 };
 
 RTCVideoEncoder::Impl::Impl(media::GpuVideoAcceleratorFactories* gpu_factories,
                             webrtc::VideoCodecType video_codec_type)
     : gpu_factories_(gpu_factories),
-      async_waiter_(NULL),
-      async_retval_(NULL),
-      input_next_frame_(NULL),
+      async_waiter_(nullptr),
+      async_retval_(nullptr),
+      failed_timestamp_match_(false),
+      input_next_frame_(nullptr),
       input_next_frame_keyframe_(false),
       output_buffers_free_count_(0),
       last_capture_time_ms_(-1),
       encoded_image_callback_(nullptr),
       video_codec_type_(video_codec_type),
       status_(WEBRTC_VIDEO_CODEC_UNINITIALIZED) {
   thread_checker_.DetachFromThread();
   // Picture ID should start on a random number.
   picture_id_ = static_cast<uint16_t>(base::RandInt(0, 0x7FFF));
 }
@@ skipping 168 matching lines @@
     video_encoder_->UseOutputBitstreamBuffer(media::BitstreamBuffer(
         i, output_buffers_[i]->handle(), output_buffers_[i]->mapped_size()));
     output_buffers_free_count_++;
   }
   DCHECK_EQ(GetStatus(), WEBRTC_VIDEO_CODEC_UNINITIALIZED);
   SetStatus(WEBRTC_VIDEO_CODEC_OK);
   SignalAsyncWaiter(WEBRTC_VIDEO_CODEC_OK);
 }
 
 void RTCVideoEncoder::Impl::BitstreamBufferReady(int32_t bitstream_buffer_id,
-    size_t payload_size,
-    bool key_frame,
-    base::TimeDelta timestamp) {
+                                                 size_t payload_size,
+                                                 bool key_frame,
+                                                 base::TimeDelta timestamp) {
   DVLOG(3) << "Impl::BitstreamBufferReady(): bitstream_buffer_id="
            << bitstream_buffer_id << ", payload_size=" << payload_size
            << ", key_frame=" << key_frame
            << ", timestamp ms=" << timestamp.InMilliseconds();
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (bitstream_buffer_id < 0 ||
       bitstream_buffer_id >= static_cast<int>(output_buffers_.size())) {
     LogAndNotifyError(FROM_HERE, "invalid bitstream_buffer_id",
                       media::VideoEncodeAccelerator::kPlatformFailureError);
     return;
   }
   base::SharedMemory* output_buffer =
       output_buffers_[bitstream_buffer_id].get();
   if (payload_size > output_buffer->mapped_size()) {
     LogAndNotifyError(FROM_HERE, "invalid payload_size",
                       media::VideoEncodeAccelerator::kPlatformFailureError);
     return;
   }
   output_buffers_free_count_--;
 
   // Derive the capture time in ms from system clock. Make sure that it is
   // greater than the last.
   const int64_t capture_time_us = rtc::TimeMicros();
   int64_t capture_time_ms =
       capture_time_us / base::Time::kMicrosecondsPerMillisecond;
   capture_time_ms = std::max(capture_time_ms, last_capture_time_ms_ + 1);
   last_capture_time_ms_ = capture_time_ms;
 
-  // Fallback to the current time if encoder does not provide timestamp.
-  const int64_t encoder_time_us =
-      timestamp.is_zero() ? capture_time_us : timestamp.InMicroseconds();
-
-  // Derive the RTP timestamp (in 90KHz ticks).  It can wrap around, get the
-  // lower 32 bits.
-  const uint32_t rtp_timestamp = static_cast<uint32_t>(
-      encoder_time_us * 90 / base::Time::kMicrosecondsPerMillisecond);
+  // Find RTP timestamp by going through |pending_timestamps_|. Derive it from
+  // capture time otherwise.
+  base::Optional<uint32_t> rtp_timestamp;
+  if (!timestamp.is_zero() && !failed_timestamp_match_) {
+    // Pop timestamps until we have a match.
+    while (!pending_timestamps_.empty()) {
+      const auto& front_timestamps = pending_timestamps_.front();
+      if (front_timestamps.media_timestamp_in_microseconds ==
+          timestamp.InMicroseconds()) {
+        rtp_timestamp = front_timestamps.rtp_timestamp;
+        pending_timestamps_.pop_front();
+        break;
+      }
+      pending_timestamps_.pop_front();
+    }
+    DCHECK(rtp_timestamp.has_value());
+  }
+  if (!rtp_timestamp.has_value()) {
+    failed_timestamp_match_ = true;
+    pending_timestamps_.clear();
+    // RTP timestamp can wrap around. Get the lower 32 bits.
+    rtp_timestamp = static_cast<uint32_t>(
+        capture_time_us * 90 / base::Time::kMicrosecondsPerMillisecond);
+  }
 
   webrtc::EncodedImage image(
       reinterpret_cast<uint8_t*>(output_buffer->memory()), payload_size,
       output_buffer->mapped_size());
   image._encodedWidth = input_visible_size_.width();
   image._encodedHeight = input_visible_size_.height();
-  image._timeStamp = rtp_timestamp;
+  image._timeStamp = rtp_timestamp.value();
   image.capture_time_ms_ = capture_time_ms;
   image._frameType =
       (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta);
   image._completeFrame = true;
 
   ReturnEncodedImage(image, bitstream_buffer_id, picture_id_);
   // Picture ID must wrap after reaching the maximum.
   picture_id_ = (picture_id_ + 1) & 0x7FFF;
 }
 
@@ skipping 99 matching lines @@
                           frame->visible_rect().width(),
                           frame->visible_rect().height(),
                           libyuv::kFilterBox)) {
       LogAndNotifyError(FROM_HERE, "Failed to copy buffer",
                         media::VideoEncodeAccelerator::kPlatformFailureError);
       return;
     }
   }
   frame->AddDestructionObserver(media::BindToCurrentLoop(
       base::Bind(&RTCVideoEncoder::Impl::EncodeFrameFinished, this, index)));
+  if (!failed_timestamp_match_) {
+    DCHECK(std::find_if(pending_timestamps_.begin(), pending_timestamps_.end(),
+                        [&frame](const RTCTimestamps& entry) {
+                          return entry.media_timestamp_in_microseconds ==
+                                 frame->timestamp().InMicroseconds();
+                        }) == pending_timestamps_.end());
+    pending_timestamps_.emplace_back(frame->timestamp(),
+                                     next_frame->timestamp());
+  }
   video_encoder_->Encode(frame, next_frame_keyframe);
   input_buffers_free_.pop_back();
   SignalAsyncWaiter(WEBRTC_VIDEO_CODEC_OK);
 }
 
 void RTCVideoEncoder::Impl::EncodeFrameFinished(int index) {
   DVLOG(3) << "Impl::EncodeFrameFinished(): index=" << index;
   DCHECK(thread_checker_.CalledOnValidThread());
   DCHECK_GE(index, 0);
   DCHECK_LT(index, static_cast<int>(input_buffers_.size()));
@@ skipping 257 matching lines @@
   UMA_HISTOGRAM_BOOLEAN("Media.RTCVideoEncoderInitEncodeSuccess",
                         init_retval == WEBRTC_VIDEO_CODEC_OK);
   if (init_retval == WEBRTC_VIDEO_CODEC_OK) {
     UMA_HISTOGRAM_ENUMERATION("Media.RTCVideoEncoderProfile",
                               profile,
                               media::VIDEO_CODEC_PROFILE_MAX + 1);
   }
 }
 
 }  // namespace content