RTC Video Encoder: Use capturer timestamp

content/renderer/media/rtc_video_encoder.cc

 // Copyright 2013 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/rtc_video_encoder.h"
 
 #include <string.h>
 
 #include "base/bind.h"
 #include "base/location.h"
@@ skipping 131 matching lines @@
   int32_t GetStatus() const;
 
   webrtc::VideoCodecType video_codec_type() { return video_codec_type_; }
 
   // media::VideoEncodeAccelerator::Client implementation.
   void RequireBitstreamBuffers(unsigned int input_count,
                                const gfx::Size& input_coded_size,
                                size_t output_buffer_size) override;
   void BitstreamBufferReady(int32_t bitstream_buffer_id,
                             size_t payload_size,
-                            bool key_frame) override;
+                            bool key_frame,
+                            base::TimeDelta timestamp) override;
   void NotifyError(media::VideoEncodeAccelerator::Error error) override;
 
  private:
   friend class base::RefCountedThreadSafe<Impl>;
 
   enum {
     kInputBufferExtraCount = 1,  // The number of input buffers allocated, more
                                  // than what is requested by
                                  // VEA::RequireBitstreamBuffers().
     kOutputBufferCount = 3,
@@ skipping 279 matching lines @@
     video_encoder_->UseOutputBitstreamBuffer(media::BitstreamBuffer(
         i, output_buffers_[i]->handle(), output_buffers_[i]->mapped_size()));
     output_buffers_free_count_++;
   }
   DCHECK_EQ(GetStatus(), WEBRTC_VIDEO_CODEC_UNINITIALIZED);
   SetStatus(WEBRTC_VIDEO_CODEC_OK);
   SignalAsyncWaiter(WEBRTC_VIDEO_CODEC_OK);
 }
 
 void RTCVideoEncoder::Impl::BitstreamBufferReady(int32_t bitstream_buffer_id,
-                                                 size_t payload_size,
-                                                 bool key_frame) {
+    size_t payload_size,
+    bool key_frame,
+    base::TimeDelta timestamp) {
   DVLOG(3) << "Impl::BitstreamBufferReady(): "
               "bitstream_buffer_id=" << bitstream_buffer_id
            << ", payload_size=" << payload_size
            << ", key_frame=" << key_frame;
   DCHECK(thread_checker_.CalledOnValidThread());
 
   if (bitstream_buffer_id < 0 ||
       bitstream_buffer_id >= static_cast<int>(output_buffers_.size())) {
     LogAndNotifyError(FROM_HERE, "invalid bitstream_buffer_id",
                       media::VideoEncodeAccelerator::kPlatformFailureError);
     return;
   }
   base::SharedMemory* output_buffer = output_buffers_[bitstream_buffer_id];
   if (payload_size > output_buffer->mapped_size()) {
     LogAndNotifyError(FROM_HERE, "invalid payload_size",
                       media::VideoEncodeAccelerator::kPlatformFailureError);
     return;
   }
   output_buffers_free_count_--;
 
-  // Use webrtc timestamps to ensure correct RTP sender behavior.
-  // TODO(hshi): obtain timestamp from the capturer, see crbug.com/350106.
-  const int64_t capture_time_us = rtc::TimeMicros();
+  const int64_t capture_time_us = timestamp.InMicroseconds();

[Pawel Osciak, 2016/05/23 07:15:52]

Please remove this line and at l.489:

image.capture_time_ms_ = timestamp.InMilliseconds();

[shenghao, 2016/05/24 10:50:19]

Done.

 
   // Derive the capture time (in ms) and RTP timestamp (in 90KHz ticks).
   const int64_t capture_time_ms = capture_time_us / 1000;

[Pawel Osciak, 2016/05/23 07:15:52]

This could also be removed and l.480 modified accordingly.

[shenghao, 2016/05/24 10:50:19]

Done.

   const uint32_t rtp_timestamp =

[pbos, 2016/05/20 12:15:51]

This is the timestamp that should be copied over for webrtc's usage.

[shenghao, 2016/05/24 10:50:19]

Talked offline. We would calculate rtp_timestamp from ntp_timestamp here.

       static_cast<uint32_t>(capture_time_us * 90 / 1000);
 
   webrtc::EncodedImage image(
       reinterpret_cast<uint8_t*>(output_buffer->memory()), payload_size,
       output_buffer->mapped_size());
   image._encodedWidth = input_visible_size_.width();
   image._encodedHeight = input_visible_size_.height();
   image._timeStamp = rtp_timestamp;
   image.capture_time_ms_ = capture_time_ms;
   image._frameType =
@@ skipping 70 matching lines @@
     requires_copy = true;
   }
 
   if (requires_copy) {
     base::SharedMemory* input_buffer = input_buffers_[index];
     frame = media::VideoFrame::WrapExternalSharedMemory(
         media::PIXEL_FORMAT_I420, input_frame_coded_size_,
         gfx::Rect(input_visible_size_), input_visible_size_,
         reinterpret_cast<uint8_t*>(input_buffer->memory()),
         input_buffer->mapped_size(), input_buffer->handle(), 0,
-        base::TimeDelta());
+        base::TimeDelta::FromMilliseconds(next_frame->ntp_time_ms()));

[pbos, 2016/05/20 12:15:51]

This needs to copy over next_frame->timestamp(), not the NTP time. We need the
RTP timestamps to match.

Be careful about rounding errors as well, we use this timestamp to match
incoming frames with outgoing frames.

[shenghao, 2016/05/24 10:50:19]

Talked offline. We agreed to use ntp_time_ms()

     if (!frame.get()) {
       LogAndNotifyError(FROM_HERE, "failed to create frame",
                         media::VideoEncodeAccelerator::kPlatformFailureError);
       return;
     }
     // Do a strided copy of the input frame to match the input requirements for
     // the encoder.
     // TODO(sheu): support zero-copy from WebRTC.  http://crbug.com/269312
     if (libyuv::I420Copy(next_frame->video_frame_buffer()->DataY(),
                          next_frame->video_frame_buffer()->StrideY(),
@@ skipping 273 matching lines @@
   UMA_HISTOGRAM_BOOLEAN("Media.RTCVideoEncoderInitEncodeSuccess",
                         init_retval == WEBRTC_VIDEO_CODEC_OK);
   if (init_retval == WEBRTC_VIDEO_CODEC_OK) {
     UMA_HISTOGRAM_ENUMERATION("Media.RTCVideoEncoderProfile",
                               profile,
                               media::VIDEO_CODEC_PROFILE_MAX + 1);
   }
 }
 
 }  // namespace content