Preliminary change for new rtc rendering algorithm

content/renderer/media/webrtc/media_stream_remote_video_source.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/renderer/media/webrtc/media_stream_remote_video_source.h"
 
 #include "base/bind.h"
 #include "base/callback_helpers.h"
 #include "base/location.h"
 #include "base/threading/thread_checker.h"
 #include "base/trace_event/trace_event.h"
 #include "content/renderer/media/webrtc/track_observer.h"
 #include "media/base/bind_to_current_loop.h"
 #include "media/base/video_frame.h"
 #include "media/base/video_util.h"
 #include "third_party/libjingle/source/talk/media/base/videoframe.h"
+#include "third_party/webrtc/system_wrappers/interface/tick_util.h"
 
 namespace content {
 
 // Internal class used for receiving frames from the webrtc track on a
 // libjingle thread and forward it to the IO-thread.
 class MediaStreamRemoteVideoSource::RemoteVideoSourceDelegate
     : public base::RefCountedThreadSafe<RemoteVideoSourceDelegate>,
       public webrtc::VideoRendererInterface {
  public:
   RemoteVideoSourceDelegate(
@@ skipping 13 matching lines @@
       const scoped_refptr<media::VideoFrame>& video_frame);
 
  private:
   // Bound to the render thread.
   base::ThreadChecker thread_checker_;
 
   scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_;
 
   // |frame_callback_| is accessed on the IO thread.
   VideoCaptureDeliverFrameCB frame_callback_;
+
+  // WebRTC Chromium timestamp diff
+  int64_t time_diff_us_;
 };
 
 MediaStreamRemoteVideoSource::RemoteVideoSourceDelegate::
     RemoteVideoSourceDelegate(
         scoped_refptr<base::SingleThreadTaskRunner> io_task_runner,
         const VideoCaptureDeliverFrameCB& new_frame_callback)
     : io_task_runner_(io_task_runner), frame_callback_(new_frame_callback) {
+  time_diff_us_ =

[miu, 2015/08/04 04:35:07]

Yes, this can be a tricky problem.  There's two differences between clocks: 1)
the offset, 2) the rate (i.e., one clock runs faster than the other).

I would say this is an okay stub for now, but don't be surprised if there are
performance issues on some platforms.  I'd suggest two things:

1. Put a TODO comment here with a crbug to revisit this issue.

2. Add a SetRealTimeClock() function here:
https://code.google.com/p/chromium/codesearch#chromium/src/third_party/webrtc...

Then, wherever Chromium initializes libwebrtc, it could provide an
implementation of the webrtc::Clock interface that is based on calling
Chromium's base::TimeTicks::Now().  That way, ALL the code in both Chromium and
WebRTC are using the same reference clock!  :)

[qiangchen, 2015/08/04 16:35:25]

Did the TODO. While, adding SetRealTimeClock() cannot be done in this CL,
because that file is in WebRTC. And besides, in WebRTC, many places just refer
to the static method TickTime::Now(), instead of
GetRealTimeClock()->TimestampInMicrosecond(). Thus it is not that easy to make
ALL the code use same clock.

[miu, 2015/08/04 20:37:32]

Oh, yeah.  I did mean in another CL.  ;)
I'd argue this is something important to fix in WebRTC, so I hope you or someone
else will pursue it aggressively.  The ability to use the SimulatedClock for
simulations is a huge win.  We did this for Cast Streaming a while back, and can
easily test algorithm changes on our desktops by simulating a 5-minute session
that only takes about 20 seconds in real-word time to run.

+      (base::TimeTicks::Now() - base::TimeTicks()).InMicroseconds() -
+      webrtc::TickTime::MicrosecondTimestamp();
 }
 
 MediaStreamRemoteVideoSource::
 RemoteVideoSourceDelegate::~RemoteVideoSourceDelegate() {
 }
 
 void MediaStreamRemoteVideoSource::RemoteVideoSourceDelegate::RenderFrame(
     const cricket::VideoFrame* incoming_frame) {
-  TRACE_EVENT0("webrtc", "RemoteVideoSourceDelegate::RenderFrame");
+  base::TimeTicks render_time =
+      base::TimeTicks() +
+      base::TimeDelta::FromMicroseconds(incoming_frame->GetTimeStamp() / 1000 +
+                                        time_diff_us_);
+
+  TRACE_EVENT1("webrtc", "RemoteVideoSourceDelegate::RenderFrame",
+               "Ideal Render Instant", render_time.ToInternalValue());
+
   base::TimeDelta timestamp = base::TimeDelta::FromMicroseconds(
       incoming_frame->GetElapsedTime() / rtc::kNumNanosecsPerMicrosec);
 
   scoped_refptr<media::VideoFrame> video_frame;
   if (incoming_frame->GetNativeHandle() != NULL) {
     video_frame =
         static_cast<media::VideoFrame*>(incoming_frame->GetNativeHandle());
     video_frame->set_timestamp(timestamp);
   } else {
     const cricket::VideoFrame* frame =
@@ skipping 13 matching lines @@
     video_frame = media::VideoFrame::WrapExternalYuvData(
         media::PIXEL_FORMAT_YV12, size, gfx::Rect(size), size,
         frame->GetYPitch(), frame->GetUPitch(), frame->GetVPitch(),
         const_cast<uint8_t*>(frame->GetYPlane()),
         const_cast<uint8_t*>(frame->GetUPlane()),
         const_cast<uint8_t*>(frame->GetVPlane()), timestamp);
     video_frame->AddDestructionObserver(
         base::Bind(&base::DeletePointer<cricket::VideoFrame>, frame->Copy()));
   }
 
+  video_frame->metadata()->SetTimeTicks(
+      media::VideoFrameMetadata::REFERENCE_TIME, render_time);
+
   io_task_runner_->PostTask(
       FROM_HERE, base::Bind(&RemoteVideoSourceDelegate::DoRenderFrameOnIOThread,
                             this, video_frame));
 }
 
 void MediaStreamRemoteVideoSource::
 RemoteVideoSourceDelegate::DoRenderFrameOnIOThread(
     const scoped_refptr<media::VideoFrame>& video_frame) {
   DCHECK(io_task_runner_->BelongsToCurrentThread());
   TRACE_EVENT0("webrtc", "RemoteVideoSourceDelegate::DoRenderFrameOnIOThread");
@@ skipping 66 matching lines @@
     case webrtc::MediaStreamTrackInterface::kEnded:
       SetReadyState(blink::WebMediaStreamSource::ReadyStateEnded);
       break;
     default:
       NOTREACHED();
       break;
   }
 }
 
 }  // namespace content