Account for expected capture and encode latency in WebrtcFrameSchedulerSimple

remoting/protocol/webrtc_frame_scheduler_simple.cc

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "remoting/protocol/webrtc_frame_scheduler_simple.h"
 
 #include <algorithm>
 
 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
 
 namespace remoting {
 namespace protocol {
 
 namespace {
 
+// Number of samples used to estimate processing time for the next frame.
+const int kStatsWindow = 5;
+
 const int kTargetFrameRate = 30;
 constexpr base::TimeDelta kTargetFrameInterval =
     base::TimeDelta::FromMilliseconds(1000 / kTargetFrameRate);
 
 // Target quantizer at which stop the encoding top-off.
 const int kTargetQuantizerForVp8TopOff = 30;
 
 }  // namespace
 
-WebrtcFrameSchedulerSimple::WebrtcFrameSchedulerSimple() {}
+WebrtcFrameSchedulerSimple::WebrtcFrameSchedulerSimple()
+    : frame_processing_delay_us_(kStatsWindow) {}
 WebrtcFrameSchedulerSimple::~WebrtcFrameSchedulerSimple() {}
 
 void WebrtcFrameSchedulerSimple::Start(const base::Closure& capture_callback) {
   capture_callback_ = capture_callback;
   ScheduleNextFrame();
 }
 
 void WebrtcFrameSchedulerSimple::Pause(bool pause) {
   paused_ = pause;
   if (paused_) {
@@ skipping 39 matching lines @@
 
 void WebrtcFrameSchedulerSimple::OnFrameEncoded(
     const WebrtcVideoEncoder::EncodedFrame& encoded_frame,
     const webrtc::EncodedImageCallback::Result& send_result) {
   if (encoded_frame.data.empty()) {
     top_off_is_active_ = false;
     ScheduleNextFrame();
     return;
   }
 
+  frame_processing_delay_us_.Record(
+      (base::TimeTicks::Now() - last_capture_started_time_).InMicroseconds());
+
   // Top-off until the target quantizer value is reached.
   top_off_is_active_ = encoded_frame.quantizer > kTargetQuantizerForVp8TopOff;
 
   // Capture next frame after we finish sending the current one.
   const double kKiloBitsPerByte = 8.0 / 1000.0;
   base::TimeDelta expected_send_delay = base::TimeDelta::FromSecondsD(
       encoded_frame.data.size() * kKiloBitsPerByte / target_bitrate_kbps_);
   last_frame_send_finish_time_ = base::TimeTicks::Now() + expected_send_delay;
 
   ScheduleNextFrame();
 }
 
 void WebrtcFrameSchedulerSimple::ScheduleNextFrame() {
   if (paused_)
     return;
 
   base::TimeTicks now = base::TimeTicks::Now();
   base::TimeDelta delay;
 
   // If this is not the first frame then capture next frame after the previous
   // one has finished sending.
   if (!last_frame_send_finish_time_.is_null()) {
-    delay = std::max(base::TimeDelta(), last_frame_send_finish_time_ - now);
+    base::TimeDelta expected_processing_time =
+        base::TimeDelta::FromMicroseconds(frame_processing_delay_us_.Max());

[Irfan, 2016/09/27 18:04:51]

assigning expected value to Max() seems confusing. Do we want average or median
here ?

[Sergey Ulanov, 2016/09/27 18:49:09]

I think we want to approximate the upper bound of the expected capture/encode
time. This will make it more likely the frame will be ready to sent right after
the previous frame. I actually first tried Average() here, and Max() works
better in tests (at least with current BWE logic).

+    delay = std::max(base::TimeDelta(), last_frame_send_finish_time_ -
+                                            expected_processing_time - now);
   }
 
   // Cap interval between frames to kTargetFrameInterval.
   if (!last_capture_started_time_.is_null()) {
     delay = std::max(delay,
                      last_capture_started_time_ + kTargetFrameInterval - now);
   }
 
   capture_timer_.Start(FROM_HERE, delay,
                        base::Bind(&WebrtcFrameSchedulerSimple::CaptureNextFrame,
                                   base::Unretained(this)));
 }
 
 void WebrtcFrameSchedulerSimple::CaptureNextFrame() {
   last_capture_started_time_ = base::TimeTicks::Now();
   capture_callback_.Run();
 }
 
 }  // namespace protocol
 }  // namespace remoting