Don't use barcodes in ProtocolPerfTests

remoting/test/protocol_perftest.cc

 // Copyright 2014 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include <numeric>
 #include <utility>
 
 #include "base/base64.h"
 #include "base/files/file_util.h"
 #include "base/macros.h"
@@ skipping 145 matching lines @@
     if (!done.is_null())
       done.Run();
   }
 
   protocol::FrameConsumer::PixelFormat GetPixelFormat() override {
     return FORMAT_BGRA;
   }
 
   // FrameStatsConsumer interface.
   void OnVideoFrameStats(const protocol::FrameStats& frame_stats) override {
+    // Ignore store stats for empty frames.
+    if (!frame_stats.host_stats.frame_size)
+      return;
+
     frame_stats_.push_back(frame_stats);
 
     if (waiting_frame_stats_loop_ &&
         frame_stats_.size() >= num_expected_frame_stats_) {
       waiting_frame_stats_loop_->Quit();
     }
   }
 
   // HostStatusObserver interface.
   void OnClientConnected(const std::string& jid) override {
+    if (event_timestamp_source_) {
+      auto& session = host_->client_sessions_for_tests().front();
+      session->SetEventTimestampsSourceForTests(
+          std::move(event_timestamp_source_));
+    }
+
     message_loop_.task_runner()->PostTask(
         FROM_HERE, base::Bind(&ProtocolPerfTest::OnHostConnectedMainThread,
                               base::Unretained(this)));
   }
 
  protected:
   void WaitConnected() {
     client_connected_ = false;
     host_connected_ = false;
 
@@ skipping 167 matching lines @@
   base::MessageLoopForIO message_loop_;
 
   scoped_refptr<FakeNetworkDispatcher> fake_network_dispatcher_;
 
   base::Thread host_thread_;
   base::Thread capture_thread_;
   base::Thread encode_thread_;
   base::Thread decode_thread_;
   std::unique_ptr<FakeDesktopEnvironmentFactory> desktop_environment_factory_;
 
+  scoped_refptr<protocol::InputEventTimestampsSource> event_timestamp_source_;
+
   FakeCursorShapeStub cursor_shape_stub_;
 
   std::unique_ptr<protocol::CandidateSessionConfig> protocol_config_;
 
   std::unique_ptr<FakeSignalStrategy> host_signaling_;
   std::unique_ptr<FakeSignalStrategy> client_signaling_;
 
   std::unique_ptr<ChromotingHost> host_;
   std::unique_ptr<ClientContext> client_context_;
   std::unique_ptr<SoftwareVideoRenderer> video_renderer_;
@@ skipping 56 matching lines @@
         NetworkPerformanceParams(100000, 30000, 130, 5, 0.00),
         NetworkPerformanceParams(100000, 200000, 130, 5, 0.00)));
 
 // TotalLatency[Ice|Webrtc] tests measure video latency in the case when the
 // whole screen is updated occasionally. It's intended to simulate the case when
 // user actions (e.g. Alt-Tab, click on the task bar) cause whole screen to be
 // updated.
 void ProtocolPerfTest::MeasureTotalLatency(bool use_webrtc) {
   scoped_refptr<test::CyclicFrameGenerator> frame_generator =
       test::CyclicFrameGenerator::Create();
-  frame_generator->set_draw_barcode(true);
-
   desktop_environment_factory_->set_frame_generator(
       base::Bind(&test::CyclicFrameGenerator::GenerateFrame, frame_generator));
+  event_timestamp_source_ = frame_generator;
 
   StartHostAndClient(use_webrtc);
   ASSERT_NO_FATAL_FAILURE(WaitConnected());
 
-  int skipped_frames = 0;
-  while (skipped_frames < 10) {
-    std::unique_ptr<webrtc::DesktopFrame> frame = ReceiveFrame();
-    test::CyclicFrameGenerator::ChangeInfoList changes =
-        frame_generator->GetChangeList(frame.get());
-    skipped_frames += changes.size();
+  int warm_up_frames = 0;
+  int total_frames = 0;
+
+  base::TimeTicks start_time = base::TimeTicks::Now();
+  const base::TimeDelta kWarmUpTime = base::TimeDelta::FromSeconds(2);
+  while ((base::TimeTicks::Now() - start_time) < kWarmUpTime) {
+    ReceiveFrame();
+    ++warm_up_frames;
+    ++total_frames;
   }

[Jamie, 2016/10/14 23:26:07]

Please add a comment explaining why warm-up frames are important.

[Sergey Ulanov, 2016/10/17 22:40:26]

Done. Also simplified this code to avoid duplicated while loop.

 
-  base::TimeDelta total_latency_big_frames;
-  int big_frame_count = 0;
-  base::TimeDelta total_latency_small_frames;
-  int small_frame_count = 0;
+  start_time = base::TimeTicks::Now();
+  const base::TimeDelta kTestTime = base::TimeDelta::FromSeconds(5);
+  while ((base::TimeTicks::Now() - start_time) < kTestTime) {
+    ReceiveFrame();
+    ++total_frames;
+  }
 
-  while (big_frame_count + small_frame_count < 30) {
-    std::unique_ptr<webrtc::DesktopFrame> frame = ReceiveFrame();
-    base::TimeTicks frame_received_time = base::TimeTicks::Now();
+  WaitFrameStats(total_frames);
+
+  int big_update_count = 0;
+  base::TimeDelta total_latency_big_updates;
+  int small_update_count = 0;
+  base::TimeDelta total_latency_small_updates;
+  for (int i = 0; i < total_frames; ++i) {
+    const protocol::FrameStats& stats = frame_stats_[i];
+    if (stats.host_stats.latest_event_timestamp.is_null())
+      continue;

[Jamie, 2016/10/14 23:26:07]

Under what circumstances do we expect this to be null?

[Sergey Ulanov, 2016/10/17 22:40:26]

Actually it should never be null because
CyclicFrameGenerator::TakeLastEventTimestamps() never returns null. Replaced
with CHECK().
Also updated ClientSession to keep video stream paused until OnClientConnected()
is called. This ensures that the default EventsTimestampsSource is not used for
the first frame (previously the first frame was captured before
OnClientConnected()).

     test::CyclicFrameGenerator::ChangeInfoList changes =
-        frame_generator->GetChangeList(frame.get());
+        frame_generator->GetChangeList(stats.host_stats.latest_event_timestamp);
+    if (i < warm_up_frames)
+      continue;
+
     for (auto& change_info : changes) {
-      base::TimeDelta latency = frame_received_time - change_info.timestamp;
+      base::TimeDelta latency =
+          stats.client_stats.time_rendered - change_info.timestamp;
       switch (change_info.type) {
         case test::CyclicFrameGenerator::ChangeType::NO_CHANGES:
           NOTREACHED();
           break;
         case test::CyclicFrameGenerator::ChangeType::FULL:
-          total_latency_big_frames += latency;
-          ++big_frame_count;
+          total_latency_big_updates += latency;
+          ++big_update_count;
           break;
         case test::CyclicFrameGenerator::ChangeType::CURSOR:
-          total_latency_small_frames += latency;
-          ++small_frame_count;
+          total_latency_small_updates += latency;
+          ++small_update_count;
           break;
       }
     }
   }
 
-  CHECK(big_frame_count);
-  VLOG(0) << "Average latency for big frames: "
-          << (total_latency_big_frames / big_frame_count).InMillisecondsF();
+  CHECK(big_update_count);
+  VLOG(0) << "Average latency for big updates: "
+          << (total_latency_big_updates / big_update_count).InMillisecondsF()
+          << " " << big_update_count;
 
-  if (small_frame_count) {
+  if (small_update_count) {
     VLOG(0)
-        << "Average latency for small frames: "
-        << (total_latency_small_frames / small_frame_count).InMillisecondsF();
+        << "Average latency for small updates: "
+        << (total_latency_small_updates / small_update_count).InMillisecondsF()
+        << " " << small_update_count;
   }
 }
 
 TEST_P(ProtocolPerfTest, TotalLatencyIce) {
   MeasureTotalLatency(false);
 }
 
 TEST_P(ProtocolPerfTest, TotalLatencyWebrtc) {
   MeasureTotalLatency(true);
 }
 
 // ScrollPerformance[Ice|Webrtc] tests simulate whole screen being scrolled
 // continuously. They measure FPS and video latency.
 void ProtocolPerfTest::MeasureScrollPerformance(bool use_webrtc) {
   scoped_refptr<test::ScrollFrameGenerator> frame_generator =
       new test::ScrollFrameGenerator();
-
   desktop_environment_factory_->set_frame_generator(
       base::Bind(&test::ScrollFrameGenerator::GenerateFrame, frame_generator));
+  event_timestamp_source_ = frame_generator;
 
   StartHostAndClient(use_webrtc);
   ASSERT_NO_FATAL_FAILURE(WaitConnected());
 
   int warm_up_frames = 0;
 
   base::TimeTicks start_time = base::TimeTicks::Now();
   const base::TimeDelta kWarmUpTime = base::TimeDelta::FromSeconds(2);
   while ((base::TimeTicks::Now() - start_time) < kWarmUpTime) {
     ReceiveFrame();
     ++warm_up_frames;
   }
 
   client_socket_factory_->ResetStats();
 
   // Run the test for 2 seconds.
   const base::TimeDelta kTestTime = base::TimeDelta::FromSeconds(2);
 
   int num_frames = 0;
-  base::TimeDelta latency_sum;
   start_time = base::TimeTicks::Now();
   while ((base::TimeTicks::Now() - start_time) < kTestTime) {
     std::unique_ptr<webrtc::DesktopFrame> frame = ReceiveFrame();
     ++num_frames;
-    latency_sum += frame_generator->GetFrameLatency(*frame);
   }
 
   base::TimeDelta total_time = (base::TimeTicks::Now() - start_time);
 
   WaitFrameStats(warm_up_frames + num_frames);
 
   int total_size =
       std::accumulate(frame_stats_.begin() + warm_up_frames,
                       frame_stats_.begin() + warm_up_frames + num_frames, 0,
                       [](int sum, const protocol::FrameStats& stats) {
                         return sum + stats.host_stats.frame_size;
                       });
 
+  base::TimeDelta latency_sum = std::accumulate(
+      frame_stats_.begin() + warm_up_frames,
+      frame_stats_.begin() + warm_up_frames + num_frames, base::TimeDelta(),
+      [](base::TimeDelta sum, const protocol::FrameStats& stats) {
+        return sum + (stats.client_stats.time_rendered -
+                      stats.host_stats.latest_event_timestamp);
+      });
+
   VLOG(0) << "FPS: " << num_frames / total_time.InSecondsF();
   VLOG(0) << "Average latency: " << latency_sum.InMillisecondsF() / num_frames
           << " ms";
   VLOG(0) << "Total size: " << total_size << " bytes";
   VLOG(0) << "Bandwidth utilization: "
           << 100 * total_size / (total_time.InSecondsF() * GetParam().bandwidth)
           << "%";
   VLOG(0) << "Network buffer delay (bufferbloat), average: "
           << client_socket_factory_->average_buffer_delay().InMilliseconds()
           << " ms,  max:"
           << client_socket_factory_->max_buffer_delay().InMilliseconds()
           << " ms";
   VLOG(0) << "Packet drop rate: " << client_socket_factory_->drop_rate();
 }
 
 TEST_P(ProtocolPerfTest, ScrollPerformanceIce) {
   MeasureScrollPerformance(false);
 }
 
 TEST_P(ProtocolPerfTest, ScrollPerformanceWebrtc) {
   MeasureScrollPerformance(true);
 }
 
 }  // namespace remoting