Track telemetry benchmark cycle time

telemetry/telemetry/internal/story_runner_unittest.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import json
 import math
 import os
+import shutil
 import StringIO
 import sys
+import tempfile
 import unittest
 
 from py_utils import cloud_storage  # pylint: disable=import-error
 
 from telemetry import benchmark
+from telemetry import page
 from telemetry.core import exceptions
 from telemetry.core import util
 from telemetry import decorators
 from telemetry.internal.actions import page_action
 from telemetry.internal.results import page_test_results
 from telemetry.internal.results import results_options
 from telemetry.internal import story_runner
 from telemetry.internal.util import exception_formatter as ex_formatter_module
 from telemetry.page import page as page_module
 from telemetry.page import legacy_page_test
 from telemetry import story as story_module
+from telemetry.testing import fakes
 from telemetry.testing import options_for_unittests
 from telemetry.testing import system_stub
 import mock
 from telemetry.value import failure
 from telemetry.value import improvement_direction
 from telemetry.value import list_of_scalar_values
 from telemetry.value import scalar
 from telemetry.value import skip
 from telemetry.value import summary as summary_module
 from telemetry.web_perf import story_test
 from telemetry.web_perf import timeline_based_measurement
 from telemetry.wpr import archive_info
 
 # This linter complains if we define classes nested inside functions.
 # pylint: disable=bad-super-call
 
 # pylint: disable=too-many-lines
 
-
 class FakePlatform(object):
   def CanMonitorThermalThrottling(self):
     return False
 
   def GetOSName(self):
     pass
 
   def WaitForTemperature(self, _):
     pass
 
   def GetDeviceTypeName(self):
     return "GetDeviceTypeName"
 
-
 class TestSharedState(story_module.SharedState):
 
   _platform = FakePlatform()
 
   @classmethod
   def SetTestPlatform(cls, platform):
     cls._platform = platform
 
   def __init__(self, test, options, story_set):
     super(TestSharedState, self).__init__(
@@ skipping 76 matching lines @@
 def SetupStorySet(allow_multiple_story_states, story_state_list):
   if allow_multiple_story_states:
     story_set = MixedStateStorySet()
   else:
     story_set = story_module.StorySet()
   for i, story_state in enumerate(story_state_list):
     story_set.AddStory(DummyLocalStory(story_state,
                                        name='story%d' % i))
   return story_set
 
+class FakeBenchmark(benchmark.Benchmark):
+  @classmethod
+  def Name(cls):
+    return 'fake'
+
+  test = DummyTest
+
+  def page_set(self):
+    story_set = story_module.StorySet()
+
+    example_page = page.Page(
+        'https://www.example.com',
+        startup_url='https://www.example.com', page_set=story_set)
+    return story_set
+
 
 def _GetOptionForUnittest():
   options = options_for_unittests.GetCopy()
   options.output_formats = ['none']
   options.suppress_gtest_report = False
   parser = options.CreateParser()
   story_runner.AddCommandLineArgs(parser)
   options.MergeDefaultValues(parser.get_default_values())
   story_runner.ProcessCommandLineArgs(parser, options)
   return options
@@ skipping 865 matching lines @@
       mock.call.test.WillRunStory(root_mock.state.platform),
       mock.call.state.WillRunStory(root_mock.story),
       mock.call.state.CanRunStory(root_mock.story),
       mock.call.state.RunStory(root_mock.results),
       mock.call.test.Measure(root_mock.state.platform, root_mock.results),
       mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results),
       mock.call.results.AddValue(FailureValueMatcher('foo')),
       mock.call.state.DidRunStory(root_mock.results),
       mock.call.test.DidRunStory(root_mock.state.platform)
     ])
+
+  def testRunBenchmarkTimeDuration(self):
+    fake_benchmark = FakeBenchmark()
+    options = fakes._FakeBrowserFinderOptions()

[nednguyen, 2017/03/23 23:26:52]

Use options = fakes.CreateBrowserFinderOptions() instead

[martiniss, 2017/03/24 17:28:36]

Done.

+    options.upload_results = None
+    options.suppress_gtest_report = False
+    options.results_label = None
+    options.use_live_sites = False
+    options.max_failures = 100
+    options.pageset_repeat = 1
+    options.output_formats = ['chartjson']
+
+    with mock.patch('telemetry.internal.story_runner.time.time') as time_patch:
+      # 3, because telemetry code asks for the time at some point
+      time_patch.side_effect = [1, 0, 61]
+      tmp_path = tempfile.mkdtemp()
+
+      try:
+        options.output_dir = tmp_path
+        story_runner.RunBenchmark(fake_benchmark, options)
+        with open(os.path.join(tmp_path, 'results-chart.json')) as f:
+          data = json.load(f)
+
+        self.assertEqual(len(data['charts']), 1)
+        charts = data['charts']
+        self.assertIn('BenchmarkDuration', charts)
+        duration = charts['BenchmarkDuration']
+        self.assertIn("summary", duration)
+        summary = duration['summary']
+        duration = summary['value']
+        self.assertTrue(abs(duration - 1) < 0.001)

[nednguyen, 2017/03/23 23:26:52]

nits: self.assertAlmostEqual(duration, 1)

[martiniss, 2017/03/24 17:28:36]

Done.

+      finally:
+        shutil.rmtree(tmp_path)