Track telemetry benchmark cycle time

telemetry/telemetry/internal/story_runner.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 import optparse
 import os
 import subprocess
 import sys
 import time
 
 import py_utils
 from py_utils import cloud_storage  # pylint: disable=import-error
 
 from telemetry.core import exceptions
 from telemetry import decorators
 from telemetry.internal.actions import page_action
 from telemetry.internal.browser import browser_finder
 from telemetry.internal.results import results_options
 from telemetry.internal.util import exception_formatter
 from telemetry import page
 from telemetry.page import legacy_page_test
 from telemetry import story as story_module
 from telemetry.util import wpr_modes
 from telemetry.value import failure
 from telemetry.value import skip
+from telemetry.value import scalar
 from telemetry.web_perf import story_test
 
 
 class ArchiveError(Exception):
   pass
 
 
 def AddCommandLineArgs(parser):
   story_module.StoryFilter.AddCommandLineArgs(parser)
   results_options.AddResultsOptions(parser)
@@ skipping 257 matching lines @@
         story.display_name)
 
 
 def RunBenchmark(benchmark, finder_options):
   """Run this test with the given options.
 
   Returns:
     The number of failure values (up to 254) or 255 if there is an uncaught
     exception.
   """
+  start = time.time()
   benchmark.CustomizeBrowserOptions(finder_options.browser_options)
 
   benchmark_metadata = benchmark.GetMetadata()
   possible_browser = browser_finder.FindBrowser(finder_options)
   if not possible_browser:
     print ('Cannot find browser of type %s. To list out all '
            'available browsers, rerun your command with '
            '--browser=list' %  finder_options.browser_options.browser_type)
     return 1
   if (possible_browser and
     not decorators.IsBenchmarkEnabled(benchmark, possible_browser)):
     print '%s is disabled on the selected browser' % benchmark.Name()
     if finder_options.run_disabled_tests:
       print 'Running benchmark anyway due to: --also-run-disabled-tests'
     else:
       print 'Try --also-run-disabled-tests to force the benchmark to run.'
       # If chartjson is specified, this will print a dict indicating the
       # benchmark name and disabled state.
       with results_options.CreateResults(
           benchmark_metadata, finder_options,
           benchmark.ValueCanBeAddedPredicate, benchmark_enabled=False
           ) as results:
         results.PrintSummary()
       # When a disabled benchmark is run we now want to return success since
       # we are no longer filtering these out in the buildbot recipes.
-      return 0
+      return 17
 
   pt = benchmark.CreatePageTest(finder_options)
   pt.__name__ = benchmark.__class__.__name__
 
   disabled_attr_name = decorators.DisabledAttributeName(benchmark)
   # pylint: disable=protected-access
   pt._disabled_strings = getattr(benchmark, disabled_attr_name, set())
   if hasattr(benchmark, '_enabled_strings'):
     # pylint: disable=protected-access
     pt._enabled_strings = benchmark._enabled_strings
@@ skipping 19 matching lines @@
       benchmark_metadata, finder_options,
       benchmark.ValueCanBeAddedPredicate, benchmark_enabled=True) as results:
     try:
       Run(pt, stories, finder_options, results, benchmark.max_failures,
           should_tear_down_state_after_each_story_run,
           benchmark.ShouldTearDownStateAfterEachStorySetRun())
       return_code = min(254, len(results.failures))
     except Exception:
       exception_formatter.PrintFormattedException()
       return_code = 255
+    finally:
+      duration = time.time() - start

[nednguyen, 2017/03/17 20:49:32]

I mean duration should be after the finally block in line 381. That makes sure
that if the resulst uploading takes too long, the timing is tracked

 
     try:
       if finder_options.upload_results:
         bucket = finder_options.upload_bucket
         if bucket in cloud_storage.BUCKET_ALIASES:
           bucket = cloud_storage.BUCKET_ALIASES[bucket]
         results.UploadTraceFilesToCloud(bucket)
         results.UploadProfilingFilesToCloud(bucket)
     finally:
+      results.AddSummaryValue(scalar.ScalarValue(
+          None, 'BenchmarkDuration', 'minutes', duration / 60))

[nednguyen, 2017/03/17 20:55:08]

nits: divide by 60.0

[martiniss, 2017/03/23 23:22:21]

Done

       results.PrintSummary()
   return return_code
 
 
 def _UpdateAndCheckArchives(archive_data_file, wpr_archive_info,
                             filtered_stories):
   """Verifies that all stories are local or have WPR archives.
 
   Logs warnings and returns False if any are missing.
   """
@@ skipping 70 matching lines @@
     logging.warning('Device is thermally throttled before running '
                     'performance tests, results will vary.')
 
 
 def _CheckThermalThrottling(platform):
   if not platform.CanMonitorThermalThrottling():
     return
   if platform.HasBeenThermallyThrottled():
     logging.warning('Device has been thermally throttled during '
                     'performance tests, results will vary.')