[Telemetry] Add Benchmark.ShouldTearDownStateAfterEachStorySetRun

telemetry/telemetry/benchmark.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import optparse
 
 from telemetry import decorators
 from telemetry.internal import story_runner
 from telemetry.internal.util import command_line
 from telemetry.page import legacy_page_test
@@ skipping 82 matching lines @@
   @property
   def max_failures(self):
     return self._max_failures
 
   @classmethod
   def Name(cls):
     return '%s.%s' % (cls.__module__.split('.')[-1], cls.__name__)
 
   @classmethod
   def ShouldTearDownStateAfterEachStoryRun(cls):
-    """Override this method to tear down state after each story run.
+    """Override to specify whether to tear down state after each story run.
 
     Tearing down all states after each story run, e.g., clearing profiles,
     stopping the browser, stopping local server, etc. So the browser will not be
     reused among multiple stories. This is particularly useful to get the
     startup part of launching the browser in each story.
 
     This should only be used by TimelineBasedMeasurement (TBM) benchmarks, but
     not by PageTest based benchmarks.
     """
     return False
 
   @classmethod
+  def ShouldTearDownStateAfterEachStorySetRun(cls):
+    """Override to specify whether to tear down state after each story set run.
+
+    Defaults to True in order to reset the state and make individual story set
+    repeats more independent of each other. The intended effect is to average
+    out noise in measurements between repeats.
+
+    Long running benchmarks willing to stess test the browser and have it run
+    for long periods of time may switch this value to False.
+
+    This should only be used by TimelineBasedMeasurement (TBM) benchmarks, but
+    not by PageTest based benchmarks.
+    """
+    # TODO(perezju): Switch default value after any benchmarks requiring the
+    # old behaviour are updated accordingly.
+    # https://github.com/catapult-project/catapult/issues/2294#issuecomment-228306977

[nednguyen, 2016/06/24 16:11:55]

this may cause a lint check failure. Feel free to add s.t similar to
https://cs.chromium.org/chromium/src/tools/perf/pylintrc?rcl=0&l=27

+    return False
+
+  @classmethod
   def AddCommandLineArgs(cls, parser):
     group = optparse.OptionGroup(parser, '%s test options' % cls.Name())
     cls.AddBenchmarkCommandLineArgs(group)
 
     if cls.HasTraceRerunDebugOption():
       group.add_option(
           '--rerun-with-debug-trace',
           action='store_true',
           help='Rerun option that enables more extensive tracing.')
 
@@ skipping 116 matching lines @@
       raise NotImplementedError('This test has no "page_set" attribute.')
     return self.page_set()  # pylint: disable=not-callable
 
 
 def AddCommandLineArgs(parser):
   story_runner.AddCommandLineArgs(parser)
 
 
 def ProcessCommandLineArgs(parser, args):
   story_runner.ProcessCommandLineArgs(parser, args)