[Telemetry] Add Benchmark.ShouldTearDownStateAfterEachStorySetRun

telemetry/telemetry/benchmark.py

Index: telemetry/telemetry/benchmark.py
diff --git a/telemetry/telemetry/benchmark.py b/telemetry/telemetry/benchmark.py
index c67bad5c4a1953b0bef6beb48e9f7e13dc34454c..0e7a184b2bf39892af09bd71c48ec017bf656d1c 100644
--- a/telemetry/telemetry/benchmark.py
+++ b/telemetry/telemetry/benchmark.py
@@ -100,7 +100,7 @@ class Benchmark(command_line.Command):
 
   @classmethod
   def ShouldTearDownStateAfterEachStoryRun(cls):
-    """Override this method to tear down state after each story run.
+    """Override to specify whether to tear down state after each story run.
 
     Tearing down all states after each story run, e.g., clearing profiles,
     stopping the browser, stopping local server, etc. So the browser will not be
@@ -113,6 +113,25 @@ class Benchmark(command_line.Command):
     return False
 
   @classmethod
+  def ShouldTearDownStateAfterEachStorySetRun(cls):
+    """Override to specify whether to tear down state after each story set run.
+
+    Defaults to True in order to reset the state and make individual story set
+    repeats more independent of each other. The intended effect is to average
+    out noise in measurements between repeats.
+
+    Long running benchmarks willing to stess test the browser and have it run
+    for long periods of time may switch this value to False.
+
+    This should only be used by TimelineBasedMeasurement (TBM) benchmarks, but
+    not by PageTest based benchmarks.
+    """
+    # TODO(perezju): Switch default value after any benchmarks requiring the
+    # old behaviour are updated accordingly.
+    # https://github.com/catapult-project/catapult/issues/2294#issuecomment-228306977

[nednguyen, 2016/06/24 16:11:55]

this may cause a lint check failure. Feel free to add s.t similar to
https://cs.chromium.org/chromium/src/tools/perf/pylintrc?rcl=0&l=27

+    return False
+
+  @classmethod
   def AddCommandLineArgs(cls, parser):
     group = optparse.OptionGroup(parser, '%s test options' % cls.Name())
     cls.AddBenchmarkCommandLineArgs(group)