[Telemetry] Add Benchmark.ShouldTearDownStateAfterEachStorySetRun

telemetry/telemetry/benchmark.py

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import optparse
 
 from telemetry import decorators
 from telemetry.internal import story_runner
 from telemetry.internal.util import command_line
 from telemetry.page import legacy_page_test
@@ skipping 82 matching lines @@
   @property
   def max_failures(self):
     return self._max_failures
 
   @classmethod
   def Name(cls):
     return '%s.%s' % (cls.__module__.split('.')[-1], cls.__name__)
 
   @classmethod
   def ShouldTearDownStateAfterEachStoryRun(cls):
-    """Override this method to tear down state after each story run.
+    """Override to specify whether to tear down state after each story run.
 
     Tearing down all states after each story run, e.g., clearing profiles,
     stopping the browser, stopping local server, etc. So the browser will not be
     reused among multiple stories. This is particularly useful to get the
     startup part of launching the browser in each story.
 
     This should only be used by TimelineBasedMeasurement (TBM) benchmarks, but
     not by PageTest based benchmarks.
     """
     return False
 
   @classmethod
+  def ShouldTearDownStateAfterEachStorySetRun(cls):
+    """Override to specify whether to tear down state after each story set run.
+
+    Defaults to True in order to reset the state and make individual story set
+    repeats more independent of each other. The intended effect is to average
+    out noise in measurements between repeats.
+
+    Long running benchmarks willing to stess test the browser and have it run
+    for long periods of time may switch this value to False.
+
+    This should only be used by TimelineBasedMeasurement (TBM) benchmarks, but
+    not by PageTest based benchmarks.
+    """
+    return True

[nednguyen, 2016/06/24 14:44:50]

This will cause a behavior change to all existing benchmarks that's using
pageset-repeat. I think we should make this False for now, then send an email to
telemetry-announce@ telling people about this, then switch this to True.

[perezju, 2016/06/24 15:12:36]

ok, sgtm

+
+  @classmethod
   def AddCommandLineArgs(cls, parser):
     group = optparse.OptionGroup(parser, '%s test options' % cls.Name())
     cls.AddBenchmarkCommandLineArgs(group)
 
     if cls.HasTraceRerunDebugOption():
       group.add_option(
           '--rerun-with-debug-trace',
           action='store_true',
           help='Rerun option that enables more extensive tracing.')
 
@@ skipping 116 matching lines @@
       raise NotImplementedError('This test has no "page_set" attribute.')
     return self.page_set()  # pylint: disable=not-callable
 
 
 def AddCommandLineArgs(parser):
   story_runner.AddCommandLineArgs(parser)
 
 
 def ProcessCommandLineArgs(parser, args):
   story_runner.ProcessCommandLineArgs(parser, args)