Allow pyauto performance tests in perf.py to run as an autotest.

chrome/test/functional/perf.py

 #!/usr/bin/python
 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+"""Basic pyauto performance tests.
+
+By default, each test in this file runs a single experimental trial (which
+consists of a simple action that is measured for performance, such as opening
+a new tab).  An environment variable 'PYAUTO_PERF_TEST_NUM_TRIALS' can
+optionally be set, which must be an integer >= 0.  If set, this refers to the
+number of additional experimental trials to run for each test after the initial
+trial.  The average and standard deviation of the additional trials will be
+computed.
+"""
+
 import logging
 import math
 import os
 import time
 
 import pyauto_functional  # Must be imported before pyauto.
 import pyauto
 
 
 class PerfTest(pyauto.PyUITest):
   """Basic performance tests."""
 
-  _NUM_ITERATIONS = 50
+  def setUp(self):
+    """Performs necessary setup work before running each test."""

[Nirnimesh, 2011/08/11 18:26:36]

So if it's not defined in the environment, the test will not run anything?
That doesn't sound right. It should do the default 50, when nothing has been
provided, and then the environment var can be used to tweak it.

[dennis_jeffrey, 2011/08/11 19:29:32]

No, if that variable is not defined in the environment, it would have run a
single trial only (the default behavior).
Changed the code based on our offline discussion about how many iterations each
test should run in the default case.

+    self._num_trials = 0
+    if 'PYAUTO_PERF_TEST_NUM_TRIALS' in os.environ:
+      try:
+        self._num_trials = int(os.environ['PYAUTO_PERF_TEST_NUM_TRIALS'])

[Nirnimesh, 2011/08/11 18:26:36]

That's a env var no one will remember. Use NUM_ITERATIONS

[dennis_jeffrey, 2011/08/11 19:29:32]

Done.

+        if self._num_trials < 0:
+          raise ValueError('Environment variable PYAUTO_PERF_TEST_NUM_TRIALS '
+                           'must be an integer >= 0.')
+      except ValueError, e:
+        self.fail('Error processing environment variable: %s' % e)
+    pyauto.PyUITest.setUp(self)
 
   def _MeasureElapsedTime(self, python_command, num_invocations):
     """Measures time (in msec) to execute a python command one or more times.
 
     Args:
       python_command: A callable.
       num_invocations: An integer number of times to invoke the given command.
 
     Returns:
       The time required to execute the python command the specified number of
@@ skipping 22 matching lines @@
 
   def _PrintSummaryResults(self, first_val, num_iter, values, units):
     """Logs summary measurement information.
 
     Args:
       first_val: A numeric measurement value for a single initial trial.
       num_iter: An integer number of iterations used for multiple trials.
       values: A list of numeric value measurements.
       units: A string specifying the units for the specified measurements.
     """
-    avg, std_dev = self._AvgAndStdDev(values)
-    logging.debug('First trial: %.2f %s', first_val, units)
-    logging.info('Number of iterations: %d', num_iter)
-    for val in values:
-      logging.info('  %.2f %s', val, units)
-    logging.info('  --------------------------')
-    logging.info('  Average: %.2f %s', avg, units)
-    logging.info('  Std dev: %.2f %s', std_dev, units)
+    logging.debug('Single trial: %.2f %s', first_val, units)
+    if self._num_trials:
+      avg, std_dev = self._AvgAndStdDev(values)
+      logging.info('Number of iterations: %d', num_iter)
+      for val in values:
+        logging.info('  %.2f %s', val, units)
+      logging.info('  --------------------------')
+      logging.info('  Average: %.2f %s', avg, units)
+      logging.info('  Std dev: %.2f %s', std_dev, units)
 
   def _RunNewTabTest(self, open_tab_command, num_tabs=1):
     """Runs a perf test that involves opening new tab(s).
 
     This helper function can be called from different tests to do perf testing
     with different types of tabs.  It is assumed that the |open_tab_command|
     will open up a single tab.
 
     Args:
       open_tab_command: A callable that will open a single tab.
       num_tabs: The number of tabs to open, i.e., the number of times to invoke
                 the |open_tab_command|.
     """
     assert callable(open_tab_command)
     orig_elapsed = self._MeasureElapsedTime(open_tab_command, num_tabs)
     self.assertEqual(1 + num_tabs, self.GetTabCount(),
                      msg='Did not open %d new tab(s).' % num_tabs)
     for _ in range(num_tabs):
       self.GetBrowserWindow(0).GetTab(1).Close(True)
 
     timings = []
-    for _ in range(self._NUM_ITERATIONS):
+    for _ in range(self._num_trials):
       elapsed = self._MeasureElapsedTime(open_tab_command, num_tabs)
       self.assertEqual(1 + num_tabs, self.GetTabCount(),
                        msg='Did not open %d new tab(s).' % num_tabs)
       for _ in range(num_tabs):
         self.GetBrowserWindow(0).GetTab(1).Close(True)
       timings.append(elapsed)
 
-    self._PrintSummaryResults(orig_elapsed, self._NUM_ITERATIONS, timings, 'ms')
+    self._PrintSummaryResults(orig_elapsed, self._num_trials, timings, 'ms')
 
   def testNewTab(self):
     """Measures time to open a new tab."""
     self._RunNewTabTest(lambda: self.AppendTab(pyauto.GURL('chrome://newtab')))
 
   def testNewTabPdf(self):
     """Measures time to open a new tab navigated to a PDF file."""
     url = self.GetFileURLForDataPath('pyauto_private', 'pdf', 'TechCrunch.pdf')
     self._RunNewTabTest(lambda: self.AppendTab(pyauto.GURL(url)))
 
@@ skipping 28 matching lines @@
       val = self.ExecuteJavascript(js, 0, 1)
       score = val[val.find(':') + 2:]
       self.GetBrowserWindow(0).GetTab(1).Close(True)
       return int(score)
 
     orig_score = _RunSingleV8BenchmarkSuite()
     self.assertEqual(1, self.GetTabCount(),
                      msg='Did not clean up after running benchmark suite.')
 
     scores = []
-    for _ in range(self._NUM_ITERATIONS):
+    for _ in range(self._num_trials):
       score = _RunSingleV8BenchmarkSuite()
       self.assertEqual(1, self.GetTabCount(),
                        msg='Did not clean up after running benchmark suite.')
       scores.append(score)
 
-    self._PrintSummaryResults(orig_score, self._NUM_ITERATIONS, scores, 'score')
+    self._PrintSummaryResults(orig_score, self._num_trials, scores, 'score')
 
 
 if __name__ == '__main__':
   pyauto_functional.Main()