Update _OutputPerfGraphValue to handle Chrome.

functional/perf.py

 #!/usr/bin/env python
 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Basic pyauto performance tests.
 
 For tests that need to be run for multiple iterations (e.g., so that average
 and standard deviation values can be reported), the default number of iterations
 run for each of these tests is specified by |_DEFAULT_NUM_ITERATIONS|.
@@ skipping 49 matching lines @@
   def setUp(self):
     """Performs necessary setup work before running each test."""
     self._num_iterations = self._DEFAULT_NUM_ITERATIONS
     if 'NUM_ITERATIONS' in os.environ:
       self._num_iterations = int(os.environ['NUM_ITERATIONS'])
     self._max_timeout_count = self._DEFAULT_MAX_TIMEOUT_COUNT
     if 'MAX_TIMEOUT_COUNT' in os.environ:
       self._max_timeout_count = int(os.environ['MAX_TIMEOUT_COUNT'])
     self._timeout_count = 0
     pyauto.PyUITest.setUp(self)
+    print ('CHROME VERSION: ' +
+           self.GetBrowserInfo()['properties']['ChromeVersion'])

[dennis_jeffrey, 2011/12/07 22:25:29]

This is not actually needed right now, until we submit the change that makes use
of it in the buildbot code (and even then, we may ultimately find out that we
don't need this print statement at all).  I recommend we remove this change for
now, until we're sure that we will need it.  If you still want to keep it,
please add a TODO above this line indicating that we need to remember to either
make use of this output value (in the buildbot test output parsing code), or
else remove this statement if it turns out that it's not needed.

[chrisphan, 2011/12/07 22:34:13]

Done.

 
   def _AppendTab(self, url):
     """Appends a tab and increments a counter if the automation call times out.
 
     Args:
       url: The string url to which the appended tab should be navigated.
     """
     if not self.AppendTab(pyauto.GURL(url)):
       self._timeout_count += 1
 
@@ skipping 26 matching lines @@
     """
     avg = 0.0
     std_dev = 0.0
     if values:
       avg = float(sum(values)) / len(values)
       if len(values) > 1:
         temp_vals = [math.pow(x - avg, 2) for x in values]
         std_dev = math.sqrt(sum(temp_vals) / (len(temp_vals) - 1))
     return avg, std_dev
 
-  def _OutputPerfGraphValue(self, description, value):
+  def _OutputPerfGraphValue(self, description, value, units,
+                            graph_name='Default-Graph'):
     """Outputs a performance value to have it graphed on the performance bots.
 
-    Only used for ChromeOS.  The performance bots have a 30-character limit on
-    the length of the description for a performance value.  Any characters
-    beyond that are truncated before results are stored in the autotest
+    The output format differs, depending on whether the current platform is
+    Chrome desktop or ChromeOS.
+
+

[dennis_jeffrey, 2011/12/07 22:25:29]

remove one of these blank lines

[chrisphan, 2011/12/07 22:34:13]

Done.

+    For ChromeOS, the performance bots have a 30-character limit on the length
+    of the key associated with a performance value.  A key on ChromeOS is
+    considered to be of the form "units_description" (for example,
+    "milliseconds_NewTabPage"), and is created from the |units| and
+    |description| passed as input to this function.  Any characters beyond the
+    length 30 limit are truncated before results are stored in the autotest
     database.
 
     Args:
-      description: A string description of the performance value.  The string
-                   should be of the form "units_identifier" (for example,
-                   "milliseconds_NewTabPage").  This description will be
-                   truncated to 30 characters when stored in the autotest
-                   database.
+      description: A string description of the performance value.  Should not
+                   include spaces.
       value: A numeric value representing a single performance measurement.
+      units: A string representing the units of the performance value.  Should
+             not include spaces.
+      graph_name: A string name for the graph associated with this performance
+                  value.  Only used on Chrome desktop.
+
     """
     if self.IsChromeOS():
-      if len(description) > 30:
+      perf_key = '%s_%s' % (units, description)
+      if len(perf_key) > 30:
         logging.warning('The description "%s" will be truncated to "%s" '
                         '(length 30) when added to the autotest database.',
-                        description, description[:30])
-      print '\n%s(\'%s\', %.2f)%s' % (self._PERF_OUTPUT_MARKER_PRE, description,
-                                      value, self._PERF_OUTPUT_MARKER_POST)
+                        perf_key, perf_key[:30])
+      print '\n%s(\'%s\', %.2f)%s' % (self._PERF_OUTPUT_MARKER_PRE,
+                                      perf_key, value,
+                                      self._PERF_OUTPUT_MARKER_POST)
       sys.stdout.flush()
+    else:
+      pyauto_utils.PrintPerfResult(graph_name, description, value, units)
 
-  def _PrintSummaryResults(self, description, values, units):
+  def _PrintSummaryResults(self, description, values, units,
+                            graph_name='Default-Graph'):

[dennis_jeffrey, 2011/12/07 22:25:29]

indent this line by 1 fewer space

[chrisphan, 2011/12/07 22:34:13]

Done.

     """Logs summary measurement information.
 
     This function computes and outputs the average and standard deviation of
     the specified list of value measurements.  It also invokes
     _OutputPerfGraphValue() with the computed *average* value, to ensure the
     average value can be plotted in a performance graph.
 
     Args:
       description: A string description for the specified results.
       values: A list of numeric value measurements.
       units: A string specifying the units for the specified measurements.

[dennis_jeffrey, 2011/12/07 22:25:29]

add a description for the new "graph_name" argument

[chrisphan, 2011/12/07 22:34:13]

Done.

     """
     logging.info('Results for: ' + description)
     if values:
       avg, std_dev = self._AvgAndStdDev(values)
       logging.info('Number of iterations: %d', len(values))
       for val in values:
         logging.info('  %.2f %s', val, units)
       logging.info('  --------------------------')
       logging.info('  Average: %.2f %s', avg, units)
       logging.info('  Std dev: %.2f %s', std_dev, units)
-      self._OutputPerfGraphValue('%s_%s' % (units, description), avg)
+      self._OutputPerfGraphValue(description, avg, units, graph_name)

[dennis_jeffrey, 2011/12/07 22:25:29]

pass the last argument as a named argument: graph_name=graph_name

[chrisphan, 2011/12/07 22:34:13]

Done.

     else:
       logging.info('No results to report.')
 
   def _RunNewTabTest(self, description, open_tab_command, num_tabs=1):
     """Runs a perf test that involves opening new tab(s).
 
     This helper function can be called from different tests to do perf testing
     with different types of tabs.  It is assumed that the |open_tab_command|
     will open up a single tab.
 
@@ skipping 13 matching lines @@
       # Only count the timing measurement if no automation call timed out.
       if self._timeout_count == orig_timeout_count:
         timings.append(elapsed_time)
       self.assertTrue(self._timeout_count <= self._max_timeout_count,
                       msg='Test exceeded automation timeout threshold.')
       self.assertEqual(1 + num_tabs, self.GetTabCount(),
                        msg='Did not open %d new tab(s).' % num_tabs)
       for _ in range(num_tabs):
         self.GetBrowserWindow(0).GetTab(1).Close(True)
 
-    self._PrintSummaryResults(description, timings, 'milliseconds')
+    self._PrintSummaryResults(description, timings, 'milliseconds',
+                              description)
 
   def _LoginToGoogleAccount(self):
     """Logs in to a testing Google account."""
     creds = self.GetPrivateInfo()['test_google_account']
     test_utils.GoogleAccountsLogin(self, creds['username'], creds['password'])
     self.NavigateToURL('about:blank')  # Clear the existing tab.
 
   def _GetCPUUsage(self):
     """Returns machine's CPU usage.
 
@@ skipping 112 matching lines @@
 
     timings = {}
     for _ in xrange(self._num_iterations):
       result_dict = _RunBenchmarkOnce(url)
       for key, val in result_dict.items():
         timings.setdefault(key, []).append(val)
 
     for key, val in timings.items():
       print
       if key == 'final_score':
-        self._PrintSummaryResults('V8Benchmark', val, 'score')
+        self._PrintSummaryResults('V8Benchmark', val, 'score',
+                                  'V8Benchmark-final')
       else:
-        self._PrintSummaryResults('V8Benchmark-%s' % key, val, 'score')
+        self._PrintSummaryResults('V8Benchmark-%s' % key, val, 'score',
+                                  'V8Benchmark-individual')
 
   def testSunSpider(self):
     """Runs the SunSpider javascript benchmark suite."""
     url = self.GetFileURLForDataPath('sunspider', 'sunspider-driver.html')
     self.assertTrue(self.AppendTab(pyauto.GURL(url)),
                     msg='Failed to append tab for SunSpider benchmark suite.')
 
     js_is_done = """
         var done = false;
         if (document.getElementById("console"))
           done = true;
         window.domAutomationController.send(JSON.stringify(done));
     """
     self.assertTrue(
         self.WaitUntil(
             lambda: self.ExecuteJavascript(js_is_done, tab_index=1) == 'true',
             timeout=300, retry_sleep=1),
         msg='Timed out when waiting for SunSpider benchmark score.')
 
     js_get_results = """
         window.domAutomationController.send(
             document.getElementById("console").innerHTML);
     """
     # Append '<br>' to the result to simplify regular expression matching.
     results = self.ExecuteJavascript(js_get_results, tab_index=1) + '<br>'
     total = re.search('Total:\s*([\d.]+)ms', results).group(1)
     logging.info('Total: %.2f ms' % float(total))
-    self._OutputPerfGraphValue('ms_SunSpider-total', float(total))
+    self._OutputPerfGraphValue('SunSpider', float(total), 'ms',

[dennis_jeffrey, 2011/12/07 22:25:29]

please change the first argument to "SunSpider-total", otherwise this will
change the "perf_key" that gets created on ChromeOS.

[chrisphan, 2011/12/07 22:34:13]

Done.

+                               graph_name='SunSpider-total')
 
     for match_category in re.finditer('\s\s(\w+):\s*([\d.]+)ms.+?<br><br>',
                                       results):
       category_name = match_category.group(1)
       category_result = match_category.group(2)
       logging.info('Benchmark "%s": %.2f ms', category_name,
                    float(category_result))
-      self._OutputPerfGraphValue('ms_SunSpider-%s' % category_name,
-                                 float(category_result))
+      self._OutputPerfGraphValue('SunSpider-' + category_name,
+                                 float(category_result), 'ms',
+                                 graph_name='SunSpider-individual')
+
       for match_result in re.finditer('<br>\s\s\s\s([\w-]+):\s*([\d.]+)ms',
                                       match_category.group(0)):
         result_name = match_result.group(1)
         result_value = match_result.group(2)
         logging.info('  Result "%s-%s": %.2f ms', category_name, result_name,
                      float(result_value))
         self._OutputPerfGraphValue(
-            'ms_SunSpider-%s-%s' % (category_name, result_name),
-            float(result_value))
+            'SunSpider-%s-%s' % (category_name, result_name),
+            float(result_value), 'ms',
+            graph_name='SunSpider-individual')
 
   def testDromaeoSuite(self):
     """Measures results from Dromaeo benchmark suite."""
     url = self.GetFileURLForDataPath('dromaeo', 'index.html')
     self.assertTrue(self.AppendTab(pyauto.GURL(url + '?dromaeo')),
                     msg='Failed to append tab for Dromaeo benchmark suite.')
 
     js_is_ready = """
         var val = document.getElementById('pause').value;
         window.domAutomationController.send(val);
@@ skipping 36 matching lines @@
                 group_results['sub_groups'][sub_name] =
                     $(this).text().match(/: ([\d.]+)/)[1]
             });
             result['all_results'][group_name] = group_results;
         });
         window.domAutomationController.send(JSON.stringify(result));
     """
     results = eval(self.ExecuteJavascript(js_get_results, tab_index=1))
     total_result = results['total_result']
     logging.info('Total result: ' + total_result)
-    self._OutputPerfGraphValue('runsPerSec_Dromaeo-total', float(total_result))
-
+    self._OutputPerfGraphValue(
+        'Dromaeo-total',
+        float(total_result), 'runsPerSec',
+        graph_name='Dromaeo-total')
+    
     for group_name, group in results['all_results'].iteritems():
       logging.info('Benchmark "%s": %s', group_name, group['result'])
       self._OutputPerfGraphValue(
-          'runsPerSec_Dromaeo-%s' % group_name.replace(' ', ''),
-          float(group['result']))
-
+          'Dromaeo-' + group_name.replace(' ', ''),
+          float(group['result']), 'runsPerSec',
+          graph_name='Dromaeo-individual')
       for benchmark_name, benchmark_score in group['sub_groups'].iteritems():
         logging.info('  Result "%s": %s', benchmark_name, benchmark_score)
 
 
 class LiveWebappLoadTest(BasePerfTest):
   """Tests that involve performance measurements of live webapps.
 
   These tests connect to live webpages (e.g., Gmail, Calendar, Docs) and are
   therefore subject to network conditions.  These tests are meant to generate
   "ball-park" numbers only (to see roughly how long things take to occur from a
@@ skipping 129 matching lines @@
     time.sleep(60)
     total_video_frames = self._GetVideoFrames() - init_video_frames
     total_dropped_frames = self._GetVideoDroppedFrames() - init_dropped_frames
     cpu_usage_end = self._GetCPUUsage()
     fraction_non_idle_time = \
         self._GetFractionNonIdleCPUTime(cpu_usage_start, cpu_usage_end)
     # Counting extrapolation for utilization to play the video.
     extrapolation_value = fraction_non_idle_time * \
         (total_video_frames + total_dropped_frames) / total_video_frames
     logging.info('Netflix CPU extrapolation: %.2f' % extrapolation_value)
-    self._OutputPerfGraphValue('extrapolation_NetflixCPUExtrapolation',
-                               extrapolation_value)
+    self._OutputPerfGraphValue(
+        'NetflixCPUExtrapolation',
+        extrapolation_value, 'extrapolation',
+        graph_name='NetflixCPUExtrapolation')
 
 
 class YoutubePerfTest(BasePerfTest, YoutubeTestHelper):
   """Test Youtube video performance."""
 
   def __init__(self, methodName='runTest', **kwargs):
     pyauto.PyUITest.__init__(self, methodName, **kwargs)
     YoutubeTestHelper.__init__(self, self)
 
   def _VerifyVideoTotalBytes(self):
@@ skipping 72 matching lines @@
     total_dropped_frames = self.GetVideoDroppedFrames() - init_dropped_frames
     cpu_usage_end = self._GetCPUUsage()
     
     fraction_non_idle_time = self._GetFractionNonIdleCPUTime(
         cpu_usage_start, cpu_usage_end)
     total_frames = total_shown_frames + total_dropped_frames
     # Counting extrapolation for utilization to play the video.
     extrapolation_value = (fraction_non_idle_time *
                            (total_frames / total_shown_frames))
     logging.info('Youtube CPU extrapolation: %.2f' % extrapolation_value)
-    self._OutputPerfGraphValue('extrapolation_YoutubeCPUExtrapolation',
-                               extrapolation_value)
+    self._OutputPerfGraphValue(
+        'YoutubeCPUExtrapolation',
+        extrapolation_value, 'extrapolation',
+        graph_name='YoutubeCPUExtrapolation')
 
 
 class WebGLTest(BasePerfTest):
   """Tests for WebGL performance."""
 
   def _RunWebGLTest(self, url, description):
     """Measures FPS using a specified WebGL demo.
 
     Args:
       url: The string URL that, once loaded, will run the WebGL demo (default
@@ skipping 426 matching lines @@
         window.domAutomationController.send(
             JSON.stringify(final_average_fps));
     """
     self.assertTrue(
         self.WaitUntil(
             lambda: self.ExecuteJavascript(js, tab_index=1) != '-1',
             timeout=300, expect_retval=True, retry_sleep=1),
         msg='Timed out when waiting for test result.')
     result = float(self.ExecuteJavascript(js, tab_index=1))
     logging.info('Result for %s: %.2f FPS (average)', description, result)
-    self._OutputPerfGraphValue('%s_%s' % ('FPS', description), result)
+    self._OutputPerfGraphValue(
+        description,
+        result, 'FPS',
+        graph_name=description)
 
   def testFlashGaming(self):
     """Runs a simple flash gaming benchmark test."""
     webpage_url = self.GetHttpURLForDataPath('pyauto_private', 'flash',
                                              'FlashGamingTest2.html')
     self._RunFlashTestForAverageFPS(webpage_url, 'FlashGaming')
 
   def testFlashText(self):
     """Runs a simple flash text benchmark test."""
     webpage_url = self.GetHttpURLForDataPath('pyauto_private', 'flash',
@@ skipping 27 matching lines @@
     result = eval(self.ExecuteJavascript(js_result, tab_index=1))
     for benchmark in result:
       mflops = float(result[benchmark][0])
       mem = float(result[benchmark][1])
       if benchmark.endswith('_mflops'):
         benchmark = benchmark[:benchmark.find('_mflops')]
       logging.info('Results for ScimarkGui_' + benchmark + ':')
       logging.info('  %.2f MFLOPS', mflops)
       logging.info('  %.2f MB', mem)
       self._OutputPerfGraphValue(
-          '%s_ScimarkGui-%s-MFLOPS' % ('MFLOPS', benchmark), mflops)
+          'ScimarkGui-%s-MFLOPS' % benchmark,
+          mflops, 'MFLOPS',
+          graph_name='ScimarkGui')
       self._OutputPerfGraphValue(
-          '%s_ScimarkGui-%s-Mem' % ('MB', benchmark), mem)
+          'ScimarkGui-%s-Mem' % benchmark,
+          mem, 'MB',
+          graph_name='ScimarkGui')
 
 
 class LiveGamePerfTest(BasePerfTest):
   """Tests to measure performance of live gaming webapps."""
 
   def ExtraChromeFlags(self):
     """Ensures Chrome is launched with custom flags.
 
     Returns:
       A list of extra flags to pass to Chrome when it is launched.
@@ skipping 28 matching lines @@
 
     # Let the app run for 1 minute.
     time.sleep(60)
 
     cpu_usage_end = self._GetCPUUsage()
     fraction_non_idle_time = self._GetFractionNonIdleCPUTime(
         cpu_usage_start, cpu_usage_end)
 
     logging.info('Fraction of CPU time spent non-idle: %.2f' %
                  fraction_non_idle_time)
-    self._OutputPerfGraphValue('Fraction_%sCpuBusy' % description,
-                               fraction_non_idle_time)
+    self._OutputPerfGraphValue(
+        description + 'CpuBusy',
+        fraction_non_idle_time, 'Fraction',
+        graph_name='CpuBusy')
     snapshotter = perf_snapshot.PerformanceSnapshotter()
     snapshot = snapshotter.HeapSnapshot()[0]
     v8_heap_size = snapshot['total_heap_size'] / (1024.0 * 1024.0)
     logging.info('Total v8 heap size: %.2f MB' % v8_heap_size)
-    self._OutputPerfGraphValue('MB_%sV8HeapSize' % description, v8_heap_size)
+    self._OutputPerfGraphValue(
+        description + 'V8HeapSize',
+        v8_heap_size, 'MB',
+        graph_name='V8HeapSize')
 
   def testAngryBirds(self):
     """Measures performance for Angry Birds."""
     self._RunLiveGamePerfTest('http://chrome.angrybirds.com', 'Angry Birds',
                               'AngryBirds')
 
 
 class PerfTestServerRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
   """Request handler for the local performance test server."""
 
@@ skipping 220 matching lines @@
     """Identifies the port number to which the server is currently bound.
 
     Returns:
       The numeric port number to which the server is currently bound.
     """
     return self._server.server_address[1]
 
 
 if __name__ == '__main__':
   pyauto_functional.Main()