Final touches for running smoketests.

tools/testing/perf_testing/create_graph.py

 #!/usr/bin/python
 
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import datetime
 import math
 try:
   from matplotlib.font_manager import FontProperties
@@ skipping 15 matching lines @@
 
 DART_INSTALL_LOCATION = os.path.join(os.path.dirname(os.path.abspath(__file__)),
     '..', '..', '..')
 V8_MEAN = 'V8 Mean'
 FROG_MEAN = 'frog Mean'
 COMMAND_LINE = 'commandline'
 V8 = 'v8'
 FROG = 'frog'
 V8_AND_FROG = [V8, FROG]
 CORRECTNESS = 'Percent passing'
-BENCHMARKS = ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 
-    'BinaryTrees', 'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 
-    'Loop', 'Permute', 'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 
-    'Tak', 'Takl', 'Towers', 'TreeSort']
 COLORS = ['blue', 'green', 'red', 'cyan', 'magenta', 'black']
 GRAPH_OUT_DIR = 'graphs'
 SLEEP_TIME = 200
 PERFBOT_MODE = False
 VERBOSE = False
 
 """First, some utility methods."""
 
 def RunCmd(cmd_list, outfile=None, append=False):
   """Run the specified command and print out any output to stdout.
   Args:
     cmd_list a list of strings that make up the command to run
     outfile a string indicating the name of the file that we should write stdout
        to 
     append True if we want to append to the file instead of overwriting it"""
   if VERBOSE:
     print ' '.join(cmd_list)
   out = subprocess.PIPE
   if outfile:
     mode = 'w'
     if append:
       mode = 'a'
     out = open(outfile, mode)
-  p = subprocess.Popen(cmd_list, stdout = out, stderr = subprocess.PIPE)
+  p = ''
+  if platform.system() == 'Windows':
+    # On Windows, shell must be true to get the correct environment variables.
+    p = subprocess.Popen(cmd_list, stdout = out, stderr = subprocess.PIPE,
+        shell=True)
+  else:
+    p = subprocess.Popen(cmd_list, stdout = out, stderr = subprocess.PIPE)
   output, not_used = p.communicate();
   if output:
     print output
   return output
 
 def TimeCmd(cmd):
   """Determine the amount of (real) time it takes to execute a given command."""
   start = time.time()
   RunCmd(cmd)
   return time.time() - start
@@ skipping 68 matching lines @@
   if platform.system() == 'Windows':
     browsers += ['ie']
   return browsers
 
 def GetVersions():
   if not PERFBOT_MODE:
     return [FROG]
   else:
     return V8_AND_FROG
 
+def GetBenchmarks():
+  if not PERFBOT_MODE:
+    return ['Smoketest']
+  else:
+    return ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 'BinaryTrees',
+    'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 'Loop', 'Permute',
+    'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 'Tak', 'Takl', 'Towers',
+    'TreeSort']
+
 def UploadToAppEngine():
   """Upload our results to our appengine server."""
   # TODO(efortuna): This is the most basic way to get the data up 
   # for others to view. Revisit this once we're serving nicer graphs (Google
   # Chart Tools) and from multiple perfbots and once we're in a position to
   # organize the data in a useful manner(!!).
   os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing',
       'perf_testing'))
   shutil.rmtree(os.path.join('appengine', 'static', 'graphs'), 
       ignore_errors=True)
@@ skipping 127 matching lines @@
             out.write('<td>%f</td>' % perf)
           out.write('</tr>\n<tr><td> %s</td>' % label_2)
           for perf in dict_2:
             out.write('<td>%f</td>' % perf)
           out.write('</tr> </table>')
 
   def CalculateGeometricMean(self, platform, frog_or_v8, svn_revision):
     """Calculate the aggregate geometric mean for V8 and frog benchmark sets,
     given two benchmark dictionaries."""
     geo_mean = 0
-    for benchmark in BENCHMARKS:
+    for benchmark in GetBenchmarks():
       geo_mean += math.log(self.values_dict[platform][frog_or_v8][benchmark][
           len(self.values_dict[platform][frog_or_v8][benchmark]) - 1])
  
     mean = V8_MEAN
     if frog_or_v8 == FROG:
        mean = FROG_MEAN
     self.values_dict[platform][frog_or_v8][mean] += \
-        [math.pow(math.e, geo_mean / len(BENCHMARKS))]
+        [math.pow(math.e, geo_mean / len(GetBenchmarks()))]
     self.revision_dict[platform][frog_or_v8][mean] += [svn_revision]
 
   def Run(self):
     """Run the benchmarks/tests from the command line and plot the 
     results."""
     if PERFBOT_MODE:
       plt.cla() # cla = clear current axes
     os.chdir(DART_INSTALL_LOCATION)
     EnsureOutputDirectory(self.result_folder_name)
     EnsureOutputDirectory(GRAPH_OUT_DIR)
     self.RunTests()
     os.chdir(os.path.join('tools', 'testing', 'perf_testing'))
     
     # TODO(efortuna): You will want to make this only use a subset of the files
     # eventually.
     files = os.listdir(self.result_folder_name)
 
     for afile in files:
       if not afile.startswith('.'):
         self.ProcessFile(afile)
 
     if PERFBOT_MODE:
       self.PlotResults('%s.png' % self.result_folder_name)
 
 class PerformanceTestRunner(TestRunner):
   """Super class for all performance testing."""
   def __init__(self, result_folder_name, platform_list, platform_type):
     super(PerformanceTestRunner, self).__init__(result_folder_name, 
-        platform_list, GetVersions(), BENCHMARKS)
+        platform_list, GetVersions(), GetBenchmarks())
     self.platform_list = platform_list
     self.platform_type = platform_type
 
   def PlotAllPerf(self, png_filename):
     """Create a plot that shows the performance changes of individual benchmarks
     run by V8 and generated by frog, over svn history."""
-    for benchmark in BENCHMARKS:
+    for benchmark in GetBenchmarks():
       self.StyleAndSavePerfPlot(
           'Performance of %s over time on the %s' % (benchmark,
           self.platform_type), 'Speed (bigger = better)', 16, 14, 'lower left', 
           benchmark + png_filename, self.platform_list, GetVersions(), 
           [benchmark])
 
   def PlotAvgPerf(self, png_filename):
     """Generate a plot that shows the performance changes of the geomentric mean
     of V8 and frog benchmark performance over svn history."""
     (title, y_axis, size_x, size_y, loc, filename) = \
@@ skipping 30 matching lines @@
     f = open(os.path.join(self.result_folder_name, afile))
     tabulate_data = False
     revision_num = 0
     for line in f.readlines():
       if 'Revision' in line:
         revision_num = int(line.split()[1])
       elif 'Benchmark' in line:
         tabulate_data = True
       elif tabulate_data:
         tokens = line.split()
-        if len(tokens) < 4 or tokens[0] not in BENCHMARKS:
+        if len(tokens) < 4 or tokens[0] not in GetBenchmarks():
           #Done tabulating data.
           break
         v8_value = float(tokens[1])
         frog_value = float(tokens[3])
         if v8_value == 0 or frog_value == 0:
           #Then there was an error when this performance test was run. Do not 
           #count it in our numbers.
           return
         benchmark = tokens[0]
         self.revision_dict[COMMAND_LINE][V8][benchmark] += [revision_num]
@@ skipping 21 matching lines @@
   def __init__(self, result_folder_name):
     super(BrowserPerformanceTestRunner, self).__init__(
         result_folder_name, GetBrowsers(), 'browser')
 
   def RunTests(self):
     """Run a performance test in the browser."""
       # For the smoke test, just run a simple test, not the actual benchmarks to
       # ensure we haven't broken the Firefox DOM.
 
     os.chdir('frog')
-    RunCmd(['python', os.path.join('benchmarks', 'make_web_benchmarks.py')])
+    if PERFBOT_MODE:
+      RunCmd(['python', os.path.join('benchmarks', 'make_web_benchmarks.py')])
+    else:
+      RunCmd(['./minfrog', '--out=../tools/testing/perf_testing/smoketest/' + \
+      'smoketest_frog.js', '--libdir=%s/lib' % os.getcwd(),
+      '--compile-only', '../tools/testing/perf_testing/smoketest/' + \
+      'dartWebBase.dart'])
     os.chdir('..')
 
     for browser in GetBrowsers():
       for version in GetVersions():
         self.trace_file = os.path.join('tools', 'testing', 'perf_testing', 
             self.result_folder_name,
             'perf-%s-%s-%s' % (self.cur_time, browser, version))
         self.AddSvnRevisionToTrace(self.trace_file)
-        bench_page = 'benchmark_page'
+        file_path = os.path.join(os.getcwd(), 'internal', 'browserBenchmarks', 
+            'benchmark_page_%s.html' % version)
         if not PERFBOT_MODE:
-          bench_page = 'smoketest'
-          pass
+          file_path = os.path.join(os.getcwd(), 'tools', 'testing',
+              'perf_testing', 'smoketest', 'smoketest_%s.html' % version)
         RunCmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'), 
-            '--out', os.path.join(os.getcwd(), 'internal', 'browserBenchmarks', 
-            '%s_%s.html' % (bench_page, version)), '--browser', browser, 
+            '--out', file_path, '--browser', browser, 
             '--timeout', '600', '--perf'], self.trace_file, append=True)
 
   def ProcessFile(self, afile):
     """Comb through the html to find the performance results."""
     parts = afile.split('-')
     browser = parts[2] 
     version = parts[3]
     f = open(os.path.join(self.result_folder_name, afile))
     lines = f.readlines()
     line = ''
@@ skipping 272 matching lines @@
       if HasNewCode():
         RunTestSequence(cl, size, language, perf)
       else:
         time.sleep(SLEEP_TIME)
   else:
     RunTestSequence(cl, size, language, perf)
 
 if __name__ == '__main__':
   main()