Add Safari to the list of browsers we test on a regular basis.

tools/testing/perf_testing/create_graph.py

 #!/usr/bin/python
 
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import datetime
 import math
 try:
   from matplotlib.font_manager import FontProperties
@@ skipping 67 matching lines @@
   Args:
     failed_once True if we have attempted to build this once before, and we've
       failed, indicating the build is broken.
   Returns:
     err_code = 1 if there was a problem building two times in a row."""
   os.chdir(DART_INSTALL_LOCATION)
   #Revert our newly built minfrog to prevent conflicts when we update
   RunCmd(['svn', 'revert',  os.path.join(os.getcwd(), 'frog', 'minfrog')])
 
   RunCmd(['gclient', 'sync'])
-  lines = RunCmd([os.path.join('.', 'tools', 'build.py'), '-m', 'release'])
-  os.chdir('frog')
-  lines += RunCmd([os.path.join('..', 'tools', 'build.py'), '-m', 
-      'debug,release'])
-  os.chdir('..')
+  #TODO(efortuna): Temporary fix to get IE data without requiring Dart to build
+  # on Windows. Take this out once we have SDKs or can build on Windows.
+  if platform.system() != 'Windows':
+    lines = RunCmd([os.path.join('.', 'tools', 'build.py'), '-m', 'release'])
+    os.chdir('frog')
+    lines += RunCmd([os.path.join('..', 'tools', 'build.py'), '-m', 
+        'debug,release'])
+    os.chdir('..')
   
-  for line in lines:
-    if 'BUILD FAILED' in lines:
-      if failed_once:
-        # Someone checked in a broken build! Just stop trying to make it work
-        # and wait for the next hour to try again.
-        print 'Broken Build'
-        return 1
-      #Remove the xcode directory and attempt to build again. If it still
-      #fails, abort, and try again next hour.
-      out_dir = 'out'
-      if platform.system() == 'Darwin':
-        out_dir = 'xcodebuild'
-      shutil.rmtree(os.path.join(os.getcwd(), out_dir, 'Release_ia32'))
-      shutil.rmtree(os.path.join(os.getcwd(), 'frog', out_dir, 
-          'Debug_ia32'))
-      shutil.rmtree(os.path.join(os.getcwd(), 'frog', out_dir, 
-          'Release_ia32'))
-      SyncAndBuild(True)
+    for line in lines:
+      if 'BUILD FAILED' in lines:
+        if failed_once:
+          # Someone checked in a broken build! Just stop trying to make it work
+          # and wait for the next hour to try again.
+          print 'Broken Build'
+          return 1
+        #Remove the xcode directory and attempt to build again. If it still
+        #fails, abort, and try again next hour.
+        out_dir = 'out'
+        if platform.system() == 'Darwin':
+          out_dir = 'xcodebuild'
+        shutil.rmtree(os.path.join(os.getcwd(), out_dir, 'Release_ia32'))
+        shutil.rmtree(os.path.join(os.getcwd(), 'frog', out_dir, 
+            'Debug_ia32'))
+        shutil.rmtree(os.path.join(os.getcwd(), 'frog', out_dir, 
+            'Release_ia32'))
+        SyncAndBuild(True)
   return 0
 
 def EnsureOutputDirectory(dir_name):
   """Test that the listed directory name exists, and if not, create one for
   our output to be placed.
   Args:
     dir_name the directory we will create if it does not exist."""
   dir_path = os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing',
       'perf_testing', dir_name)
   if not os.path.exists(dir_path):
     os.mkdir(dir_path)
     print 'Creating output directory ', dir_path
 
 def HasNewCode():
   """Tests if there are any newer versions of files on the server."""
   os.chdir(DART_INSTALL_LOCATION)    
   results = RunCmd(['svn', 'st', '-u'])
   for line in results:
     if '*' in line:
       return True
   return False
 
 def GetBrowsers():
   if not PERFBOT_MODE:
     # Only Firefox (and Chrome, but we have Dump Render Tree) works in Linux
     return ['ff']
-  browsers = ['ff', 'chrome']
+  browsers = ['ff', 'chrome', 'safari']
   if platform.system() == 'Windows':
     browsers += ['ie']
   return browsers
 
 def GetVersions():
   if not PERFBOT_MODE:
     return [FROG]
   else:
     return V8_AND_FROG
 
@@ skipping 24 matching lines @@
          performance results will be stored.
          platform_list a list containing the platform(s) that our data has been
             run on. (command line, firefox, chrome, etc)
          v8_and_or_frog_list a list specifying whether we hold data about Frog
             generated code, plain JS code (v8), or a combination of both.
          values_list a list containing the type of data we will be graphing
             (benchmarks, percentage passing, etc)"""
     self.result_folder_name = result_folder_name
     # cur_time is used as a timestamp of when this performance test was run.
     self.cur_time = str(time.mktime(datetime.datetime.now().timetuple()))
-    self.browser_color = {'chrome': 'green', 'ie': 'blue', 'ff': 'red'}
+    self.browser_color = {'chrome': 'green', 'ie': 'blue', 'ff': 'red',
+        'safari':'black'}
     self.values_list = values_list
     self.platform_list = platform_list
     self.revision_dict = dict()
     self.values_dict = dict()
     self.color_index = 0
     for platform in platform_list:
       self.revision_dict[platform] = dict()
       self.values_dict[platform] = dict()
       for f in v8_and_or_frog_list:
         self.revision_dict[platform][f] = dict()
@@ skipping 102 matching lines @@
       geo_mean += math.log(self.values_dict[platform][frog_or_v8][benchmark][
           len(self.values_dict[platform][frog_or_v8][benchmark]) - 1])
  
     mean = V8_MEAN
     if frog_or_v8 == FROG:
        mean = FROG_MEAN
     self.values_dict[platform][frog_or_v8][mean] += \
         [math.pow(math.e, geo_mean / len(BENCHMARKS))]
     self.revision_dict[platform][frog_or_v8][mean] += [svn_revision]
 
-  def Cleanup(self):
-      pass
-
   def Run(self):
     """Run the benchmarks/tests from the command line and plot the 
     results."""
     if PERFBOT_MODE:
       plt.cla() # cla = clear current axes
     os.chdir(DART_INSTALL_LOCATION)
     EnsureOutputDirectory(self.result_folder_name)
     EnsureOutputDirectory(GRAPH_OUT_DIR)
     self.RunTests()
     os.chdir(os.path.join('tools', 'testing', 'perf_testing'))
     
     # TODO(efortuna): You will want to make this only use a subset of the files
     # eventually.
     files = os.listdir(self.result_folder_name)
 
     for afile in files:
       if not afile.startswith('.'):
         self.ProcessFile(afile)
 
     if PERFBOT_MODE:
       self.PlotResults('%s.png' % self.result_folder_name)
-    self.Cleanup();
 
 class PerformanceTestRunner(TestRunner):
   """Super class for all performance testing."""
   def __init__(self, result_folder_name, platform_list, platform_type):
     super(PerformanceTestRunner, self).__init__(result_folder_name, 
         platform_list, GetVersions(), BENCHMARKS)
     self.platform_list = platform_list
     self.platform_type = platform_type
 
   def PlotAllPerf(self, png_filename):
@@ skipping 81 matching lines @@
 
 class BrowserPerformanceTestRunner(PerformanceTestRunner):
   """Runs performance tests, in the browser."""
 
   def __init__(self, result_folder_name):
     super(BrowserPerformanceTestRunner, self).__init__(
         result_folder_name, GetBrowsers(), 'browser')
 
   def RunTests(self):
     """Run a performance test in the browser."""
+      # For the smoke test, just run a simple test, not the actual benchmarks to
+      # ensure we haven't broken the Firefox DOM.
+
     os.chdir('frog')
     RunCmd(['python', os.path.join('benchmarks', 'make_web_benchmarks.py')])
     os.chdir('..')
 
     for browser in GetBrowsers():
       for version in GetVersions():
         self.trace_file = os.path.join('tools', 'testing', 'perf_testing', 
             self.result_folder_name,
             'perf-%s-%s-%s' % (self.cur_time, browser, version))
         self.AddSvnRevisionToTrace(self.trace_file)
+        bench_page = 'benchmark_page'
+        if not PERFBOT_MODE:
+          bench_page = 'smoketest'
+          pass
         RunCmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'), 
-            '--out', os.path.join(os.getcwd(), 'internal', 'browserBenchmarks',
-            'benchmark_page_%s.html' % version), '--browser', browser, 
+            '--out', os.path.join(os.getcwd(), 'internal', 'browserBenchmarks', 
+            '%s_%s.html' % (bench_page, version)), '--browser', browser, 
             '--timeout', '600', '--perf'], self.trace_file, append=True)
 
   def ProcessFile(self, afile):
     """Comb through the html to find the performance results."""
     parts = afile.split('-')
     browser = parts[2] 
     version = parts[3]
     f = open(os.path.join(self.result_folder_name, afile))
     lines = f.readlines()
     line = ''
@@ skipping 39 matching lines @@
       os.remove(os.path.join(self.result_folder_name, afile))
     else:
       self.CalculateGeometricMean(browser, version, revision_num)
 
   def WriteHtml(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2, 
       cleanFile=False):
       #TODO(efortuna)
       pass
 
       
-  def Cleanup(self):
-    # Kill the zombie chromedriver processes.
-    RunCmd(['killall', 'chromedriver'])
-
-
 class BrowserCorrectnessTestRunner(TestRunner):
   def __init__(self, test_type, result_folder_name):
     super(BrowserCorrectnessTestRunner, self).__init__(result_folder_name,
         GetBrowsers(), [FROG], [CORRECTNESS])
     self.test_type = test_type
 
   def RunTests(self):
     """Run a test of the latest svn revision."""
     for browser in GetBrowsers():
       current_file = 'correctness%s-%s' % (self.cur_time, browser)
@@ skipping 38 matching lines @@
     f.close()
 
   def PlotResults(self, png_filename):
     first_time = True
     for browser in GetBrowsers():
       self.StyleAndSavePerfPlot('Percentage of language tests passing in '
           'different browsers', '% of tests passed', 8, 8, 'lower left', 
           png_filename, [browser], [FROG], [CORRECTNESS], first_time) 
       first_time = False
 
-  def Cleanup(self):
-    # Kill the zombie chromedriver processes.
-    RunCmd(['killall', 'chromedriver'])
 
 class CompileTimeAndSizeTestRunner(TestRunner):
   """Run tests to determine how long minfrog takes to compile, and the compiled 
   file output size of some benchmarking files."""
   def __init__(self, result_folder_name):
     super(CompileTimeAndSizeTestRunner, self).__init__(result_folder_name,
         [COMMAND_LINE], [FROG], ['Compiling on Dart VM', 'Bootstrapping',
         'minfrog', 'swarm', 'total'])
     self.failure_threshold = {'Compiling on Dart VM' : 1, 'Bootstrapping' : .5, 
         'minfrog' : 100, 'swarm' : 100, 'total' : 100}
@@ skipping 109 matching lines @@
   parser.add_option('--language', '-l', dest = 'language', 
       help = 'Run the language correctness tests', 
       action = 'store_true', default = False)
   parser.add_option('--browser-perf', '-b', dest = 'perf',
       help = 'Run the browser performance tests',
       action = 'store_true', default = False)
   parser.add_option('--forever', '-f', dest = 'continuous',
       help = 'Run this script forever, always checking for the next svn '
       'checkin', action = 'store_true', default = False)
   parser.add_option('--perfbot', '-p', dest = 'perfbot',
-      help = "Run in perfbot mode. (Generate plots, and remove trace files)", 
+      help = "Run in perfbot mode. (Generate plots, and keep trace files)", 
       action = 'store_true', default = False)
   parser.add_option('--verbose', '-v', dest = 'verbose',
       help = 'Print extra debug output', action = 'store_true', default = False)
 
   args, ignored = parser.parse_args()
   if not (args.cl or args.size or args.language or args.perf):
     args.cl = args.size = args.language = args.perf = True
   return (args.cl, args.size, args.language, args.perf, args.continuous,
       args.perfbot, args.verbose)
 
@@ skipping 25 matching lines @@
       if HasNewCode():
         RunTestSequence(cl, size, language, perf)
       else:
         time.sleep(SLEEP_TIME)
   else:
     RunTestSequence(cl, size, language, perf)
 
 if __name__ == '__main__':
   main()