Make browser "sometest" like a regular test run by test.dart/py instead of

tools/testing/perf_testing/create_graph.py

 #!/usr/bin/python
 
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import datetime
 import math
-try:
-  from matplotlib.font_manager import FontProperties
-  import matplotlib.pyplot as plt
-except ImportError:
-  print 'Warning: no matplotlib. ' + \
-      'Please ignore if you are running buildbot smoketests.'
+from matplotlib.font_manager import FontProperties
+import matplotlib.pyplot as plt
 import optparse
 import os
 from os.path import dirname, abspath
 import platform
 import shutil
 import subprocess
 import time
 import traceback
 import sys
 
@@ skipping 10 matching lines @@
 V8_MEAN = 'V8 Mean'
 FROG_MEAN = 'frog Mean'
 COMMAND_LINE = 'commandline'
 V8 = 'v8'
 FROG = 'frog'
 V8_AND_FROG = [V8, FROG]
 CORRECTNESS = 'Percent passing'
 COLORS = ['blue', 'green', 'red', 'cyan', 'magenta', 'black']
 GRAPH_OUT_DIR = 'graphs'
 SLEEP_TIME = 200
-PERFBOT_MODE = False
 VERBOSE = False
 HAS_SHELL = False
 if platform.system() == 'Windows':
   # On Windows, shell must be true to get the correct environment variables.
   HAS_SHELL = True
 
 """First, some utility methods."""
 
 def run_cmd(cmd_list, outfile=None, append=False):
   """Run the specified command and print out any output to stdout.
@@ skipping 67 matching lines @@
 def has_new_code():
   """Tests if there are any newer versions of files on the server."""
   os.chdir(DART_INSTALL_LOCATION)    
   results = run_cmd(['svn', 'st', '-u'])
   for line in results:
     if '*' in line:
       return True
   return False
 
 def get_browsers():
-  if not PERFBOT_MODE:
-    # Only Firefox (and Chrome, but we have Dump Render Tree) works in Linux
-    return ['ff']
-  browsers = ['ff', 'chrome', 'safari']
+  browsers = ['ff', 'chrome']
+  if platform.system() == 'Darwin':
+    browsers += ['safari']
   if platform.system() == 'Windows':
     browsers += ['ie']
   return browsers
 
 def get_versions():
-  if not PERFBOT_MODE:
-    return [FROG]
-  else:
-    return V8_AND_FROG
+  return V8_AND_FROG
 
 def get_benchmarks():
-  if not PERFBOT_MODE:
-    return ['Smoketest']
-  else:
-    return ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 'BinaryTrees',
-    'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 'Loop', 'Permute',
-    'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 'Tak', 'Takl', 'Towers',
-    'TreeSort']
+  return ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 'BinaryTrees',
+  'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 'Loop', 'Permute',
+  'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 'Tak', 'Takl', 'Towers',
+  'TreeSort']
 
 def get_os_directory():
   """Specifies the name of the directory for the testing build of dart, which
   has yet a different naming convention from utils.getBuildRoot(...)."""
   if platform.system() == 'Windows':
     return 'windows'
   elif platform.system() == 'Darwin':
     return 'macos'
   else:
     return 'linux'
@@ skipping 152 matching lines @@
     mean = V8_MEAN
     if frog_or_v8 == FROG:
        mean = FROG_MEAN
     self.values_dict[platform][frog_or_v8][mean] += \
         [math.pow(math.e, geo_mean / len(get_benchmarks()))]
     self.revision_dict[platform][frog_or_v8][mean] += [svn_revision]
 
   def run(self):
     """Run the benchmarks/tests from the command line and plot the 
     results."""
-    if PERFBOT_MODE:
-      plt.cla() # cla = clear current axes
+    plt.cla() # cla = clear current axes
     os.chdir(DART_INSTALL_LOCATION)
     ensure_output_directory(self.result_folder_name)
     ensure_output_directory(GRAPH_OUT_DIR)
     self.run_tests()
     os.chdir(os.path.join('tools', 'testing', 'perf_testing'))
     
     # TODO(efortuna): You will want to make this only use a subset of the files
     # eventually.
     files = os.listdir(self.result_folder_name)
 
     for afile in files:
       if not afile.startswith('.'):
         self.process_file(afile)
 
-    if PERFBOT_MODE:
-      self.plot_results('%s.png' % self.result_folder_name)
+    self.plot_results('%s.png' % self.result_folder_name)
 
 class PerformanceTestRunner(TestRunner):
   """Super class for all performance testing."""
   def __init__(self, result_folder_name, platform_list, platform_type):
     super(PerformanceTestRunner, self).__init__(result_folder_name, 
         platform_list, get_versions(), get_benchmarks())
     self.platform_list = platform_list
     self.platform_type = platform_type
 
   def plot_all_perf(self, png_filename):
@@ skipping 78 matching lines @@
 
 class BrowserPerformanceTestRunner(PerformanceTestRunner):
   """Runs performance tests, in the browser."""
 
   def __init__(self, result_folder_name):
     super(BrowserPerformanceTestRunner, self).__init__(
         result_folder_name, get_browsers(), 'browser')
 
   def run_tests(self):
     """Run a performance test in the browser."""
-      # For the smoke test, just run a simple test, not the actual benchmarks to
-      # ensure we haven't broken the Firefox DOM.
 
     os.chdir('frog')
-    if PERFBOT_MODE:
-      run_cmd(['python', os.path.join('benchmarks', 'make_web_benchmarks.py')])
-    else:
-      run_cmd(['./minfrog', '--out=../tools/testing/perf_testing/smoketest/' + \
-      'smoketest_frog.js', '--libdir=%s/lib' % os.getcwd(),
-      '--compile-only', '../tools/testing/perf_testing/smoketest/' + \
-      'dartWebBase.dart'])
+    run_cmd(['python', os.path.join('benchmarks', 'make_web_benchmarks.py')])
     os.chdir('..')
 
     for browser in get_browsers():
       for version in get_versions():
         self.trace_file = os.path.join('tools', 'testing', 'perf_testing', 
             self.result_folder_name,
             'perf-%s-%s-%s' % (self.cur_time, browser, version))
         self.add_svn_revision_to_trace(self.trace_file)
         file_path = os.path.join(os.getcwd(), 'internal', 'browserBenchmarks', 
             'benchmark_page_%s.html' % version)
-        if not PERFBOT_MODE:
-          file_path = os.path.join(os.getcwd(), 'tools', 'testing',
-              'perf_testing', 'smoketest', 'smoketest_%s.html' % version)
         run_cmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'), 
             '--out', file_path, '--browser', browser, 
             '--timeout', '600', '--perf'], self.trace_file, append=True)
 
   def process_file(self, afile):
     """Comb through the html to find the performance results."""
     parts = afile.split('-')
     browser = parts[2] 
     version = parts[3]
     f = open(os.path.join(self.result_folder_name, afile))
     lines = f.readlines()
     line = ''
     i = 0
     revision_num = 0
     while '<div id="results">' not in line and i < len(lines):
       if 'Revision' in line:
         revision_num = int(line.split()[1])
       line = lines[i]
       i += 1
 
     if i >= len(lines) or revision_num == 0:
-      # Then this run did not complete. Ignore this tracefile. or in the case of
-      # the smoke test, report an error.
-      if not PERFBOT_MODE:
-        print 'FAIL %s %s' % (browser, version)
-        os.remove(os.path.join(self.result_folder_name, afile))
+      # Then this run did not complete. Ignore this tracefile.
       return
 
     line = lines[i]
     i += 1
     results = []
     if line.find('<br>') > -1:
       results = line.split('<br>')
     else:
       results = line.split('<br />')
     for result in results:
       name_and_score = result.split(':')
       if len(name_and_score) < 2: 
         break
       name = name_and_score[0].strip()
       score = name_and_score[1].strip()
       if version == V8:
         bench_dict = self.values_dict[browser][V8]
       else:
         bench_dict = self.values_dict[browser][FROG]
       bench_dict[name] += [float(score)]
       self.revision_dict[browser][version][name] += [revision_num]
 
     f.close()
-    if not PERFBOT_MODE:
-      print 'PASS'
-      os.remove(os.path.join(self.result_folder_name, afile))
-    else:
-      self.calculate_geometric_mean(browser, version, revision_num)
+    self.calculate_geometric_mean(browser, version, revision_num)
 
   def write_html(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2, 
       cleanFile=False):
       #TODO(efortuna)
       pass
 
       
 class BrowserCorrectnessTestRunner(TestRunner):
   def __init__(self, test_type, result_folder_name):
     super(BrowserCorrectnessTestRunner, self).__init__(result_folder_name,
         get_browsers(), [FROG], [CORRECTNESS])
     self.test_type = test_type
 
   def run_tests(self):
     """run a test of the latest svn revision."""
-    the_os = get_os_directory()
+    system = get_os_directory()
     suffix = ''
     if platform.system() == 'Windows':
       suffix = '.exe'
     for browser in get_browsers():
       current_file = 'correctness%s-%s' % (self.cur_time, browser)
       self.trace_file = os.path.join('tools', 'testing',
           'perf_testing', self.result_folder_name, current_file)
       self.add_svn_revision_to_trace(self.trace_file)
       dart_sdk = os.path.join(os.getcwd(), utils.GetBuildRoot(utils.GuessOS(),
           'release', 'ia32'), 'dart-sdk')
-      run_cmd([os.path.join('.', 'tools', 'testing', 'bin', the_os, 
+      run_cmd([os.path.join('.', 'tools', 'testing', 'bin', system, 
           'dart' + suffix), os.path.join('tools', 'test.dart'), 
           '--component=webdriver', '--flag=%s,--frog=%s,--froglib=%s' % \
           (browser, os.path.join(dart_sdk, 'bin', 'frogc'), 
           os.path.join(dart_sdk, 'lib')), '--report',
           '--timeout=20', '--progress=color', '--mode=release', '-j1',
           self.test_type], self.trace_file, append=True)
 
   def process_file(self, afile):
     """Given a trace file, extract all the relevant information out of it to
     determine the number of correctly passing tests.
@@ skipping 154 matching lines @@
       action = 'store_true', default = False)
   parser.add_option('--language', '-l', dest = 'language', 
       help = 'Run the language correctness tests', 
       action = 'store_true', default = False)
   parser.add_option('--browser-perf', '-b', dest = 'perf',
       help = 'Run the browser performance tests',
       action = 'store_true', default = False)
   parser.add_option('--forever', '-f', dest = 'continuous',
       help = 'Run this script forever, always checking for the next svn '
       'checkin', action = 'store_true', default = False)
-  parser.add_option('--perfbot', '-p', dest = 'perfbot',
-      help = "Run in perfbot mode. (Generate plots, and keep trace files)", 
-      action = 'store_true', default = False)
   parser.add_option('--verbose', '-v', dest = 'verbose',
       help = 'Print extra debug output', action = 'store_true', default = False)
 
   args, ignored = parser.parse_args()
   if not (args.cl or args.size or args.language or args.perf):
     args.cl = args.size = args.language = args.perf = True
   return (args.cl, args.size, args.language, args.perf, args.continuous,
-      args.perfbot, args.verbose)
+      args.verbose)
 
 def run_test_sequence(cl, size, language, perf):
-  if PERFBOT_MODE:
-    # The buildbot already builds and syncs to a specific revision. Don't fight
-    # with it or replicate work.
-    if sync_and_build() == 1:
-      return # The build is broken.
+  # The buildbot already builds and syncs to a specific revision. Don't fight
+  # with it or replicate work.
+  if sync_and_build() == 1:
+    return # The build is broken.
   if cl:
     CommandLinePerformanceTestRunner('cl-results').run()
   if size:
     CompileTimeAndSizeTestRunner('code-time-size').run()
   if language:
     BrowserCorrectnessTestRunner('language', 'browser-correctness').run()
   if perf:
     BrowserPerformanceTestRunner('browser-perf').run()
 
-  if PERFBOT_MODE:
-    # TODO(efortuna): Temporarily disabled until you make a safe way to provide
-    # your username/password for the uploading process.
-    #upload_to_app_engine()
-    pass
+  # TODO(efortuna): Temporarily disabled until you make a safe way to provide
+  # your username/password for the uploading process.
+  #upload_to_app_engine()
 
 def main():
-  global PERFBOT_MODE, VERBOSE
-  (cl, size, language, perf, continuous, perfbot, verbose) = parse_args()
-  PERFBOT_MODE = perfbot
+  global VERBOSE
+  (cl, size, language, perf, continuous, verbose) = parse_args()
   VERBOSE = verbose
   if continuous:
     while True:
       if has_new_code():
         run_test_sequence(cl, size, language, perf)
       else:
         time.sleep(SLEEP_TIME)
   else:
     run_test_sequence(cl, size, language, perf)
 
 if __name__ == '__main__':
   main()