Adding the performance and browser benchmarking script.

tools/testing/perf_testing/create_graph.py

+#!/usr/bin/python
+
+# Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import datetime
+import math
+from matplotlib.font_manager import FontProperties
+import matplotlib.pyplot as plt
+import os
+import platform
+import shutil
+import subprocess
+import time
+import traceback
+
+"""This script is run hourly to track performance and correctness progress of 

[Siggi Cherem (dart-lang), 2011/11/23 20:00:29]

could we trigger it on each revision change?

[Emily Fortuna, 2011/11/30 00:37:13]

Done.

+different svn revisions."""
+
+class TestRunner:
+  """The base clas to provide shared code for different tests we will run and
+  graph."""
+  DART_INSTALL_LOCATION = '/Users/efortuna/perfChart/dart'

[Siggi Cherem (dart-lang), 2011/11/23 20:00:29]

make it independent - you can derive it from the currently file path (__file__)

[Emily Fortuna, 2011/11/30 00:37:13]

Done.

+  GCLIENT_LOCATION = '/Users/efortuna/depot_tools/gclient'
+  # True if we have attempted to build this once before, and we've failed, to
+  # indicate if we think the build is likely broken.
+  failed_once = False
+  V8_MEAN = 'V8 Mean'
+  FROG_MEAN = 'frog Mean'

[Siggi Cherem (dart-lang), 2011/11/23 20:00:29]

(nit) I'd move all these constants to the top level, make failed_once a field
initialized in __init__ like all other fields.

[Emily Fortuna, 2011/11/30 00:37:13]

Done.

+
+  def __init__(self, result_folder_name):
+    """Args:
+         result_folder_name the name of the folder where a tracefile of
+         performance results will be stored."""
+    self.result_folder_name = result_folder_name
+    # cur_time is used as a timestamp of when this performance test was run.
+    self.cur_time = str(time.mktime(datetime.datetime.now().timetuple()))
+    self.browsers = ['ff', 'chrome']
+    if platform.system() == 'Windows':
+      self.browsers += ['ie']
+    self.browser_color = {'chrome': 'green', 'ie': 'blue', 'ff': 'red'}
+    self.benchmarks = ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 
+        'BinaryTrees', 'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 
+        'Loop', 'Permute', 'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 
+        'Tak', 'Takl', 'Towers', 'TreeSort']
+  
+  def style_and_save_perf_plot(self, chart_title, y_axis_label, size_x, size_y, 

[Siggi Cherem (dart-lang), 2011/11/23 20:00:29]

(nit): we tend to use a CamelCase style for functions

[Emily Fortuna, 2011/11/30 00:37:13]

Done.

+      legend_loc, filename):
+    """Sets style preferences for chart boilerplate that is consistent across 
+    all charts, and saves the chart as a png.
+    Args:
+      size_x the size of the printed chart, in inches, in the horizontal 
+        direction
+      size_y the size of the printed chart, in inches in the vertical direction
+      legend_loc the location of the legend in on the chart. See suitable
+        arguments for the loc argument in matplotlib
+      filename the filename that we want to save the resulting chart as"""
+    plt.xlabel('Revision Number')
+    plt.ylabel(y_axis_label)
+    plt.title(chart_title)
+    fontP = FontProperties()
+    fontP.set_size('small')
+    plt.legend(loc=legend_loc, prop = fontP)
+
+    fig = plt.gcf()
+    fig.set_size_inches(size_x, size_y)
+    fig.savefig(filename) 
+
+  @staticmethod

[Siggi Cherem (dart-lang), 2011/11/23 20:00:29]

you can also move this definition to the top-level. See 'Execute' in many of the
scripts in testing/...

[Emily Fortuna, 2011/11/30 00:37:13]

Done.

+  def run_cmd(string):
+    """Run the specified command and print out any output to stdout.
+    Args:
+      string the command to run"""
+    p = subprocess.Popen(string, shell = True, stdout = subprocess.PIPE, 
+        stderr = subprocess.STDOUT, close_fds=True)
+    lines = p.stdout.readlines()
+    for line in lines:
+      print line,
+    return lines
+
+  @staticmethod
+  def sync_and_build():
+    """Make sure we have the latest version of of the repo, and build it. We
+    begin and end standing in TestRunner.DART_INSTALL_LOCATION.
+    Args:
+      should_sync is true if we want to sync to the latest repo version, rather
+        than simply moving to the correct directory."""
+    os.chdir(TestRunner.DART_INSTALL_LOCATION)
+    #Remove our newly built frogsh to prevent conflicts when we update
+    try:
+      os.remove(os.getcwd() + os.path.join('frog', 'frogsh'))

[Siggi Cherem (dart-lang), 2011/11/23 20:00:29]

or 'svn revert'?

[Emily Fortuna, 2011/11/30 00:37:13]

Done.

+    except OSError:
+      pass
+
+    TestRunner.run_cmd('%s sync' % TestRunner.GCLIENT_LOCATION)
+    lines = TestRunner.run_cmd('%s -m release' % os.path.join('.', 'tools', 
+        'build.py'))
+    os.chdir('frog')
+    lines += TestRunner.run_cmd('%s -m debug,release' % os.path.join('..', 
+        'tools', 'build.py'))
+    os.chdir('..')
+    
+    for line in lines:
+      if '** BUILD FAILED **' in lines:
+        if TestRunner.failed_once:
+          # Someone checked in a broken build! Just stop trying to make it work
+          # and wait for the next hour to try again.
+          print 'FAILED ONCEEEEEE'
+          sys.exit(0)
+        #Remove the xcode directory and attempt to build again. If it still
+        #fails, abort, and try again next hour.
+        out_dir = 'out'
+        if platform.system() == 'Darwin':
+          out_dir = 'xcodebuild'
+        os.remove(os.getcwd() + os.path.join('dart', out_dir, 'Release_ia32'))
+        os.remove(os.getcwd() + os.path.join('dart', 'frog', out_dir, 
+            'Release_ia32'))
+        os.remove(os.getcwd() + os.path.join('dart', 'frog', out_dir, 
+            'Debug_ia32'))
+        TestRunner.failed_once = True
+        TestRunner.sync_and_build()
+
+  @staticmethod
+  def ensure_output_directory(dir_name):
+    """Test that the listed directory name exists, and if not, create one for
+    our output to be placed."""
+    dir_path = os.path.join(TestRunner.DART_INSTALL_LOCATION, 'tools', 'testing',

[Siggi Cherem (dart-lang), 2011/11/23 20:00:29]

80

[Emily Fortuna, 2011/11/30 00:37:13]

Done.

+        'perf_testing', dir_name)
+    if not os.path.exists(dir_path):
+      os.makedir(dir_path)
+      print 'Creating output directory ', dir_path
+
+  def add_svn_revision_to_trace(self, outfile):
+    """Add the svn version number to the provided tracefile."""
+    p = subprocess.Popen('svn info ', shell = True, 
+          stdout = subprocess.PIPE, stderr = subprocess.STDOUT, close_fds=True)
+
+    lines = p.stdout.readlines()
+    for line in lines:
+      if 'Revision' in line:
+        TestRunner.run_cmd('echo "%s" > %s' % (line.strip(), outfile))
+
+  
+  def write_html(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2, 
+      cleanFile=False):
+    """Adds an html table to the webpage to display the data values. This method
+    will be removed when we have a nicer way to display data values."""
+    #TODO(efortuna): Take this method out when have finalized where the data is
+    # going to be displayed.
+    f = ''
+    out = ''
+    if cleanFile:
+      f = open('template.html')
+    else:
+      shutil.copy('index.html', 'temp.html')
+      f = open('temp.html')
+    out = open('index.html', 'w')
+    inTable = False
+    for line in f.readlines():
+      if not inTable:
+        out.write(line)
+      if delimiter in line:
+        inTable = not inTable
+        if inTable:
+          out.write('<table border="1"> <tr> <td> svn revision </td>')
+          for revision in rev_nums:
+            out.write('<td>%d</td>' % revision)
+          out.write('</tr>\n<tr><td> %s</td>' % label_1)
+          for perf in dict_1:
+            out.write('<td>%f</td>' % perf)
+          out.write('</tr>\n<tr><td> %s</td>' % label_2)
+          for perf in dict_2:
+            out.write('<td>%f</td>' % perf)
+          out.write('</tr> </table>')
+
+  def calculate_geometric_mean(self, bench_dict_1, bench_dict_2):
+    """Calculate the aggregate geometric mean for V8 and frog benchmark sets,
+    given two benchmark dictionaries."""
+    frog_geo_mean = 0
+    v8_geo_mean = 0
+    for benchmark in self.benchmarks:
+      v8_geo_mean += math.log(bench_dict_1[benchmark][
+          len(bench_dict_1[benchmark]) - 1])
+      frog_geo_mean += math.log(bench_dict_2[benchmark][
+          len(bench_dict_2[benchmark]) - 1]) 
+  
+    bench_dict_1[TestRunner.V8_MEAN] += \
+        [math.pow(math.e, v8_geo_mean / len(self.benchmarks))]
+    bench_dict_2[TestRunner.FROG_MEAN] += \
+        [math.pow(math.e, frog_geo_mean / len(self.benchmarks))]
+
+  def run(self):
+    """Run the benchmarks/tests from the command line and plot the 
+    results."""
+    plt.cla() # cla = clear current axes
+    os.chdir(TestRunner.DART_INSTALL_LOCATION)
+    TestRunner.ensure_output_directory(self.result_folder_name)
+    self.run_tests()
+    os.chdir(os.path.join('tools', 'testing', 'perf_testing'))
+    
+    # TODO(efortuna): You will want to make this only use a subset of the files
+    # eventually.
+    files = os.listdir(self.result_folder_name)
+
+    for afile in files:
+      if not afile.startswith('.'):
+        self.process_file(afile)
+    self.plot_results('%s.png' % self.result_folder_name)
+
+class CommandLinePerformanceTestRunner(TestRunner):
+  """Run performance tests from the command line."""
+
+  def __init__(self, result_folder_name):
+    TestRunner.__init__(self, result_folder_name)

[Siggi Cherem (dart-lang), 2011/11/23 20:00:29]

use 'super'? (there are some examples in architecture.py, you might need to make
the baseclass (TestRunner) extend from 'object')

[Emily Fortuna, 2011/11/30 00:37:13]

Done.

+    self.revision_nums = dict() 
+    self.bench_dict_js = dict()
+    self.bench_dict_frog = dict()
+    for benchmark in self.benchmarks:
+      self.bench_dict_js[benchmark] = []
+      self.bench_dict_frog[benchmark] = []
+    self.bench_dict_js[TestRunner.V8_MEAN] = []
+    self.bench_dict_frog[TestRunner.FROG_MEAN] = []
+    self.revision_nums['js'] = []
+    self.revision_nums['frog'] = []
+
+  def plot_all_perf(self, png_filename):
+    """Create a plot that shows the performance changes of individual benchmarks
+    run by V8 and generated by frog, over svn history."""
+    markers = ['o', '.', 's', 'v', '>', '<', '^']
+    #mfc = marker face color. Bad naming convention to match the named parameter
+    # in matplotlib.plot(...). 
+    mfc = ['red', 'yellow', 'black']
+    bench_style = dict()
+    i = 0
+    j = 0
+    for benchmark in self.benchmarks:
+      bench_style[benchmark] = (markers[i], mfc[j])
+      i += 1
+      if i >= len(markers):
+        j += 1

[Siggi Cherem (dart-lang), 2011/11/23 20:00:29]

what if you have > than 21 benchmarks?

[Emily Fortuna, 2011/11/30 00:37:13]

Done.

+        i=0

[Siggi Cherem (dart-lang), 2011/11/23 20:00:29]

Alternatively you could write the index calculations more declaratively:

for i in range(len(self.benchmarks)):

  bench_style[benchmark] = (markers[i % len(markers)], mfc[(i / len(markers)) %
len(mfc])]

[Emily Fortuna, 2011/11/30 00:37:13]

Done.

+
+    for benchmark in self.benchmarks:
+      style = bench_style[benchmark]

[Siggi Cherem (dart-lang), 2011/11/23 20:00:29]

instead of using style[0] below, you could also use the tuple deconstructor
here:

bench_marker, bench_mfc = bench_style[benchmark]

[Emily Fortuna, 2011/11/30 00:37:13]

Done.

+      plt.plot(self.revision_nums['js'], self.bench_dict_js[benchmark], 
+          color='blue', marker=style[0], mfc=style[1], 
+          label='%s-js' % benchmark)
+      plt.plot(self.revision_nums['frog'], self.bench_dict_frog[benchmark], 
+          color='green', marker=style[0], mfc=style[1], 
+          label='%s-frog' % benchmark)
+
+    self.style_and_save_perf_plot('Performance of benchmarks over time', 
+        'Speed (bigger = better)', 16, 14, 'lower left', png_filename)
+
+  def plot_avg_perf(self, png_filename):
+    """Generate a plot that shows the performance changes of the geomentric mean
+    of V8 and frog benchmark performance over svn history."""
+    plt.cla() # cla = clear current axes
+    plt.plot(self.revision_nums['js'], 
+        self.bench_dict_js[TestRunner.V8_MEAN], color = 'blue', 
+        label=TestRunner.V8_MEAN, linewidth=2.0)
+    plt.plot(self.revision_nums['frog'], 
+        self.bench_dict_frog[TestRunner.FROG_MEAN], color='green',
+        label=PerformanceTestRunner.FROG_MEAN, linewidth=2.0)
+    self.style_and_save_perf_plot('Geometric Mean of benchmark performance', 
+        'Speed (bigger = better)', 16, 5, 'center', png_filename)
+    self.write_html('table', 
+        self.revision_nums['js'], 
+        'V8 mean', self.bench_dict_js[TestRunner.V8_MEAN],
+        'Frog mean', self.bench_dict_frog[TestRunner.FROG_MEAN], 
+        True)
+
+  def plot_results(self, png_filename):
+    self.plot_all_perf(png_filename)
+    self.plot_avg_perf('2' + png_filename)
+
+  def process_file(self, afile):
+    """Pull all the relevant information out of a given tracefile.
+
+    Args:
+      afile is the filename string we will be processing."""
+    f = open(os.path.join(self.result_folder_name, afile))
+    tabulate_data = False
+    for line in f.readlines():
+      if 'Revision' in line:
+        revision_num = int(line.split()[1])
+        self.revision_nums['js'] += [revision_num]
+        self.revision_nums['frog'] += [revision_num]
+      elif 'Benchmark' in line:
+        tabulate_data = True
+      elif tabulate_data:
+        tokens = line.split()
+        if len(tokens) < 4 or tokens[0] not in self.benchmarks:
+          #Done tabulating data.
+          break
+        v8_value = float(tokens[1])
+        frog_value = float(tokens[3])
+        if v8_value == 0 or frog_value == 0:
+          #Then there was an error when this performance test was run. Do not 
+          #count it in our numbers.
+          self.revision_nums['js'].pop()
+          self.revision_nums['frog'].pop()
+          return
+        self.bench_dict_js[tokens[0]] += [v8_value]
+        self.bench_dict_frog[tokens[0]] += [frog_value]
+    f.close()
+
+    self.calculate_geometric_mean(self.bench_dict_js, self.bench_dict_frog)
+  
+  def run_tests(self):
+    """Run a performance test on our updated system."""
+    os.chdir('frog')
+    file_path = os.path.join('..', 'tools', 'testing', 'perf_testing',
+        self.result_folder_name, 'result')
+    TestRunner.run_cmd('python %s > %s%s' % (os.path.join('benchmarks', 
+        'perf_tests.py'), file_path, self.cur_time))
+    os.chdir('..')
+
+
+class BrowserPerformanceTestRunner(TestRunner):
+  """Runs performance tests, in the browser."""
+
+  def __init__(self, result_folder_name):
+    TestRunner.__init__(self, result_folder_name)
+    self.revision_nums = dict()
+    self.bench_dict_js = dict()
+    self.bench_dict_frog = dict()
+    for browser in self.browsers:
+      self.revision_nums[browser] = dict()
+      self.bench_dict_js[browser] = dict()
+      self.bench_dict_frog[browser] = dict()
+      for benchmark in self.benchmarks:
+        self.bench_dict_js[browser][benchmark] = []
+        self.bench_dict_frog[browser][benchmark] = []
+      self.bench_dict_js[browser][TestRunner.V8_MEAN] = []
+      self.bench_dict_frog[browser][TestRunner.FROG_MEAN] = []
+      self.revision_nums[browser]['js'] = []
+      self.revision_nums[browser]['frog'] = []
+
+  def run_tests(self):
+    """Run a performance test in the browser."""
+    os.chdir('frog')
+    self.run_cmd('python benchmarks/make_web_benchmarks.py')
+    os.chdir('..')
+
+    for browser in self.browsers:
+      for version in ['js', 'frog']:
+        self.add_svn_revision_to_trace(os.path.join('tools', 'testing',
+            'perf_testing', self.result_folder_name, 
+            'perf-%s-%s-%s' % (self.cur_time, browser, version)))
+        TestRunner.run_cmd('python %s %s %s 1000 perfTest >> %s' % 
+            (os.path.join('tools', 'testing', 'run_selenium.py'),
+            os.path.join(os.getcwd(), 'internal', 'browserBenchmarks',
+            'benchmark_page_%s.html' % version), 
+            browser, 
+            os.path.join('tools', 'testing', 'perf_testing', 
+            self.result_folder_name, 'perf-%s-%s-%s' % (self.cur_time, browser, 
+            version))))
+
+  def process_file(self, afile):
+    """Comb through the html to find the performance results."""
+    parts = afile.split('-')
+    browser = parts[2] 
+    version = parts[3]
+    f = open(os.path.join(self.result_folder_name, afile))
+    lines = f.readlines()
+    line = ''
+    i = 0
+    while '<div id="results">' not in line:
+      line = lines[i]
+      i += 1
+
+    line = lines[i]
+    i += 1
+    results = []
+    if line.find('<br>') > -1:
+      results = line.split('<br>')
+    else:
+      results = line.split('<br />')
+    for result in results:
+      name_and_score = result.split(':')
+      if len(name_and_score) < 2: 
+        break
+      name = name_and_score[0].strip()
+      score = name_and_score[1].strip()
+      if version == 'js':
+        bench_dict = self.bench_dict_js[browser]
+      else:
+        bench_dict = self.bench_dict_frog[browser]
+      bench_dict[name] += [score]
+
+    while i < len(lines):
+      if 'Revision' in line:
+        self.revision_nums[browser][version] = int(line.split()[1])
+      line = lines[i]
+      i += 1
+    f.close()
+    self.calculate_geometric_mean(bench_dict_js[browser], 
+        bench_dict_frog[browser])    
+
+  def plot_results(self, png_filename):
+    """Generate a plot that shows the performance changes of the geomentric mean
+    of V8 and frog benchmark performance over svn history."""
+    plt.cla() # cla = clear current axes
+    for browser in self.browsers:
+      plt.cla() # cla = clear current axes
+      plt.plot(self.revision_nums[browser]['js'], 
+          self.bench_dict_js[browser][TestRunner.V8_MEAN], 
+          color = self.browser_color[browser], 
+          label=TestRunner.V8_MEAN + '-' + browser, marker='s', linewidth=2.0)
+      plt.plot(self.revision_nums[browser]['frog'], 
+          self.bench_dict_frog[browser][TestRunner.FROG_MEAN], 
+          color=self.browser_color[browser], marker='o', 
+          label=TestRunner.FROG_MEAN + '-' + browser, linewidth=2.0)
+      self.style_and_save_perf_plot('Geometric Mean of benchmark performance', 
+          'Speed (bigger = better)', 16, 5, 'center', png_filename)
+      #self.write_html('table', 
+      #    self.revision_nums['js'], 
+      #    'V8 mean', self.bench_dict_js[TestRunner.V8_MEAN],
+      #    'Frog mean', self.bench_dict_frog[TestRunner.FROG_MEAN], 
+      #    True)
+
+
+  def write_html(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2, 
+      cleanFile=False):
+      #TODO(efortuna)
+      pass
+
+class BrowserCorrectnessTestRunner(TestRunner):
+  def __init__(self, test_type, result_folder_name):
+    TestRunner.__init__(self, result_folder_name)
+    self.test_type = test_type
+    
+    self.passing = dict()
+    self.revision_nums = dict()
+    for browser in self.browsers:
+      self.passing[browser] = []
+      self.revision_nums[browser] = []
+
+  def run_tests(self):
+    """Run a test of the latest svn revision."""
+    for browser in self.browsers:
+      current_file = 'correctness%s-%s' % (self.cur_time, browser)
+      current_file_path = os.path.join('tools', 'testing',
+          'perf_testing', self.result_folder_name, current_file)
+      self.add_svn_revision_to_trace(current_file_path)
+      TestRunner.run_cmd(os.path.join('.', 'tools', 'test.py') +
+          ' --component=webdriver --flag=%s --report --timeout=20 ' % browser +
+          '--progress=color --mode=release -j1 %s >>' % self.test_type + 
+          current_file_path)
+
+  def process_file(self, afile):
+    """Given a trace file, extract all the relevant information out of it to
+    determine the number of correctly passing tests.
+    
+    Arguments:
+      afile the filename string"""
+    browser = afile.rpartition('-')[2]
+    f = open(os.path.join(self.result_folder_name, afile))
+    revision_num = 1
+    lines = f.readlines()
+    total_tests = 0
+    num_failed = 0
+    expect_fail = 0
+    for line in lines:
+      if 'Total:' in line:
+        total_tests = int(line.split()[1])
+      if 'will be skipped' in line:
+        total_tests -= int(line.split()[1])
+      if 'we should fix' in line:
+        expect_fail += int(line.split()[1])
+      if 'Revision' in line:
+        revision_num = int(line.split()[1])
+      if '--- TIMEOUT ---' in line or 'FAIL:' in line or 'PASS' in line:
+        # (A printed out 'PASS' indicates we incorrectly passed a negative
+        # test.)
+        num_failed += 1
+    
+    self.revision_nums[browser] += [revision_num]
+    self.passing[browser] += [100.0 * (((float)(total_tests - (expect_fail + 
+        num_failed))) /total_tests)]
+    f.close()
+
+  def plot_results(self, png_filename):
+    for browser in self.browsers:
+      plt.plot(self.revision_nums[browser], self.passing[browser], 
+          color=self.browser_color[browser], label=browser)
+    self.style_and_save_perf_plot('Percentage of language tests passing in '
+        'different browsers', '% of tests passed', 8, 8, 'lower left', 
+        png_filename) 
+
+
+class CompileTimeAndSizeTestRunner(TestRunner):
+  """Run tests to determine how long frogsh takes to compile, and the compiled 
+  file output size of some benchmarking files."""
+  def __init__(self, result_folder_name):
+    TestRunner.__init__(self, result_folder_name)
+    self.metrics = ['Compiling on Dart VM', 'Bootstrapping', 'frogsh', 'swarm', 
+        'total']
+    self.revision_nums = [] 
+    self.metric_dict = dict()
+    self.failure_threshold = {'Compiling on Dart VM' : 1, 'Bootstrapping' : .5, 
+        'frogsh' : 100, 'swarm' : 100, 'total' : 100}
+    for metric in self.metrics:
+      self.metric_dict[metric] = []
+
+  def time_cmd(self, cmd):
+    start = time.time()
+    self.run_cmd(cmd)
+    return time.time() - start
+
+  def run_tests(self):
+    os.chdir('frog')
+    current_file_path = os.path.join('..', 'tools', 'testing', 'perf_testing', 
+        self.result_folder_name, self.result_folder_name + self.cur_time)
+    
+    self.add_svn_revision_to_trace(current_file_path)    
+
+    elapsed = self.time_cmd(os.path.join('.', 'frog.py') + 
+        ' --vm_flags="--compile_all --enable_type_checks --enable_asserts" --'
+        ' --compile_all --enable_type_checks --out=frogsh frog.dart')
+    self.run_cmd('echo "%f Compiling on Dart VM in checked mode in' % elapsed +
+        ' seconds" >> %s' % current_file_path)
+    self.run_cmd('chmod +x frogsh')
+    elapsed = self.time_cmd(os.path.join('.', 'frogsh') + ' --out=frogsh '
+        '--enable_type_checks frog.dart --enable_type_checks ' + 
+        os.path.join('tests', 'hello.dart'))
+    if elapsed < self.failure_threshold['Bootstrapping']:
+      #frogsh didn't compile correctly. Stop testing now, because subsequent
+      #numbers will be meaningless.
+      return
+    size = os.path.getsize('frogsh')
+    self.run_cmd('echo "%f Bootstrapping time in seconds in checked mode"' % 
+        elapsed + ' >> %s' % current_file_path)
+    self.run_cmd('echo "%d Generated checked frogsh size "' % size + 
+        ' >> %s' % current_file_path)
+
+    self.run_cmd(os.path.join('.', 'frogsh') + ' --out=swarm-result '
+        '--compile-only ' + os.path.join('..', 'client', 'samples', 'swarm',
+        'swarm.dart'))
+    swarm_size = 0
+    try:
+      swarm_size = os.path.getsize('swarm-result')
+    except OSError:
+      pass #If compilation failed, continue on running other tests.
+
+    self.run_cmd(os.path.join('.', 'frogsh') + ' --out=total-result '
+        '--compile-only ' + os.path.join('..', 'client', 'samples', 'total',
+        'src', 'Total.dart'))
+    total_size = 0
+    try:
+      total_size = os.path.getsize('total-result')
+    except OSError:
+      pass #If compilation failed, continue on running other tests.
+
+    self.run_cmd('echo "%d Generated checked swarm size "' % swarm_size +
+        ' >> %s' % current_file_path) 
+    
+    self.run_cmd('echo "%d Generated checked total size "' % total_size +
+        ' >> %s' % current_file_path) 
+    os.chdir('..')
+
+  def process_file(self, afile):
+    """Pull all the relevant information out of a given tracefile.
+
+    Args:
+      afile is the filename string we will be processing."""
+    f = open(os.path.join(self.result_folder_name, afile))
+    tabulate_data = False
+    had_version_number = False
+    for line in f.readlines():
+      tokens = line.split()
+      if 'Revision' in line:
+        had_version_number = True
+        revision_num = int(line.split()[1])
+        self.revision_nums += [revision_num]
+      else:
+        for metric in self.metrics:
+          if metric in line:
+            num = tokens[0]
+            if num.find('.') == -1:
+              num = int(num)
+            else:
+              num = float(num)
+            self.metric_dict[metric] += [num]
+
+    for metric in self.metrics:
+      # Fill 0 if compilation failed.
+      if len(self.revision_nums) != len(self.metric_dict[metric]) \
+          or self.metric_dict[metric][-1] < self.failure_threshold[metric]: 
+        if len(self.revision_nums) == len(self.metric_dict[metric]) and \
+            self.metric_dict[metric][-1] < self.failure_threshold[metric]:
+          self.metric_dict[metric].pop()
+        self.metric_dict[metric] += [0]
+    f.close()
+
+  def plot_results(self, png_filename):
+    plt.cla() # cla = clear current axes
+    plt.plot(self.revision_nums, self.metric_dict['swarm'], 
+        color = 'blue', label='swarm')
+    plt.plot(self.revision_nums, self.metric_dict['total'], 
+        color = 'red', label='total')
+    self.style_and_save_perf_plot('Compiled Swarm and Total Sizes', 
+        'Size (in bytes)', 10, 10, 'center', png_filename)
+    self.write_html('foo', self.revision_nums, 'swarm size', 
+        self.metric_dict['swarm'], 'total size', self.metric_dict['total'])
+
+    plt.cla() # cla = clear current axes
+    plt.plot(self.revision_nums, self.metric_dict['frogsh'], 
+        color = 'green', label='frogsh')
+    self.style_and_save_perf_plot('Compiled frogsh Sizes', 
+        'Size (in bytes)', 10, 10, 'center', '2' + png_filename)
+    self.write_html('bar', self.revision_nums, 'frogsh size', 
+        self.metric_dict['frogsh'], '', [])
+
+    plt.cla() # cla = clear current axes
+    plt.plot(self.revision_nums, self.metric_dict['Bootstrapping'],
+        color='green', label='bootstrapping')
+    plt.plot(self.revision_nums, self.metric_dict['Compiling on Dart VM'],
+        color='red', label='compiling with Dart VM')
+    self.style_and_save_perf_plot('Time to compile and bootstrap', 
+        'Seconds', 10, 10, 'center', '3' + png_filename)
+    self.write_html('baz', self.revision_nums, 'Bootstrapping', 
+        self.metric_dict['Bootstrapping'], 'Compiling on Dart VM',
+        self.metric_dict['Compiling on Dart VM'])
+  
+
+def main():
+  TestRunner.sync_and_build()
+  CommandLinePerformanceTestRunner('cl-results').run()
+  CompileTimeAndSizeTestRunner('code-time-size').run()
+  BrowserCorrectnessTestRunner('language', 'browser-correctness').run()
+  BrowserPerformanceTestRunner('browser-perf').run()
+
+if __name__ == '__main__':
+  main()
+