| OLD | NEW |
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 | 2 |
| 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 4 # for details. All rights reserved. Use of this source code is governed by a | 4 # for details. All rights reserved. Use of this source code is governed by a |
| 5 # BSD-style license that can be found in the LICENSE file. | 5 # BSD-style license that can be found in the LICENSE file. |
| 6 | 6 |
| 7 import datetime | 7 import datetime |
| 8 import math | 8 import math |
| 9 try: | 9 try: |
| 10 from matplotlib.font_manager import FontProperties | 10 from matplotlib.font_manager import FontProperties |
| (...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 44 SLEEP_TIME = 200 | 44 SLEEP_TIME = 200 |
| 45 PERFBOT_MODE = False | 45 PERFBOT_MODE = False |
| 46 VERBOSE = False | 46 VERBOSE = False |
| 47 HAS_SHELL = False | 47 HAS_SHELL = False |
| 48 if platform.system() == 'Windows': | 48 if platform.system() == 'Windows': |
| 49 # On Windows, shell must be true to get the correct environment variables. | 49 # On Windows, shell must be true to get the correct environment variables. |
| 50 HAS_SHELL = True | 50 HAS_SHELL = True |
| 51 | 51 |
| 52 """First, some utility methods.""" | 52 """First, some utility methods.""" |
| 53 | 53 |
| 54 def RunCmd(cmd_list, outfile=None, append=False): | 54 def run_cmd(cmd_list, outfile=None, append=False): |
| 55 """Run the specified command and print out any output to stdout. | 55 """Run the specified command and print out any output to stdout. |
| 56 Args: | 56 Args: |
| 57 cmd_list a list of strings that make up the command to run | 57 cmd_list a list of strings that make up the command to run |
| 58 outfile a string indicating the name of the file that we should write stdout | 58 outfile a string indicating the name of the file that we should write stdout |
| 59 to | 59 to |
| 60 append True if we want to append to the file instead of overwriting it""" | 60 append True if we want to append to the file instead of overwriting it""" |
| 61 if VERBOSE: | 61 if VERBOSE: |
| 62 print ' '.join(cmd_list) | 62 print ' '.join(cmd_list) |
| 63 out = subprocess.PIPE | 63 out = subprocess.PIPE |
| 64 if outfile: | 64 if outfile: |
| 65 mode = 'w' | 65 mode = 'w' |
| 66 if append: | 66 if append: |
| 67 mode = 'a' | 67 mode = 'a' |
| 68 out = open(outfile, mode) | 68 out = open(outfile, mode) |
| 69 p = subprocess.Popen(cmd_list, stdout = out, stderr = subprocess.PIPE, | 69 p = subprocess.Popen(cmd_list, stdout = out, stderr = subprocess.PIPE, |
| 70 shell=HAS_SHELL) | 70 shell=HAS_SHELL) |
| 71 output, not_used = p.communicate(); | 71 output, not_used = p.communicate(); |
| 72 if output: | 72 if output: |
| 73 print output | 73 print output |
| 74 return output | 74 return output |
| 75 | 75 |
| 76 def TimeCmd(cmd): | 76 def time_cmd(cmd): |
| 77 """Determine the amount of (real) time it takes to execute a given command.""" | 77 """Determine the amount of (real) time it takes to execute a given command.""" |
| 78 start = time.time() | 78 start = time.time() |
| 79 RunCmd(cmd) | 79 run_cmd(cmd) |
| 80 return time.time() - start | 80 return time.time() - start |
| 81 | 81 |
| 82 def SyncAndBuild(failed_once=False): | 82 def sync_and_build(failed_once=False): |
| 83 """Make sure we have the latest version of of the repo, and build it. We | 83 """Make sure we have the latest version of of the repo, and build it. We |
| 84 begin and end standing in DART_INSTALL_LOCATION. | 84 begin and end standing in DART_INSTALL_LOCATION. |
| 85 Args: | 85 Args: |
| 86 failed_once True if we have attempted to build this once before, and we've | 86 failed_once True if we have attempted to build this once before, and we've |
| 87 failed, indicating the build is broken. | 87 failed, indicating the build is broken. |
| 88 Returns: | 88 Returns: |
| 89 err_code = 1 if there was a problem building two times in a row.""" | 89 err_code = 1 if there was a problem building two times in a row.""" |
| 90 os.chdir(DART_INSTALL_LOCATION) | 90 os.chdir(DART_INSTALL_LOCATION) |
| 91 #Revert our newly built minfrog to prevent conflicts when we update | 91 #Revert our newly built minfrog to prevent conflicts when we update |
| 92 RunCmd(['svn', 'revert', os.path.join(os.getcwd(), 'frog', 'minfrog')]) | 92 run_cmd(['svn', 'revert', os.path.join(os.getcwd(), 'frog', 'minfrog')]) |
| 93 | 93 |
| 94 RunCmd(['gclient', 'sync']) | 94 run_cmd(['gclient', 'sync']) |
| 95 # TODO(efortuna): building the sdk locally is a band-aid until all build | 95 # TODO(efortuna): building the sdk locally is a band-aid until all build |
| 96 # platform SDKs are hosted in Google storage. Pull from https://sandbox. | 96 # platform SDKs are hosted in Google storage. Pull from https://sandbox. |
| 97 # google.com/storage/?arg=dart-dump-render-tree#dart-dump-render-tree%2Fsdk | 97 # google.com/storage/?arg=dart-dump-render-tree#dart-dump-render-tree%2Fsdk |
| 98 # eventually. | 98 # eventually. |
| 99 # TODO(efortuna): Currently always building ia32 architecture because we don't | 99 # TODO(efortuna): Currently always building ia32 architecture because we don't |
| 100 # have test statistics for what's passing on x64. Eliminate arch specification | 100 # have test statistics for what's passing on x64. Eliminate arch specification |
| 101 # when we have tests running on x64, too. | 101 # when we have tests running on x64, too. |
| 102 lines = RunCmd([os.path.join('.', 'tools', 'build.py'), '-m', 'release', | 102 lines = run_cmd([os.path.join('.', 'tools', 'build.py'), '-m', 'release', |
| 103 '--arch=ia32', 'create_sdk']) | 103 '--arch=ia32', 'create_sdk']) |
| 104 | 104 |
| 105 for line in lines: | 105 for line in lines: |
| 106 if 'BUILD FAILED' in lines: | 106 if 'BUILD FAILED' in lines: |
| 107 if failed_once: | 107 if failed_once: |
| 108 # Someone checked in a broken build! Just stop trying to make it work | 108 # Someone checked in a broken build! Just stop trying to make it work |
| 109 # and wait to try again. | 109 # and wait to try again. |
| 110 print 'Broken Build' | 110 print 'Broken Build' |
| 111 return 1 | 111 return 1 |
| 112 #Remove the output directory and attempt to build again. If it still | 112 #Remove the output directory and attempt to build again. If it still |
| 113 #fails, abort, and try again in a little bit. | 113 #fails, abort, and try again in a little bit. |
| 114 shutil.rmtree(os.path.join(os.getcwd(), | 114 shutil.rmtree(os.path.join(os.getcwd(), |
| 115 utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32'))) | 115 utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32'))) |
| 116 SyncAndBuild(True) | 116 sync_and_build(True) |
| 117 return 0 | 117 return 0 |
| 118 | 118 |
| 119 def EnsureOutputDirectory(dir_name): | 119 def ensure_output_directory(dir_name): |
| 120 """Test that the listed directory name exists, and if not, create one for | 120 """Test that the listed directory name exists, and if not, create one for |
| 121 our output to be placed. | 121 our output to be placed. |
| 122 Args: | 122 Args: |
| 123 dir_name the directory we will create if it does not exist.""" | 123 dir_name the directory we will create if it does not exist.""" |
| 124 dir_path = os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', | 124 dir_path = os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', |
| 125 'perf_testing', dir_name) | 125 'perf_testing', dir_name) |
| 126 if not os.path.exists(dir_path): | 126 if not os.path.exists(dir_path): |
| 127 os.mkdir(dir_path) | 127 os.mkdir(dir_path) |
| 128 print 'Creating output directory ', dir_path | 128 print 'Creating output directory ', dir_path |
| 129 | 129 |
| 130 def HasNewCode(): | 130 def has_new_code(): |
| 131 """Tests if there are any newer versions of files on the server.""" | 131 """Tests if there are any newer versions of files on the server.""" |
| 132 os.chdir(DART_INSTALL_LOCATION) | 132 os.chdir(DART_INSTALL_LOCATION) |
| 133 results = RunCmd(['svn', 'st', '-u']) | 133 results = run_cmd(['svn', 'st', '-u']) |
| 134 for line in results: | 134 for line in results: |
| 135 if '*' in line: | 135 if '*' in line: |
| 136 return True | 136 return True |
| 137 return False | 137 return False |
| 138 | 138 |
| 139 def GetBrowsers(): | 139 def get_browsers(): |
| 140 if not PERFBOT_MODE: | 140 if not PERFBOT_MODE: |
| 141 # Only Firefox (and Chrome, but we have Dump Render Tree) works in Linux | 141 # Only Firefox (and Chrome, but we have Dump Render Tree) works in Linux |
| 142 return ['ff'] | 142 return ['ff'] |
| 143 browsers = ['ff', 'chrome', 'safari'] | 143 browsers = ['ff', 'chrome', 'safari'] |
| 144 if platform.system() == 'Windows': | 144 if platform.system() == 'Windows': |
| 145 browsers += ['ie'] | 145 browsers += ['ie'] |
| 146 return browsers | 146 return browsers |
| 147 | 147 |
| 148 def GetVersions(): | 148 def get_versions(): |
| 149 if not PERFBOT_MODE: | 149 if not PERFBOT_MODE: |
| 150 return [FROG] | 150 return [FROG] |
| 151 else: | 151 else: |
| 152 return V8_AND_FROG | 152 return V8_AND_FROG |
| 153 | 153 |
| 154 def GetBenchmarks(): | 154 def get_benchmarks(): |
| 155 if not PERFBOT_MODE: | 155 if not PERFBOT_MODE: |
| 156 return ['Smoketest'] | 156 return ['Smoketest'] |
| 157 else: | 157 else: |
| 158 return ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 'BinaryTrees', | 158 return ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 'BinaryTrees', |
| 159 'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 'Loop', 'Permute', | 159 'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 'Loop', 'Permute', |
| 160 'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 'Tak', 'Takl', 'Towers', | 160 'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 'Tak', 'Takl', 'Towers', |
| 161 'TreeSort'] | 161 'TreeSort'] |
| 162 | 162 |
| 163 def UploadToAppEngine(): | 163 def upload_to_app_engine(): |
| 164 """Upload our results to our appengine server.""" | 164 """Upload our results to our appengine server.""" |
| 165 # TODO(efortuna): This is the most basic way to get the data up | 165 # TODO(efortuna): This is the most basic way to get the data up |
| 166 # for others to view. Revisit this once we're serving nicer graphs (Google | 166 # for others to view. Revisit this once we're serving nicer graphs (Google |
| 167 # Chart Tools) and from multiple perfbots and once we're in a position to | 167 # Chart Tools) and from multiple perfbots and once we're in a position to |
| 168 # organize the data in a useful manner(!!). | 168 # organize the data in a useful manner(!!). |
| 169 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', | 169 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', |
| 170 'perf_testing')) | 170 'perf_testing')) |
| 171 shutil.rmtree(os.path.join('appengine', 'static', 'graphs'), | 171 shutil.rmtree(os.path.join('appengine', 'static', 'graphs'), |
| 172 ignore_errors=True) | 172 ignore_errors=True) |
| 173 shutil.copytree('graphs', os.path.join('appengine', 'static', 'graphs')) | 173 shutil.copytree('graphs', os.path.join('appengine', 'static', 'graphs')) |
| 174 shutil.copyfile('index.html', os.path.join('appengine', 'static', | 174 shutil.copyfile('index.html', os.path.join('appengine', 'static', |
| 175 'index.html')) | 175 'index.html')) |
| 176 RunCmd(['../../../third_party/appengine-python/1.5.4/appcfg.py', 'update', | 176 run_cmd(['../../../third_party/appengine-python/1.5.4/appcfg.py', 'update', |
| 177 'appengine/']) | 177 'appengine/']) |
| 178 | 178 |
| 179 class TestRunner(object): | 179 class TestRunner(object): |
| 180 """The base clas to provide shared code for different tests we will run and | 180 """The base clas to provide shared code for different tests we will run and |
| 181 graph.""" | 181 graph.""" |
| 182 | 182 |
| 183 def __init__(self, result_folder_name, platform_list, v8_and_or_frog_list, | 183 def __init__(self, result_folder_name, platform_list, v8_and_or_frog_list, |
| 184 values_list): | 184 values_list): |
| 185 """Args: | 185 """Args: |
| 186 result_folder_name the name of the folder where a tracefile of | 186 result_folder_name the name of the folder where a tracefile of |
| (...skipping 23 matching lines...) Expand all Loading... |
| 210 for val in values_list: | 210 for val in values_list: |
| 211 self.revision_dict[platform][f][val] = [] | 211 self.revision_dict[platform][f][val] = [] |
| 212 self.values_dict[platform][f][val] = [] | 212 self.values_dict[platform][f][val] = [] |
| 213 if V8 in v8_and_or_frog_list: | 213 if V8 in v8_and_or_frog_list: |
| 214 self.revision_dict[platform][V8][V8_MEAN] = [] | 214 self.revision_dict[platform][V8][V8_MEAN] = [] |
| 215 self.values_dict[platform][V8][V8_MEAN] = [] | 215 self.values_dict[platform][V8][V8_MEAN] = [] |
| 216 if FROG in v8_and_or_frog_list: | 216 if FROG in v8_and_or_frog_list: |
| 217 self.revision_dict[platform][FROG][FROG_MEAN] = [] | 217 self.revision_dict[platform][FROG][FROG_MEAN] = [] |
| 218 self.values_dict[platform][FROG][FROG_MEAN] = [] | 218 self.values_dict[platform][FROG][FROG_MEAN] = [] |
| 219 | 219 |
| 220 def GetColor(self): | 220 def get_color(self): |
| 221 color = COLORS[self.color_index] | 221 color = COLORS[self.color_index] |
| 222 self.color_index = (self.color_index + 1) % len(COLORS) | 222 self.color_index = (self.color_index + 1) % len(COLORS) |
| 223 return color | 223 return color |
| 224 | 224 |
| 225 def StyleAndSavePerfPlot(self, chart_title, y_axis_label, size_x, size_y, | 225 def syle_and_save_perf_plot(self, chart_title, y_axis_label, size_x, size_y, |
| 226 legend_loc, filename, platform_list, v8_and_or_frog_list, values_list, | 226 legend_loc, filename, platform_list, v8_and_or_frog_list, values_list, |
| 227 should_clear_axes=True): | 227 should_clear_axes=True): |
| 228 """Sets style preferences for chart boilerplate that is consistent across | 228 """Sets style preferences for chart boilerplate that is consistent across |
| 229 all charts, and saves the chart as a png. | 229 all charts, and saves the chart as a png. |
| 230 Args: | 230 Args: |
| 231 size_x the size of the printed chart, in inches, in the horizontal | 231 size_x the size of the printed chart, in inches, in the horizontal |
| 232 direction | 232 direction |
| 233 size_y the size of the printed chart, in inches in the vertical direction | 233 size_y the size of the printed chart, in inches in the vertical direction |
| 234 legend_loc the location of the legend in on the chart. See suitable | 234 legend_loc the location of the legend in on the chart. See suitable |
| 235 arguments for the loc argument in matplotlib | 235 arguments for the loc argument in matplotlib |
| 236 filename the filename that we want to save the resulting chart as | 236 filename the filename that we want to save the resulting chart as |
| 237 platform_list a list containing the platform(s) that our data has been run | 237 platform_list a list containing the platform(s) that our data has been run |
| 238 on. (command line, firefox, chrome, etc) | 238 on. (command line, firefox, chrome, etc) |
| 239 values_list a list containing the type of data we will be graphing | 239 values_list a list containing the type of data we will be graphing |
| 240 (performance, percentage passing, etc) | 240 (performance, percentage passing, etc) |
| 241 should_clear_axes True if we want to create a fresh graph, instead of | 241 should_clear_axes True if we want to create a fresh graph, instead of |
| 242 plotting additional lines on the current graph.""" | 242 plotting additional lines on the current graph.""" |
| 243 if should_clear_axes: | 243 if should_clear_axes: |
| 244 plt.cla() # cla = clear current axes | 244 plt.cla() # cla = clear current axes |
| 245 for platform in platform_list: | 245 for platform in platform_list: |
| 246 for f in v8_and_or_frog_list: | 246 for f in v8_and_or_frog_list: |
| 247 for val in values_list: | 247 for val in values_list: |
| 248 plt.plot(self.revision_dict[platform][f][val], | 248 plt.plot(self.revision_dict[platform][f][val], |
| 249 self.values_dict[platform][f][val], | 249 self.values_dict[platform][f][val], |
| 250 color=self.GetColor(), label='%s-%s-%s' % (platform, f, val)) | 250 color=self.get_color(), label='%s-%s-%s' % (platform, f, val)) |
| 251 | 251 |
| 252 plt.xlabel('Revision Number') | 252 plt.xlabel('Revision Number') |
| 253 plt.ylabel(y_axis_label) | 253 plt.ylabel(y_axis_label) |
| 254 plt.title(chart_title) | 254 plt.title(chart_title) |
| 255 fontP = FontProperties() | 255 fontP = FontProperties() |
| 256 fontP.set_size('small') | 256 fontP.set_size('small') |
| 257 plt.legend(loc=legend_loc, prop = fontP) | 257 plt.legend(loc=legend_loc, prop = fontP) |
| 258 | 258 |
| 259 fig = plt.gcf() | 259 fig = plt.gcf() |
| 260 fig.set_size_inches(size_x, size_y) | 260 fig.set_size_inches(size_x, size_y) |
| 261 fig.savefig(os.path.join(GRAPH_OUT_DIR, filename)) | 261 fig.savefig(os.path.join(GRAPH_OUT_DIR, filename)) |
| 262 | 262 |
| 263 def AddSvnRevisionToTrace(self, outfile): | 263 def add_svn_revision_to_trace(self, outfile): |
| 264 """Add the svn version number to the provided tracefile.""" | 264 """Add the svn version number to the provided tracefile.""" |
| 265 p = subprocess.Popen(['svn', 'info'], stdout = subprocess.PIPE, | 265 p = subprocess.Popen(['svn', 'info'], stdout = subprocess.PIPE, |
| 266 stderr = subprocess.STDOUT, shell = HAS_SHELL) | 266 stderr = subprocess.STDOUT, shell = HAS_SHELL) |
| 267 output, not_used = p.communicate() | 267 output, not_used = p.communicate() |
| 268 for line in output.split('\n'): | 268 for line in output.split('\n'): |
| 269 if 'Revision' in line: | 269 if 'Revision' in line: |
| 270 RunCmd(['echo', line.strip()], outfile) | 270 run_cmd(['echo', line.strip()], outfile) |
| 271 | 271 |
| 272 def WriteHtml(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2, | 272 def write_html(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2, |
| 273 cleanFile=False): | 273 cleanFile=False): |
| 274 """Adds an html table to the webpage to display the data values. This method | 274 """Adds an html table to the webpage to display the data values. This method |
| 275 will be removed when we have a nicer way to display data values.""" | 275 will be removed when we have a nicer way to display data values.""" |
| 276 #TODO(efortuna): fix this. | 276 #TODO(efortuna): fix this. |
| 277 return | 277 return |
| 278 #TODO(efortuna): Take this method out when have finalized where the data is | 278 #TODO(efortuna): Take this method out when have finalized where the data is |
| 279 # going to be displayed. | 279 # going to be displayed. |
| 280 f = '' | 280 f = '' |
| 281 out = '' | 281 out = '' |
| 282 if cleanFile: | 282 if cleanFile: |
| (...skipping 13 matching lines...) Expand all Loading... |
| 296 for revision in rev_nums: | 296 for revision in rev_nums: |
| 297 out.write('<td>%d</td>' % revision) | 297 out.write('<td>%d</td>' % revision) |
| 298 out.write('</tr>\n<tr><td> %s</td>' % label_1) | 298 out.write('</tr>\n<tr><td> %s</td>' % label_1) |
| 299 for perf in dict_1: | 299 for perf in dict_1: |
| 300 out.write('<td>%f</td>' % perf) | 300 out.write('<td>%f</td>' % perf) |
| 301 out.write('</tr>\n<tr><td> %s</td>' % label_2) | 301 out.write('</tr>\n<tr><td> %s</td>' % label_2) |
| 302 for perf in dict_2: | 302 for perf in dict_2: |
| 303 out.write('<td>%f</td>' % perf) | 303 out.write('<td>%f</td>' % perf) |
| 304 out.write('</tr> </table>') | 304 out.write('</tr> </table>') |
| 305 | 305 |
| 306 def CalculateGeometricMean(self, platform, frog_or_v8, svn_revision): | 306 def calculate_geometric_mean(self, platform, frog_or_v8, svn_revision): |
| 307 """Calculate the aggregate geometric mean for V8 and frog benchmark sets, | 307 """Calculate the aggregate geometric mean for V8 and frog benchmark sets, |
| 308 given two benchmark dictionaries.""" | 308 given two benchmark dictionaries.""" |
| 309 geo_mean = 0 | 309 geo_mean = 0 |
| 310 for benchmark in GetBenchmarks(): | 310 for benchmark in get_benchmarks(): |
| 311 geo_mean += math.log(self.values_dict[platform][frog_or_v8][benchmark][ | 311 geo_mean += math.log(self.values_dict[platform][frog_or_v8][benchmark][ |
| 312 len(self.values_dict[platform][frog_or_v8][benchmark]) - 1]) | 312 len(self.values_dict[platform][frog_or_v8][benchmark]) - 1]) |
| 313 | 313 |
| 314 mean = V8_MEAN | 314 mean = V8_MEAN |
| 315 if frog_or_v8 == FROG: | 315 if frog_or_v8 == FROG: |
| 316 mean = FROG_MEAN | 316 mean = FROG_MEAN |
| 317 self.values_dict[platform][frog_or_v8][mean] += \ | 317 self.values_dict[platform][frog_or_v8][mean] += \ |
| 318 [math.pow(math.e, geo_mean / len(GetBenchmarks()))] | 318 [math.pow(math.e, geo_mean / len(get_benchmarks()))] |
| 319 self.revision_dict[platform][frog_or_v8][mean] += [svn_revision] | 319 self.revision_dict[platform][frog_or_v8][mean] += [svn_revision] |
| 320 | 320 |
| 321 def Run(self): | 321 def run(self): |
| 322 """Run the benchmarks/tests from the command line and plot the | 322 """Run the benchmarks/tests from the command line and plot the |
| 323 results.""" | 323 results.""" |
| 324 if PERFBOT_MODE: | 324 if PERFBOT_MODE: |
| 325 plt.cla() # cla = clear current axes | 325 plt.cla() # cla = clear current axes |
| 326 os.chdir(DART_INSTALL_LOCATION) | 326 os.chdir(DART_INSTALL_LOCATION) |
| 327 EnsureOutputDirectory(self.result_folder_name) | 327 ensure_output_directory(self.result_folder_name) |
| 328 EnsureOutputDirectory(GRAPH_OUT_DIR) | 328 ensure_output_directory(GRAPH_OUT_DIR) |
| 329 self.RunTests() | 329 self.run_tests() |
| 330 os.chdir(os.path.join('tools', 'testing', 'perf_testing')) | 330 os.chdir(os.path.join('tools', 'testing', 'perf_testing')) |
| 331 | 331 |
| 332 # TODO(efortuna): You will want to make this only use a subset of the files | 332 # TODO(efortuna): You will want to make this only use a subset of the files |
| 333 # eventually. | 333 # eventually. |
| 334 files = os.listdir(self.result_folder_name) | 334 files = os.listdir(self.result_folder_name) |
| 335 | 335 |
| 336 for afile in files: | 336 for afile in files: |
| 337 if not afile.startswith('.'): | 337 if not afile.startswith('.'): |
| 338 self.ProcessFile(afile) | 338 self.process_file(afile) |
| 339 | 339 |
| 340 if PERFBOT_MODE: | 340 if PERFBOT_MODE: |
| 341 self.PlotResults('%s.png' % self.result_folder_name) | 341 self.plot_results('%s.png' % self.result_folder_name) |
| 342 | 342 |
| 343 class PerformanceTestRunner(TestRunner): | 343 class PerformanceTestRunner(TestRunner): |
| 344 """Super class for all performance testing.""" | 344 """Super class for all performance testing.""" |
| 345 def __init__(self, result_folder_name, platform_list, platform_type): | 345 def __init__(self, result_folder_name, platform_list, platform_type): |
| 346 super(PerformanceTestRunner, self).__init__(result_folder_name, | 346 super(PerformanceTestRunner, self).__init__(result_folder_name, |
| 347 platform_list, GetVersions(), GetBenchmarks()) | 347 platform_list, get_versions(), get_benchmarks()) |
| 348 self.platform_list = platform_list | 348 self.platform_list = platform_list |
| 349 self.platform_type = platform_type | 349 self.platform_type = platform_type |
| 350 | 350 |
| 351 def PlotAllPerf(self, png_filename): | 351 def plot_all_perf(self, png_filename): |
| 352 """Create a plot that shows the performance changes of individual benchmarks | 352 """Create a plot that shows the performance changes of individual benchmarks |
| 353 run by V8 and generated by frog, over svn history.""" | 353 run by V8 and generated by frog, over svn history.""" |
| 354 for benchmark in GetBenchmarks(): | 354 for benchmark in get_benchmarks(): |
| 355 self.StyleAndSavePerfPlot( | 355 self.syle_and_save_perf_plot( |
| 356 'Performance of %s over time on the %s' % (benchmark, | 356 'Performance of %s over time on the %s' % (benchmark, |
| 357 self.platform_type), 'Speed (bigger = better)', 16, 14, 'lower left', | 357 self.platform_type), 'Speed (bigger = better)', 16, 14, 'lower left', |
| 358 benchmark + png_filename, self.platform_list, GetVersions(), | 358 benchmark + png_filename, self.platform_list, get_versions(), |
| 359 [benchmark]) | 359 [benchmark]) |
| 360 | 360 |
| 361 def PlotAvgPerf(self, png_filename): | 361 def plot_avg_perf(self, png_filename): |
| 362 """Generate a plot that shows the performance changes of the geomentric mean | 362 """Generate a plot that shows the performance changes of the geomentric mean |
| 363 of V8 and frog benchmark performance over svn history.""" | 363 of V8 and frog benchmark performance over svn history.""" |
| 364 (title, y_axis, size_x, size_y, loc, filename) = \ | 364 (title, y_axis, size_x, size_y, loc, filename) = \ |
| 365 ('Geometric Mean of benchmark %s performance' % self.platform_type, | 365 ('Geometric Mean of benchmark %s performance' % self.platform_type, |
| 366 'Speed (bigger = better)', 16, 5, 'center', 'avg'+png_filename) | 366 'Speed (bigger = better)', 16, 5, 'center', 'avg'+png_filename) |
| 367 for platform in self.platform_list: | 367 for platform in self.platform_list: |
| 368 self.StyleAndSavePerfPlot(title, y_axis, size_x, size_y, loc, filename, | 368 self.syle_and_save_perf_plot(title, y_axis, size_x, size_y, loc, filename,
|
| 369 [platform], [V8], [V8_MEAN], True) | 369 [platform], [V8], [V8_MEAN], True) |
| 370 self.StyleAndSavePerfPlot(title, y_axis, size_x, size_y, loc, filename, | 370 self.syle_and_save_perf_plot(title, y_axis, size_x, size_y, loc, filename,
|
| 371 [platform], [FROG], [FROG_MEAN], False) | 371 [platform], [FROG], [FROG_MEAN], False) |
| 372 self.WriteHtml('table', | 372 self.write_html('table', |
| 373 self.revision_dict[platform][V8], | 373 self.revision_dict[platform][V8], |
| 374 'V8 mean', self.values_dict[platform][V8][V8_MEAN], | 374 'V8 mean', self.values_dict[platform][V8][V8_MEAN], |
| 375 'Frog mean', self.values_dict[platform][FROG][FROG_MEAN], | 375 'Frog mean', self.values_dict[platform][FROG][FROG_MEAN], |
| 376 True) | 376 True) |
| 377 | 377 |
| 378 def PlotResults(self, png_filename): | 378 def plot_results(self, png_filename): |
| 379 self.PlotAllPerf(png_filename) | 379 self.plot_all_perf(png_filename) |
| 380 self.PlotAvgPerf('2' + png_filename) | 380 self.plot_avg_perf('2' + png_filename) |
| 381 | 381 |
| 382 | 382 |
| 383 class CommandLinePerformanceTestRunner(PerformanceTestRunner): | 383 class CommandLinePerformanceTestRunner(PerformanceTestRunner): |
| 384 """Run performance tests from the command line.""" | 384 """Run performance tests from the command line.""" |
| 385 | 385 |
| 386 def __init__(self, result_folder_name): | 386 def __init__(self, result_folder_name): |
| 387 super(CommandLinePerformanceTestRunner, self).__init__(result_folder_name, | 387 super(CommandLinePerformanceTestRunner, self).__init__(result_folder_name, |
| 388 [COMMAND_LINE], 'command line') | 388 [COMMAND_LINE], 'command line') |
| 389 | 389 |
| 390 def ProcessFile(self, afile): | 390 def process_file(self, afile): |
| 391 """Pull all the relevant information out of a given tracefile. | 391 """Pull all the relevant information out of a given tracefile. |
| 392 | 392 |
| 393 Args: | 393 Args: |
| 394 afile is the filename string we will be processing.""" | 394 afile is the filename string we will be processing.""" |
| 395 f = open(os.path.join(self.result_folder_name, afile)) | 395 f = open(os.path.join(self.result_folder_name, afile)) |
| 396 tabulate_data = False | 396 tabulate_data = False |
| 397 revision_num = 0 | 397 revision_num = 0 |
| 398 for line in f.readlines(): | 398 for line in f.readlines(): |
| 399 if 'Revision' in line: | 399 if 'Revision' in line: |
| 400 revision_num = int(line.split()[1]) | 400 revision_num = int(line.split()[1]) |
| 401 elif 'Benchmark' in line: | 401 elif 'Benchmark' in line: |
| 402 tabulate_data = True | 402 tabulate_data = True |
| 403 elif tabulate_data: | 403 elif tabulate_data: |
| 404 tokens = line.split() | 404 tokens = line.split() |
| 405 if len(tokens) < 4 or tokens[0] not in GetBenchmarks(): | 405 if len(tokens) < 4 or tokens[0] not in get_benchmarks(): |
| 406 #Done tabulating data. | 406 #Done tabulating data. |
| 407 break | 407 break |
| 408 v8_value = float(tokens[1]) | 408 v8_value = float(tokens[1]) |
| 409 frog_value = float(tokens[3]) | 409 frog_value = float(tokens[3]) |
| 410 if v8_value == 0 or frog_value == 0: | 410 if v8_value == 0 or frog_value == 0: |
| 411 #Then there was an error when this performance test was run. Do not | 411 #Then there was an error when this performance test was run. Do not |
| 412 #count it in our numbers. | 412 #count it in our numbers. |
| 413 return | 413 return |
| 414 benchmark = tokens[0] | 414 benchmark = tokens[0] |
| 415 self.revision_dict[COMMAND_LINE][V8][benchmark] += [revision_num] | 415 self.revision_dict[COMMAND_LINE][V8][benchmark] += [revision_num] |
| 416 self.values_dict[COMMAND_LINE][V8][benchmark] += [v8_value] | 416 self.values_dict[COMMAND_LINE][V8][benchmark] += [v8_value] |
| 417 self.revision_dict[COMMAND_LINE][FROG][benchmark] += [revision_num] | 417 self.revision_dict[COMMAND_LINE][FROG][benchmark] += [revision_num] |
| 418 self.values_dict[COMMAND_LINE][FROG][benchmark] += [frog_value] | 418 self.values_dict[COMMAND_LINE][FROG][benchmark] += [frog_value] |
| 419 f.close() | 419 f.close() |
| 420 | 420 |
| 421 self.CalculateGeometricMean(COMMAND_LINE, FROG, revision_num) | 421 self.calculate_geometric_mean(COMMAND_LINE, FROG, revision_num) |
| 422 self.CalculateGeometricMean(COMMAND_LINE, V8, revision_num) | 422 self.calculate_geometric_mean(COMMAND_LINE, V8, revision_num) |
| 423 | 423 |
| 424 def RunTests(self): | 424 def run_tests(self): |
| 425 """Run a performance test on our updated system.""" | 425 """Run a performance test on our updated system.""" |
| 426 os.chdir('frog') | 426 os.chdir('frog') |
| 427 self.trace_file = os.path.join('..', 'tools', 'testing', 'perf_testing', | 427 self.trace_file = os.path.join('..', 'tools', 'testing', 'perf_testing', |
| 428 self.result_folder_name, 'result' + self.cur_time) | 428 self.result_folder_name, 'result' + self.cur_time) |
| 429 RunCmd(['python', os.path.join('benchmarks', 'perf_tests.py')], | 429 run_cmd(['python', os.path.join('benchmarks', 'perf_tests.py')], |
| 430 self.trace_file) | 430 self.trace_file) |
| 431 os.chdir('..') | 431 os.chdir('..') |
| 432 | 432 |
| 433 | 433 |
| 434 class BrowserPerformanceTestRunner(PerformanceTestRunner): | 434 class BrowserPerformanceTestRunner(PerformanceTestRunner): |
| 435 """Runs performance tests, in the browser.""" | 435 """Runs performance tests, in the browser.""" |
| 436 | 436 |
| 437 def __init__(self, result_folder_name): | 437 def __init__(self, result_folder_name): |
| 438 super(BrowserPerformanceTestRunner, self).__init__( | 438 super(BrowserPerformanceTestRunner, self).__init__( |
| 439 result_folder_name, GetBrowsers(), 'browser') | 439 result_folder_name, get_browsers(), 'browser') |
| 440 | 440 |
| 441 def RunTests(self): | 441 def run_tests(self): |
| 442 """Run a performance test in the browser.""" | 442 """Run a performance test in the browser.""" |
| 443 # For the smoke test, just run a simple test, not the actual benchmarks to | 443 # For the smoke test, just run a simple test, not the actual benchmarks to |
| 444 # ensure we haven't broken the Firefox DOM. | 444 # ensure we haven't broken the Firefox DOM. |
| 445 | 445 |
| 446 os.chdir('frog') | 446 os.chdir('frog') |
| 447 if PERFBOT_MODE: | 447 if PERFBOT_MODE: |
| 448 RunCmd(['python', os.path.join('benchmarks', 'make_web_benchmarks.py')]) | 448 run_cmd(['python', os.path.join('benchmarks', 'make_web_benchmarks.py')]) |
| 449 else: | 449 else: |
| 450 RunCmd(['./minfrog', '--out=../tools/testing/perf_testing/smoketest/' + \ | 450 run_cmd(['./minfrog', '--out=../tools/testing/perf_testing/smoketest/' + \ |
| 451 'smoketest_frog.js', '--libdir=%s/lib' % os.getcwd(), | 451 'smoketest_frog.js', '--libdir=%s/lib' % os.getcwd(), |
| 452 '--compile-only', '../tools/testing/perf_testing/smoketest/' + \ | 452 '--compile-only', '../tools/testing/perf_testing/smoketest/' + \ |
| 453 'dartWebBase.dart']) | 453 'dartWebBase.dart']) |
| 454 os.chdir('..') | 454 os.chdir('..') |
| 455 | 455 |
| 456 for browser in GetBrowsers(): | 456 for browser in get_browsers(): |
| 457 for version in GetVersions(): | 457 for version in get_versions(): |
| 458 self.trace_file = os.path.join('tools', 'testing', 'perf_testing', | 458 self.trace_file = os.path.join('tools', 'testing', 'perf_testing', |
| 459 self.result_folder_name, | 459 self.result_folder_name, |
| 460 'perf-%s-%s-%s' % (self.cur_time, browser, version)) | 460 'perf-%s-%s-%s' % (self.cur_time, browser, version)) |
| 461 self.AddSvnRevisionToTrace(self.trace_file) | 461 self.add_svn_revision_to_trace(self.trace_file) |
| 462 file_path = os.path.join(os.getcwd(), 'internal', 'browserBenchmarks', | 462 file_path = os.path.join(os.getcwd(), 'internal', 'browserBenchmarks', |
| 463 'benchmark_page_%s.html' % version) | 463 'benchmark_page_%s.html' % version) |
| 464 if not PERFBOT_MODE: | 464 if not PERFBOT_MODE: |
| 465 file_path = os.path.join(os.getcwd(), 'tools', 'testing', | 465 file_path = os.path.join(os.getcwd(), 'tools', 'testing', |
| 466 'perf_testing', 'smoketest', 'smoketest_%s.html' % version) | 466 'perf_testing', 'smoketest', 'smoketest_%s.html' % version) |
| 467 RunCmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'), | 467 run_cmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'), |
| 468 '--out', file_path, '--browser', browser, | 468 '--out', file_path, '--browser', browser, |
| 469 '--timeout', '600', '--perf'], self.trace_file, append=True) | 469 '--timeout', '600', '--perf'], self.trace_file, append=True) |
| 470 | 470 |
| 471 def ProcessFile(self, afile): | 471 def process_file(self, afile): |
| 472 """Comb through the html to find the performance results.""" | 472 """Comb through the html to find the performance results.""" |
| 473 parts = afile.split('-') | 473 parts = afile.split('-') |
| 474 browser = parts[2] | 474 browser = parts[2] |
| 475 version = parts[3] | 475 version = parts[3] |
| 476 f = open(os.path.join(self.result_folder_name, afile)) | 476 f = open(os.path.join(self.result_folder_name, afile)) |
| 477 lines = f.readlines() | 477 lines = f.readlines() |
| 478 line = '' | 478 line = '' |
| 479 i = 0 | 479 i = 0 |
| 480 revision_num = 0 | 480 revision_num = 0 |
| 481 while '<div id="results">' not in line and i < len(lines): | 481 while '<div id="results">' not in line and i < len(lines): |
| (...skipping 28 matching lines...) Expand all Loading... |
| 510 else: | 510 else: |
| 511 bench_dict = self.values_dict[browser][FROG] | 511 bench_dict = self.values_dict[browser][FROG] |
| 512 bench_dict[name] += [float(score)] | 512 bench_dict[name] += [float(score)] |
| 513 self.revision_dict[browser][version][name] += [revision_num] | 513 self.revision_dict[browser][version][name] += [revision_num] |
| 514 | 514 |
| 515 f.close() | 515 f.close() |
| 516 if not PERFBOT_MODE: | 516 if not PERFBOT_MODE: |
| 517 print 'PASS' | 517 print 'PASS' |
| 518 os.remove(os.path.join(self.result_folder_name, afile)) | 518 os.remove(os.path.join(self.result_folder_name, afile)) |
| 519 else: | 519 else: |
| 520 self.CalculateGeometricMean(browser, version, revision_num) | 520 self.calculate_geometric_mean(browser, version, revision_num) |
| 521 | 521 |
| 522 def WriteHtml(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2, | 522 def write_html(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2, |
| 523 cleanFile=False): | 523 cleanFile=False): |
| 524 #TODO(efortuna) | 524 #TODO(efortuna) |
| 525 pass | 525 pass |
| 526 | 526 |
| 527 | 527 |
| 528 class BrowserCorrectnessTestRunner(TestRunner): | 528 class BrowserCorrectnessTestRunner(TestRunner): |
| 529 def __init__(self, test_type, result_folder_name): | 529 def __init__(self, test_type, result_folder_name): |
| 530 super(BrowserCorrectnessTestRunner, self).__init__(result_folder_name, | 530 super(BrowserCorrectnessTestRunner, self).__init__(result_folder_name, |
| 531 GetBrowsers(), [FROG], [CORRECTNESS]) | 531 get_browsers(), [FROG], [CORRECTNESS]) |
| 532 self.test_type = test_type | 532 self.test_type = test_type |
| 533 | 533 |
| 534 def RunTests(self): | 534 def run_tests(self): |
| 535 """Run a test of the latest svn revision.""" | 535 """run a test of the latest svn revision.""" |
| 536 for browser in GetBrowsers(): | 536 for browser in get_browsers(): |
| 537 current_file = 'correctness%s-%s' % (self.cur_time, browser) | 537 current_file = 'correctness%s-%s' % (self.cur_time, browser) |
| 538 self.trace_file = os.path.join('tools', 'testing', | 538 self.trace_file = os.path.join('tools', 'testing', |
| 539 'perf_testing', self.result_folder_name, current_file) | 539 'perf_testing', self.result_folder_name, current_file) |
| 540 self.AddSvnRevisionToTrace(self.trace_file) | 540 self.add_svn_revision_to_trace(self.trace_file) |
| 541 dart_sdk = os.path.join(os.getcwd(), utils.GetBuildRoot(utils.GuessOS(), | 541 dart_sdk = os.path.join(os.getcwd(), utils.GetBuildRoot(utils.GuessOS(), |
| 542 'release', 'ia32'), 'dart-sdk') | 542 'release', 'ia32'), 'dart-sdk') |
| 543 RunCmd([os.path.join('.', 'tools', 'test.py'), | 543 run_cmd([os.path.join('.', 'tools', 'test.py'), |
| 544 '--component=webdriver', '--flag=%s' % browser, '--flag=--frog=%s' % \ | 544 '--component=webdriver', '--flag=%s' % browser, '--flag=--frog=%s' % \ |
| 545 os.path.join(dart_sdk, 'bin', 'frogc'), | 545 os.path.join(dart_sdk, 'bin', 'frogc'), |
| 546 '--flag=--froglib=%s' % os.path.join(dart_sdk, 'lib'), | 546 '--flag=--froglib=%s' % os.path.join(dart_sdk, 'lib'), |
| 547 '--timeout=20', '--progress=color', '--mode=release', '-j1', | 547 '--timeout=20', '--progress=color', '--mode=release', '-j1', |
| 548 self.test_type], self.trace_file, append=True) | 548 self.test_type], self.trace_file, append=True) |
| 549 | 549 |
| 550 def ProcessFile(self, afile): | 550 def process_file(self, afile): |
| 551 """Given a trace file, extract all the relevant information out of it to | 551 """Given a trace file, extract all the relevant information out of it to |
| 552 determine the number of correctly passing tests. | 552 determine the number of correctly passing tests. |
| 553 | 553 |
| 554 Arguments: | 554 Arguments: |
| 555 afile the filename string""" | 555 afile the filename string""" |
| 556 browser = afile.rpartition('-')[2] | 556 browser = afile.rpartition('-')[2] |
| 557 f = open(os.path.join(self.result_folder_name, afile)) | 557 f = open(os.path.join(self.result_folder_name, afile)) |
| 558 revision_num = 0 | 558 revision_num = 0 |
| 559 lines = f.readlines() | 559 lines = f.readlines() |
| 560 total_tests = 0 | 560 total_tests = 0 |
| (...skipping 11 matching lines...) Expand all Loading... |
| 572 if '--- TIMEOUT ---' in line or 'FAIL:' in line or 'PASS' in line: | 572 if '--- TIMEOUT ---' in line or 'FAIL:' in line or 'PASS' in line: |
| 573 # (A printed out 'PASS' indicates we incorrectly passed a negative | 573 # (A printed out 'PASS' indicates we incorrectly passed a negative |
| 574 # test.) | 574 # test.) |
| 575 num_failed += 1 | 575 num_failed += 1 |
| 576 | 576 |
| 577 self.revision_dict[browser][FROG][CORRECTNESS] += [revision_num] | 577 self.revision_dict[browser][FROG][CORRECTNESS] += [revision_num] |
| 578 self.values_dict[browser][FROG][CORRECTNESS] += [100.0 * | 578 self.values_dict[browser][FROG][CORRECTNESS] += [100.0 * |
| 579 (((float)(total_tests - (expect_fail + num_failed))) /total_tests)] | 579 (((float)(total_tests - (expect_fail + num_failed))) /total_tests)] |
| 580 f.close() | 580 f.close() |
| 581 | 581 |
| 582 def PlotResults(self, png_filename): | 582 def plot_results(self, png_filename): |
| 583 first_time = True | 583 first_time = True |
| 584 for browser in GetBrowsers(): | 584 for browser in get_browsers(): |
| 585 self.StyleAndSavePerfPlot('Percentage of language tests passing in ' | 585 self.syle_and_save_perf_plot('Percentage of language tests passing in ' |
| 586 'different browsers', '% of tests passed', 8, 8, 'lower left', | 586 'different browsers', '% of tests passed', 8, 8, 'lower left', |
| 587 png_filename, [browser], [FROG], [CORRECTNESS], first_time) | 587 png_filename, [browser], [FROG], [CORRECTNESS], first_time) |
| 588 first_time = False | 588 first_time = False |
| 589 | 589 |
| 590 | 590 |
| 591 class CompileTimeAndSizeTestRunner(TestRunner): | 591 class CompileTimeAndSizeTestRunner(TestRunner): |
| 592 """Run tests to determine how long minfrog takes to compile, and the compiled | 592 """Run tests to determine how long minfrog takes to compile, and the compiled |
| 593 file output size of some benchmarking files.""" | 593 file output size of some benchmarking files.""" |
| 594 def __init__(self, result_folder_name): | 594 def __init__(self, result_folder_name): |
| 595 super(CompileTimeAndSizeTestRunner, self).__init__(result_folder_name, | 595 super(CompileTimeAndSizeTestRunner, self).__init__(result_folder_name, |
| 596 [COMMAND_LINE], [FROG], ['Compiling on Dart VM', 'Bootstrapping', | 596 [COMMAND_LINE], [FROG], ['Compiling on Dart VM', 'Bootstrapping', |
| 597 'minfrog', 'swarm', 'total']) | 597 'minfrog', 'swarm', 'total']) |
| 598 self.failure_threshold = {'Compiling on Dart VM' : 1, 'Bootstrapping' : .5, | 598 self.failure_threshold = {'Compiling on Dart VM' : 1, 'Bootstrapping' : .5, |
| 599 'minfrog' : 100, 'swarm' : 100, 'total' : 100} | 599 'minfrog' : 100, 'swarm' : 100, 'total' : 100} |
| 600 | 600 |
| 601 def RunTests(self): | 601 def run_tests(self): |
| 602 os.chdir('frog') | 602 os.chdir('frog') |
| 603 self.trace_file = os.path.join('..', 'tools', 'testing', 'perf_testing', | 603 self.trace_file = os.path.join('..', 'tools', 'testing', 'perf_testing', |
| 604 self.result_folder_name, self.result_folder_name + self.cur_time) | 604 self.result_folder_name, self.result_folder_name + self.cur_time) |
| 605 | 605 |
| 606 self.AddSvnRevisionToTrace(self.trace_file) | 606 self.add_svn_revision_to_trace(self.trace_file) |
| 607 | 607 |
| 608 elapsed = TimeCmd([os.path.join('.', 'frog.py'), '--', | 608 elapsed = time_cmd([os.path.join('.', 'frog.py'), '--', |
| 609 '--out=minfrog', 'minfrog.dart']) | 609 '--out=minfrog', 'minfrog.dart']) |
| 610 RunCmd(['echo', '%f Compiling on Dart VM in production mode in seconds' | 610 run_cmd(['echo', '%f Compiling on Dart VM in production mode in seconds' |
| 611 % elapsed], self.trace_file, append=True) | 611 % elapsed], self.trace_file, append=True) |
| 612 elapsed = TimeCmd([os.path.join('.', 'minfrog'), '--out=minfrog', | 612 elapsed = time_cmd([os.path.join('.', 'minfrog'), '--out=minfrog', |
| 613 'minfrog.dart', os.path.join('tests', 'hello.dart')]) | 613 'minfrog.dart', os.path.join('tests', 'hello.dart')]) |
| 614 if elapsed < self.failure_threshold['Bootstrapping']: | 614 if elapsed < self.failure_threshold['Bootstrapping']: |
| 615 #minfrog didn't compile correctly. Stop testing now, because subsequent | 615 #minfrog didn't compile correctly. Stop testing now, because subsequent |
| 616 #numbers will be meaningless. | 616 #numbers will be meaningless. |
| 617 return | 617 return |
| 618 size = os.path.getsize('minfrog') | 618 size = os.path.getsize('minfrog') |
| 619 RunCmd(['echo', '%f Bootstrapping time in seconds in production mode' % | 619 run_cmd(['echo', '%f Bootstrapping time in seconds in production mode' % |
| 620 elapsed], self.trace_file, append=True) | 620 elapsed], self.trace_file, append=True) |
| 621 RunCmd(['echo', '%d Generated checked minfrog size' % size], | 621 run_cmd(['echo', '%d Generated checked minfrog size' % size], |
| 622 self.trace_file, append=True) | 622 self.trace_file, append=True) |
| 623 | 623 |
| 624 RunCmd([os.path.join('.', 'minfrog'), ' --out=swarm-result ', | 624 run_cmd([os.path.join('.', 'minfrog'), ' --out=swarm-result ', |
| 625 '--compile-only', os.path.join('..', 'client', 'samples', 'swarm', | 625 '--compile-only', os.path.join('..', 'client', 'samples', 'swarm', |
| 626 'swarm.dart')]) | 626 'swarm.dart')]) |
| 627 swarm_size = 0 | 627 swarm_size = 0 |
| 628 try: | 628 try: |
| 629 swarm_size = os.path.getsize('swarm-result') | 629 swarm_size = os.path.getsize('swarm-result') |
| 630 except OSError: | 630 except OSError: |
| 631 pass #If compilation failed, continue on running other tests. | 631 pass #If compilation failed, continue on running other tests. |
| 632 | 632 |
| 633 RunCmd([os.path.join('.', 'minfrog'), '--out=total-result', | 633 run_cmd([os.path.join('.', 'minfrog'), '--out=total-result', |
| 634 '--compile-only', os.path.join('..', 'client', 'samples', 'total', | 634 '--compile-only', os.path.join('..', 'client', 'samples', 'total', |
| 635 'src', 'Total.dart')]) | 635 'src', 'Total.dart')]) |
| 636 total_size = 0 | 636 total_size = 0 |
| 637 try: | 637 try: |
| 638 total_size = os.path.getsize('total-result') | 638 total_size = os.path.getsize('total-result') |
| 639 except OSError: | 639 except OSError: |
| 640 pass #If compilation failed, continue on running other tests. | 640 pass #If compilation failed, continue on running other tests. |
| 641 | 641 |
| 642 RunCmd(['echo', '%d Generated checked swarm size' % swarm_size], | 642 run_cmd(['echo', '%d Generated checked swarm size' % swarm_size], |
| 643 self.trace_file, append=True) | 643 self.trace_file, append=True) |
| 644 | 644 |
| 645 RunCmd(['echo', '%d Generated checked total size' % total_size], | 645 run_cmd(['echo', '%d Generated checked total size' % total_size], |
| 646 self.trace_file, append=True) | 646 self.trace_file, append=True) |
| 647 os.chdir('..') | 647 os.chdir('..') |
| 648 | 648 |
| 649 def ProcessFile(self, afile): | 649 def process_file(self, afile): |
| 650 """Pull all the relevant information out of a given tracefile. | 650 """Pull all the relevant information out of a given tracefile. |
| 651 Args: | 651 Args: |
| 652 afile is the filename string we will be processing.""" | 652 afile is the filename string we will be processing.""" |
| 653 f = open(os.path.join(self.result_folder_name, afile)) | 653 f = open(os.path.join(self.result_folder_name, afile)) |
| 654 tabulate_data = False | 654 tabulate_data = False |
| 655 revision_num = 0 | 655 revision_num = 0 |
| 656 for line in f.readlines(): | 656 for line in f.readlines(): |
| 657 tokens = line.split() | 657 tokens = line.split() |
| 658 if 'Revision' in line: | 658 if 'Revision' in line: |
| 659 revision_num = int(line.split()[1]) | 659 revision_num = int(line.split()[1]) |
| (...skipping 13 matching lines...) Expand all Loading... |
| 673 self.revision_dict[COMMAND_LINE][FROG][metric].pop() | 673 self.revision_dict[COMMAND_LINE][FROG][metric].pop() |
| 674 self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num] | 674 self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num] |
| 675 # Fill 0 if compilation failed. | 675 # Fill 0 if compilation failed. |
| 676 if self.values_dict[COMMAND_LINE][FROG][metric][-1] < \ | 676 if self.values_dict[COMMAND_LINE][FROG][metric][-1] < \ |
| 677 self.failure_threshold[metric]: | 677 self.failure_threshold[metric]: |
| 678 self.values_dict[COMMAND_LINE][FROG][metric] += [0] | 678 self.values_dict[COMMAND_LINE][FROG][metric] += [0] |
| 679 self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num] | 679 self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num] |
| 680 | 680 |
| 681 f.close() | 681 f.close() |
| 682 | 682 |
| 683 def PlotResults(self, png_filename): | 683 def plot_results(self, png_filename): |
| 684 self.StyleAndSavePerfPlot('Compiled minfrog Sizes', | 684 self.syle_and_save_perf_plot('Compiled minfrog Sizes', |
| 685 'Size (in bytes)', 10, 10, 'center', png_filename, [COMMAND_LINE], | 685 'Size (in bytes)', 10, 10, 'center', png_filename, [COMMAND_LINE], |
| 686 [FROG], ['swarm', 'total', 'minfrog']) | 686 [FROG], ['swarm', 'total', 'minfrog']) |
| 687 self.WriteHtml('bar', self.revision_dict[COMMAND_LINE][FROG]['minfrog'], | 687 self.write_html('bar', self.revision_dict[COMMAND_LINE][FROG]['minfrog'], |
| 688 'minfrog size', self.values_dict[COMMAND_LINE][FROG]['minfrog'], '', []) | 688 'minfrog size', self.values_dict[COMMAND_LINE][FROG]['minfrog'], '', []) |
| 689 | 689 |
| 690 self.StyleAndSavePerfPlot('Time to compile and bootstrap', | 690 self.syle_and_save_perf_plot('Time to compile and bootstrap', |
| 691 'Seconds', 10, 10, 'center', '2' + png_filename, [COMMAND_LINE], [FROG], | 691 'Seconds', 10, 10, 'center', '2' + png_filename, [COMMAND_LINE], [FROG], |
| 692 ['Bootstrapping', 'Compiling on Dart VM']) | 692 ['Bootstrapping', 'Compiling on Dart VM']) |
| 693 self.WriteHtml('baz', | 693 self.write_html('baz', |
| 694 self.revision_dict[COMMAND_LINE][FROG]['Bootstrapping'], | 694 self.revision_dict[COMMAND_LINE][FROG]['Bootstrapping'], |
| 695 'Bootstrapping', self.values_dict[COMMAND_LINE][FROG]['Bootstrapping'], | 695 'Bootstrapping', self.values_dict[COMMAND_LINE][FROG]['Bootstrapping'], |
| 696 'Compiling on Dart VM', | 696 'Compiling on Dart VM', |
| 697 self.values_dict[COMMAND_LINE][FROG]['Compiling on Dart VM']) | 697 self.values_dict[COMMAND_LINE][FROG]['Compiling on Dart VM']) |
| 698 | 698 |
| 699 | 699 |
| 700 def ParseArgs(): | 700 def parse_args(): |
| 701 parser = optparse.OptionParser() | 701 parser = optparse.OptionParser() |
| 702 parser.add_option('--command-line', '-c', dest='cl', | 702 parser.add_option('--command-line', '-c', dest='cl', |
| 703 help = 'Run the command line tests', | 703 help = 'Run the command line tests', |
| 704 action = 'store_true', default = False) | 704 action = 'store_true', default = False) |
| 705 parser.add_option('--size-time', '-s', dest = 'size', | 705 parser.add_option('--size-time', '-s', dest = 'size', |
| 706 help = 'Run the code size and timing tests', | 706 help = 'Run the code size and timing tests', |
| 707 action = 'store_true', default = False) | 707 action = 'store_true', default = False) |
| 708 parser.add_option('--language', '-l', dest = 'language', | 708 parser.add_option('--language', '-l', dest = 'language', |
| 709 help = 'Run the language correctness tests', | 709 help = 'Run the language correctness tests', |
| 710 action = 'store_true', default = False) | 710 action = 'store_true', default = False) |
| 711 parser.add_option('--browser-perf', '-b', dest = 'perf', | 711 parser.add_option('--browser-perf', '-b', dest = 'perf', |
| 712 help = 'Run the browser performance tests', | 712 help = 'Run the browser performance tests', |
| 713 action = 'store_true', default = False) | 713 action = 'store_true', default = False) |
| 714 parser.add_option('--forever', '-f', dest = 'continuous', | 714 parser.add_option('--forever', '-f', dest = 'continuous', |
| 715 help = 'Run this script forever, always checking for the next svn ' | 715 help = 'Run this script forever, always checking for the next svn ' |
| 716 'checkin', action = 'store_true', default = False) | 716 'checkin', action = 'store_true', default = False) |
| 717 parser.add_option('--perfbot', '-p', dest = 'perfbot', | 717 parser.add_option('--perfbot', '-p', dest = 'perfbot', |
| 718 help = "Run in perfbot mode. (Generate plots, and keep trace files)", | 718 help = "Run in perfbot mode. (Generate plots, and keep trace files)", |
| 719 action = 'store_true', default = False) | 719 action = 'store_true', default = False) |
| 720 parser.add_option('--verbose', '-v', dest = 'verbose', | 720 parser.add_option('--verbose', '-v', dest = 'verbose', |
| 721 help = 'Print extra debug output', action = 'store_true', default = False) | 721 help = 'Print extra debug output', action = 'store_true', default = False) |
| 722 | 722 |
| 723 args, ignored = parser.parse_args() | 723 args, ignored = parser.parse_args() |
| 724 if not (args.cl or args.size or args.language or args.perf): | 724 if not (args.cl or args.size or args.language or args.perf): |
| 725 args.cl = args.size = args.language = args.perf = True | 725 args.cl = args.size = args.language = args.perf = True |
| 726 return (args.cl, args.size, args.language, args.perf, args.continuous, | 726 return (args.cl, args.size, args.language, args.perf, args.continuous, |
| 727 args.perfbot, args.verbose) | 727 args.perfbot, args.verbose) |
| 728 | 728 |
| 729 def RunTestSequence(cl, size, language, perf): | 729 def run_test_sequence(cl, size, language, perf): |
| 730 if PERFBOT_MODE: | 730 if PERFBOT_MODE: |
| 731 # The buildbot already builds and syncs to a specific revision. Don't fight | 731 # The buildbot already builds and syncs to a specific revision. Don't fight |
| 732 # with it or replicate work. | 732 # with it or replicate work. |
| 733 if SyncAndBuild() == 1: | 733 if sync_and_build() == 1: |
| 734 return # The build is broken. | 734 return # The build is broken. |
| 735 if cl: | 735 if cl: |
| 736 CommandLinePerformanceTestRunner('cl-results').Run() | 736 CommandLinePerformanceTestRunner('cl-results').run() |
| 737 if size: | 737 if size: |
| 738 CompileTimeAndSizeTestRunner('code-time-size').Run() | 738 CompileTimeAndSizeTestRunner('code-time-size').run() |
| 739 if language: | 739 if language: |
| 740 BrowserCorrectnessTestRunner('language', 'browser-correctness').Run() | 740 BrowserCorrectnessTestRunner('language', 'browser-correctness').run() |
| 741 if perf: | 741 if perf: |
| 742 BrowserPerformanceTestRunner('browser-perf').Run() | 742 BrowserPerformanceTestRunner('browser-perf').run() |
| 743 | 743 |
| 744 if PERFBOT_MODE: | 744 if PERFBOT_MODE: |
| 745 UploadToAppEngine() | 745 upload_to_app_engine() |
| 746 | 746 |
| 747 def main(): | 747 def main(): |
| 748 global PERFBOT_MODE, VERBOSE | 748 global PERFBOT_MODE, VERBOSE |
| 749 (cl, size, language, perf, continuous, perfbot, verbose) = ParseArgs() | 749 (cl, size, language, perf, continuous, perfbot, verbose) = parse_args() |
| 750 PERFBOT_MODE = perfbot | 750 PERFBOT_MODE = perfbot |
| 751 VERBOSE = verbose | 751 VERBOSE = verbose |
| 752 if continuous: | 752 if continuous: |
| 753 while True: | 753 while True: |
| 754 if HasNewCode(): | 754 if has_new_code(): |
| 755 RunTestSequence(cl, size, language, perf) | 755 run_test_sequence(cl, size, language, perf) |
| 756 else: | 756 else: |
| 757 time.sleep(SLEEP_TIME) | 757 time.sleep(SLEEP_TIME) |
| 758 else: | 758 else: |
| 759 RunTestSequence(cl, size, language, perf) | 759 run_test_sequence(cl, size, language, perf) |
| 760 | 760 |
| 761 if __name__ == '__main__': | 761 if __name__ == '__main__': |
| 762 main() | 762 main() |
| 763 | 763 |
| OLD | NEW |