Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(393)

Side by Side Diff: tools/testing/perf_testing/create_graph.py

Issue 9024022: Fixing capitalization with style guide as per jmesserly's comments. (Closed) Base URL: http://dart.googlecode.com/svn/branches/bleeding_edge/dart/
Patch Set: '' Created 9 years ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/python 1 #!/usr/bin/python
2 2
3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
4 # for details. All rights reserved. Use of this source code is governed by a 4 # for details. All rights reserved. Use of this source code is governed by a
5 # BSD-style license that can be found in the LICENSE file. 5 # BSD-style license that can be found in the LICENSE file.
6 6
7 import datetime 7 import datetime
8 import math 8 import math
9 try: 9 try:
10 from matplotlib.font_manager import FontProperties 10 from matplotlib.font_manager import FontProperties
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after
44 SLEEP_TIME = 200 44 SLEEP_TIME = 200
45 PERFBOT_MODE = False 45 PERFBOT_MODE = False
46 VERBOSE = False 46 VERBOSE = False
47 HAS_SHELL = False 47 HAS_SHELL = False
48 if platform.system() == 'Windows': 48 if platform.system() == 'Windows':
49 # On Windows, shell must be true to get the correct environment variables. 49 # On Windows, shell must be true to get the correct environment variables.
50 HAS_SHELL = True 50 HAS_SHELL = True
51 51
52 """First, some utility methods.""" 52 """First, some utility methods."""
53 53
54 def RunCmd(cmd_list, outfile=None, append=False): 54 def run_cmd(cmd_list, outfile=None, append=False):
55 """Run the specified command and print out any output to stdout. 55 """Run the specified command and print out any output to stdout.
56 Args: 56 Args:
57 cmd_list a list of strings that make up the command to run 57 cmd_list a list of strings that make up the command to run
58 outfile a string indicating the name of the file that we should write stdout 58 outfile a string indicating the name of the file that we should write stdout
59 to 59 to
60 append True if we want to append to the file instead of overwriting it""" 60 append True if we want to append to the file instead of overwriting it"""
61 if VERBOSE: 61 if VERBOSE:
62 print ' '.join(cmd_list) 62 print ' '.join(cmd_list)
63 out = subprocess.PIPE 63 out = subprocess.PIPE
64 if outfile: 64 if outfile:
65 mode = 'w' 65 mode = 'w'
66 if append: 66 if append:
67 mode = 'a' 67 mode = 'a'
68 out = open(outfile, mode) 68 out = open(outfile, mode)
69 p = subprocess.Popen(cmd_list, stdout = out, stderr = subprocess.PIPE, 69 p = subprocess.Popen(cmd_list, stdout = out, stderr = subprocess.PIPE,
70 shell=HAS_SHELL) 70 shell=HAS_SHELL)
71 output, not_used = p.communicate(); 71 output, not_used = p.communicate();
72 if output: 72 if output:
73 print output 73 print output
74 return output 74 return output
75 75
76 def TimeCmd(cmd): 76 def time_cmd(cmd):
77 """Determine the amount of (real) time it takes to execute a given command.""" 77 """Determine the amount of (real) time it takes to execute a given command."""
78 start = time.time() 78 start = time.time()
79 RunCmd(cmd) 79 run_cmd(cmd)
80 return time.time() - start 80 return time.time() - start
81 81
82 def SyncAndBuild(failed_once=False): 82 def sync_and_build(failed_once=False):
83 """Make sure we have the latest version of of the repo, and build it. We 83 """Make sure we have the latest version of of the repo, and build it. We
84 begin and end standing in DART_INSTALL_LOCATION. 84 begin and end standing in DART_INSTALL_LOCATION.
85 Args: 85 Args:
86 failed_once True if we have attempted to build this once before, and we've 86 failed_once True if we have attempted to build this once before, and we've
87 failed, indicating the build is broken. 87 failed, indicating the build is broken.
88 Returns: 88 Returns:
89 err_code = 1 if there was a problem building two times in a row.""" 89 err_code = 1 if there was a problem building two times in a row."""
90 os.chdir(DART_INSTALL_LOCATION) 90 os.chdir(DART_INSTALL_LOCATION)
91 #Revert our newly built minfrog to prevent conflicts when we update 91 #Revert our newly built minfrog to prevent conflicts when we update
92 RunCmd(['svn', 'revert', os.path.join(os.getcwd(), 'frog', 'minfrog')]) 92 run_cmd(['svn', 'revert', os.path.join(os.getcwd(), 'frog', 'minfrog')])
93 93
94 RunCmd(['gclient', 'sync']) 94 run_cmd(['gclient', 'sync'])
95 # TODO(efortuna): building the sdk locally is a band-aid until all build 95 # TODO(efortuna): building the sdk locally is a band-aid until all build
96 # platform SDKs are hosted in Google storage. Pull from https://sandbox. 96 # platform SDKs are hosted in Google storage. Pull from https://sandbox.
97 # google.com/storage/?arg=dart-dump-render-tree#dart-dump-render-tree%2Fsdk 97 # google.com/storage/?arg=dart-dump-render-tree#dart-dump-render-tree%2Fsdk
98 # eventually. 98 # eventually.
99 # TODO(efortuna): Currently always building ia32 architecture because we don't 99 # TODO(efortuna): Currently always building ia32 architecture because we don't
100 # have test statistics for what's passing on x64. Eliminate arch specification 100 # have test statistics for what's passing on x64. Eliminate arch specification
101 # when we have tests running on x64, too. 101 # when we have tests running on x64, too.
102 lines = RunCmd([os.path.join('.', 'tools', 'build.py'), '-m', 'release', 102 lines = run_cmd([os.path.join('.', 'tools', 'build.py'), '-m', 'release',
103 '--arch=ia32', 'create_sdk']) 103 '--arch=ia32', 'create_sdk'])
104 104
105 for line in lines: 105 for line in lines:
106 if 'BUILD FAILED' in lines: 106 if 'BUILD FAILED' in lines:
107 if failed_once: 107 if failed_once:
108 # Someone checked in a broken build! Just stop trying to make it work 108 # Someone checked in a broken build! Just stop trying to make it work
109 # and wait to try again. 109 # and wait to try again.
110 print 'Broken Build' 110 print 'Broken Build'
111 return 1 111 return 1
112 #Remove the output directory and attempt to build again. If it still 112 #Remove the output directory and attempt to build again. If it still
113 #fails, abort, and try again in a little bit. 113 #fails, abort, and try again in a little bit.
114 shutil.rmtree(os.path.join(os.getcwd(), 114 shutil.rmtree(os.path.join(os.getcwd(),
115 utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32'))) 115 utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32')))
116 SyncAndBuild(True) 116 sync_and_build(True)
117 return 0 117 return 0
118 118
119 def EnsureOutputDirectory(dir_name): 119 def ensure_output_directory(dir_name):
120 """Test that the listed directory name exists, and if not, create one for 120 """Test that the listed directory name exists, and if not, create one for
121 our output to be placed. 121 our output to be placed.
122 Args: 122 Args:
123 dir_name the directory we will create if it does not exist.""" 123 dir_name the directory we will create if it does not exist."""
124 dir_path = os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', 124 dir_path = os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing',
125 'perf_testing', dir_name) 125 'perf_testing', dir_name)
126 if not os.path.exists(dir_path): 126 if not os.path.exists(dir_path):
127 os.mkdir(dir_path) 127 os.mkdir(dir_path)
128 print 'Creating output directory ', dir_path 128 print 'Creating output directory ', dir_path
129 129
130 def HasNewCode(): 130 def has_new_code():
131 """Tests if there are any newer versions of files on the server.""" 131 """Tests if there are any newer versions of files on the server."""
132 os.chdir(DART_INSTALL_LOCATION) 132 os.chdir(DART_INSTALL_LOCATION)
133 results = RunCmd(['svn', 'st', '-u']) 133 results = run_cmd(['svn', 'st', '-u'])
134 for line in results: 134 for line in results:
135 if '*' in line: 135 if '*' in line:
136 return True 136 return True
137 return False 137 return False
138 138
139 def GetBrowsers(): 139 def get_browsers():
140 if not PERFBOT_MODE: 140 if not PERFBOT_MODE:
141 # Only Firefox (and Chrome, but we have Dump Render Tree) works in Linux 141 # Only Firefox (and Chrome, but we have Dump Render Tree) works in Linux
142 return ['ff'] 142 return ['ff']
143 browsers = ['ff', 'chrome', 'safari'] 143 browsers = ['ff', 'chrome', 'safari']
144 if platform.system() == 'Windows': 144 if platform.system() == 'Windows':
145 browsers += ['ie'] 145 browsers += ['ie']
146 return browsers 146 return browsers
147 147
148 def GetVersions(): 148 def get_versions():
149 if not PERFBOT_MODE: 149 if not PERFBOT_MODE:
150 return [FROG] 150 return [FROG]
151 else: 151 else:
152 return V8_AND_FROG 152 return V8_AND_FROG
153 153
154 def GetBenchmarks(): 154 def get_benchmarks():
155 if not PERFBOT_MODE: 155 if not PERFBOT_MODE:
156 return ['Smoketest'] 156 return ['Smoketest']
157 else: 157 else:
158 return ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 'BinaryTrees', 158 return ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 'BinaryTrees',
159 'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 'Loop', 'Permute', 159 'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 'Loop', 'Permute',
160 'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 'Tak', 'Takl', 'Towers', 160 'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 'Tak', 'Takl', 'Towers',
161 'TreeSort'] 161 'TreeSort']
162 162
163 def UploadToAppEngine(): 163 def upload_to_app_engine():
164 """Upload our results to our appengine server.""" 164 """Upload our results to our appengine server."""
165 # TODO(efortuna): This is the most basic way to get the data up 165 # TODO(efortuna): This is the most basic way to get the data up
166 # for others to view. Revisit this once we're serving nicer graphs (Google 166 # for others to view. Revisit this once we're serving nicer graphs (Google
167 # Chart Tools) and from multiple perfbots and once we're in a position to 167 # Chart Tools) and from multiple perfbots and once we're in a position to
168 # organize the data in a useful manner(!!). 168 # organize the data in a useful manner(!!).
169 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', 169 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing',
170 'perf_testing')) 170 'perf_testing'))
171 shutil.rmtree(os.path.join('appengine', 'static', 'graphs'), 171 shutil.rmtree(os.path.join('appengine', 'static', 'graphs'),
172 ignore_errors=True) 172 ignore_errors=True)
173 shutil.copytree('graphs', os.path.join('appengine', 'static', 'graphs')) 173 shutil.copytree('graphs', os.path.join('appengine', 'static', 'graphs'))
174 shutil.copyfile('index.html', os.path.join('appengine', 'static', 174 shutil.copyfile('index.html', os.path.join('appengine', 'static',
175 'index.html')) 175 'index.html'))
176 RunCmd(['../../../third_party/appengine-python/1.5.4/appcfg.py', 'update', 176 run_cmd(['../../../third_party/appengine-python/1.5.4/appcfg.py', 'update',
177 'appengine/']) 177 'appengine/'])
178 178
179 class TestRunner(object): 179 class TestRunner(object):
180 """The base clas to provide shared code for different tests we will run and 180 """The base clas to provide shared code for different tests we will run and
181 graph.""" 181 graph."""
182 182
183 def __init__(self, result_folder_name, platform_list, v8_and_or_frog_list, 183 def __init__(self, result_folder_name, platform_list, v8_and_or_frog_list,
184 values_list): 184 values_list):
185 """Args: 185 """Args:
186 result_folder_name the name of the folder where a tracefile of 186 result_folder_name the name of the folder where a tracefile of
(...skipping 23 matching lines...) Expand all
210 for val in values_list: 210 for val in values_list:
211 self.revision_dict[platform][f][val] = [] 211 self.revision_dict[platform][f][val] = []
212 self.values_dict[platform][f][val] = [] 212 self.values_dict[platform][f][val] = []
213 if V8 in v8_and_or_frog_list: 213 if V8 in v8_and_or_frog_list:
214 self.revision_dict[platform][V8][V8_MEAN] = [] 214 self.revision_dict[platform][V8][V8_MEAN] = []
215 self.values_dict[platform][V8][V8_MEAN] = [] 215 self.values_dict[platform][V8][V8_MEAN] = []
216 if FROG in v8_and_or_frog_list: 216 if FROG in v8_and_or_frog_list:
217 self.revision_dict[platform][FROG][FROG_MEAN] = [] 217 self.revision_dict[platform][FROG][FROG_MEAN] = []
218 self.values_dict[platform][FROG][FROG_MEAN] = [] 218 self.values_dict[platform][FROG][FROG_MEAN] = []
219 219
220 def GetColor(self): 220 def get_color(self):
221 color = COLORS[self.color_index] 221 color = COLORS[self.color_index]
222 self.color_index = (self.color_index + 1) % len(COLORS) 222 self.color_index = (self.color_index + 1) % len(COLORS)
223 return color 223 return color
224 224
225 def StyleAndSavePerfPlot(self, chart_title, y_axis_label, size_x, size_y, 225 def syle_and_save_perf_plot(self, chart_title, y_axis_label, size_x, size_y,
226 legend_loc, filename, platform_list, v8_and_or_frog_list, values_list, 226 legend_loc, filename, platform_list, v8_and_or_frog_list, values_list,
227 should_clear_axes=True): 227 should_clear_axes=True):
228 """Sets style preferences for chart boilerplate that is consistent across 228 """Sets style preferences for chart boilerplate that is consistent across
229 all charts, and saves the chart as a png. 229 all charts, and saves the chart as a png.
230 Args: 230 Args:
231 size_x the size of the printed chart, in inches, in the horizontal 231 size_x the size of the printed chart, in inches, in the horizontal
232 direction 232 direction
233 size_y the size of the printed chart, in inches in the vertical direction 233 size_y the size of the printed chart, in inches in the vertical direction
234 legend_loc the location of the legend in on the chart. See suitable 234 legend_loc the location of the legend in on the chart. See suitable
235 arguments for the loc argument in matplotlib 235 arguments for the loc argument in matplotlib
236 filename the filename that we want to save the resulting chart as 236 filename the filename that we want to save the resulting chart as
237 platform_list a list containing the platform(s) that our data has been run 237 platform_list a list containing the platform(s) that our data has been run
238 on. (command line, firefox, chrome, etc) 238 on. (command line, firefox, chrome, etc)
239 values_list a list containing the type of data we will be graphing 239 values_list a list containing the type of data we will be graphing
240 (performance, percentage passing, etc) 240 (performance, percentage passing, etc)
241 should_clear_axes True if we want to create a fresh graph, instead of 241 should_clear_axes True if we want to create a fresh graph, instead of
242 plotting additional lines on the current graph.""" 242 plotting additional lines on the current graph."""
243 if should_clear_axes: 243 if should_clear_axes:
244 plt.cla() # cla = clear current axes 244 plt.cla() # cla = clear current axes
245 for platform in platform_list: 245 for platform in platform_list:
246 for f in v8_and_or_frog_list: 246 for f in v8_and_or_frog_list:
247 for val in values_list: 247 for val in values_list:
248 plt.plot(self.revision_dict[platform][f][val], 248 plt.plot(self.revision_dict[platform][f][val],
249 self.values_dict[platform][f][val], 249 self.values_dict[platform][f][val],
250 color=self.GetColor(), label='%s-%s-%s' % (platform, f, val)) 250 color=self.get_color(), label='%s-%s-%s' % (platform, f, val))
251 251
252 plt.xlabel('Revision Number') 252 plt.xlabel('Revision Number')
253 plt.ylabel(y_axis_label) 253 plt.ylabel(y_axis_label)
254 plt.title(chart_title) 254 plt.title(chart_title)
255 fontP = FontProperties() 255 fontP = FontProperties()
256 fontP.set_size('small') 256 fontP.set_size('small')
257 plt.legend(loc=legend_loc, prop = fontP) 257 plt.legend(loc=legend_loc, prop = fontP)
258 258
259 fig = plt.gcf() 259 fig = plt.gcf()
260 fig.set_size_inches(size_x, size_y) 260 fig.set_size_inches(size_x, size_y)
261 fig.savefig(os.path.join(GRAPH_OUT_DIR, filename)) 261 fig.savefig(os.path.join(GRAPH_OUT_DIR, filename))
262 262
263 def AddSvnRevisionToTrace(self, outfile): 263 def add_svn_revision_to_trace(self, outfile):
264 """Add the svn version number to the provided tracefile.""" 264 """Add the svn version number to the provided tracefile."""
265 p = subprocess.Popen(['svn', 'info'], stdout = subprocess.PIPE, 265 p = subprocess.Popen(['svn', 'info'], stdout = subprocess.PIPE,
266 stderr = subprocess.STDOUT, shell = HAS_SHELL) 266 stderr = subprocess.STDOUT, shell = HAS_SHELL)
267 output, not_used = p.communicate() 267 output, not_used = p.communicate()
268 for line in output.split('\n'): 268 for line in output.split('\n'):
269 if 'Revision' in line: 269 if 'Revision' in line:
270 RunCmd(['echo', line.strip()], outfile) 270 run_cmd(['echo', line.strip()], outfile)
271 271
272 def WriteHtml(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2, 272 def write_html(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2,
273 cleanFile=False): 273 cleanFile=False):
274 """Adds an html table to the webpage to display the data values. This method 274 """Adds an html table to the webpage to display the data values. This method
275 will be removed when we have a nicer way to display data values.""" 275 will be removed when we have a nicer way to display data values."""
276 #TODO(efortuna): fix this. 276 #TODO(efortuna): fix this.
277 return 277 return
278 #TODO(efortuna): Take this method out when have finalized where the data is 278 #TODO(efortuna): Take this method out when have finalized where the data is
279 # going to be displayed. 279 # going to be displayed.
280 f = '' 280 f = ''
281 out = '' 281 out = ''
282 if cleanFile: 282 if cleanFile:
(...skipping 13 matching lines...) Expand all
296 for revision in rev_nums: 296 for revision in rev_nums:
297 out.write('<td>%d</td>' % revision) 297 out.write('<td>%d</td>' % revision)
298 out.write('</tr>\n<tr><td> %s</td>' % label_1) 298 out.write('</tr>\n<tr><td> %s</td>' % label_1)
299 for perf in dict_1: 299 for perf in dict_1:
300 out.write('<td>%f</td>' % perf) 300 out.write('<td>%f</td>' % perf)
301 out.write('</tr>\n<tr><td> %s</td>' % label_2) 301 out.write('</tr>\n<tr><td> %s</td>' % label_2)
302 for perf in dict_2: 302 for perf in dict_2:
303 out.write('<td>%f</td>' % perf) 303 out.write('<td>%f</td>' % perf)
304 out.write('</tr> </table>') 304 out.write('</tr> </table>')
305 305
306 def CalculateGeometricMean(self, platform, frog_or_v8, svn_revision): 306 def calculate_geometric_mean(self, platform, frog_or_v8, svn_revision):
307 """Calculate the aggregate geometric mean for V8 and frog benchmark sets, 307 """Calculate the aggregate geometric mean for V8 and frog benchmark sets,
308 given two benchmark dictionaries.""" 308 given two benchmark dictionaries."""
309 geo_mean = 0 309 geo_mean = 0
310 for benchmark in GetBenchmarks(): 310 for benchmark in get_benchmarks():
311 geo_mean += math.log(self.values_dict[platform][frog_or_v8][benchmark][ 311 geo_mean += math.log(self.values_dict[platform][frog_or_v8][benchmark][
312 len(self.values_dict[platform][frog_or_v8][benchmark]) - 1]) 312 len(self.values_dict[platform][frog_or_v8][benchmark]) - 1])
313 313
314 mean = V8_MEAN 314 mean = V8_MEAN
315 if frog_or_v8 == FROG: 315 if frog_or_v8 == FROG:
316 mean = FROG_MEAN 316 mean = FROG_MEAN
317 self.values_dict[platform][frog_or_v8][mean] += \ 317 self.values_dict[platform][frog_or_v8][mean] += \
318 [math.pow(math.e, geo_mean / len(GetBenchmarks()))] 318 [math.pow(math.e, geo_mean / len(get_benchmarks()))]
319 self.revision_dict[platform][frog_or_v8][mean] += [svn_revision] 319 self.revision_dict[platform][frog_or_v8][mean] += [svn_revision]
320 320
321 def Run(self): 321 def run(self):
322 """Run the benchmarks/tests from the command line and plot the 322 """Run the benchmarks/tests from the command line and plot the
323 results.""" 323 results."""
324 if PERFBOT_MODE: 324 if PERFBOT_MODE:
325 plt.cla() # cla = clear current axes 325 plt.cla() # cla = clear current axes
326 os.chdir(DART_INSTALL_LOCATION) 326 os.chdir(DART_INSTALL_LOCATION)
327 EnsureOutputDirectory(self.result_folder_name) 327 ensure_output_directory(self.result_folder_name)
328 EnsureOutputDirectory(GRAPH_OUT_DIR) 328 ensure_output_directory(GRAPH_OUT_DIR)
329 self.RunTests() 329 self.run_tests()
330 os.chdir(os.path.join('tools', 'testing', 'perf_testing')) 330 os.chdir(os.path.join('tools', 'testing', 'perf_testing'))
331 331
332 # TODO(efortuna): You will want to make this only use a subset of the files 332 # TODO(efortuna): You will want to make this only use a subset of the files
333 # eventually. 333 # eventually.
334 files = os.listdir(self.result_folder_name) 334 files = os.listdir(self.result_folder_name)
335 335
336 for afile in files: 336 for afile in files:
337 if not afile.startswith('.'): 337 if not afile.startswith('.'):
338 self.ProcessFile(afile) 338 self.process_file(afile)
339 339
340 if PERFBOT_MODE: 340 if PERFBOT_MODE:
341 self.PlotResults('%s.png' % self.result_folder_name) 341 self.plot_results('%s.png' % self.result_folder_name)
342 342
343 class PerformanceTestRunner(TestRunner): 343 class PerformanceTestRunner(TestRunner):
344 """Super class for all performance testing.""" 344 """Super class for all performance testing."""
345 def __init__(self, result_folder_name, platform_list, platform_type): 345 def __init__(self, result_folder_name, platform_list, platform_type):
346 super(PerformanceTestRunner, self).__init__(result_folder_name, 346 super(PerformanceTestRunner, self).__init__(result_folder_name,
347 platform_list, GetVersions(), GetBenchmarks()) 347 platform_list, get_versions(), get_benchmarks())
348 self.platform_list = platform_list 348 self.platform_list = platform_list
349 self.platform_type = platform_type 349 self.platform_type = platform_type
350 350
351 def PlotAllPerf(self, png_filename): 351 def plot_all_perf(self, png_filename):
352 """Create a plot that shows the performance changes of individual benchmarks 352 """Create a plot that shows the performance changes of individual benchmarks
353 run by V8 and generated by frog, over svn history.""" 353 run by V8 and generated by frog, over svn history."""
354 for benchmark in GetBenchmarks(): 354 for benchmark in get_benchmarks():
355 self.StyleAndSavePerfPlot( 355 self.syle_and_save_perf_plot(
356 'Performance of %s over time on the %s' % (benchmark, 356 'Performance of %s over time on the %s' % (benchmark,
357 self.platform_type), 'Speed (bigger = better)', 16, 14, 'lower left', 357 self.platform_type), 'Speed (bigger = better)', 16, 14, 'lower left',
358 benchmark + png_filename, self.platform_list, GetVersions(), 358 benchmark + png_filename, self.platform_list, get_versions(),
359 [benchmark]) 359 [benchmark])
360 360
361 def PlotAvgPerf(self, png_filename): 361 def plot_avg_perf(self, png_filename):
362 """Generate a plot that shows the performance changes of the geomentric mean 362 """Generate a plot that shows the performance changes of the geomentric mean
363 of V8 and frog benchmark performance over svn history.""" 363 of V8 and frog benchmark performance over svn history."""
364 (title, y_axis, size_x, size_y, loc, filename) = \ 364 (title, y_axis, size_x, size_y, loc, filename) = \
365 ('Geometric Mean of benchmark %s performance' % self.platform_type, 365 ('Geometric Mean of benchmark %s performance' % self.platform_type,
366 'Speed (bigger = better)', 16, 5, 'center', 'avg'+png_filename) 366 'Speed (bigger = better)', 16, 5, 'center', 'avg'+png_filename)
367 for platform in self.platform_list: 367 for platform in self.platform_list:
368 self.StyleAndSavePerfPlot(title, y_axis, size_x, size_y, loc, filename, 368 self.syle_and_save_perf_plot(title, y_axis, size_x, size_y, loc, filename,
369 [platform], [V8], [V8_MEAN], True) 369 [platform], [V8], [V8_MEAN], True)
370 self.StyleAndSavePerfPlot(title, y_axis, size_x, size_y, loc, filename, 370 self.syle_and_save_perf_plot(title, y_axis, size_x, size_y, loc, filename,
371 [platform], [FROG], [FROG_MEAN], False) 371 [platform], [FROG], [FROG_MEAN], False)
372 self.WriteHtml('table', 372 self.write_html('table',
373 self.revision_dict[platform][V8], 373 self.revision_dict[platform][V8],
374 'V8 mean', self.values_dict[platform][V8][V8_MEAN], 374 'V8 mean', self.values_dict[platform][V8][V8_MEAN],
375 'Frog mean', self.values_dict[platform][FROG][FROG_MEAN], 375 'Frog mean', self.values_dict[platform][FROG][FROG_MEAN],
376 True) 376 True)
377 377
378 def PlotResults(self, png_filename): 378 def plot_results(self, png_filename):
379 self.PlotAllPerf(png_filename) 379 self.plot_all_perf(png_filename)
380 self.PlotAvgPerf('2' + png_filename) 380 self.plot_avg_perf('2' + png_filename)
381 381
382 382
383 class CommandLinePerformanceTestRunner(PerformanceTestRunner): 383 class CommandLinePerformanceTestRunner(PerformanceTestRunner):
384 """Run performance tests from the command line.""" 384 """Run performance tests from the command line."""
385 385
386 def __init__(self, result_folder_name): 386 def __init__(self, result_folder_name):
387 super(CommandLinePerformanceTestRunner, self).__init__(result_folder_name, 387 super(CommandLinePerformanceTestRunner, self).__init__(result_folder_name,
388 [COMMAND_LINE], 'command line') 388 [COMMAND_LINE], 'command line')
389 389
390 def ProcessFile(self, afile): 390 def process_file(self, afile):
391 """Pull all the relevant information out of a given tracefile. 391 """Pull all the relevant information out of a given tracefile.
392 392
393 Args: 393 Args:
394 afile is the filename string we will be processing.""" 394 afile is the filename string we will be processing."""
395 f = open(os.path.join(self.result_folder_name, afile)) 395 f = open(os.path.join(self.result_folder_name, afile))
396 tabulate_data = False 396 tabulate_data = False
397 revision_num = 0 397 revision_num = 0
398 for line in f.readlines(): 398 for line in f.readlines():
399 if 'Revision' in line: 399 if 'Revision' in line:
400 revision_num = int(line.split()[1]) 400 revision_num = int(line.split()[1])
401 elif 'Benchmark' in line: 401 elif 'Benchmark' in line:
402 tabulate_data = True 402 tabulate_data = True
403 elif tabulate_data: 403 elif tabulate_data:
404 tokens = line.split() 404 tokens = line.split()
405 if len(tokens) < 4 or tokens[0] not in GetBenchmarks(): 405 if len(tokens) < 4 or tokens[0] not in get_benchmarks():
406 #Done tabulating data. 406 #Done tabulating data.
407 break 407 break
408 v8_value = float(tokens[1]) 408 v8_value = float(tokens[1])
409 frog_value = float(tokens[3]) 409 frog_value = float(tokens[3])
410 if v8_value == 0 or frog_value == 0: 410 if v8_value == 0 or frog_value == 0:
411 #Then there was an error when this performance test was run. Do not 411 #Then there was an error when this performance test was run. Do not
412 #count it in our numbers. 412 #count it in our numbers.
413 return 413 return
414 benchmark = tokens[0] 414 benchmark = tokens[0]
415 self.revision_dict[COMMAND_LINE][V8][benchmark] += [revision_num] 415 self.revision_dict[COMMAND_LINE][V8][benchmark] += [revision_num]
416 self.values_dict[COMMAND_LINE][V8][benchmark] += [v8_value] 416 self.values_dict[COMMAND_LINE][V8][benchmark] += [v8_value]
417 self.revision_dict[COMMAND_LINE][FROG][benchmark] += [revision_num] 417 self.revision_dict[COMMAND_LINE][FROG][benchmark] += [revision_num]
418 self.values_dict[COMMAND_LINE][FROG][benchmark] += [frog_value] 418 self.values_dict[COMMAND_LINE][FROG][benchmark] += [frog_value]
419 f.close() 419 f.close()
420 420
421 self.CalculateGeometricMean(COMMAND_LINE, FROG, revision_num) 421 self.calculate_geometric_mean(COMMAND_LINE, FROG, revision_num)
422 self.CalculateGeometricMean(COMMAND_LINE, V8, revision_num) 422 self.calculate_geometric_mean(COMMAND_LINE, V8, revision_num)
423 423
424 def RunTests(self): 424 def run_tests(self):
425 """Run a performance test on our updated system.""" 425 """Run a performance test on our updated system."""
426 os.chdir('frog') 426 os.chdir('frog')
427 self.trace_file = os.path.join('..', 'tools', 'testing', 'perf_testing', 427 self.trace_file = os.path.join('..', 'tools', 'testing', 'perf_testing',
428 self.result_folder_name, 'result' + self.cur_time) 428 self.result_folder_name, 'result' + self.cur_time)
429 RunCmd(['python', os.path.join('benchmarks', 'perf_tests.py')], 429 run_cmd(['python', os.path.join('benchmarks', 'perf_tests.py')],
430 self.trace_file) 430 self.trace_file)
431 os.chdir('..') 431 os.chdir('..')
432 432
433 433
434 class BrowserPerformanceTestRunner(PerformanceTestRunner): 434 class BrowserPerformanceTestRunner(PerformanceTestRunner):
435 """Runs performance tests, in the browser.""" 435 """Runs performance tests, in the browser."""
436 436
437 def __init__(self, result_folder_name): 437 def __init__(self, result_folder_name):
438 super(BrowserPerformanceTestRunner, self).__init__( 438 super(BrowserPerformanceTestRunner, self).__init__(
439 result_folder_name, GetBrowsers(), 'browser') 439 result_folder_name, get_browsers(), 'browser')
440 440
441 def RunTests(self): 441 def run_tests(self):
442 """Run a performance test in the browser.""" 442 """Run a performance test in the browser."""
443 # For the smoke test, just run a simple test, not the actual benchmarks to 443 # For the smoke test, just run a simple test, not the actual benchmarks to
444 # ensure we haven't broken the Firefox DOM. 444 # ensure we haven't broken the Firefox DOM.
445 445
446 os.chdir('frog') 446 os.chdir('frog')
447 if PERFBOT_MODE: 447 if PERFBOT_MODE:
448 RunCmd(['python', os.path.join('benchmarks', 'make_web_benchmarks.py')]) 448 run_cmd(['python', os.path.join('benchmarks', 'make_web_benchmarks.py')])
449 else: 449 else:
450 RunCmd(['./minfrog', '--out=../tools/testing/perf_testing/smoketest/' + \ 450 run_cmd(['./minfrog', '--out=../tools/testing/perf_testing/smoketest/' + \
451 'smoketest_frog.js', '--libdir=%s/lib' % os.getcwd(), 451 'smoketest_frog.js', '--libdir=%s/lib' % os.getcwd(),
452 '--compile-only', '../tools/testing/perf_testing/smoketest/' + \ 452 '--compile-only', '../tools/testing/perf_testing/smoketest/' + \
453 'dartWebBase.dart']) 453 'dartWebBase.dart'])
454 os.chdir('..') 454 os.chdir('..')
455 455
456 for browser in GetBrowsers(): 456 for browser in get_browsers():
457 for version in GetVersions(): 457 for version in get_versions():
458 self.trace_file = os.path.join('tools', 'testing', 'perf_testing', 458 self.trace_file = os.path.join('tools', 'testing', 'perf_testing',
459 self.result_folder_name, 459 self.result_folder_name,
460 'perf-%s-%s-%s' % (self.cur_time, browser, version)) 460 'perf-%s-%s-%s' % (self.cur_time, browser, version))
461 self.AddSvnRevisionToTrace(self.trace_file) 461 self.add_svn_revision_to_trace(self.trace_file)
462 file_path = os.path.join(os.getcwd(), 'internal', 'browserBenchmarks', 462 file_path = os.path.join(os.getcwd(), 'internal', 'browserBenchmarks',
463 'benchmark_page_%s.html' % version) 463 'benchmark_page_%s.html' % version)
464 if not PERFBOT_MODE: 464 if not PERFBOT_MODE:
465 file_path = os.path.join(os.getcwd(), 'tools', 'testing', 465 file_path = os.path.join(os.getcwd(), 'tools', 'testing',
466 'perf_testing', 'smoketest', 'smoketest_%s.html' % version) 466 'perf_testing', 'smoketest', 'smoketest_%s.html' % version)
467 RunCmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'), 467 run_cmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'),
468 '--out', file_path, '--browser', browser, 468 '--out', file_path, '--browser', browser,
469 '--timeout', '600', '--perf'], self.trace_file, append=True) 469 '--timeout', '600', '--perf'], self.trace_file, append=True)
470 470
471 def ProcessFile(self, afile): 471 def process_file(self, afile):
472 """Comb through the html to find the performance results.""" 472 """Comb through the html to find the performance results."""
473 parts = afile.split('-') 473 parts = afile.split('-')
474 browser = parts[2] 474 browser = parts[2]
475 version = parts[3] 475 version = parts[3]
476 f = open(os.path.join(self.result_folder_name, afile)) 476 f = open(os.path.join(self.result_folder_name, afile))
477 lines = f.readlines() 477 lines = f.readlines()
478 line = '' 478 line = ''
479 i = 0 479 i = 0
480 revision_num = 0 480 revision_num = 0
481 while '<div id="results">' not in line and i < len(lines): 481 while '<div id="results">' not in line and i < len(lines):
(...skipping 28 matching lines...) Expand all
510 else: 510 else:
511 bench_dict = self.values_dict[browser][FROG] 511 bench_dict = self.values_dict[browser][FROG]
512 bench_dict[name] += [float(score)] 512 bench_dict[name] += [float(score)]
513 self.revision_dict[browser][version][name] += [revision_num] 513 self.revision_dict[browser][version][name] += [revision_num]
514 514
515 f.close() 515 f.close()
516 if not PERFBOT_MODE: 516 if not PERFBOT_MODE:
517 print 'PASS' 517 print 'PASS'
518 os.remove(os.path.join(self.result_folder_name, afile)) 518 os.remove(os.path.join(self.result_folder_name, afile))
519 else: 519 else:
520 self.CalculateGeometricMean(browser, version, revision_num) 520 self.calculate_geometric_mean(browser, version, revision_num)
521 521
522 def WriteHtml(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2, 522 def write_html(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2,
523 cleanFile=False): 523 cleanFile=False):
524 #TODO(efortuna) 524 #TODO(efortuna)
525 pass 525 pass
526 526
527 527
528 class BrowserCorrectnessTestRunner(TestRunner): 528 class BrowserCorrectnessTestRunner(TestRunner):
529 def __init__(self, test_type, result_folder_name): 529 def __init__(self, test_type, result_folder_name):
530 super(BrowserCorrectnessTestRunner, self).__init__(result_folder_name, 530 super(BrowserCorrectnessTestRunner, self).__init__(result_folder_name,
531 GetBrowsers(), [FROG], [CORRECTNESS]) 531 get_browsers(), [FROG], [CORRECTNESS])
532 self.test_type = test_type 532 self.test_type = test_type
533 533
534 def RunTests(self): 534 def run_tests(self):
535 """Run a test of the latest svn revision.""" 535 """run a test of the latest svn revision."""
536 for browser in GetBrowsers(): 536 for browser in get_browsers():
537 current_file = 'correctness%s-%s' % (self.cur_time, browser) 537 current_file = 'correctness%s-%s' % (self.cur_time, browser)
538 self.trace_file = os.path.join('tools', 'testing', 538 self.trace_file = os.path.join('tools', 'testing',
539 'perf_testing', self.result_folder_name, current_file) 539 'perf_testing', self.result_folder_name, current_file)
540 self.AddSvnRevisionToTrace(self.trace_file) 540 self.add_svn_revision_to_trace(self.trace_file)
541 dart_sdk = os.path.join(os.getcwd(), utils.GetBuildRoot(utils.GuessOS(), 541 dart_sdk = os.path.join(os.getcwd(), utils.GetBuildRoot(utils.GuessOS(),
542 'release', 'ia32'), 'dart-sdk') 542 'release', 'ia32'), 'dart-sdk')
543 RunCmd([os.path.join('.', 'tools', 'test.py'), 543 run_cmd([os.path.join('.', 'tools', 'test.py'),
544 '--component=webdriver', '--flag=%s' % browser, '--flag=--frog=%s' % \ 544 '--component=webdriver', '--flag=%s' % browser, '--flag=--frog=%s' % \
545 os.path.join(dart_sdk, 'bin', 'frogc'), 545 os.path.join(dart_sdk, 'bin', 'frogc'),
546 '--flag=--froglib=%s' % os.path.join(dart_sdk, 'lib'), 546 '--flag=--froglib=%s' % os.path.join(dart_sdk, 'lib'),
547 '--timeout=20', '--progress=color', '--mode=release', '-j1', 547 '--timeout=20', '--progress=color', '--mode=release', '-j1',
548 self.test_type], self.trace_file, append=True) 548 self.test_type], self.trace_file, append=True)
549 549
550 def ProcessFile(self, afile): 550 def process_file(self, afile):
551 """Given a trace file, extract all the relevant information out of it to 551 """Given a trace file, extract all the relevant information out of it to
552 determine the number of correctly passing tests. 552 determine the number of correctly passing tests.
553 553
554 Arguments: 554 Arguments:
555 afile the filename string""" 555 afile the filename string"""
556 browser = afile.rpartition('-')[2] 556 browser = afile.rpartition('-')[2]
557 f = open(os.path.join(self.result_folder_name, afile)) 557 f = open(os.path.join(self.result_folder_name, afile))
558 revision_num = 0 558 revision_num = 0
559 lines = f.readlines() 559 lines = f.readlines()
560 total_tests = 0 560 total_tests = 0
(...skipping 11 matching lines...) Expand all
572 if '--- TIMEOUT ---' in line or 'FAIL:' in line or 'PASS' in line: 572 if '--- TIMEOUT ---' in line or 'FAIL:' in line or 'PASS' in line:
573 # (A printed out 'PASS' indicates we incorrectly passed a negative 573 # (A printed out 'PASS' indicates we incorrectly passed a negative
574 # test.) 574 # test.)
575 num_failed += 1 575 num_failed += 1
576 576
577 self.revision_dict[browser][FROG][CORRECTNESS] += [revision_num] 577 self.revision_dict[browser][FROG][CORRECTNESS] += [revision_num]
578 self.values_dict[browser][FROG][CORRECTNESS] += [100.0 * 578 self.values_dict[browser][FROG][CORRECTNESS] += [100.0 *
579 (((float)(total_tests - (expect_fail + num_failed))) /total_tests)] 579 (((float)(total_tests - (expect_fail + num_failed))) /total_tests)]
580 f.close() 580 f.close()
581 581
582 def PlotResults(self, png_filename): 582 def plot_results(self, png_filename):
583 first_time = True 583 first_time = True
584 for browser in GetBrowsers(): 584 for browser in get_browsers():
585 self.StyleAndSavePerfPlot('Percentage of language tests passing in ' 585 self.syle_and_save_perf_plot('Percentage of language tests passing in '
586 'different browsers', '% of tests passed', 8, 8, 'lower left', 586 'different browsers', '% of tests passed', 8, 8, 'lower left',
587 png_filename, [browser], [FROG], [CORRECTNESS], first_time) 587 png_filename, [browser], [FROG], [CORRECTNESS], first_time)
588 first_time = False 588 first_time = False
589 589
590 590
591 class CompileTimeAndSizeTestRunner(TestRunner): 591 class CompileTimeAndSizeTestRunner(TestRunner):
592 """Run tests to determine how long minfrog takes to compile, and the compiled 592 """Run tests to determine how long minfrog takes to compile, and the compiled
593 file output size of some benchmarking files.""" 593 file output size of some benchmarking files."""
594 def __init__(self, result_folder_name): 594 def __init__(self, result_folder_name):
595 super(CompileTimeAndSizeTestRunner, self).__init__(result_folder_name, 595 super(CompileTimeAndSizeTestRunner, self).__init__(result_folder_name,
596 [COMMAND_LINE], [FROG], ['Compiling on Dart VM', 'Bootstrapping', 596 [COMMAND_LINE], [FROG], ['Compiling on Dart VM', 'Bootstrapping',
597 'minfrog', 'swarm', 'total']) 597 'minfrog', 'swarm', 'total'])
598 self.failure_threshold = {'Compiling on Dart VM' : 1, 'Bootstrapping' : .5, 598 self.failure_threshold = {'Compiling on Dart VM' : 1, 'Bootstrapping' : .5,
599 'minfrog' : 100, 'swarm' : 100, 'total' : 100} 599 'minfrog' : 100, 'swarm' : 100, 'total' : 100}
600 600
601 def RunTests(self): 601 def run_tests(self):
602 os.chdir('frog') 602 os.chdir('frog')
603 self.trace_file = os.path.join('..', 'tools', 'testing', 'perf_testing', 603 self.trace_file = os.path.join('..', 'tools', 'testing', 'perf_testing',
604 self.result_folder_name, self.result_folder_name + self.cur_time) 604 self.result_folder_name, self.result_folder_name + self.cur_time)
605 605
606 self.AddSvnRevisionToTrace(self.trace_file) 606 self.add_svn_revision_to_trace(self.trace_file)
607 607
608 elapsed = TimeCmd([os.path.join('.', 'frog.py'), '--', 608 elapsed = time_cmd([os.path.join('.', 'frog.py'), '--',
609 '--out=minfrog', 'minfrog.dart']) 609 '--out=minfrog', 'minfrog.dart'])
610 RunCmd(['echo', '%f Compiling on Dart VM in production mode in seconds' 610 run_cmd(['echo', '%f Compiling on Dart VM in production mode in seconds'
611 % elapsed], self.trace_file, append=True) 611 % elapsed], self.trace_file, append=True)
612 elapsed = TimeCmd([os.path.join('.', 'minfrog'), '--out=minfrog', 612 elapsed = time_cmd([os.path.join('.', 'minfrog'), '--out=minfrog',
613 'minfrog.dart', os.path.join('tests', 'hello.dart')]) 613 'minfrog.dart', os.path.join('tests', 'hello.dart')])
614 if elapsed < self.failure_threshold['Bootstrapping']: 614 if elapsed < self.failure_threshold['Bootstrapping']:
615 #minfrog didn't compile correctly. Stop testing now, because subsequent 615 #minfrog didn't compile correctly. Stop testing now, because subsequent
616 #numbers will be meaningless. 616 #numbers will be meaningless.
617 return 617 return
618 size = os.path.getsize('minfrog') 618 size = os.path.getsize('minfrog')
619 RunCmd(['echo', '%f Bootstrapping time in seconds in production mode' % 619 run_cmd(['echo', '%f Bootstrapping time in seconds in production mode' %
620 elapsed], self.trace_file, append=True) 620 elapsed], self.trace_file, append=True)
621 RunCmd(['echo', '%d Generated checked minfrog size' % size], 621 run_cmd(['echo', '%d Generated checked minfrog size' % size],
622 self.trace_file, append=True) 622 self.trace_file, append=True)
623 623
624 RunCmd([os.path.join('.', 'minfrog'), ' --out=swarm-result ', 624 run_cmd([os.path.join('.', 'minfrog'), ' --out=swarm-result ',
625 '--compile-only', os.path.join('..', 'client', 'samples', 'swarm', 625 '--compile-only', os.path.join('..', 'client', 'samples', 'swarm',
626 'swarm.dart')]) 626 'swarm.dart')])
627 swarm_size = 0 627 swarm_size = 0
628 try: 628 try:
629 swarm_size = os.path.getsize('swarm-result') 629 swarm_size = os.path.getsize('swarm-result')
630 except OSError: 630 except OSError:
631 pass #If compilation failed, continue on running other tests. 631 pass #If compilation failed, continue on running other tests.
632 632
633 RunCmd([os.path.join('.', 'minfrog'), '--out=total-result', 633 run_cmd([os.path.join('.', 'minfrog'), '--out=total-result',
634 '--compile-only', os.path.join('..', 'client', 'samples', 'total', 634 '--compile-only', os.path.join('..', 'client', 'samples', 'total',
635 'src', 'Total.dart')]) 635 'src', 'Total.dart')])
636 total_size = 0 636 total_size = 0
637 try: 637 try:
638 total_size = os.path.getsize('total-result') 638 total_size = os.path.getsize('total-result')
639 except OSError: 639 except OSError:
640 pass #If compilation failed, continue on running other tests. 640 pass #If compilation failed, continue on running other tests.
641 641
642 RunCmd(['echo', '%d Generated checked swarm size' % swarm_size], 642 run_cmd(['echo', '%d Generated checked swarm size' % swarm_size],
643 self.trace_file, append=True) 643 self.trace_file, append=True)
644 644
645 RunCmd(['echo', '%d Generated checked total size' % total_size], 645 run_cmd(['echo', '%d Generated checked total size' % total_size],
646 self.trace_file, append=True) 646 self.trace_file, append=True)
647 os.chdir('..') 647 os.chdir('..')
648 648
649 def ProcessFile(self, afile): 649 def process_file(self, afile):
650 """Pull all the relevant information out of a given tracefile. 650 """Pull all the relevant information out of a given tracefile.
651 Args: 651 Args:
652 afile is the filename string we will be processing.""" 652 afile is the filename string we will be processing."""
653 f = open(os.path.join(self.result_folder_name, afile)) 653 f = open(os.path.join(self.result_folder_name, afile))
654 tabulate_data = False 654 tabulate_data = False
655 revision_num = 0 655 revision_num = 0
656 for line in f.readlines(): 656 for line in f.readlines():
657 tokens = line.split() 657 tokens = line.split()
658 if 'Revision' in line: 658 if 'Revision' in line:
659 revision_num = int(line.split()[1]) 659 revision_num = int(line.split()[1])
(...skipping 13 matching lines...) Expand all
673 self.revision_dict[COMMAND_LINE][FROG][metric].pop() 673 self.revision_dict[COMMAND_LINE][FROG][metric].pop()
674 self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num] 674 self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num]
675 # Fill 0 if compilation failed. 675 # Fill 0 if compilation failed.
676 if self.values_dict[COMMAND_LINE][FROG][metric][-1] < \ 676 if self.values_dict[COMMAND_LINE][FROG][metric][-1] < \
677 self.failure_threshold[metric]: 677 self.failure_threshold[metric]:
678 self.values_dict[COMMAND_LINE][FROG][metric] += [0] 678 self.values_dict[COMMAND_LINE][FROG][metric] += [0]
679 self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num] 679 self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num]
680 680
681 f.close() 681 f.close()
682 682
683 def PlotResults(self, png_filename): 683 def plot_results(self, png_filename):
684 self.StyleAndSavePerfPlot('Compiled minfrog Sizes', 684 self.syle_and_save_perf_plot('Compiled minfrog Sizes',
685 'Size (in bytes)', 10, 10, 'center', png_filename, [COMMAND_LINE], 685 'Size (in bytes)', 10, 10, 'center', png_filename, [COMMAND_LINE],
686 [FROG], ['swarm', 'total', 'minfrog']) 686 [FROG], ['swarm', 'total', 'minfrog'])
687 self.WriteHtml('bar', self.revision_dict[COMMAND_LINE][FROG]['minfrog'], 687 self.write_html('bar', self.revision_dict[COMMAND_LINE][FROG]['minfrog'],
688 'minfrog size', self.values_dict[COMMAND_LINE][FROG]['minfrog'], '', []) 688 'minfrog size', self.values_dict[COMMAND_LINE][FROG]['minfrog'], '', [])
689 689
690 self.StyleAndSavePerfPlot('Time to compile and bootstrap', 690 self.syle_and_save_perf_plot('Time to compile and bootstrap',
691 'Seconds', 10, 10, 'center', '2' + png_filename, [COMMAND_LINE], [FROG], 691 'Seconds', 10, 10, 'center', '2' + png_filename, [COMMAND_LINE], [FROG],
692 ['Bootstrapping', 'Compiling on Dart VM']) 692 ['Bootstrapping', 'Compiling on Dart VM'])
693 self.WriteHtml('baz', 693 self.write_html('baz',
694 self.revision_dict[COMMAND_LINE][FROG]['Bootstrapping'], 694 self.revision_dict[COMMAND_LINE][FROG]['Bootstrapping'],
695 'Bootstrapping', self.values_dict[COMMAND_LINE][FROG]['Bootstrapping'], 695 'Bootstrapping', self.values_dict[COMMAND_LINE][FROG]['Bootstrapping'],
696 'Compiling on Dart VM', 696 'Compiling on Dart VM',
697 self.values_dict[COMMAND_LINE][FROG]['Compiling on Dart VM']) 697 self.values_dict[COMMAND_LINE][FROG]['Compiling on Dart VM'])
698 698
699 699
700 def ParseArgs(): 700 def parse_args():
701 parser = optparse.OptionParser() 701 parser = optparse.OptionParser()
702 parser.add_option('--command-line', '-c', dest='cl', 702 parser.add_option('--command-line', '-c', dest='cl',
703 help = 'Run the command line tests', 703 help = 'Run the command line tests',
704 action = 'store_true', default = False) 704 action = 'store_true', default = False)
705 parser.add_option('--size-time', '-s', dest = 'size', 705 parser.add_option('--size-time', '-s', dest = 'size',
706 help = 'Run the code size and timing tests', 706 help = 'Run the code size and timing tests',
707 action = 'store_true', default = False) 707 action = 'store_true', default = False)
708 parser.add_option('--language', '-l', dest = 'language', 708 parser.add_option('--language', '-l', dest = 'language',
709 help = 'Run the language correctness tests', 709 help = 'Run the language correctness tests',
710 action = 'store_true', default = False) 710 action = 'store_true', default = False)
711 parser.add_option('--browser-perf', '-b', dest = 'perf', 711 parser.add_option('--browser-perf', '-b', dest = 'perf',
712 help = 'Run the browser performance tests', 712 help = 'Run the browser performance tests',
713 action = 'store_true', default = False) 713 action = 'store_true', default = False)
714 parser.add_option('--forever', '-f', dest = 'continuous', 714 parser.add_option('--forever', '-f', dest = 'continuous',
715 help = 'Run this script forever, always checking for the next svn ' 715 help = 'Run this script forever, always checking for the next svn '
716 'checkin', action = 'store_true', default = False) 716 'checkin', action = 'store_true', default = False)
717 parser.add_option('--perfbot', '-p', dest = 'perfbot', 717 parser.add_option('--perfbot', '-p', dest = 'perfbot',
718 help = "Run in perfbot mode. (Generate plots, and keep trace files)", 718 help = "Run in perfbot mode. (Generate plots, and keep trace files)",
719 action = 'store_true', default = False) 719 action = 'store_true', default = False)
720 parser.add_option('--verbose', '-v', dest = 'verbose', 720 parser.add_option('--verbose', '-v', dest = 'verbose',
721 help = 'Print extra debug output', action = 'store_true', default = False) 721 help = 'Print extra debug output', action = 'store_true', default = False)
722 722
723 args, ignored = parser.parse_args() 723 args, ignored = parser.parse_args()
724 if not (args.cl or args.size or args.language or args.perf): 724 if not (args.cl or args.size or args.language or args.perf):
725 args.cl = args.size = args.language = args.perf = True 725 args.cl = args.size = args.language = args.perf = True
726 return (args.cl, args.size, args.language, args.perf, args.continuous, 726 return (args.cl, args.size, args.language, args.perf, args.continuous,
727 args.perfbot, args.verbose) 727 args.perfbot, args.verbose)
728 728
729 def RunTestSequence(cl, size, language, perf): 729 def run_test_sequence(cl, size, language, perf):
730 if PERFBOT_MODE: 730 if PERFBOT_MODE:
731 # The buildbot already builds and syncs to a specific revision. Don't fight 731 # The buildbot already builds and syncs to a specific revision. Don't fight
732 # with it or replicate work. 732 # with it or replicate work.
733 if SyncAndBuild() == 1: 733 if sync_and_build() == 1:
734 return # The build is broken. 734 return # The build is broken.
735 if cl: 735 if cl:
736 CommandLinePerformanceTestRunner('cl-results').Run() 736 CommandLinePerformanceTestRunner('cl-results').run()
737 if size: 737 if size:
738 CompileTimeAndSizeTestRunner('code-time-size').Run() 738 CompileTimeAndSizeTestRunner('code-time-size').run()
739 if language: 739 if language:
740 BrowserCorrectnessTestRunner('language', 'browser-correctness').Run() 740 BrowserCorrectnessTestRunner('language', 'browser-correctness').run()
741 if perf: 741 if perf:
742 BrowserPerformanceTestRunner('browser-perf').Run() 742 BrowserPerformanceTestRunner('browser-perf').run()
743 743
744 if PERFBOT_MODE: 744 if PERFBOT_MODE:
745 UploadToAppEngine() 745 upload_to_app_engine()
746 746
747 def main(): 747 def main():
748 global PERFBOT_MODE, VERBOSE 748 global PERFBOT_MODE, VERBOSE
749 (cl, size, language, perf, continuous, perfbot, verbose) = ParseArgs() 749 (cl, size, language, perf, continuous, perfbot, verbose) = parse_args()
750 PERFBOT_MODE = perfbot 750 PERFBOT_MODE = perfbot
751 VERBOSE = verbose 751 VERBOSE = verbose
752 if continuous: 752 if continuous:
753 while True: 753 while True:
754 if HasNewCode(): 754 if has_new_code():
755 RunTestSequence(cl, size, language, perf) 755 run_test_sequence(cl, size, language, perf)
756 else: 756 else:
757 time.sleep(SLEEP_TIME) 757 time.sleep(SLEEP_TIME)
758 else: 758 else:
759 RunTestSequence(cl, size, language, perf) 759 run_test_sequence(cl, size, language, perf)
760 760
761 if __name__ == '__main__': 761 if __name__ == '__main__':
762 main() 762 main()
763 763
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698