OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Run Performance Test Bisect Tool | 6 """Run Performance Test Bisect Tool |
7 | 7 |
8 This script is used by a trybot to run the src/tools/bisect-perf-regression.py | 8 This script is used by a try bot to run the src/tools/bisect-perf-regression.py |
9 script with the parameters specified in run-bisect-perf-regression.cfg. It will | 9 script with the parameters specified in run-bisect-perf-regression.cfg. It will |
10 check out a copy of the depot in a subdirectory 'bisect' of the working | 10 check out a copy of the depot in a subdirectory 'bisect' of the working |
11 directory provided, and run the bisect-perf-regression.py script there. | 11 directory provided, and run the bisect-perf-regression.py script there. |
12 | 12 |
13 """ | 13 """ |
14 | 14 |
15 import imp | 15 import imp |
16 import optparse | 16 import optparse |
17 import os | 17 import os |
18 import platform | 18 import platform |
(...skipping 194 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
213 opts_dict['build_preference'] = 'ninja' | 213 opts_dict['build_preference'] = 'ninja' |
214 opts_dict['output_buildbot_annotations'] = True | 214 opts_dict['output_buildbot_annotations'] = True |
215 | 215 |
216 if '--browser=cros' in config['command']: | 216 if '--browser=cros' in config['command']: |
217 opts_dict['target_platform'] = 'cros' | 217 opts_dict['target_platform'] = 'cros' |
218 | 218 |
219 if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]: | 219 if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]: |
220 opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV] | 220 opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV] |
221 opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV] | 221 opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV] |
222 else: | 222 else: |
223 raise RuntimeError('Cros build selected, but BISECT_CROS_IP or' | 223 raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or' |
224 'BISECT_CROS_BOARD undefined.') | 224 'BISECT_CROS_BOARD undefined.') |
225 elif 'android' in config['command']: | 225 elif 'android' in config['command']: |
226 if 'android-chrome-shell' in config['command']: | 226 if 'android-chrome-shell' in config['command']: |
227 opts_dict['target_platform'] = 'android' | 227 opts_dict['target_platform'] = 'android' |
228 elif 'android-chrome' in config['command']: | 228 elif 'android-chrome' in config['command']: |
229 opts_dict['target_platform'] = 'android-chrome' | 229 opts_dict['target_platform'] = 'android-chrome' |
230 else: | 230 else: |
231 opts_dict['target_platform'] = 'android' | 231 opts_dict['target_platform'] = 'android' |
232 | 232 |
233 return bisect.BisectOptions.FromDict(opts_dict) | 233 return bisect.BisectOptions.FromDict(opts_dict) |
(...skipping 30 matching lines...) Expand all Loading... |
264 opts.command, opts.metric, reset_on_first_run=True, results_label='Patch') | 264 opts.command, opts.metric, reset_on_first_run=True, results_label='Patch') |
265 | 265 |
266 if results_with_patch[1]: | 266 if results_with_patch[1]: |
267 raise RuntimeError('Patched version failed to run performance test.') | 267 raise RuntimeError('Patched version failed to run performance test.') |
268 | 268 |
269 bisect_utils.OutputAnnotationStepClosed() | 269 bisect_utils.OutputAnnotationStepClosed() |
270 | 270 |
271 bisect_utils.OutputAnnotationStepStart('Reverting Patch') | 271 bisect_utils.OutputAnnotationStepStart('Reverting Patch') |
272 # TODO: When this is re-written to recipes, this should use bot_update's | 272 # TODO: When this is re-written to recipes, this should use bot_update's |
273 # revert mechanism to fully revert the client. But for now, since we know that | 273 # revert mechanism to fully revert the client. But for now, since we know that |
274 # the perf trybot currently only supports src/ and src/third_party/WebKit, we | 274 # the perf try bot currently only supports src/ and src/third_party/WebKit, we |
275 # simply reset those two directories. | 275 # simply reset those two directories. |
276 bisect_utils.CheckRunGit(['reset', '--hard']) | 276 bisect_utils.CheckRunGit(['reset', '--hard']) |
277 bisect_utils.CheckRunGit(['reset', '--hard'], | 277 bisect_utils.CheckRunGit(['reset', '--hard'], |
278 os.path.join('third_party', 'WebKit')) | 278 os.path.join('third_party', 'WebKit')) |
279 bisect_utils.OutputAnnotationStepClosed() | 279 bisect_utils.OutputAnnotationStepClosed() |
280 | 280 |
281 bisect_utils.OutputAnnotationStepStart('Building Without Patch') | 281 bisect_utils.OutputAnnotationStepStart('Building Without Patch') |
282 | 282 |
283 if bisect_utils.RunGClient(['runhooks']): | 283 if bisect_utils.RunGClient(['runhooks']): |
284 raise RuntimeError('Failed to run gclient runhooks') | 284 raise RuntimeError('Failed to run gclient runhooks') |
285 | 285 |
286 if not b.BuildCurrentRevision('chromium'): | 286 if not b.BuildCurrentRevision('chromium'): |
287 raise RuntimeError('Unpatched version failed to build.') | 287 raise RuntimeError('Unpatched version failed to build.') |
288 | 288 |
289 bisect_utils.OutputAnnotationStepClosed() | 289 bisect_utils.OutputAnnotationStepClosed() |
290 bisect_utils.OutputAnnotationStepStart('Running Without Patch') | 290 bisect_utils.OutputAnnotationStepStart('Running Without Patch') |
291 | 291 |
292 results_without_patch = b.RunPerformanceTestAndParseResults( | 292 results_without_patch = b.RunPerformanceTestAndParseResults( |
293 opts.command, opts.metric, upload_on_last_run=True, results_label='ToT') | 293 opts.command, opts.metric, upload_on_last_run=True, results_label='ToT') |
294 | 294 |
295 if results_without_patch[1]: | 295 if results_without_patch[1]: |
296 raise RuntimeError('Unpatched version failed to run performance test.') | 296 raise RuntimeError('Unpatched version failed to run performance test.') |
297 | 297 |
298 # Find the link to the cloud stored results file. | 298 # Find the link to the cloud stored results file. |
299 output = results_without_patch[2] | 299 output = results_without_patch[2] |
300 cloud_file_link = [t for t in output.splitlines() | 300 cloud_file_link = [t for t in output.splitlines() |
301 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t] | 301 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t] |
302 if cloud_file_link: | 302 if cloud_file_link: |
303 # What we're getting here is basically "View online at http://..." so parse | 303 # What we're getting here is basically "View online at http://..." so parse |
304 # out just the url portion. | 304 # out just the URL portion. |
305 cloud_file_link = cloud_file_link[0] | 305 cloud_file_link = cloud_file_link[0] |
306 cloud_file_link = [t for t in cloud_file_link.split(' ') | 306 cloud_file_link = [t for t in cloud_file_link.split(' ') |
307 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t] | 307 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t] |
308 assert cloud_file_link, "Couldn't parse url from output." | 308 assert cloud_file_link, 'Couldn\'t parse URL from output.' |
309 cloud_file_link = cloud_file_link[0] | 309 cloud_file_link = cloud_file_link[0] |
310 else: | 310 else: |
311 cloud_file_link = '' | 311 cloud_file_link = '' |
312 | 312 |
313 # Calculate the % difference in the means of the 2 runs. | 313 # Calculate the % difference in the means of the 2 runs. |
314 percent_diff_in_means = None | 314 percent_diff_in_means = None |
315 std_err = None | 315 std_err = None |
316 if (results_with_patch[0].has_key('mean') and | 316 if (results_with_patch[0].has_key('mean') and |
317 results_with_patch[0].has_key('values')): | 317 results_with_patch[0].has_key('values')): |
318 percent_diff_in_means = (results_with_patch[0]['mean'] / | 318 percent_diff_in_means = (results_with_patch[0]['mean'] / |
(...skipping 116 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
435 if config.get('gs_bucket') and platform.release() == 'XP': | 435 if config.get('gs_bucket') and platform.release() == 'XP': |
436 print ('Goma doesn\'t have a win32 binary, therefore it is not supported ' | 436 print ('Goma doesn\'t have a win32 binary, therefore it is not supported ' |
437 'on Windows XP platform. Please refer to crbug.com/330900.') | 437 'on Windows XP platform. Please refer to crbug.com/330900.') |
438 path_to_goma = None | 438 path_to_goma = None |
439 cmd.append('--use_goma') | 439 cmd.append('--use_goma') |
440 | 440 |
441 if path_to_extra_src: | 441 if path_to_extra_src: |
442 cmd.extend(['--extra_src', path_to_extra_src]) | 442 cmd.extend(['--extra_src', path_to_extra_src]) |
443 | 443 |
444 # These flags are used to download build archives from cloud storage if | 444 # These flags are used to download build archives from cloud storage if |
445 # available, otherwise will post a try_job_http request to build it on | 445 # available, otherwise will post a try_job_http request to build it on the |
446 # tryserver. | 446 # try server. |
447 if config.get('gs_bucket'): | 447 if config.get('gs_bucket'): |
448 if config.get('builder_host') and config.get('builder_port'): | 448 if config.get('builder_host') and config.get('builder_port'): |
449 cmd.extend(['--gs_bucket', config['gs_bucket'], | 449 cmd.extend(['--gs_bucket', config['gs_bucket'], |
450 '--builder_host', config['builder_host'], | 450 '--builder_host', config['builder_host'], |
451 '--builder_port', config['builder_port'] | 451 '--builder_port', config['builder_port'] |
452 ]) | 452 ]) |
453 else: | 453 else: |
454 print ('Error: Specified gs_bucket, but missing builder_host or ' | 454 print ('Error: Specified gs_bucket, but missing builder_host or ' |
455 'builder_port information in config.') | 455 'builder_port information in config.') |
456 return 1 | 456 return 1 |
(...skipping 19 matching lines...) Expand all Loading... |
476 print | 476 print |
477 for k, v in config.iteritems(): | 477 for k, v in config.iteritems(): |
478 print ' %s : %s' % (k, v) | 478 print ' %s : %s' % (k, v) |
479 print | 479 print |
480 bisect_utils.OutputAnnotationStepClosed() | 480 bisect_utils.OutputAnnotationStepClosed() |
481 | 481 |
482 | 482 |
483 def _OptionParser(): | 483 def _OptionParser(): |
484 """Returns the options parser for run-bisect-perf-regression.py.""" | 484 """Returns the options parser for run-bisect-perf-regression.py.""" |
485 usage = ('%prog [options] [-- chromium-options]\n' | 485 usage = ('%prog [options] [-- chromium-options]\n' |
486 'Used by a trybot to run the bisection script using the parameters' | 486 'Used by a try bot to run the bisection script using the parameters' |
487 ' provided in the run-bisect-perf-regression.cfg file.') | 487 ' provided in the run-bisect-perf-regression.cfg file.') |
488 parser = optparse.OptionParser(usage=usage) | 488 parser = optparse.OptionParser(usage=usage) |
489 parser.add_option('-w', '--working_directory', | 489 parser.add_option('-w', '--working_directory', |
490 type='str', | 490 type='str', |
491 help='A working directory to supply to the bisection ' | 491 help='A working directory to supply to the bisection ' |
492 'script, which will use it as the location to checkout ' | 492 'script, which will use it as the location to checkout ' |
493 'a copy of the chromium depot.') | 493 'a copy of the chromium depot.') |
494 parser.add_option('-p', '--path_to_goma', | 494 parser.add_option('-p', '--path_to_goma', |
495 type='str', | 495 type='str', |
496 help='Path to goma directory. If this is supplied, goma ' | 496 help='Path to goma directory. If this is supplied, goma ' |
(...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
565 config, current_dir, opts.path_to_goma) | 565 config, current_dir, opts.path_to_goma) |
566 | 566 |
567 print ('Error: Could not load config file. Double check your changes to ' | 567 print ('Error: Could not load config file. Double check your changes to ' |
568 'run-bisect-perf-regression.cfg or run-perf-test.cfg for syntax ' | 568 'run-bisect-perf-regression.cfg or run-perf-test.cfg for syntax ' |
569 'errors.\n') | 569 'errors.\n') |
570 return 1 | 570 return 1 |
571 | 571 |
572 | 572 |
573 if __name__ == '__main__': | 573 if __name__ == '__main__': |
574 sys.exit(main()) | 574 sys.exit(main()) |
OLD | NEW |