| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Run Performance Test Bisect Tool | 6 """Run Performance Test Bisect Tool |
| 7 | 7 |
| 8 This script is used by a trybot to run the src/tools/bisect-perf-regression.py | 8 This script is used by a try bot to run the src/tools/bisect-perf-regression.py |
| 9 script with the parameters specified in run-bisect-perf-regression.cfg. It will | 9 script with the parameters specified in run-bisect-perf-regression.cfg. It will |
| 10 check out a copy of the depot in a subdirectory 'bisect' of the working | 10 check out a copy of the depot in a subdirectory 'bisect' of the working |
| 11 directory provided, and run the bisect-perf-regression.py script there. | 11 directory provided, and run the bisect-perf-regression.py script there. |
| 12 | 12 |
| 13 """ | 13 """ |
| 14 | 14 |
| 15 import imp | 15 import imp |
| 16 import optparse | 16 import optparse |
| 17 import os | 17 import os |
| 18 import platform | 18 import platform |
| (...skipping 194 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 213 opts_dict['build_preference'] = 'ninja' | 213 opts_dict['build_preference'] = 'ninja' |
| 214 opts_dict['output_buildbot_annotations'] = True | 214 opts_dict['output_buildbot_annotations'] = True |
| 215 | 215 |
| 216 if '--browser=cros' in config['command']: | 216 if '--browser=cros' in config['command']: |
| 217 opts_dict['target_platform'] = 'cros' | 217 opts_dict['target_platform'] = 'cros' |
| 218 | 218 |
| 219 if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]: | 219 if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]: |
| 220 opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV] | 220 opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV] |
| 221 opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV] | 221 opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV] |
| 222 else: | 222 else: |
| 223 raise RuntimeError('Cros build selected, but BISECT_CROS_IP or' | 223 raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or' |
| 224 'BISECT_CROS_BOARD undefined.') | 224 'BISECT_CROS_BOARD undefined.') |
| 225 elif 'android' in config['command']: | 225 elif 'android' in config['command']: |
| 226 if 'android-chrome-shell' in config['command']: | 226 if 'android-chrome-shell' in config['command']: |
| 227 opts_dict['target_platform'] = 'android' | 227 opts_dict['target_platform'] = 'android' |
| 228 elif 'android-chrome' in config['command']: | 228 elif 'android-chrome' in config['command']: |
| 229 opts_dict['target_platform'] = 'android-chrome' | 229 opts_dict['target_platform'] = 'android-chrome' |
| 230 else: | 230 else: |
| 231 opts_dict['target_platform'] = 'android' | 231 opts_dict['target_platform'] = 'android' |
| 232 | 232 |
| 233 return bisect.BisectOptions.FromDict(opts_dict) | 233 return bisect.BisectOptions.FromDict(opts_dict) |
| (...skipping 30 matching lines...) Expand all Loading... |
| 264 opts.command, opts.metric, reset_on_first_run=True, results_label='Patch') | 264 opts.command, opts.metric, reset_on_first_run=True, results_label='Patch') |
| 265 | 265 |
| 266 if results_with_patch[1]: | 266 if results_with_patch[1]: |
| 267 raise RuntimeError('Patched version failed to run performance test.') | 267 raise RuntimeError('Patched version failed to run performance test.') |
| 268 | 268 |
| 269 bisect_utils.OutputAnnotationStepClosed() | 269 bisect_utils.OutputAnnotationStepClosed() |
| 270 | 270 |
| 271 bisect_utils.OutputAnnotationStepStart('Reverting Patch') | 271 bisect_utils.OutputAnnotationStepStart('Reverting Patch') |
| 272 # TODO: When this is re-written to recipes, this should use bot_update's | 272 # TODO: When this is re-written to recipes, this should use bot_update's |
| 273 # revert mechanism to fully revert the client. But for now, since we know that | 273 # revert mechanism to fully revert the client. But for now, since we know that |
| 274 # the perf trybot currently only supports src/ and src/third_party/WebKit, we | 274 # the perf try bot currently only supports src/ and src/third_party/WebKit, we |
| 275 # simply reset those two directories. | 275 # simply reset those two directories. |
| 276 bisect_utils.CheckRunGit(['reset', '--hard']) | 276 bisect_utils.CheckRunGit(['reset', '--hard']) |
| 277 bisect_utils.CheckRunGit(['reset', '--hard'], | 277 bisect_utils.CheckRunGit(['reset', '--hard'], |
| 278 os.path.join('third_party', 'WebKit')) | 278 os.path.join('third_party', 'WebKit')) |
| 279 bisect_utils.OutputAnnotationStepClosed() | 279 bisect_utils.OutputAnnotationStepClosed() |
| 280 | 280 |
| 281 bisect_utils.OutputAnnotationStepStart('Building Without Patch') | 281 bisect_utils.OutputAnnotationStepStart('Building Without Patch') |
| 282 | 282 |
| 283 if bisect_utils.RunGClient(['runhooks']): | 283 if bisect_utils.RunGClient(['runhooks']): |
| 284 raise RuntimeError('Failed to run gclient runhooks') | 284 raise RuntimeError('Failed to run gclient runhooks') |
| 285 | 285 |
| 286 if not b.BuildCurrentRevision('chromium'): | 286 if not b.BuildCurrentRevision('chromium'): |
| 287 raise RuntimeError('Unpatched version failed to build.') | 287 raise RuntimeError('Unpatched version failed to build.') |
| 288 | 288 |
| 289 bisect_utils.OutputAnnotationStepClosed() | 289 bisect_utils.OutputAnnotationStepClosed() |
| 290 bisect_utils.OutputAnnotationStepStart('Running Without Patch') | 290 bisect_utils.OutputAnnotationStepStart('Running Without Patch') |
| 291 | 291 |
| 292 results_without_patch = b.RunPerformanceTestAndParseResults( | 292 results_without_patch = b.RunPerformanceTestAndParseResults( |
| 293 opts.command, opts.metric, upload_on_last_run=True, results_label='ToT') | 293 opts.command, opts.metric, upload_on_last_run=True, results_label='ToT') |
| 294 | 294 |
| 295 if results_without_patch[1]: | 295 if results_without_patch[1]: |
| 296 raise RuntimeError('Unpatched version failed to run performance test.') | 296 raise RuntimeError('Unpatched version failed to run performance test.') |
| 297 | 297 |
| 298 # Find the link to the cloud stored results file. | 298 # Find the link to the cloud stored results file. |
| 299 output = results_without_patch[2] | 299 output = results_without_patch[2] |
| 300 cloud_file_link = [t for t in output.splitlines() | 300 cloud_file_link = [t for t in output.splitlines() |
| 301 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t] | 301 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t] |
| 302 if cloud_file_link: | 302 if cloud_file_link: |
| 303 # What we're getting here is basically "View online at http://..." so parse | 303 # What we're getting here is basically "View online at http://..." so parse |
| 304 # out just the url portion. | 304 # out just the URL portion. |
| 305 cloud_file_link = cloud_file_link[0] | 305 cloud_file_link = cloud_file_link[0] |
| 306 cloud_file_link = [t for t in cloud_file_link.split(' ') | 306 cloud_file_link = [t for t in cloud_file_link.split(' ') |
| 307 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t] | 307 if 'storage.googleapis.com/chromium-telemetry/html-results/' in t] |
| 308 assert cloud_file_link, "Couldn't parse url from output." | 308 assert cloud_file_link, 'Couldn\'t parse URL from output.' |
| 309 cloud_file_link = cloud_file_link[0] | 309 cloud_file_link = cloud_file_link[0] |
| 310 else: | 310 else: |
| 311 cloud_file_link = '' | 311 cloud_file_link = '' |
| 312 | 312 |
| 313 # Calculate the % difference in the means of the 2 runs. | 313 # Calculate the % difference in the means of the 2 runs. |
| 314 percent_diff_in_means = None | 314 percent_diff_in_means = None |
| 315 std_err = None | 315 std_err = None |
| 316 if (results_with_patch[0].has_key('mean') and | 316 if (results_with_patch[0].has_key('mean') and |
| 317 results_with_patch[0].has_key('values')): | 317 results_with_patch[0].has_key('values')): |
| 318 percent_diff_in_means = (results_with_patch[0]['mean'] / | 318 percent_diff_in_means = (results_with_patch[0]['mean'] / |
| (...skipping 116 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 435 if config.get('gs_bucket') and platform.release() == 'XP': | 435 if config.get('gs_bucket') and platform.release() == 'XP': |
| 436 print ('Goma doesn\'t have a win32 binary, therefore it is not supported ' | 436 print ('Goma doesn\'t have a win32 binary, therefore it is not supported ' |
| 437 'on Windows XP platform. Please refer to crbug.com/330900.') | 437 'on Windows XP platform. Please refer to crbug.com/330900.') |
| 438 path_to_goma = None | 438 path_to_goma = None |
| 439 cmd.append('--use_goma') | 439 cmd.append('--use_goma') |
| 440 | 440 |
| 441 if path_to_extra_src: | 441 if path_to_extra_src: |
| 442 cmd.extend(['--extra_src', path_to_extra_src]) | 442 cmd.extend(['--extra_src', path_to_extra_src]) |
| 443 | 443 |
| 444 # These flags are used to download build archives from cloud storage if | 444 # These flags are used to download build archives from cloud storage if |
| 445 # available, otherwise will post a try_job_http request to build it on | 445 # available, otherwise will post a try_job_http request to build it on the |
| 446 # tryserver. | 446 # try server. |
| 447 if config.get('gs_bucket'): | 447 if config.get('gs_bucket'): |
| 448 if config.get('builder_host') and config.get('builder_port'): | 448 if config.get('builder_host') and config.get('builder_port'): |
| 449 cmd.extend(['--gs_bucket', config['gs_bucket'], | 449 cmd.extend(['--gs_bucket', config['gs_bucket'], |
| 450 '--builder_host', config['builder_host'], | 450 '--builder_host', config['builder_host'], |
| 451 '--builder_port', config['builder_port'] | 451 '--builder_port', config['builder_port'] |
| 452 ]) | 452 ]) |
| 453 else: | 453 else: |
| 454 print ('Error: Specified gs_bucket, but missing builder_host or ' | 454 print ('Error: Specified gs_bucket, but missing builder_host or ' |
| 455 'builder_port information in config.') | 455 'builder_port information in config.') |
| 456 return 1 | 456 return 1 |
| (...skipping 19 matching lines...) Expand all Loading... |
| 476 print | 476 print |
| 477 for k, v in config.iteritems(): | 477 for k, v in config.iteritems(): |
| 478 print ' %s : %s' % (k, v) | 478 print ' %s : %s' % (k, v) |
| 479 print | 479 print |
| 480 bisect_utils.OutputAnnotationStepClosed() | 480 bisect_utils.OutputAnnotationStepClosed() |
| 481 | 481 |
| 482 | 482 |
| 483 def _OptionParser(): | 483 def _OptionParser(): |
| 484 """Returns the options parser for run-bisect-perf-regression.py.""" | 484 """Returns the options parser for run-bisect-perf-regression.py.""" |
| 485 usage = ('%prog [options] [-- chromium-options]\n' | 485 usage = ('%prog [options] [-- chromium-options]\n' |
| 486 'Used by a trybot to run the bisection script using the parameters' | 486 'Used by a try bot to run the bisection script using the parameters' |
| 487 ' provided in the run-bisect-perf-regression.cfg file.') | 487 ' provided in the run-bisect-perf-regression.cfg file.') |
| 488 parser = optparse.OptionParser(usage=usage) | 488 parser = optparse.OptionParser(usage=usage) |
| 489 parser.add_option('-w', '--working_directory', | 489 parser.add_option('-w', '--working_directory', |
| 490 type='str', | 490 type='str', |
| 491 help='A working directory to supply to the bisection ' | 491 help='A working directory to supply to the bisection ' |
| 492 'script, which will use it as the location to checkout ' | 492 'script, which will use it as the location to checkout ' |
| 493 'a copy of the chromium depot.') | 493 'a copy of the chromium depot.') |
| 494 parser.add_option('-p', '--path_to_goma', | 494 parser.add_option('-p', '--path_to_goma', |
| 495 type='str', | 495 type='str', |
| 496 help='Path to goma directory. If this is supplied, goma ' | 496 help='Path to goma directory. If this is supplied, goma ' |
| (...skipping 68 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 565 config, current_dir, opts.path_to_goma) | 565 config, current_dir, opts.path_to_goma) |
| 566 | 566 |
| 567 print ('Error: Could not load config file. Double check your changes to ' | 567 print ('Error: Could not load config file. Double check your changes to ' |
| 568 'run-bisect-perf-regression.cfg or run-perf-test.cfg for syntax ' | 568 'run-bisect-perf-regression.cfg or run-perf-test.cfg for syntax ' |
| 569 'errors.\n') | 569 'errors.\n') |
| 570 return 1 | 570 return 1 |
| 571 | 571 |
| 572 | 572 |
| 573 if __name__ == '__main__': | 573 if __name__ == '__main__': |
| 574 sys.exit(main()) | 574 sys.exit(main()) |
| OLD | NEW |