| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Run Performance Test Bisect Tool | 6 """Run Performance Test Bisect Tool |
| 7 | 7 |
| 8 This script is used by a try bot to run the src/tools/bisect-perf-regression.py | 8 This script is used by a try bot to run the bisect script with the parameters |
| 9 script with the parameters specified in src/tools/auto_bisect/bisect.cfg. | 9 specified in the bisect config file. It checks out a copy of the depot in |
| 10 It will check out a copy of the depot in a subdirectory 'bisect' of the working | 10 a subdirectory 'bisect' of the working directory provided, annd runs the |
| 11 directory provided, and run the bisect-perf-regression.py script there. | 11 bisect scrip there. |
| 12 """ | 12 """ |
| 13 | 13 |
| 14 import imp | |
| 15 import optparse | 14 import optparse |
| 16 import os | 15 import os |
| 17 import platform | 16 import platform |
| 18 import subprocess | 17 import subprocess |
| 19 import sys | 18 import sys |
| 20 import traceback | 19 import traceback |
| 21 | 20 |
| 21 from auto_bisect import bisect_perf_regression |
| 22 from auto_bisect import bisect_utils | 22 from auto_bisect import bisect_utils |
| 23 from auto_bisect import math_utils | 23 from auto_bisect import math_utils |
| 24 | 24 |
| 25 bisect = imp.load_source('bisect-perf-regression', | |
| 26 os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), | |
| 27 'bisect-perf-regression.py')) | |
| 28 | |
| 29 | |
| 30 CROS_BOARD_ENV = 'BISECT_CROS_BOARD' | 25 CROS_BOARD_ENV = 'BISECT_CROS_BOARD' |
| 31 CROS_IP_ENV = 'BISECT_CROS_IP' | 26 CROS_IP_ENV = 'BISECT_CROS_IP' |
| 32 | 27 |
| 33 # Default config file paths, relative to this script. | 28 SCRIPT_DIR = os.path.dirname(__file__) |
| 34 BISECT_REGRESSION_CONFIG = os.path.join('auto_bisect', 'bisect.cfg') | 29 SRC_DIR = os.path.join(SCRIPT_DIR, os.path.pardir) |
| 35 RUN_TEST_CONFIG = 'run-perf-test.cfg' | 30 BISECT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'auto_bisect', 'bisect.cfg') |
| 36 WEBKIT_RUN_TEST_CONFIG = os.path.join( | 31 RUN_TEST_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'run-perf-test.cfg') |
| 37 '..', 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg') | 32 WEBKIT_RUN_TEST_CONFIG_PATH = os.path.join( |
| 33 SRC_DIR, 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg') |
| 34 BISECT_SCRIPT_DIR = os.path.join(SCRIPT_DIR, 'auto_bisect') |
| 38 | 35 |
| 39 | 36 |
| 40 class Goma(object): | 37 class Goma(object): |
| 41 | 38 |
| 42 def __init__(self, path_to_goma): | 39 def __init__(self, path_to_goma): |
| 43 self._abs_path_to_goma = None | 40 self._abs_path_to_goma = None |
| 44 self._abs_path_to_goma_file = None | 41 self._abs_path_to_goma_file = None |
| 45 if not path_to_goma: | 42 if not path_to_goma: |
| 46 return | 43 return |
| 47 self._abs_path_to_goma = os.path.abspath(path_to_goma) | 44 self._abs_path_to_goma = os.path.abspath(path_to_goma) |
| (...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 223 raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or' | 220 raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or' |
| 224 'BISECT_CROS_BOARD undefined.') | 221 'BISECT_CROS_BOARD undefined.') |
| 225 elif 'android' in config['command']: | 222 elif 'android' in config['command']: |
| 226 if 'android-chrome-shell' in config['command']: | 223 if 'android-chrome-shell' in config['command']: |
| 227 opts_dict['target_platform'] = 'android' | 224 opts_dict['target_platform'] = 'android' |
| 228 elif 'android-chrome' in config['command']: | 225 elif 'android-chrome' in config['command']: |
| 229 opts_dict['target_platform'] = 'android-chrome' | 226 opts_dict['target_platform'] = 'android-chrome' |
| 230 else: | 227 else: |
| 231 opts_dict['target_platform'] = 'android' | 228 opts_dict['target_platform'] = 'android' |
| 232 | 229 |
| 233 return bisect.BisectOptions.FromDict(opts_dict) | 230 return bisect_perf_regression.BisectOptions.FromDict(opts_dict) |
| 234 | 231 |
| 235 | 232 |
| 236 def _RunPerformanceTest(config, path_to_file): | 233 def _RunPerformanceTest(config): |
| 237 """Runs a performance test with and without the current patch. | 234 """Runs a performance test with and without the current patch. |
| 238 | 235 |
| 239 Args: | 236 Args: |
| 240 config: Contents of the config file, a dictionary. | 237 config: Contents of the config file, a dictionary. |
| 241 path_to_file: Path to the bisect-perf-regression.py script. | |
| 242 | 238 |
| 243 Attempts to build and run the current revision with and without the | 239 Attempts to build and run the current revision with and without the |
| 244 current patch, with the parameters passed in. | 240 current patch, with the parameters passed in. |
| 245 """ | 241 """ |
| 246 # Bisect script expects to be run from the src directory | 242 # Bisect script expects to be run from the src directory |
| 247 os.chdir(os.path.join(path_to_file, '..')) | 243 os.chdir(SRC_DIR) |
| 248 | 244 |
| 249 bisect_utils.OutputAnnotationStepStart('Building With Patch') | 245 bisect_utils.OutputAnnotationStepStart('Building With Patch') |
| 250 | 246 |
| 251 opts = _CreateBisectOptionsFromConfig(config) | 247 opts = _CreateBisectOptionsFromConfig(config) |
| 252 b = bisect.BisectPerformanceMetrics(None, opts) | 248 b = bisect_perf_regression.BisectPerformanceMetrics(None, opts) |
| 253 | 249 |
| 254 if bisect_utils.RunGClient(['runhooks']): | 250 if bisect_utils.RunGClient(['runhooks']): |
| 255 raise RuntimeError('Failed to run gclient runhooks') | 251 raise RuntimeError('Failed to run gclient runhooks') |
| 256 | 252 |
| 257 if not b.BuildCurrentRevision('chromium'): | 253 if not b.BuildCurrentRevision('chromium'): |
| 258 raise RuntimeError('Patched version failed to build.') | 254 raise RuntimeError('Patched version failed to build.') |
| 259 | 255 |
| 260 bisect_utils.OutputAnnotationStepClosed() | 256 bisect_utils.OutputAnnotationStepClosed() |
| 261 bisect_utils.OutputAnnotationStepStart('Running With Patch') | 257 bisect_utils.OutputAnnotationStepStart('Running With Patch') |
| 262 | 258 |
| (...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 332 print ' %s %s %s' % ('No Patch'.center(10, ' '), | 328 print ' %s %s %s' % ('No Patch'.center(10, ' '), |
| 333 ('%.02f' % results_without_patch[0]['mean']).center(20, ' '), | 329 ('%.02f' % results_without_patch[0]['mean']).center(20, ' '), |
| 334 ('%.02f' % results_without_patch[0]['std_err']).center(20, ' ')) | 330 ('%.02f' % results_without_patch[0]['std_err']).center(20, ' ')) |
| 335 if cloud_file_link: | 331 if cloud_file_link: |
| 336 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link) | 332 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link) |
| 337 bisect_utils.OutputAnnotationStepClosed() | 333 bisect_utils.OutputAnnotationStepClosed() |
| 338 elif cloud_file_link: | 334 elif cloud_file_link: |
| 339 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link) | 335 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link) |
| 340 | 336 |
| 341 | 337 |
| 342 def _SetupAndRunPerformanceTest(config, path_to_file, path_to_goma): | 338 def _SetupAndRunPerformanceTest(config, path_to_goma): |
| 343 """Attempts to build and run the current revision with and without the | 339 """Attempts to build and run the current revision with and without the |
| 344 current patch, with the parameters passed in. | 340 current patch, with the parameters passed in. |
| 345 | 341 |
| 346 Args: | 342 Args: |
| 347 config: The config read from run-perf-test.cfg. | 343 config: The config read from run-perf-test.cfg. |
| 348 path_to_file: Path to the bisect-perf-regression.py script. | |
| 349 path_to_goma: Path to goma directory. | 344 path_to_goma: Path to goma directory. |
| 350 | 345 |
| 351 Returns: | 346 Returns: |
| 352 The exit code of bisect-perf-regression.py: 0 on success, otherwise 1. | 347 An exit code: 0 on success, otherwise 1. |
| 353 """ | 348 """ |
| 354 if platform.release() == 'XP': | 349 if platform.release() == 'XP': |
| 355 print 'Windows XP is not supported for perf try jobs because it lacks ' | 350 print 'Windows XP is not supported for perf try jobs because it lacks ' |
| 356 print 'goma support. Please refer to crbug.com/330900.' | 351 print 'goma support. Please refer to crbug.com/330900.' |
| 357 return 1 | 352 return 1 |
| 358 try: | 353 try: |
| 359 with Goma(path_to_goma) as _: | 354 with Goma(path_to_goma) as _: |
| 360 config['use_goma'] = bool(path_to_goma) | 355 config['use_goma'] = bool(path_to_goma) |
| 361 if config['use_goma']: | 356 if config['use_goma']: |
| 362 config['goma_dir'] = os.path.abspath(path_to_goma) | 357 config['goma_dir'] = os.path.abspath(path_to_goma) |
| 363 _RunPerformanceTest(config, path_to_file) | 358 _RunPerformanceTest(config) |
| 364 return 0 | 359 return 0 |
| 365 except RuntimeError, e: | 360 except RuntimeError, e: |
| 366 bisect_utils.OutputAnnotationStepClosed() | 361 bisect_utils.OutputAnnotationStepClosed() |
| 367 _OutputFailedResults('Error: %s' % e.message) | 362 _OutputFailedResults('Error: %s' % e.message) |
| 368 return 1 | 363 return 1 |
| 369 | 364 |
| 370 | 365 |
| 371 def _RunBisectionScript( | 366 def _RunBisectionScript( |
| 372 config, working_directory, path_to_file, path_to_goma, path_to_extra_src, | 367 config, working_directory, path_to_goma, path_to_extra_src, dry_run): |
| 373 dry_run): | 368 """Attempts to execute the bisect script with the given parameters. |
| 374 """Attempts to execute bisect-perf-regression.py with the given parameters. | |
| 375 | 369 |
| 376 Args: | 370 Args: |
| 377 config: A dict containing the parameters to pass to the script. | 371 config: A dict containing the parameters to pass to the script. |
| 378 working_directory: A working directory to provide to the | 372 working_directory: A working directory to provide to the bisect script, |
| 379 bisect-perf-regression.py script, where it will store it's own copy of | 373 where it will store it's own copy of the depot. |
| 380 the depot. | |
| 381 path_to_file: Path to the bisect-perf-regression.py script. | |
| 382 path_to_goma: Path to goma directory. | 374 path_to_goma: Path to goma directory. |
| 383 path_to_extra_src: Path to extra source file. | 375 path_to_extra_src: Path to extra source file. |
| 384 dry_run: Do a dry run, skipping sync, build, and performance testing steps. | 376 dry_run: Do a dry run, skipping sync, build, and performance testing steps. |
| 385 | 377 |
| 386 Returns: | 378 Returns: |
| 387 An exit status code: 0 on success, otherwise 1. | 379 An exit status code: 0 on success, otherwise 1. |
| 388 """ | 380 """ |
| 389 _PrintConfigStep(config) | 381 _PrintConfigStep(config) |
| 390 | 382 |
| 391 cmd = ['python', os.path.join(path_to_file, 'bisect-perf-regression.py'), | 383 cmd = ['python', os.path.join(BISECT_SCRIPT_DIR, 'bisect_perf_regression.py'), |
| 392 '-c', config['command'], | 384 '-c', config['command'], |
| 393 '-g', config['good_revision'], | 385 '-g', config['good_revision'], |
| 394 '-b', config['bad_revision'], | 386 '-b', config['bad_revision'], |
| 395 '-m', config['metric'], | 387 '-m', config['metric'], |
| 396 '--working_directory', working_directory, | 388 '--working_directory', working_directory, |
| 397 '--output_buildbot_annotations'] | 389 '--output_buildbot_annotations'] |
| 398 | 390 |
| 399 if config.get('metric'): | 391 if config.get('metric'): |
| 400 cmd.extend(['-m', config['metric']]) | 392 cmd.extend(['-m', config['metric']]) |
| 401 | 393 |
| (...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 462 | 454 |
| 463 if dry_run: | 455 if dry_run: |
| 464 cmd.extend(['--debug_ignore_build', '--debug_ignore_sync', | 456 cmd.extend(['--debug_ignore_build', '--debug_ignore_sync', |
| 465 '--debug_ignore_perf_test']) | 457 '--debug_ignore_perf_test']) |
| 466 cmd = [str(c) for c in cmd] | 458 cmd = [str(c) for c in cmd] |
| 467 | 459 |
| 468 with Goma(path_to_goma) as _: | 460 with Goma(path_to_goma) as _: |
| 469 return_code = subprocess.call(cmd) | 461 return_code = subprocess.call(cmd) |
| 470 | 462 |
| 471 if return_code: | 463 if return_code: |
| 472 print ('Error: bisect-perf-regression.py returned with error %d\n' | 464 print ('Error: bisect_perf_regression.py returned with error %d\n' |
| 473 % return_code) | 465 % return_code) |
| 474 | 466 |
| 475 return return_code | 467 return return_code |
| 476 | 468 |
| 477 | 469 |
| 478 def _PrintConfigStep(config): | 470 def _PrintConfigStep(config): |
| 479 """Prints out the given config, along with Buildbot annotations.""" | 471 """Prints out the given config, along with Buildbot annotations.""" |
| 480 bisect_utils.OutputAnnotationStepStart('Config') | 472 bisect_utils.OutputAnnotationStepStart('Config') |
| 481 print | 473 print |
| 482 for k, v in config.iteritems(): | 474 for k, v in config.iteritems(): |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 518 return parser | 510 return parser |
| 519 | 511 |
| 520 | 512 |
| 521 def main(): | 513 def main(): |
| 522 """Entry point for run-bisect-perf-regression.py. | 514 """Entry point for run-bisect-perf-regression.py. |
| 523 | 515 |
| 524 Reads the config file, and then tries to either bisect a regression or | 516 Reads the config file, and then tries to either bisect a regression or |
| 525 just run a performance test, depending on the particular config parameters | 517 just run a performance test, depending on the particular config parameters |
| 526 specified in the config file. | 518 specified in the config file. |
| 527 """ | 519 """ |
| 528 | |
| 529 parser = _OptionParser() | 520 parser = _OptionParser() |
| 530 opts, _ = parser.parse_args() | 521 opts, _ = parser.parse_args() |
| 531 | 522 |
| 532 current_dir = os.path.abspath(os.path.dirname(sys.argv[0])) | |
| 533 | |
| 534 # Use the default config file path unless one was specified. | 523 # Use the default config file path unless one was specified. |
| 535 config_path = os.path.join(current_dir, BISECT_REGRESSION_CONFIG) | 524 config_path = BISECT_CONFIG_PATH |
| 536 if opts.path_to_config: | 525 if opts.path_to_config: |
| 537 config_path = opts.path_to_config | 526 config_path = opts.path_to_config |
| 538 config = _LoadConfigFile(config_path) | 527 config = _LoadConfigFile(config_path) |
| 539 | 528 |
| 540 # Check if the config is valid for running bisect job. | 529 # Check if the config is valid for running bisect job. |
| 541 config_is_valid = _ValidateBisectConfigFile(config) | 530 config_is_valid = _ValidateBisectConfigFile(config) |
| 542 | 531 |
| 543 if config and config_is_valid: | 532 if config and config_is_valid: |
| 544 if not opts.working_directory: | 533 if not opts.working_directory: |
| 545 print 'Error: missing required parameter: --working_directory\n' | 534 print 'Error: missing required parameter: --working_directory\n' |
| 546 parser.print_help() | 535 parser.print_help() |
| 547 return 1 | 536 return 1 |
| 548 | 537 |
| 549 return _RunBisectionScript( | 538 return _RunBisectionScript( |
| 550 config, opts.working_directory, current_dir, | 539 config, opts.working_directory, opts.path_to_goma, opts.extra_src, |
| 551 opts.path_to_goma, opts.extra_src, opts.dry_run) | 540 opts.dry_run) |
| 552 | 541 |
| 553 # If it wasn't valid for running a bisect, then maybe the user wanted | 542 # If it wasn't valid for running a bisect, then maybe the user wanted |
| 554 # to run a perf test instead of a bisect job. Try reading any possible | 543 # to run a perf test instead of a bisect job. Try reading any possible |
| 555 # perf test config files. | 544 # perf test config files. |
| 556 perf_cfg_files = [RUN_TEST_CONFIG, WEBKIT_RUN_TEST_CONFIG] | 545 perf_cfg_files = [RUN_TEST_CONFIG_PATH, WEBKIT_RUN_TEST_CONFIG_PATH] |
| 557 for current_perf_cfg_file in perf_cfg_files: | 546 for current_perf_cfg_file in perf_cfg_files: |
| 558 if opts.path_to_config: | 547 if opts.path_to_config: |
| 559 path_to_perf_cfg = opts.path_to_config | 548 path_to_perf_cfg = opts.path_to_config |
| 560 else: | 549 else: |
| 561 path_to_perf_cfg = os.path.join( | 550 path_to_perf_cfg = os.path.join( |
| 562 os.path.abspath(os.path.dirname(sys.argv[0])), | 551 os.path.abspath(os.path.dirname(sys.argv[0])), |
| 563 current_perf_cfg_file) | 552 current_perf_cfg_file) |
| 564 | 553 |
| 565 config = _LoadConfigFile(path_to_perf_cfg) | 554 config = _LoadConfigFile(path_to_perf_cfg) |
| 566 config_is_valid = _ValidatePerfConfigFile(config) | 555 config_is_valid = _ValidatePerfConfigFile(config) |
| 567 | 556 |
| 568 if config and config_is_valid: | 557 if config and config_is_valid: |
| 569 return _SetupAndRunPerformanceTest( | 558 return _SetupAndRunPerformanceTest(config, opts.path_to_goma) |
| 570 config, current_dir, opts.path_to_goma) | |
| 571 | 559 |
| 572 print ('Error: Could not load config file. Double check your changes to ' | 560 print ('Error: Could not load config file. Double check your changes to ' |
| 573 'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax ' | 561 'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax errors.\n') |
| 574 'errors.\n') | |
| 575 return 1 | 562 return 1 |
| 576 | 563 |
| 577 | 564 |
| 578 if __name__ == '__main__': | 565 if __name__ == '__main__': |
| 579 sys.exit(main()) | 566 sys.exit(main()) |
| OLD | NEW |