OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Run Performance Test Bisect Tool | 6 """Run Performance Test Bisect Tool |
7 | 7 |
8 This script is used by a try bot to run the src/tools/bisect-perf-regression.py | 8 This script is used by a try bot to run the bisect script with the parameters |
9 script with the parameters specified in src/tools/auto_bisect/bisect.cfg. | 9 specified in the bisect config file. It checks out a copy of the depot in |
10 It will check out a copy of the depot in a subdirectory 'bisect' of the working | 10 a subdirectory 'bisect' of the working directory provided, annd runs the |
11 directory provided, and run the bisect-perf-regression.py script there. | 11 bisect scrip there. |
12 """ | 12 """ |
13 | 13 |
14 import imp | |
15 import optparse | 14 import optparse |
16 import os | 15 import os |
17 import platform | 16 import platform |
18 import subprocess | 17 import subprocess |
19 import sys | 18 import sys |
20 import traceback | 19 import traceback |
21 | 20 |
| 21 from auto_bisect import bisect_perf_regression |
22 from auto_bisect import bisect_utils | 22 from auto_bisect import bisect_utils |
23 from auto_bisect import math_utils | 23 from auto_bisect import math_utils |
24 | 24 |
25 bisect = imp.load_source('bisect-perf-regression', | |
26 os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])), | |
27 'bisect-perf-regression.py')) | |
28 | |
29 | |
30 CROS_BOARD_ENV = 'BISECT_CROS_BOARD' | 25 CROS_BOARD_ENV = 'BISECT_CROS_BOARD' |
31 CROS_IP_ENV = 'BISECT_CROS_IP' | 26 CROS_IP_ENV = 'BISECT_CROS_IP' |
32 | 27 |
33 # Default config file paths, relative to this script. | 28 SCRIPT_DIR = os.path.dirname(__file__) |
34 BISECT_REGRESSION_CONFIG = os.path.join('auto_bisect', 'bisect.cfg') | 29 SRC_DIR = os.path.join(SCRIPT_DIR, os.path.pardir) |
35 RUN_TEST_CONFIG = 'run-perf-test.cfg' | 30 BISECT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'auto_bisect', 'bisect.cfg') |
36 WEBKIT_RUN_TEST_CONFIG = os.path.join( | 31 RUN_TEST_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'run-perf-test.cfg') |
37 '..', 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg') | 32 WEBKIT_RUN_TEST_CONFIG_PATH = os.path.join( |
| 33 SRC_DIR, 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg') |
| 34 BISECT_SCRIPT_DIR = os.path.join(SCRIPT_DIR, 'auto_bisect') |
38 | 35 |
39 | 36 |
40 class Goma(object): | 37 class Goma(object): |
41 | 38 |
42 def __init__(self, path_to_goma): | 39 def __init__(self, path_to_goma): |
43 self._abs_path_to_goma = None | 40 self._abs_path_to_goma = None |
44 self._abs_path_to_goma_file = None | 41 self._abs_path_to_goma_file = None |
45 if not path_to_goma: | 42 if not path_to_goma: |
46 return | 43 return |
47 self._abs_path_to_goma = os.path.abspath(path_to_goma) | 44 self._abs_path_to_goma = os.path.abspath(path_to_goma) |
(...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
223 raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or' | 220 raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or' |
224 'BISECT_CROS_BOARD undefined.') | 221 'BISECT_CROS_BOARD undefined.') |
225 elif 'android' in config['command']: | 222 elif 'android' in config['command']: |
226 if 'android-chrome-shell' in config['command']: | 223 if 'android-chrome-shell' in config['command']: |
227 opts_dict['target_platform'] = 'android' | 224 opts_dict['target_platform'] = 'android' |
228 elif 'android-chrome' in config['command']: | 225 elif 'android-chrome' in config['command']: |
229 opts_dict['target_platform'] = 'android-chrome' | 226 opts_dict['target_platform'] = 'android-chrome' |
230 else: | 227 else: |
231 opts_dict['target_platform'] = 'android' | 228 opts_dict['target_platform'] = 'android' |
232 | 229 |
233 return bisect.BisectOptions.FromDict(opts_dict) | 230 return bisect_perf_regression.BisectOptions.FromDict(opts_dict) |
234 | 231 |
235 | 232 |
236 def _RunPerformanceTest(config, path_to_file): | 233 def _RunPerformanceTest(config): |
237 """Runs a performance test with and without the current patch. | 234 """Runs a performance test with and without the current patch. |
238 | 235 |
239 Args: | 236 Args: |
240 config: Contents of the config file, a dictionary. | 237 config: Contents of the config file, a dictionary. |
241 path_to_file: Path to the bisect-perf-regression.py script. | |
242 | 238 |
243 Attempts to build and run the current revision with and without the | 239 Attempts to build and run the current revision with and without the |
244 current patch, with the parameters passed in. | 240 current patch, with the parameters passed in. |
245 """ | 241 """ |
246 # Bisect script expects to be run from the src directory | 242 # Bisect script expects to be run from the src directory |
247 os.chdir(os.path.join(path_to_file, '..')) | 243 os.chdir(SRC_DIR) |
248 | 244 |
249 bisect_utils.OutputAnnotationStepStart('Building With Patch') | 245 bisect_utils.OutputAnnotationStepStart('Building With Patch') |
250 | 246 |
251 opts = _CreateBisectOptionsFromConfig(config) | 247 opts = _CreateBisectOptionsFromConfig(config) |
252 b = bisect.BisectPerformanceMetrics(None, opts) | 248 b = bisect_perf_regression.BisectPerformanceMetrics(None, opts) |
253 | 249 |
254 if bisect_utils.RunGClient(['runhooks']): | 250 if bisect_utils.RunGClient(['runhooks']): |
255 raise RuntimeError('Failed to run gclient runhooks') | 251 raise RuntimeError('Failed to run gclient runhooks') |
256 | 252 |
257 if not b.BuildCurrentRevision('chromium'): | 253 if not b.BuildCurrentRevision('chromium'): |
258 raise RuntimeError('Patched version failed to build.') | 254 raise RuntimeError('Patched version failed to build.') |
259 | 255 |
260 bisect_utils.OutputAnnotationStepClosed() | 256 bisect_utils.OutputAnnotationStepClosed() |
261 bisect_utils.OutputAnnotationStepStart('Running With Patch') | 257 bisect_utils.OutputAnnotationStepStart('Running With Patch') |
262 | 258 |
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
332 print ' %s %s %s' % ('No Patch'.center(10, ' '), | 328 print ' %s %s %s' % ('No Patch'.center(10, ' '), |
333 ('%.02f' % results_without_patch[0]['mean']).center(20, ' '), | 329 ('%.02f' % results_without_patch[0]['mean']).center(20, ' '), |
334 ('%.02f' % results_without_patch[0]['std_err']).center(20, ' ')) | 330 ('%.02f' % results_without_patch[0]['std_err']).center(20, ' ')) |
335 if cloud_file_link: | 331 if cloud_file_link: |
336 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link) | 332 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link) |
337 bisect_utils.OutputAnnotationStepClosed() | 333 bisect_utils.OutputAnnotationStepClosed() |
338 elif cloud_file_link: | 334 elif cloud_file_link: |
339 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link) | 335 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link) |
340 | 336 |
341 | 337 |
342 def _SetupAndRunPerformanceTest(config, path_to_file, path_to_goma): | 338 def _SetupAndRunPerformanceTest(config, path_to_goma): |
343 """Attempts to build and run the current revision with and without the | 339 """Attempts to build and run the current revision with and without the |
344 current patch, with the parameters passed in. | 340 current patch, with the parameters passed in. |
345 | 341 |
346 Args: | 342 Args: |
347 config: The config read from run-perf-test.cfg. | 343 config: The config read from run-perf-test.cfg. |
348 path_to_file: Path to the bisect-perf-regression.py script. | |
349 path_to_goma: Path to goma directory. | 344 path_to_goma: Path to goma directory. |
350 | 345 |
351 Returns: | 346 Returns: |
352 The exit code of bisect-perf-regression.py: 0 on success, otherwise 1. | 347 An exit code: 0 on success, otherwise 1. |
353 """ | 348 """ |
354 if platform.release() == 'XP': | 349 if platform.release() == 'XP': |
355 print 'Windows XP is not supported for perf try jobs because it lacks ' | 350 print 'Windows XP is not supported for perf try jobs because it lacks ' |
356 print 'goma support. Please refer to crbug.com/330900.' | 351 print 'goma support. Please refer to crbug.com/330900.' |
357 return 1 | 352 return 1 |
358 try: | 353 try: |
359 with Goma(path_to_goma) as _: | 354 with Goma(path_to_goma) as _: |
360 config['use_goma'] = bool(path_to_goma) | 355 config['use_goma'] = bool(path_to_goma) |
361 if config['use_goma']: | 356 if config['use_goma']: |
362 config['goma_dir'] = os.path.abspath(path_to_goma) | 357 config['goma_dir'] = os.path.abspath(path_to_goma) |
363 _RunPerformanceTest(config, path_to_file) | 358 _RunPerformanceTest(config) |
364 return 0 | 359 return 0 |
365 except RuntimeError, e: | 360 except RuntimeError, e: |
366 bisect_utils.OutputAnnotationStepClosed() | 361 bisect_utils.OutputAnnotationStepClosed() |
367 _OutputFailedResults('Error: %s' % e.message) | 362 _OutputFailedResults('Error: %s' % e.message) |
368 return 1 | 363 return 1 |
369 | 364 |
370 | 365 |
371 def _RunBisectionScript( | 366 def _RunBisectionScript( |
372 config, working_directory, path_to_file, path_to_goma, path_to_extra_src, | 367 config, working_directory, path_to_goma, path_to_extra_src, dry_run): |
373 dry_run): | 368 """Attempts to execute the bisect script with the given parameters. |
374 """Attempts to execute bisect-perf-regression.py with the given parameters. | |
375 | 369 |
376 Args: | 370 Args: |
377 config: A dict containing the parameters to pass to the script. | 371 config: A dict containing the parameters to pass to the script. |
378 working_directory: A working directory to provide to the | 372 working_directory: A working directory to provide to the bisect script, |
379 bisect-perf-regression.py script, where it will store it's own copy of | 373 where it will store it's own copy of the depot. |
380 the depot. | |
381 path_to_file: Path to the bisect-perf-regression.py script. | |
382 path_to_goma: Path to goma directory. | 374 path_to_goma: Path to goma directory. |
383 path_to_extra_src: Path to extra source file. | 375 path_to_extra_src: Path to extra source file. |
384 dry_run: Do a dry run, skipping sync, build, and performance testing steps. | 376 dry_run: Do a dry run, skipping sync, build, and performance testing steps. |
385 | 377 |
386 Returns: | 378 Returns: |
387 An exit status code: 0 on success, otherwise 1. | 379 An exit status code: 0 on success, otherwise 1. |
388 """ | 380 """ |
389 _PrintConfigStep(config) | 381 _PrintConfigStep(config) |
390 | 382 |
391 cmd = ['python', os.path.join(path_to_file, 'bisect-perf-regression.py'), | 383 cmd = ['python', os.path.join(BISECT_SCRIPT_DIR, 'bisect_perf_regression.py'), |
392 '-c', config['command'], | 384 '-c', config['command'], |
393 '-g', config['good_revision'], | 385 '-g', config['good_revision'], |
394 '-b', config['bad_revision'], | 386 '-b', config['bad_revision'], |
395 '-m', config['metric'], | 387 '-m', config['metric'], |
396 '--working_directory', working_directory, | 388 '--working_directory', working_directory, |
397 '--output_buildbot_annotations'] | 389 '--output_buildbot_annotations'] |
398 | 390 |
399 if config.get('metric'): | 391 if config.get('metric'): |
400 cmd.extend(['-m', config['metric']]) | 392 cmd.extend(['-m', config['metric']]) |
401 | 393 |
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
462 | 454 |
463 if dry_run: | 455 if dry_run: |
464 cmd.extend(['--debug_ignore_build', '--debug_ignore_sync', | 456 cmd.extend(['--debug_ignore_build', '--debug_ignore_sync', |
465 '--debug_ignore_perf_test']) | 457 '--debug_ignore_perf_test']) |
466 cmd = [str(c) for c in cmd] | 458 cmd = [str(c) for c in cmd] |
467 | 459 |
468 with Goma(path_to_goma) as _: | 460 with Goma(path_to_goma) as _: |
469 return_code = subprocess.call(cmd) | 461 return_code = subprocess.call(cmd) |
470 | 462 |
471 if return_code: | 463 if return_code: |
472 print ('Error: bisect-perf-regression.py returned with error %d\n' | 464 print ('Error: bisect_perf_regression.py returned with error %d\n' |
473 % return_code) | 465 % return_code) |
474 | 466 |
475 return return_code | 467 return return_code |
476 | 468 |
477 | 469 |
478 def _PrintConfigStep(config): | 470 def _PrintConfigStep(config): |
479 """Prints out the given config, along with Buildbot annotations.""" | 471 """Prints out the given config, along with Buildbot annotations.""" |
480 bisect_utils.OutputAnnotationStepStart('Config') | 472 bisect_utils.OutputAnnotationStepStart('Config') |
481 print | 473 print |
482 for k, v in config.iteritems(): | 474 for k, v in config.iteritems(): |
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
518 return parser | 510 return parser |
519 | 511 |
520 | 512 |
521 def main(): | 513 def main(): |
522 """Entry point for run-bisect-perf-regression.py. | 514 """Entry point for run-bisect-perf-regression.py. |
523 | 515 |
524 Reads the config file, and then tries to either bisect a regression or | 516 Reads the config file, and then tries to either bisect a regression or |
525 just run a performance test, depending on the particular config parameters | 517 just run a performance test, depending on the particular config parameters |
526 specified in the config file. | 518 specified in the config file. |
527 """ | 519 """ |
528 | |
529 parser = _OptionParser() | 520 parser = _OptionParser() |
530 opts, _ = parser.parse_args() | 521 opts, _ = parser.parse_args() |
531 | 522 |
532 current_dir = os.path.abspath(os.path.dirname(sys.argv[0])) | |
533 | |
534 # Use the default config file path unless one was specified. | 523 # Use the default config file path unless one was specified. |
535 config_path = os.path.join(current_dir, BISECT_REGRESSION_CONFIG) | 524 config_path = BISECT_CONFIG_PATH |
536 if opts.path_to_config: | 525 if opts.path_to_config: |
537 config_path = opts.path_to_config | 526 config_path = opts.path_to_config |
538 config = _LoadConfigFile(config_path) | 527 config = _LoadConfigFile(config_path) |
539 | 528 |
540 # Check if the config is valid for running bisect job. | 529 # Check if the config is valid for running bisect job. |
541 config_is_valid = _ValidateBisectConfigFile(config) | 530 config_is_valid = _ValidateBisectConfigFile(config) |
542 | 531 |
543 if config and config_is_valid: | 532 if config and config_is_valid: |
544 if not opts.working_directory: | 533 if not opts.working_directory: |
545 print 'Error: missing required parameter: --working_directory\n' | 534 print 'Error: missing required parameter: --working_directory\n' |
546 parser.print_help() | 535 parser.print_help() |
547 return 1 | 536 return 1 |
548 | 537 |
549 return _RunBisectionScript( | 538 return _RunBisectionScript( |
550 config, opts.working_directory, current_dir, | 539 config, opts.working_directory, opts.path_to_goma, opts.extra_src, |
551 opts.path_to_goma, opts.extra_src, opts.dry_run) | 540 opts.dry_run) |
552 | 541 |
553 # If it wasn't valid for running a bisect, then maybe the user wanted | 542 # If it wasn't valid for running a bisect, then maybe the user wanted |
554 # to run a perf test instead of a bisect job. Try reading any possible | 543 # to run a perf test instead of a bisect job. Try reading any possible |
555 # perf test config files. | 544 # perf test config files. |
556 perf_cfg_files = [RUN_TEST_CONFIG, WEBKIT_RUN_TEST_CONFIG] | 545 perf_cfg_files = [RUN_TEST_CONFIG_PATH, WEBKIT_RUN_TEST_CONFIG_PATH] |
557 for current_perf_cfg_file in perf_cfg_files: | 546 for current_perf_cfg_file in perf_cfg_files: |
558 if opts.path_to_config: | 547 if opts.path_to_config: |
559 path_to_perf_cfg = opts.path_to_config | 548 path_to_perf_cfg = opts.path_to_config |
560 else: | 549 else: |
561 path_to_perf_cfg = os.path.join( | 550 path_to_perf_cfg = os.path.join( |
562 os.path.abspath(os.path.dirname(sys.argv[0])), | 551 os.path.abspath(os.path.dirname(sys.argv[0])), |
563 current_perf_cfg_file) | 552 current_perf_cfg_file) |
564 | 553 |
565 config = _LoadConfigFile(path_to_perf_cfg) | 554 config = _LoadConfigFile(path_to_perf_cfg) |
566 config_is_valid = _ValidatePerfConfigFile(config) | 555 config_is_valid = _ValidatePerfConfigFile(config) |
567 | 556 |
568 if config and config_is_valid: | 557 if config and config_is_valid: |
569 return _SetupAndRunPerformanceTest( | 558 return _SetupAndRunPerformanceTest(config, opts.path_to_goma) |
570 config, current_dir, opts.path_to_goma) | |
571 | 559 |
572 print ('Error: Could not load config file. Double check your changes to ' | 560 print ('Error: Could not load config file. Double check your changes to ' |
573 'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax ' | 561 'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax errors.\n') |
574 'errors.\n') | |
575 return 1 | 562 return 1 |
576 | 563 |
577 | 564 |
578 if __name__ == '__main__': | 565 if __name__ == '__main__': |
579 sys.exit(main()) | 566 sys.exit(main()) |
OLD | NEW |