Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(74)

Side by Side Diff: tools/run-bisect-perf-regression.py

Issue 564663002: Move bisect-perf-regression.py into auto_bisect directory. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Add telemetry to PYTHONPATH for PRESUBMIT.py. Created 6 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tools/run-bisect-manual-test.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Run Performance Test Bisect Tool 6 """Run Performance Test Bisect Tool
7 7
8 This script is used by a try bot to run the src/tools/bisect-perf-regression.py 8 This script is used by a try bot to run the bisect script with the parameters
9 script with the parameters specified in src/tools/auto_bisect/bisect.cfg. 9 specified in the bisect config file. It checks out a copy of the depot in
10 It will check out a copy of the depot in a subdirectory 'bisect' of the working 10 a subdirectory 'bisect' of the working directory provided, annd runs the
11 directory provided, and run the bisect-perf-regression.py script there. 11 bisect scrip there.
12 """ 12 """
13 13
14 import imp
15 import optparse 14 import optparse
16 import os 15 import os
17 import platform 16 import platform
18 import subprocess 17 import subprocess
19 import sys 18 import sys
20 import traceback 19 import traceback
21 20
21 from auto_bisect import bisect_perf_regression
22 from auto_bisect import bisect_utils 22 from auto_bisect import bisect_utils
23 from auto_bisect import math_utils 23 from auto_bisect import math_utils
24 24
25 bisect = imp.load_source('bisect-perf-regression',
26 os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])),
27 'bisect-perf-regression.py'))
28
29
30 CROS_BOARD_ENV = 'BISECT_CROS_BOARD' 25 CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
31 CROS_IP_ENV = 'BISECT_CROS_IP' 26 CROS_IP_ENV = 'BISECT_CROS_IP'
32 27
33 # Default config file paths, relative to this script. 28 SCRIPT_DIR = os.path.dirname(__file__)
34 BISECT_REGRESSION_CONFIG = os.path.join('auto_bisect', 'bisect.cfg') 29 SRC_DIR = os.path.join(SCRIPT_DIR, os.path.pardir)
35 RUN_TEST_CONFIG = 'run-perf-test.cfg' 30 BISECT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'auto_bisect', 'bisect.cfg')
36 WEBKIT_RUN_TEST_CONFIG = os.path.join( 31 RUN_TEST_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'run-perf-test.cfg')
37 '..', 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg') 32 WEBKIT_RUN_TEST_CONFIG_PATH = os.path.join(
33 SRC_DIR, 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg')
34 BISECT_SCRIPT_DIR = os.path.join(SCRIPT_DIR, 'auto_bisect')
38 35
39 36
40 class Goma(object): 37 class Goma(object):
41 38
42 def __init__(self, path_to_goma): 39 def __init__(self, path_to_goma):
43 self._abs_path_to_goma = None 40 self._abs_path_to_goma = None
44 self._abs_path_to_goma_file = None 41 self._abs_path_to_goma_file = None
45 if not path_to_goma: 42 if not path_to_goma:
46 return 43 return
47 self._abs_path_to_goma = os.path.abspath(path_to_goma) 44 self._abs_path_to_goma = os.path.abspath(path_to_goma)
(...skipping 175 matching lines...) Expand 10 before | Expand all | Expand 10 after
223 raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or' 220 raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or'
224 'BISECT_CROS_BOARD undefined.') 221 'BISECT_CROS_BOARD undefined.')
225 elif 'android' in config['command']: 222 elif 'android' in config['command']:
226 if 'android-chrome-shell' in config['command']: 223 if 'android-chrome-shell' in config['command']:
227 opts_dict['target_platform'] = 'android' 224 opts_dict['target_platform'] = 'android'
228 elif 'android-chrome' in config['command']: 225 elif 'android-chrome' in config['command']:
229 opts_dict['target_platform'] = 'android-chrome' 226 opts_dict['target_platform'] = 'android-chrome'
230 else: 227 else:
231 opts_dict['target_platform'] = 'android' 228 opts_dict['target_platform'] = 'android'
232 229
233 return bisect.BisectOptions.FromDict(opts_dict) 230 return bisect_perf_regression.BisectOptions.FromDict(opts_dict)
234 231
235 232
236 def _RunPerformanceTest(config, path_to_file): 233 def _RunPerformanceTest(config):
237 """Runs a performance test with and without the current patch. 234 """Runs a performance test with and without the current patch.
238 235
239 Args: 236 Args:
240 config: Contents of the config file, a dictionary. 237 config: Contents of the config file, a dictionary.
241 path_to_file: Path to the bisect-perf-regression.py script.
242 238
243 Attempts to build and run the current revision with and without the 239 Attempts to build and run the current revision with and without the
244 current patch, with the parameters passed in. 240 current patch, with the parameters passed in.
245 """ 241 """
246 # Bisect script expects to be run from the src directory 242 # Bisect script expects to be run from the src directory
247 os.chdir(os.path.join(path_to_file, '..')) 243 os.chdir(SRC_DIR)
248 244
249 bisect_utils.OutputAnnotationStepStart('Building With Patch') 245 bisect_utils.OutputAnnotationStepStart('Building With Patch')
250 246
251 opts = _CreateBisectOptionsFromConfig(config) 247 opts = _CreateBisectOptionsFromConfig(config)
252 b = bisect.BisectPerformanceMetrics(None, opts) 248 b = bisect_perf_regression.BisectPerformanceMetrics(None, opts)
253 249
254 if bisect_utils.RunGClient(['runhooks']): 250 if bisect_utils.RunGClient(['runhooks']):
255 raise RuntimeError('Failed to run gclient runhooks') 251 raise RuntimeError('Failed to run gclient runhooks')
256 252
257 if not b.BuildCurrentRevision('chromium'): 253 if not b.BuildCurrentRevision('chromium'):
258 raise RuntimeError('Patched version failed to build.') 254 raise RuntimeError('Patched version failed to build.')
259 255
260 bisect_utils.OutputAnnotationStepClosed() 256 bisect_utils.OutputAnnotationStepClosed()
261 bisect_utils.OutputAnnotationStepStart('Running With Patch') 257 bisect_utils.OutputAnnotationStepStart('Running With Patch')
262 258
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after
332 print ' %s %s %s' % ('No Patch'.center(10, ' '), 328 print ' %s %s %s' % ('No Patch'.center(10, ' '),
333 ('%.02f' % results_without_patch[0]['mean']).center(20, ' '), 329 ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
334 ('%.02f' % results_without_patch[0]['std_err']).center(20, ' ')) 330 ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
335 if cloud_file_link: 331 if cloud_file_link:
336 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link) 332 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
337 bisect_utils.OutputAnnotationStepClosed() 333 bisect_utils.OutputAnnotationStepClosed()
338 elif cloud_file_link: 334 elif cloud_file_link:
339 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link) 335 bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
340 336
341 337
342 def _SetupAndRunPerformanceTest(config, path_to_file, path_to_goma): 338 def _SetupAndRunPerformanceTest(config, path_to_goma):
343 """Attempts to build and run the current revision with and without the 339 """Attempts to build and run the current revision with and without the
344 current patch, with the parameters passed in. 340 current patch, with the parameters passed in.
345 341
346 Args: 342 Args:
347 config: The config read from run-perf-test.cfg. 343 config: The config read from run-perf-test.cfg.
348 path_to_file: Path to the bisect-perf-regression.py script.
349 path_to_goma: Path to goma directory. 344 path_to_goma: Path to goma directory.
350 345
351 Returns: 346 Returns:
352 The exit code of bisect-perf-regression.py: 0 on success, otherwise 1. 347 An exit code: 0 on success, otherwise 1.
353 """ 348 """
354 if platform.release() == 'XP': 349 if platform.release() == 'XP':
355 print 'Windows XP is not supported for perf try jobs because it lacks ' 350 print 'Windows XP is not supported for perf try jobs because it lacks '
356 print 'goma support. Please refer to crbug.com/330900.' 351 print 'goma support. Please refer to crbug.com/330900.'
357 return 1 352 return 1
358 try: 353 try:
359 with Goma(path_to_goma) as _: 354 with Goma(path_to_goma) as _:
360 config['use_goma'] = bool(path_to_goma) 355 config['use_goma'] = bool(path_to_goma)
361 if config['use_goma']: 356 if config['use_goma']:
362 config['goma_dir'] = os.path.abspath(path_to_goma) 357 config['goma_dir'] = os.path.abspath(path_to_goma)
363 _RunPerformanceTest(config, path_to_file) 358 _RunPerformanceTest(config)
364 return 0 359 return 0
365 except RuntimeError, e: 360 except RuntimeError, e:
366 bisect_utils.OutputAnnotationStepClosed() 361 bisect_utils.OutputAnnotationStepClosed()
367 _OutputFailedResults('Error: %s' % e.message) 362 _OutputFailedResults('Error: %s' % e.message)
368 return 1 363 return 1
369 364
370 365
371 def _RunBisectionScript( 366 def _RunBisectionScript(
372 config, working_directory, path_to_file, path_to_goma, path_to_extra_src, 367 config, working_directory, path_to_goma, path_to_extra_src, dry_run):
373 dry_run): 368 """Attempts to execute the bisect script with the given parameters.
374 """Attempts to execute bisect-perf-regression.py with the given parameters.
375 369
376 Args: 370 Args:
377 config: A dict containing the parameters to pass to the script. 371 config: A dict containing the parameters to pass to the script.
378 working_directory: A working directory to provide to the 372 working_directory: A working directory to provide to the bisect script,
379 bisect-perf-regression.py script, where it will store it's own copy of 373 where it will store it's own copy of the depot.
380 the depot.
381 path_to_file: Path to the bisect-perf-regression.py script.
382 path_to_goma: Path to goma directory. 374 path_to_goma: Path to goma directory.
383 path_to_extra_src: Path to extra source file. 375 path_to_extra_src: Path to extra source file.
384 dry_run: Do a dry run, skipping sync, build, and performance testing steps. 376 dry_run: Do a dry run, skipping sync, build, and performance testing steps.
385 377
386 Returns: 378 Returns:
387 An exit status code: 0 on success, otherwise 1. 379 An exit status code: 0 on success, otherwise 1.
388 """ 380 """
389 _PrintConfigStep(config) 381 _PrintConfigStep(config)
390 382
391 cmd = ['python', os.path.join(path_to_file, 'bisect-perf-regression.py'), 383 cmd = ['python', os.path.join(BISECT_SCRIPT_DIR, 'bisect_perf_regression.py'),
392 '-c', config['command'], 384 '-c', config['command'],
393 '-g', config['good_revision'], 385 '-g', config['good_revision'],
394 '-b', config['bad_revision'], 386 '-b', config['bad_revision'],
395 '-m', config['metric'], 387 '-m', config['metric'],
396 '--working_directory', working_directory, 388 '--working_directory', working_directory,
397 '--output_buildbot_annotations'] 389 '--output_buildbot_annotations']
398 390
399 if config.get('metric'): 391 if config.get('metric'):
400 cmd.extend(['-m', config['metric']]) 392 cmd.extend(['-m', config['metric']])
401 393
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after
462 454
463 if dry_run: 455 if dry_run:
464 cmd.extend(['--debug_ignore_build', '--debug_ignore_sync', 456 cmd.extend(['--debug_ignore_build', '--debug_ignore_sync',
465 '--debug_ignore_perf_test']) 457 '--debug_ignore_perf_test'])
466 cmd = [str(c) for c in cmd] 458 cmd = [str(c) for c in cmd]
467 459
468 with Goma(path_to_goma) as _: 460 with Goma(path_to_goma) as _:
469 return_code = subprocess.call(cmd) 461 return_code = subprocess.call(cmd)
470 462
471 if return_code: 463 if return_code:
472 print ('Error: bisect-perf-regression.py returned with error %d\n' 464 print ('Error: bisect_perf_regression.py returned with error %d\n'
473 % return_code) 465 % return_code)
474 466
475 return return_code 467 return return_code
476 468
477 469
478 def _PrintConfigStep(config): 470 def _PrintConfigStep(config):
479 """Prints out the given config, along with Buildbot annotations.""" 471 """Prints out the given config, along with Buildbot annotations."""
480 bisect_utils.OutputAnnotationStepStart('Config') 472 bisect_utils.OutputAnnotationStepStart('Config')
481 print 473 print
482 for k, v in config.iteritems(): 474 for k, v in config.iteritems():
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after
518 return parser 510 return parser
519 511
520 512
521 def main(): 513 def main():
522 """Entry point for run-bisect-perf-regression.py. 514 """Entry point for run-bisect-perf-regression.py.
523 515
524 Reads the config file, and then tries to either bisect a regression or 516 Reads the config file, and then tries to either bisect a regression or
525 just run a performance test, depending on the particular config parameters 517 just run a performance test, depending on the particular config parameters
526 specified in the config file. 518 specified in the config file.
527 """ 519 """
528
529 parser = _OptionParser() 520 parser = _OptionParser()
530 opts, _ = parser.parse_args() 521 opts, _ = parser.parse_args()
531 522
532 current_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
533
534 # Use the default config file path unless one was specified. 523 # Use the default config file path unless one was specified.
535 config_path = os.path.join(current_dir, BISECT_REGRESSION_CONFIG) 524 config_path = BISECT_CONFIG_PATH
536 if opts.path_to_config: 525 if opts.path_to_config:
537 config_path = opts.path_to_config 526 config_path = opts.path_to_config
538 config = _LoadConfigFile(config_path) 527 config = _LoadConfigFile(config_path)
539 528
540 # Check if the config is valid for running bisect job. 529 # Check if the config is valid for running bisect job.
541 config_is_valid = _ValidateBisectConfigFile(config) 530 config_is_valid = _ValidateBisectConfigFile(config)
542 531
543 if config and config_is_valid: 532 if config and config_is_valid:
544 if not opts.working_directory: 533 if not opts.working_directory:
545 print 'Error: missing required parameter: --working_directory\n' 534 print 'Error: missing required parameter: --working_directory\n'
546 parser.print_help() 535 parser.print_help()
547 return 1 536 return 1
548 537
549 return _RunBisectionScript( 538 return _RunBisectionScript(
550 config, opts.working_directory, current_dir, 539 config, opts.working_directory, opts.path_to_goma, opts.extra_src,
551 opts.path_to_goma, opts.extra_src, opts.dry_run) 540 opts.dry_run)
552 541
553 # If it wasn't valid for running a bisect, then maybe the user wanted 542 # If it wasn't valid for running a bisect, then maybe the user wanted
554 # to run a perf test instead of a bisect job. Try reading any possible 543 # to run a perf test instead of a bisect job. Try reading any possible
555 # perf test config files. 544 # perf test config files.
556 perf_cfg_files = [RUN_TEST_CONFIG, WEBKIT_RUN_TEST_CONFIG] 545 perf_cfg_files = [RUN_TEST_CONFIG_PATH, WEBKIT_RUN_TEST_CONFIG_PATH]
557 for current_perf_cfg_file in perf_cfg_files: 546 for current_perf_cfg_file in perf_cfg_files:
558 if opts.path_to_config: 547 if opts.path_to_config:
559 path_to_perf_cfg = opts.path_to_config 548 path_to_perf_cfg = opts.path_to_config
560 else: 549 else:
561 path_to_perf_cfg = os.path.join( 550 path_to_perf_cfg = os.path.join(
562 os.path.abspath(os.path.dirname(sys.argv[0])), 551 os.path.abspath(os.path.dirname(sys.argv[0])),
563 current_perf_cfg_file) 552 current_perf_cfg_file)
564 553
565 config = _LoadConfigFile(path_to_perf_cfg) 554 config = _LoadConfigFile(path_to_perf_cfg)
566 config_is_valid = _ValidatePerfConfigFile(config) 555 config_is_valid = _ValidatePerfConfigFile(config)
567 556
568 if config and config_is_valid: 557 if config and config_is_valid:
569 return _SetupAndRunPerformanceTest( 558 return _SetupAndRunPerformanceTest(config, opts.path_to_goma)
570 config, current_dir, opts.path_to_goma)
571 559
572 print ('Error: Could not load config file. Double check your changes to ' 560 print ('Error: Could not load config file. Double check your changes to '
573 'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax ' 561 'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax errors.\n')
574 'errors.\n')
575 return 1 562 return 1
576 563
577 564
578 if __name__ == '__main__': 565 if __name__ == '__main__':
579 sys.exit(main()) 566 sys.exit(main())
OLDNEW
« no previous file with comments | « tools/run-bisect-manual-test.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698