| OLD | NEW |
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
| 5 | 5 |
| 6 """Performance Test Bisect Tool | 6 """Performance Test Bisect Tool |
| 7 | 7 |
| 8 This script bisects a series of changelists using binary search. It starts at | 8 This script bisects a series of changelists using binary search. It starts at |
| 9 a bad revision where a performance metric has regressed, and asks for a last | 9 a bad revision where a performance metric has regressed, and asks for a last |
| 10 known-good revision. It will then binary search across this revision range by | 10 known-good revision. It will then binary search across this revision range by |
| (...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 47 import zipfile | 47 import zipfile |
| 48 | 48 |
| 49 sys.path.append(os.path.join( | 49 sys.path.append(os.path.join( |
| 50 os.path.dirname(__file__), os.path.pardir, 'telemetry')) | 50 os.path.dirname(__file__), os.path.pardir, 'telemetry')) |
| 51 | 51 |
| 52 from bisect_printer import BisectPrinter | 52 from bisect_printer import BisectPrinter |
| 53 from bisect_results import BisectResults | 53 from bisect_results import BisectResults |
| 54 from bisect_state import BisectState | 54 from bisect_state import BisectState |
| 55 import bisect_utils | 55 import bisect_utils |
| 56 import builder | 56 import builder |
| 57 import query_crbug |
| 57 import math_utils | 58 import math_utils |
| 58 import request_build | 59 import request_build |
| 59 import source_control | 60 import source_control |
| 60 from telemetry.util import cloud_storage | 61 from telemetry.util import cloud_storage |
| 61 | 62 |
| 62 # The script is in chromium/src/tools/auto_bisect. Throughout this script, | 63 # The script is in chromium/src/tools/auto_bisect. Throughout this script, |
| 63 # we use paths to other things in the chromium/src repository. | 64 # we use paths to other things in the chromium/src repository. |
| 64 | 65 |
| 65 # Possible return values from BisectPerformanceMetrics.RunTest. | 66 # Possible return values from BisectPerformanceMetrics.RunTest. |
| 66 BUILD_RESULT_SUCCEED = 0 | 67 BUILD_RESULT_SUCCEED = 0 |
| (...skipping 2060 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2127 | 2128 |
| 2128 Args: | 2129 Args: |
| 2129 command_to_run: Specify the command to execute the performance test. | 2130 command_to_run: Specify the command to execute the performance test. |
| 2130 good_revision: Number/tag of the known good revision. | 2131 good_revision: Number/tag of the known good revision. |
| 2131 bad_revision: Number/tag of the known bad revision. | 2132 bad_revision: Number/tag of the known bad revision. |
| 2132 metric: The performance metric to monitor. | 2133 metric: The performance metric to monitor. |
| 2133 | 2134 |
| 2134 Returns: | 2135 Returns: |
| 2135 A BisectResults object. | 2136 A BisectResults object. |
| 2136 """ | 2137 """ |
| 2138 |
| 2137 # Choose depot to bisect first | 2139 # Choose depot to bisect first |
| 2138 target_depot = 'chromium' | 2140 target_depot = 'chromium' |
| 2139 if self.opts.target_platform == 'android-chrome': | 2141 if self.opts.target_platform == 'android-chrome': |
| 2140 target_depot = 'android-chrome' | 2142 target_depot = 'android-chrome' |
| 2141 | 2143 |
| 2142 cwd = os.getcwd() | 2144 cwd = os.getcwd() |
| 2143 self.depot_registry.ChangeToDepotDir(target_depot) | 2145 self.depot_registry.ChangeToDepotDir(target_depot) |
| 2144 | 2146 |
| 2145 # If they passed SVN revisions, we can try match them to git SHA1 hashes. | 2147 # If they passed SVN revisions, we can try match them to git SHA1 hashes. |
| 2146 bad_revision = source_control.ResolveToRevision( | 2148 bad_revision = source_control.ResolveToRevision( |
| (...skipping 335 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2482 self.debug_ignore_perf_test = None | 2484 self.debug_ignore_perf_test = None |
| 2483 self.debug_ignore_regression_confidence = None | 2485 self.debug_ignore_regression_confidence = None |
| 2484 self.debug_fake_first_test_mean = 0 | 2486 self.debug_fake_first_test_mean = 0 |
| 2485 self.gs_bucket = None | 2487 self.gs_bucket = None |
| 2486 self.target_arch = 'ia32' | 2488 self.target_arch = 'ia32' |
| 2487 self.target_build_type = 'Release' | 2489 self.target_build_type = 'Release' |
| 2488 self.builder_host = None | 2490 self.builder_host = None |
| 2489 self.builder_port = None | 2491 self.builder_port = None |
| 2490 self.bisect_mode = bisect_utils.BISECT_MODE_MEAN | 2492 self.bisect_mode = bisect_utils.BISECT_MODE_MEAN |
| 2491 self.improvement_direction = 0 | 2493 self.improvement_direction = 0 |
| 2494 self.bug_id = '' |
| 2492 | 2495 |
| 2493 @staticmethod | 2496 @staticmethod |
| 2494 def _AddBisectOptionsGroup(parser): | 2497 def _AddBisectOptionsGroup(parser): |
| 2495 group = parser.add_argument_group('Bisect options') | 2498 group = parser.add_argument_group('Bisect options') |
| 2496 group.add_argument('-c', '--command', required=True, | 2499 group.add_argument('-c', '--command', required=True, |
| 2497 help='A command to execute your performance test at ' | 2500 help='A command to execute your performance test at ' |
| 2498 'each point in the bisection.') | 2501 'each point in the bisection.') |
| 2499 group.add_argument('-b', '--bad_revision', required=True, | 2502 group.add_argument('-b', '--bad_revision', required=True, |
| 2500 help='A bad revision to start bisection. Must be later ' | 2503 help='A bad revision to start bisection. Must be later ' |
| 2501 'than good revision. May be either a git or svn ' | 2504 'than good revision. May be either a git or svn ' |
| (...skipping 27 matching lines...) Expand all Loading... |
| 2529 help='The highest/lowest % are discarded to form a ' | 2532 help='The highest/lowest % are discarded to form a ' |
| 2530 'truncated mean. Values will be clamped to range ' | 2533 'truncated mean. Values will be clamped to range ' |
| 2531 '[0, 25]. Default value is 25 (highest/lowest 25% ' | 2534 '[0, 25]. Default value is 25 (highest/lowest 25% ' |
| 2532 'will be discarded).') | 2535 'will be discarded).') |
| 2533 group.add_argument('--bisect_mode', default=bisect_utils.BISECT_MODE_MEAN, | 2536 group.add_argument('--bisect_mode', default=bisect_utils.BISECT_MODE_MEAN, |
| 2534 choices=[bisect_utils.BISECT_MODE_MEAN, | 2537 choices=[bisect_utils.BISECT_MODE_MEAN, |
| 2535 bisect_utils.BISECT_MODE_STD_DEV, | 2538 bisect_utils.BISECT_MODE_STD_DEV, |
| 2536 bisect_utils.BISECT_MODE_RETURN_CODE], | 2539 bisect_utils.BISECT_MODE_RETURN_CODE], |
| 2537 help='The bisect mode. Choices are to bisect on the ' | 2540 help='The bisect mode. Choices are to bisect on the ' |
| 2538 'difference in mean, std_dev, or return_code.') | 2541 'difference in mean, std_dev, or return_code.') |
| 2542 group.add_argument('--bug_id', default='', |
| 2543 help='The id for the bug associated with this bisect. ' + |
| 2544 'If this number is given, bisect will attempt to ' + |
| 2545 'verify that the bug is not closed before ' |
| 2546 'starting.') |
| 2539 | 2547 |
| 2540 @staticmethod | 2548 @staticmethod |
| 2541 def _AddBuildOptionsGroup(parser): | 2549 def _AddBuildOptionsGroup(parser): |
| 2542 group = parser.add_argument_group('Build options') | 2550 group = parser.add_argument_group('Build options') |
| 2543 group.add_argument('-w', '--working_directory', | 2551 group.add_argument('-w', '--working_directory', |
| 2544 help='Path to the working directory where the script ' | 2552 help='Path to the working directory where the script ' |
| 2545 'will do an initial checkout of the chromium depot. The ' | 2553 'will do an initial checkout of the chromium depot. The ' |
| 2546 'files will be placed in a subdirectory "bisect" under ' | 2554 'files will be placed in a subdirectory "bisect" under ' |
| 2547 'working_directory and that will be used to perform the ' | 2555 'working_directory and that will be used to perform the ' |
| 2548 'bisection. This parameter is optional, if it is not ' | 2556 'bisection. This parameter is optional, if it is not ' |
| 2549 'supplied, the script will work from the current depot.') | 2557 'supplied, the script will work from the current depot.') |
| 2550 group.add_argument('--build_preference', type=str, | 2558 group.add_argument('--build_preference', |
| 2551 choices=['msvs', 'ninja', 'make'], | 2559 choices=['msvs', 'ninja', 'make'], |
| 2552 help='The preferred build system to use. On linux/mac ' | 2560 help='The preferred build system to use. On linux/mac ' |
| 2553 'the options are make/ninja. On Windows, the ' | 2561 'the options are make/ninja. On Windows, the ' |
| 2554 'options are msvs/ninja.') | 2562 'options are msvs/ninja.') |
| 2555 group.add_argument('--target_platform', type=str, default='chromium', | 2563 group.add_argument('--target_platform', default='chromium', |
| 2556 choices=['chromium', 'android', 'android-chrome'], | 2564 choices=['chromium', 'android', 'android-chrome'], |
| 2557 help='The target platform. Choices are "chromium" ' | 2565 help='The target platform. Choices are "chromium" ' |
| 2558 '(current platform), or "android". If you specify ' | 2566 '(current platform), or "android". If you specify ' |
| 2559 'something other than "chromium", you must be ' | 2567 'something other than "chromium", you must be ' |
| 2560 'properly set up to build that platform.') | 2568 'properly set up to build that platform.') |
| 2561 group.add_argument('--no_custom_deps', dest='no_custom_deps', | 2569 group.add_argument('--no_custom_deps', dest='no_custom_deps', |
| 2562 action='store_true', default=False, | 2570 action='store_true', default=False, |
| 2563 help='Run the script with custom_deps or not.') | 2571 help='Run the script with custom_deps or not.') |
| 2564 group.add_argument('--extra_src', | 2572 group.add_argument('--extra_src', |
| 2565 help='Path to a script which can be used to modify the ' | 2573 help='Path to a script which can be used to modify the ' |
| 2566 'bisect script\'s behavior.') | 2574 'bisect script\'s behavior.') |
| 2567 group.add_argument('--use_goma', action='store_true', | 2575 group.add_argument('--use_goma', action='store_true', |
| 2568 help='Add a bunch of extra threads for goma, and enable ' | 2576 help='Add a bunch of extra threads for goma, and enable ' |
| 2569 'goma') | 2577 'goma') |
| 2570 group.add_argument('--goma_dir', | 2578 group.add_argument('--goma_dir', |
| 2571 help='Path to goma tools (or system default if not ' | 2579 help='Path to goma tools (or system default if not ' |
| 2572 'specified).') | 2580 'specified).') |
| 2573 group.add_argument('--goma_threads', type=int, default='64', | 2581 group.add_argument('--goma_threads', type=int, default='64', |
| 2574 help='Number of threads for goma, only if using goma.') | 2582 help='Number of threads for goma, only if using goma.') |
| 2575 group.add_argument('--output_buildbot_annotations', action='store_true', | 2583 group.add_argument('--output_buildbot_annotations', action='store_true', |
| 2576 help='Add extra annotation output for buildbot.') | 2584 help='Add extra annotation output for buildbot.') |
| 2577 group.add_argument('--gs_bucket', default='', dest='gs_bucket', | 2585 group.add_argument('--gs_bucket', default='', dest='gs_bucket', |
| 2578 help='Name of Google Storage bucket to upload or ' | 2586 help='Name of Google Storage bucket to upload or ' |
| 2579 'download build. e.g., chrome-perf') | 2587 'download build. e.g., chrome-perf') |
| 2580 group.add_argument('--target_arch', type=str, default='ia32', | 2588 group.add_argument('--target_arch', default='ia32', |
| 2581 dest='target_arch', choices=['ia32', 'x64', 'arm'], | 2589 dest='target_arch', choices=['ia32', 'x64', 'arm'], |
| 2582 help='The target build architecture. Choices are "ia32" ' | 2590 help='The target build architecture. Choices are "ia32" ' |
| 2583 '(default), "x64" or "arm".') | 2591 '(default), "x64" or "arm".') |
| 2584 group.add_argument('--target_build_type', type=str, default='Release', | 2592 group.add_argument('--target_build_type', default='Release', |
| 2585 choices=['Release', 'Debug'], | 2593 choices=['Release', 'Debug'], |
| 2586 help='The target build type. Choices are "Release" ' | 2594 help='The target build type. Choices are "Release" ' |
| 2587 '(default), or "Debug".') | 2595 '(default), or "Debug".') |
| 2588 group.add_argument('--builder_host', dest='builder_host', | 2596 group.add_argument('--builder_host', dest='builder_host', |
| 2589 help='Host address of server to produce build by ' | 2597 help='Host address of server to produce build by ' |
| 2590 'posting try job request.') | 2598 'posting try job request.') |
| 2591 group.add_argument('--builder_port', dest='builder_port', type=int, | 2599 group.add_argument('--builder_port', dest='builder_port', type=int, |
| 2592 help='HTTP port of the server to produce build by ' | 2600 help='HTTP port of the server to produce build by ' |
| 2593 'posting try job request.') | 2601 'posting try job request.') |
| 2594 | 2602 |
| (...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2705 logging.basicConfig( | 2713 logging.basicConfig( |
| 2706 stream=logging.sys.stdout, level=logging.INFO, format=logging_format) | 2714 stream=logging.sys.stdout, level=logging.INFO, format=logging_format) |
| 2707 | 2715 |
| 2708 | 2716 |
| 2709 def main(): | 2717 def main(): |
| 2710 _ConfigureLogging() | 2718 _ConfigureLogging() |
| 2711 try: | 2719 try: |
| 2712 opts = BisectOptions() | 2720 opts = BisectOptions() |
| 2713 opts.ParseCommandLine() | 2721 opts.ParseCommandLine() |
| 2714 | 2722 |
| 2723 if opts.bug_id: |
| 2724 if opts.output_buildbot_annotations: |
| 2725 bisect_utils.OutputAnnotationStepStart('Checking Issue Tracker') |
| 2726 issue_closed = query_crbug.CheckIssueClosed(opts.bug_id) |
| 2727 if issue_closed: |
| 2728 print 'Aborting bisect because bug is closed' |
| 2729 else: |
| 2730 print 'Could not confirm bug is closed, proceeding.' |
| 2731 if opts.output_buildbot_annotations: |
| 2732 bisect_utils.OutputAnnotationStepClosed() |
| 2733 if issue_closed: |
| 2734 results = BisectResults(abort_reason='the bug is closed.') |
| 2735 bisect_test = BisectPerformanceMetrics(opts, os.getcwd()) |
| 2736 bisect_test.printer.FormatAndPrintResults(results) |
| 2737 return 0 |
| 2738 |
| 2739 |
| 2715 if opts.extra_src: | 2740 if opts.extra_src: |
| 2716 extra_src = bisect_utils.LoadExtraSrc(opts.extra_src) | 2741 extra_src = bisect_utils.LoadExtraSrc(opts.extra_src) |
| 2717 if not extra_src: | 2742 if not extra_src: |
| 2718 raise RuntimeError('Invalid or missing --extra_src.') | 2743 raise RuntimeError('Invalid or missing --extra_src.') |
| 2719 bisect_utils.AddAdditionalDepotInfo(extra_src.GetAdditionalDepotInfo()) | 2744 bisect_utils.AddAdditionalDepotInfo(extra_src.GetAdditionalDepotInfo()) |
| 2720 | 2745 |
| 2721 if opts.working_directory: | 2746 if opts.working_directory: |
| 2722 custom_deps = bisect_utils.DEFAULT_GCLIENT_CUSTOM_DEPS | 2747 custom_deps = bisect_utils.DEFAULT_GCLIENT_CUSTOM_DEPS |
| 2723 if opts.no_custom_deps: | 2748 if opts.no_custom_deps: |
| 2724 custom_deps = None | 2749 custom_deps = None |
| (...skipping 30 matching lines...) Expand all Loading... |
| 2755 # bugs. If you change this, please update the perf dashboard as well. | 2780 # bugs. If you change this, please update the perf dashboard as well. |
| 2756 bisect_utils.OutputAnnotationStepStart('Results') | 2781 bisect_utils.OutputAnnotationStepStart('Results') |
| 2757 print 'Runtime Error: %s' % e | 2782 print 'Runtime Error: %s' % e |
| 2758 if opts.output_buildbot_annotations: | 2783 if opts.output_buildbot_annotations: |
| 2759 bisect_utils.OutputAnnotationStepClosed() | 2784 bisect_utils.OutputAnnotationStepClosed() |
| 2760 return 1 | 2785 return 1 |
| 2761 | 2786 |
| 2762 | 2787 |
| 2763 if __name__ == '__main__': | 2788 if __name__ == '__main__': |
| 2764 sys.exit(main()) | 2789 sys.exit(main()) |
| OLD | NEW |