OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Performance Test Bisect Tool | 6 """Performance Test Bisect Tool |
7 | 7 |
8 This script bisects a series of changelists using binary search. It starts at | 8 This script bisects a series of changelists using binary search. It starts at |
9 a bad revision where a performance metric has regressed, and asks for a last | 9 a bad revision where a performance metric has regressed, and asks for a last |
10 known-good revision. It will then binary search across this revision range by | 10 known-good revision. It will then binary search across this revision range by |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
47 import zipfile | 47 import zipfile |
48 | 48 |
49 sys.path.append(os.path.join( | 49 sys.path.append(os.path.join( |
50 os.path.dirname(__file__), os.path.pardir, 'telemetry')) | 50 os.path.dirname(__file__), os.path.pardir, 'telemetry')) |
51 | 51 |
52 from bisect_printer import BisectPrinter | 52 from bisect_printer import BisectPrinter |
53 from bisect_results import BisectResults | 53 from bisect_results import BisectResults |
54 from bisect_state import BisectState | 54 from bisect_state import BisectState |
55 import bisect_utils | 55 import bisect_utils |
56 import builder | 56 import builder |
| 57 import query_crbug |
57 import math_utils | 58 import math_utils |
58 import request_build | 59 import request_build |
59 import source_control | 60 import source_control |
60 from telemetry.util import cloud_storage | 61 from telemetry.util import cloud_storage |
61 | 62 |
62 # The script is in chromium/src/tools/auto_bisect. Throughout this script, | 63 # The script is in chromium/src/tools/auto_bisect. Throughout this script, |
63 # we use paths to other things in the chromium/src repository. | 64 # we use paths to other things in the chromium/src repository. |
64 | 65 |
65 # Possible return values from BisectPerformanceMetrics.RunTest. | 66 # Possible return values from BisectPerformanceMetrics.RunTest. |
66 BUILD_RESULT_SUCCEED = 0 | 67 BUILD_RESULT_SUCCEED = 0 |
(...skipping 2060 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2127 | 2128 |
2128 Args: | 2129 Args: |
2129 command_to_run: Specify the command to execute the performance test. | 2130 command_to_run: Specify the command to execute the performance test. |
2130 good_revision: Number/tag of the known good revision. | 2131 good_revision: Number/tag of the known good revision. |
2131 bad_revision: Number/tag of the known bad revision. | 2132 bad_revision: Number/tag of the known bad revision. |
2132 metric: The performance metric to monitor. | 2133 metric: The performance metric to monitor. |
2133 | 2134 |
2134 Returns: | 2135 Returns: |
2135 A BisectResults object. | 2136 A BisectResults object. |
2136 """ | 2137 """ |
| 2138 |
2137 # Choose depot to bisect first | 2139 # Choose depot to bisect first |
2138 target_depot = 'chromium' | 2140 target_depot = 'chromium' |
2139 if self.opts.target_platform == 'android-chrome': | 2141 if self.opts.target_platform == 'android-chrome': |
2140 target_depot = 'android-chrome' | 2142 target_depot = 'android-chrome' |
2141 | 2143 |
2142 cwd = os.getcwd() | 2144 cwd = os.getcwd() |
2143 self.depot_registry.ChangeToDepotDir(target_depot) | 2145 self.depot_registry.ChangeToDepotDir(target_depot) |
2144 | 2146 |
2145 # If they passed SVN revisions, we can try match them to git SHA1 hashes. | 2147 # If they passed SVN revisions, we can try match them to git SHA1 hashes. |
2146 bad_revision = source_control.ResolveToRevision( | 2148 bad_revision = source_control.ResolveToRevision( |
(...skipping 335 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2482 self.debug_ignore_perf_test = None | 2484 self.debug_ignore_perf_test = None |
2483 self.debug_ignore_regression_confidence = None | 2485 self.debug_ignore_regression_confidence = None |
2484 self.debug_fake_first_test_mean = 0 | 2486 self.debug_fake_first_test_mean = 0 |
2485 self.gs_bucket = None | 2487 self.gs_bucket = None |
2486 self.target_arch = 'ia32' | 2488 self.target_arch = 'ia32' |
2487 self.target_build_type = 'Release' | 2489 self.target_build_type = 'Release' |
2488 self.builder_host = None | 2490 self.builder_host = None |
2489 self.builder_port = None | 2491 self.builder_port = None |
2490 self.bisect_mode = bisect_utils.BISECT_MODE_MEAN | 2492 self.bisect_mode = bisect_utils.BISECT_MODE_MEAN |
2491 self.improvement_direction = 0 | 2493 self.improvement_direction = 0 |
| 2494 self.bug_id = '' |
2492 | 2495 |
2493 @staticmethod | 2496 @staticmethod |
2494 def _AddBisectOptionsGroup(parser): | 2497 def _AddBisectOptionsGroup(parser): |
2495 group = parser.add_argument_group('Bisect options') | 2498 group = parser.add_argument_group('Bisect options') |
2496 group.add_argument('-c', '--command', required=True, | 2499 group.add_argument('-c', '--command', required=True, |
2497 help='A command to execute your performance test at ' | 2500 help='A command to execute your performance test at ' |
2498 'each point in the bisection.') | 2501 'each point in the bisection.') |
2499 group.add_argument('-b', '--bad_revision', required=True, | 2502 group.add_argument('-b', '--bad_revision', required=True, |
2500 help='A bad revision to start bisection. Must be later ' | 2503 help='A bad revision to start bisection. Must be later ' |
2501 'than good revision. May be either a git or svn ' | 2504 'than good revision. May be either a git or svn ' |
(...skipping 27 matching lines...) Expand all Loading... |
2529 help='The highest/lowest % are discarded to form a ' | 2532 help='The highest/lowest % are discarded to form a ' |
2530 'truncated mean. Values will be clamped to range ' | 2533 'truncated mean. Values will be clamped to range ' |
2531 '[0, 25]. Default value is 25 (highest/lowest 25% ' | 2534 '[0, 25]. Default value is 25 (highest/lowest 25% ' |
2532 'will be discarded).') | 2535 'will be discarded).') |
2533 group.add_argument('--bisect_mode', default=bisect_utils.BISECT_MODE_MEAN, | 2536 group.add_argument('--bisect_mode', default=bisect_utils.BISECT_MODE_MEAN, |
2534 choices=[bisect_utils.BISECT_MODE_MEAN, | 2537 choices=[bisect_utils.BISECT_MODE_MEAN, |
2535 bisect_utils.BISECT_MODE_STD_DEV, | 2538 bisect_utils.BISECT_MODE_STD_DEV, |
2536 bisect_utils.BISECT_MODE_RETURN_CODE], | 2539 bisect_utils.BISECT_MODE_RETURN_CODE], |
2537 help='The bisect mode. Choices are to bisect on the ' | 2540 help='The bisect mode. Choices are to bisect on the ' |
2538 'difference in mean, std_dev, or return_code.') | 2541 'difference in mean, std_dev, or return_code.') |
| 2542 group.add_argument('--bug_id', default='', |
| 2543 help='The id for the bug associated with this bisect. ' + |
| 2544 'If this number is given, bisect will attempt to ' + |
| 2545 'verify that the bug is not closed before ' |
| 2546 'starting.') |
2539 | 2547 |
2540 @staticmethod | 2548 @staticmethod |
2541 def _AddBuildOptionsGroup(parser): | 2549 def _AddBuildOptionsGroup(parser): |
2542 group = parser.add_argument_group('Build options') | 2550 group = parser.add_argument_group('Build options') |
2543 group.add_argument('-w', '--working_directory', | 2551 group.add_argument('-w', '--working_directory', |
2544 help='Path to the working directory where the script ' | 2552 help='Path to the working directory where the script ' |
2545 'will do an initial checkout of the chromium depot. The ' | 2553 'will do an initial checkout of the chromium depot. The ' |
2546 'files will be placed in a subdirectory "bisect" under ' | 2554 'files will be placed in a subdirectory "bisect" under ' |
2547 'working_directory and that will be used to perform the ' | 2555 'working_directory and that will be used to perform the ' |
2548 'bisection. This parameter is optional, if it is not ' | 2556 'bisection. This parameter is optional, if it is not ' |
2549 'supplied, the script will work from the current depot.') | 2557 'supplied, the script will work from the current depot.') |
2550 group.add_argument('--build_preference', type=str, | 2558 group.add_argument('--build_preference', |
2551 choices=['msvs', 'ninja', 'make'], | 2559 choices=['msvs', 'ninja', 'make'], |
2552 help='The preferred build system to use. On linux/mac ' | 2560 help='The preferred build system to use. On linux/mac ' |
2553 'the options are make/ninja. On Windows, the ' | 2561 'the options are make/ninja. On Windows, the ' |
2554 'options are msvs/ninja.') | 2562 'options are msvs/ninja.') |
2555 group.add_argument('--target_platform', type=str, default='chromium', | 2563 group.add_argument('--target_platform', default='chromium', |
2556 choices=['chromium', 'android', 'android-chrome'], | 2564 choices=['chromium', 'android', 'android-chrome'], |
2557 help='The target platform. Choices are "chromium" ' | 2565 help='The target platform. Choices are "chromium" ' |
2558 '(current platform), or "android". If you specify ' | 2566 '(current platform), or "android". If you specify ' |
2559 'something other than "chromium", you must be ' | 2567 'something other than "chromium", you must be ' |
2560 'properly set up to build that platform.') | 2568 'properly set up to build that platform.') |
2561 group.add_argument('--no_custom_deps', dest='no_custom_deps', | 2569 group.add_argument('--no_custom_deps', dest='no_custom_deps', |
2562 action='store_true', default=False, | 2570 action='store_true', default=False, |
2563 help='Run the script with custom_deps or not.') | 2571 help='Run the script with custom_deps or not.') |
2564 group.add_argument('--extra_src', | 2572 group.add_argument('--extra_src', |
2565 help='Path to a script which can be used to modify the ' | 2573 help='Path to a script which can be used to modify the ' |
2566 'bisect script\'s behavior.') | 2574 'bisect script\'s behavior.') |
2567 group.add_argument('--use_goma', action='store_true', | 2575 group.add_argument('--use_goma', action='store_true', |
2568 help='Add a bunch of extra threads for goma, and enable ' | 2576 help='Add a bunch of extra threads for goma, and enable ' |
2569 'goma') | 2577 'goma') |
2570 group.add_argument('--goma_dir', | 2578 group.add_argument('--goma_dir', |
2571 help='Path to goma tools (or system default if not ' | 2579 help='Path to goma tools (or system default if not ' |
2572 'specified).') | 2580 'specified).') |
2573 group.add_argument('--goma_threads', type=int, default='64', | 2581 group.add_argument('--goma_threads', type=int, default='64', |
2574 help='Number of threads for goma, only if using goma.') | 2582 help='Number of threads for goma, only if using goma.') |
2575 group.add_argument('--output_buildbot_annotations', action='store_true', | 2583 group.add_argument('--output_buildbot_annotations', action='store_true', |
2576 help='Add extra annotation output for buildbot.') | 2584 help='Add extra annotation output for buildbot.') |
2577 group.add_argument('--gs_bucket', default='', dest='gs_bucket', | 2585 group.add_argument('--gs_bucket', default='', dest='gs_bucket', |
2578 help='Name of Google Storage bucket to upload or ' | 2586 help='Name of Google Storage bucket to upload or ' |
2579 'download build. e.g., chrome-perf') | 2587 'download build. e.g., chrome-perf') |
2580 group.add_argument('--target_arch', type=str, default='ia32', | 2588 group.add_argument('--target_arch', default='ia32', |
2581 dest='target_arch', choices=['ia32', 'x64', 'arm'], | 2589 dest='target_arch', choices=['ia32', 'x64', 'arm'], |
2582 help='The target build architecture. Choices are "ia32" ' | 2590 help='The target build architecture. Choices are "ia32" ' |
2583 '(default), "x64" or "arm".') | 2591 '(default), "x64" or "arm".') |
2584 group.add_argument('--target_build_type', type=str, default='Release', | 2592 group.add_argument('--target_build_type', default='Release', |
2585 choices=['Release', 'Debug'], | 2593 choices=['Release', 'Debug'], |
2586 help='The target build type. Choices are "Release" ' | 2594 help='The target build type. Choices are "Release" ' |
2587 '(default), or "Debug".') | 2595 '(default), or "Debug".') |
2588 group.add_argument('--builder_host', dest='builder_host', | 2596 group.add_argument('--builder_host', dest='builder_host', |
2589 help='Host address of server to produce build by ' | 2597 help='Host address of server to produce build by ' |
2590 'posting try job request.') | 2598 'posting try job request.') |
2591 group.add_argument('--builder_port', dest='builder_port', type=int, | 2599 group.add_argument('--builder_port', dest='builder_port', type=int, |
2592 help='HTTP port of the server to produce build by ' | 2600 help='HTTP port of the server to produce build by ' |
2593 'posting try job request.') | 2601 'posting try job request.') |
2594 | 2602 |
(...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2705 logging.basicConfig( | 2713 logging.basicConfig( |
2706 stream=logging.sys.stdout, level=logging.INFO, format=logging_format) | 2714 stream=logging.sys.stdout, level=logging.INFO, format=logging_format) |
2707 | 2715 |
2708 | 2716 |
2709 def main(): | 2717 def main(): |
2710 _ConfigureLogging() | 2718 _ConfigureLogging() |
2711 try: | 2719 try: |
2712 opts = BisectOptions() | 2720 opts = BisectOptions() |
2713 opts.ParseCommandLine() | 2721 opts.ParseCommandLine() |
2714 | 2722 |
| 2723 if opts.bug_id: |
| 2724 if opts.output_buildbot_annotations: |
| 2725 bisect_utils.OutputAnnotationStepStart('Checking Issue Tracker') |
| 2726 issue_closed = query_crbug.CheckIssueClosed(opts.bug_id) |
| 2727 if issue_closed: |
| 2728 print 'Aborting bisect because bug is closed' |
| 2729 else: |
| 2730 print 'Could not confirm bug is closed, proceeding.' |
| 2731 if opts.output_buildbot_annotations: |
| 2732 bisect_utils.OutputAnnotationStepClosed() |
| 2733 if issue_closed: |
| 2734 results = BisectResults(abort_reason='the bug is closed.') |
| 2735 bisect_test = BisectPerformanceMetrics(opts, os.getcwd()) |
| 2736 bisect_test.printer.FormatAndPrintResults(results) |
| 2737 return 0 |
| 2738 |
| 2739 |
2715 if opts.extra_src: | 2740 if opts.extra_src: |
2716 extra_src = bisect_utils.LoadExtraSrc(opts.extra_src) | 2741 extra_src = bisect_utils.LoadExtraSrc(opts.extra_src) |
2717 if not extra_src: | 2742 if not extra_src: |
2718 raise RuntimeError('Invalid or missing --extra_src.') | 2743 raise RuntimeError('Invalid or missing --extra_src.') |
2719 bisect_utils.AddAdditionalDepotInfo(extra_src.GetAdditionalDepotInfo()) | 2744 bisect_utils.AddAdditionalDepotInfo(extra_src.GetAdditionalDepotInfo()) |
2720 | 2745 |
2721 if opts.working_directory: | 2746 if opts.working_directory: |
2722 custom_deps = bisect_utils.DEFAULT_GCLIENT_CUSTOM_DEPS | 2747 custom_deps = bisect_utils.DEFAULT_GCLIENT_CUSTOM_DEPS |
2723 if opts.no_custom_deps: | 2748 if opts.no_custom_deps: |
2724 custom_deps = None | 2749 custom_deps = None |
(...skipping 30 matching lines...) Expand all Loading... |
2755 # bugs. If you change this, please update the perf dashboard as well. | 2780 # bugs. If you change this, please update the perf dashboard as well. |
2756 bisect_utils.OutputAnnotationStepStart('Results') | 2781 bisect_utils.OutputAnnotationStepStart('Results') |
2757 print 'Runtime Error: %s' % e | 2782 print 'Runtime Error: %s' % e |
2758 if opts.output_buildbot_annotations: | 2783 if opts.output_buildbot_annotations: |
2759 bisect_utils.OutputAnnotationStepClosed() | 2784 bisect_utils.OutputAnnotationStepClosed() |
2760 return 1 | 2785 return 1 |
2761 | 2786 |
2762 | 2787 |
2763 if __name__ == '__main__': | 2788 if __name__ == '__main__': |
2764 sys.exit(main()) | 2789 sys.exit(main()) |
OLD | NEW |