| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import argparse |
| 5 import os | 6 import os |
| 6 import re | 7 import re |
| 7 import shutil | 8 import shutil |
| 8 import sys | 9 import sys |
| 9 import unittest | 10 import unittest |
| 10 | 11 |
| 11 SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir) | 12 SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir) |
| 12 sys.path.append(os.path.join(SRC, 'third_party', 'pymock')) | 13 sys.path.append(os.path.join(SRC, 'third_party', 'pymock')) |
| 13 | 14 |
| 14 import bisect_perf_regression | 15 import bisect_perf_regression |
| (...skipping 109 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 124 | 125 |
| 125 def _FakeTestResult(values, bisect_mode_is_return_code): | 126 def _FakeTestResult(values, bisect_mode_is_return_code): |
| 126 mean = 0.0 | 127 mean = 0.0 |
| 127 if bisect_mode_is_return_code: | 128 if bisect_mode_is_return_code: |
| 128 mean = 0 if (all(v == 0 for v in values)) else 1 | 129 mean = 0 if (all(v == 0 for v in values)) else 1 |
| 129 result_dict = {'mean': mean, 'std_err': 0.0, 'std_dev': 0.0, 'values': values} | 130 result_dict = {'mean': mean, 'std_err': 0.0, 'std_dev': 0.0, 'values': values} |
| 130 success_code = 0 | 131 success_code = 0 |
| 131 return (result_dict, success_code) | 132 return (result_dict, success_code) |
| 132 | 133 |
| 133 | 134 |
| 135 def _GetMockCallArg(function_mock, call_index): |
| 136 """Gets the first argument value for call at "call_index". |
| 137 |
| 138 Args: |
| 139 function_mock: A Mock object. |
| 140 call_index: The index at which the mocked function was called. |
| 141 |
| 142 Returns: |
| 143 The first argument value. |
| 144 """ |
| 145 call_args_list = function_mock.call_args_list |
| 146 if not call_args_list or len(call_args_list) <= call_index: |
| 147 return None |
| 148 args, _ = call_args_list[call_index] |
| 149 return args |
| 150 |
| 151 |
| 134 def _GetBisectPerformanceMetricsInstance(options_dict): | 152 def _GetBisectPerformanceMetricsInstance(options_dict): |
| 135 """Returns an instance of the BisectPerformanceMetrics class.""" | 153 """Returns an instance of the BisectPerformanceMetrics class.""" |
| 136 opts = bisect_perf_regression.BisectOptions.FromDict(options_dict) | 154 opts = bisect_perf_regression.BisectOptions.FromDict(options_dict) |
| 137 return bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd()) | 155 return bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd()) |
| 138 | 156 |
| 139 | 157 |
| 140 def _GetExtendedOptions(improvement_dir, fake_first, ignore_confidence=True, | 158 def _GetExtendedOptions(improvement_dir, fake_first, ignore_confidence=True, |
| 141 **extra_opts): | 159 **extra_opts): |
| 142 """Returns the a copy of the default options dict plus some options.""" | 160 """Returns the a copy of the default options dict plus some options.""" |
| 143 result = dict(DEFAULT_OPTIONS) | 161 result = dict(DEFAULT_OPTIONS) |
| 144 result.update({ | 162 result.update({ |
| 145 'improvement_direction': improvement_dir, | 163 'improvement_direction': improvement_dir, |
| 146 'debug_fake_first_test_mean': fake_first, | 164 'debug_fake_first_test_mean': fake_first, |
| 147 'debug_ignore_regression_confidence': ignore_confidence | 165 'debug_ignore_regression_confidence': ignore_confidence |
| 148 }) | 166 }) |
| 149 result.update(extra_opts) | 167 result.update(extra_opts) |
| 150 return result | 168 return result |
| 151 | 169 |
| 152 | 170 |
| 153 def _GenericDryRun(options, print_results=False): | 171 def _GenericDryRun(options, print_results=False): |
| 154 """Performs a dry run of the bisector. | 172 """Performs a dry run of the bisector. |
| 155 | 173 |
| 156 Args: | 174 Args: |
| 157 options: Dictionary containing the options for the bisect instance. | 175 options: Dictionary containing the options for the bisect instance. |
| 158 print_results: Boolean telling whether to call FormatAndPrintResults. | 176 print_results: Boolean telling whether to call FormatAndPrintResults. |
| 159 | 177 |
| 160 Returns: | 178 Returns: |
| 161 The results dictionary as returned by the bisect Run method. | 179 The results dictionary as returned by the bisect Run method. |
| 162 """ | 180 """ |
| 163 _AbortIfThereAreStagedChanges() | 181 #_AbortIfThereAreStagedChanges() |
| 164 # Disable rmtree to avoid deleting local trees. | 182 # Disable rmtree to avoid deleting local trees. |
| 165 old_rmtree = shutil.rmtree | 183 old_rmtree = shutil.rmtree |
| 166 shutil.rmtree = lambda path, on_error: None | 184 shutil.rmtree = lambda path, on_error: None |
| 167 # git reset HEAD may be run during the dry run, which removes staged changes. | 185 # git reset HEAD may be run during the dry run, which removes staged changes. |
| 168 try: | 186 try: |
| 169 bisect_instance = _GetBisectPerformanceMetricsInstance(options) | 187 bisect_instance = _GetBisectPerformanceMetricsInstance(options) |
| 170 results = bisect_instance.Run( | 188 results = bisect_instance.Run( |
| 171 bisect_instance.opts.command, bisect_instance.opts.bad_revision, | 189 bisect_instance.opts.command, bisect_instance.opts.bad_revision, |
| 172 bisect_instance.opts.good_revision, bisect_instance.opts.metric) | 190 bisect_instance.opts.good_revision, bisect_instance.opts.metric) |
| 173 | 191 |
| (...skipping 138 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 312 | 330 |
| 313 def testBisectImprovementDirectionSucceeds(self): | 331 def testBisectImprovementDirectionSucceeds(self): |
| 314 """Bisects with improvement direction matching regression range.""" | 332 """Bisects with improvement direction matching regression range.""" |
| 315 # Test result goes from 0 to 100 where lower is better | 333 # Test result goes from 0 to 100 where lower is better |
| 316 results = _GenericDryRun(_GetExtendedOptions(-1, 100)) | 334 results = _GenericDryRun(_GetExtendedOptions(-1, 100)) |
| 317 self.assertIsNone(results.error) | 335 self.assertIsNone(results.error) |
| 318 # Test result goes from 0 to -100 where higher is better | 336 # Test result goes from 0 to -100 where higher is better |
| 319 results = _GenericDryRun(_GetExtendedOptions(1, -100)) | 337 results = _GenericDryRun(_GetExtendedOptions(1, -100)) |
| 320 self.assertIsNone(results.error) | 338 self.assertIsNone(results.error) |
| 321 | 339 |
| 340 @mock.patch('urllib2.urlopen') |
| 341 def testBisectResultsPosted(self, mock_urlopen): |
| 342 """Bisects with improvement direction matching regression range.""" |
| 343 # Test result goes from 0 to 100 where lower is better |
| 344 options_dict = dict(DEFAULT_OPTIONS) |
| 345 options_dict.update({ |
| 346 'bisect_mode': bisect_utils.BISECT_MODE_MEAN, |
| 347 'try_job_id': 1234, |
| 348 }) |
| 349 opts = bisect_perf_regression.BisectOptions.FromDict(options_dict) |
| 350 results = _GenericDryRun(options_dict, True) |
| 351 bisect_perf_regression._PostBisectResults(results, opts, os.getcwd()) |
| 352 |
| 353 call_args = _GetMockCallArg(mock_urlopen, 0) |
| 354 self.assertIsNotNone(call_args) |
| 355 self.assertIn('"try_job_id": 1234', call_args[1]) |
| 356 |
| 322 def _CheckAbortsEarly(self, results, **extra_opts): | 357 def _CheckAbortsEarly(self, results, **extra_opts): |
| 323 """Returns True if the bisect job would abort early.""" | 358 """Returns True if the bisect job would abort early.""" |
| 324 global _MockResultsGenerator | 359 global _MockResultsGenerator |
| 325 _MockResultsGenerator = (r for r in results) | 360 _MockResultsGenerator = (r for r in results) |
| 326 bisect_class = bisect_perf_regression.BisectPerformanceMetrics | 361 bisect_class = bisect_perf_regression.BisectPerformanceMetrics |
| 327 original_run_tests = bisect_class.RunPerformanceTestAndParseResults | 362 original_run_tests = bisect_class.RunPerformanceTestAndParseResults |
| 328 bisect_class.RunPerformanceTestAndParseResults = _MakeMockRunTests() | 363 bisect_class.RunPerformanceTestAndParseResults = _MakeMockRunTests() |
| 329 | 364 |
| 330 try: | 365 try: |
| 331 dry_run_results = _GenericDryRun(_GetExtendedOptions( | 366 dry_run_results = _GenericDryRun(_GetExtendedOptions( |
| (...skipping 355 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 687 (None, 0)), | 722 (None, 0)), |
| 688 ] | 723 ] |
| 689 self._SetupRunGitMock(try_cmd) | 724 self._SetupRunGitMock(try_cmd) |
| 690 bisect_perf_regression._StartBuilderTryJob( | 725 bisect_perf_regression._StartBuilderTryJob( |
| 691 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name, | 726 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name, |
| 692 patch) | 727 patch) |
| 693 | 728 |
| 694 | 729 |
| 695 if __name__ == '__main__': | 730 if __name__ == '__main__': |
| 696 unittest.main() | 731 unittest.main() |
| OLD | NEW |