OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
| 5 import argparse |
5 import os | 6 import os |
6 import re | 7 import re |
7 import shutil | 8 import shutil |
8 import sys | 9 import sys |
9 import unittest | 10 import unittest |
10 | 11 |
11 SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir) | 12 SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir) |
12 sys.path.append(os.path.join(SRC, 'third_party', 'pymock')) | 13 sys.path.append(os.path.join(SRC, 'third_party', 'pymock')) |
13 | 14 |
14 import bisect_perf_regression | 15 import bisect_perf_regression |
(...skipping 109 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
124 | 125 |
125 def _FakeTestResult(values, bisect_mode_is_return_code): | 126 def _FakeTestResult(values, bisect_mode_is_return_code): |
126 mean = 0.0 | 127 mean = 0.0 |
127 if bisect_mode_is_return_code: | 128 if bisect_mode_is_return_code: |
128 mean = 0 if (all(v == 0 for v in values)) else 1 | 129 mean = 0 if (all(v == 0 for v in values)) else 1 |
129 result_dict = {'mean': mean, 'std_err': 0.0, 'std_dev': 0.0, 'values': values} | 130 result_dict = {'mean': mean, 'std_err': 0.0, 'std_dev': 0.0, 'values': values} |
130 success_code = 0 | 131 success_code = 0 |
131 return (result_dict, success_code) | 132 return (result_dict, success_code) |
132 | 133 |
133 | 134 |
| 135 def _GetMockCallArg(function_mock, call_index): |
| 136 """Gets the first argument value for call at "call_index". |
| 137 |
| 138 Args: |
| 139 function_mock: A Mock object. |
| 140 call_index: The index at which the mocked function was called. |
| 141 |
| 142 Returns: |
| 143 The first argument value. |
| 144 """ |
| 145 call_args_list = function_mock.call_args_list |
| 146 if not call_args_list or len(call_args_list) <= call_index: |
| 147 return None |
| 148 args, _ = call_args_list[call_index] |
| 149 return args |
| 150 |
| 151 |
134 def _GetBisectPerformanceMetricsInstance(options_dict): | 152 def _GetBisectPerformanceMetricsInstance(options_dict): |
135 """Returns an instance of the BisectPerformanceMetrics class.""" | 153 """Returns an instance of the BisectPerformanceMetrics class.""" |
136 opts = bisect_perf_regression.BisectOptions.FromDict(options_dict) | 154 opts = bisect_perf_regression.BisectOptions.FromDict(options_dict) |
137 return bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd()) | 155 return bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd()) |
138 | 156 |
139 | 157 |
140 def _GetExtendedOptions(improvement_dir, fake_first, ignore_confidence=True, | 158 def _GetExtendedOptions(improvement_dir, fake_first, ignore_confidence=True, |
141 **extra_opts): | 159 **extra_opts): |
142 """Returns the a copy of the default options dict plus some options.""" | 160 """Returns the a copy of the default options dict plus some options.""" |
143 result = dict(DEFAULT_OPTIONS) | 161 result = dict(DEFAULT_OPTIONS) |
144 result.update({ | 162 result.update({ |
145 'improvement_direction': improvement_dir, | 163 'improvement_direction': improvement_dir, |
146 'debug_fake_first_test_mean': fake_first, | 164 'debug_fake_first_test_mean': fake_first, |
147 'debug_ignore_regression_confidence': ignore_confidence | 165 'debug_ignore_regression_confidence': ignore_confidence |
148 }) | 166 }) |
149 result.update(extra_opts) | 167 result.update(extra_opts) |
150 return result | 168 return result |
151 | 169 |
152 | 170 |
153 def _GenericDryRun(options, print_results=False): | 171 def _GenericDryRun(options, print_results=False): |
154 """Performs a dry run of the bisector. | 172 """Performs a dry run of the bisector. |
155 | 173 |
156 Args: | 174 Args: |
157 options: Dictionary containing the options for the bisect instance. | 175 options: Dictionary containing the options for the bisect instance. |
158 print_results: Boolean telling whether to call FormatAndPrintResults. | 176 print_results: Boolean telling whether to call FormatAndPrintResults. |
159 | 177 |
160 Returns: | 178 Returns: |
161 The results dictionary as returned by the bisect Run method. | 179 The results dictionary as returned by the bisect Run method. |
162 """ | 180 """ |
163 _AbortIfThereAreStagedChanges() | 181 #_AbortIfThereAreStagedChanges() |
164 # Disable rmtree to avoid deleting local trees. | 182 # Disable rmtree to avoid deleting local trees. |
165 old_rmtree = shutil.rmtree | 183 old_rmtree = shutil.rmtree |
166 shutil.rmtree = lambda path, on_error: None | 184 shutil.rmtree = lambda path, on_error: None |
167 # git reset HEAD may be run during the dry run, which removes staged changes. | 185 # git reset HEAD may be run during the dry run, which removes staged changes. |
168 try: | 186 try: |
169 bisect_instance = _GetBisectPerformanceMetricsInstance(options) | 187 bisect_instance = _GetBisectPerformanceMetricsInstance(options) |
170 results = bisect_instance.Run( | 188 results = bisect_instance.Run( |
171 bisect_instance.opts.command, bisect_instance.opts.bad_revision, | 189 bisect_instance.opts.command, bisect_instance.opts.bad_revision, |
172 bisect_instance.opts.good_revision, bisect_instance.opts.metric) | 190 bisect_instance.opts.good_revision, bisect_instance.opts.metric) |
173 | 191 |
(...skipping 138 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
312 | 330 |
313 def testBisectImprovementDirectionSucceeds(self): | 331 def testBisectImprovementDirectionSucceeds(self): |
314 """Bisects with improvement direction matching regression range.""" | 332 """Bisects with improvement direction matching regression range.""" |
315 # Test result goes from 0 to 100 where lower is better | 333 # Test result goes from 0 to 100 where lower is better |
316 results = _GenericDryRun(_GetExtendedOptions(-1, 100)) | 334 results = _GenericDryRun(_GetExtendedOptions(-1, 100)) |
317 self.assertIsNone(results.error) | 335 self.assertIsNone(results.error) |
318 # Test result goes from 0 to -100 where higher is better | 336 # Test result goes from 0 to -100 where higher is better |
319 results = _GenericDryRun(_GetExtendedOptions(1, -100)) | 337 results = _GenericDryRun(_GetExtendedOptions(1, -100)) |
320 self.assertIsNone(results.error) | 338 self.assertIsNone(results.error) |
321 | 339 |
| 340 @mock.patch('urllib2.urlopen') |
| 341 def testBisectResultsPosted(self, mock_urlopen): |
| 342 """Bisects with improvement direction matching regression range.""" |
| 343 # Test result goes from 0 to 100 where lower is better |
| 344 options_dict = dict(DEFAULT_OPTIONS) |
| 345 options_dict.update({ |
| 346 'bisect_mode': bisect_utils.BISECT_MODE_MEAN, |
| 347 'try_job_id': 1234, |
| 348 }) |
| 349 opts = bisect_perf_regression.BisectOptions.FromDict(options_dict) |
| 350 results = _GenericDryRun(options_dict, True) |
| 351 bisect_perf_regression._PostBisectResults(results, opts, os.getcwd()) |
| 352 |
| 353 call_args = _GetMockCallArg(mock_urlopen, 0) |
| 354 self.assertIsNotNone(call_args) |
| 355 self.assertIn('"try_job_id": 1234', call_args[1]) |
| 356 |
322 def _CheckAbortsEarly(self, results, **extra_opts): | 357 def _CheckAbortsEarly(self, results, **extra_opts): |
323 """Returns True if the bisect job would abort early.""" | 358 """Returns True if the bisect job would abort early.""" |
324 global _MockResultsGenerator | 359 global _MockResultsGenerator |
325 _MockResultsGenerator = (r for r in results) | 360 _MockResultsGenerator = (r for r in results) |
326 bisect_class = bisect_perf_regression.BisectPerformanceMetrics | 361 bisect_class = bisect_perf_regression.BisectPerformanceMetrics |
327 original_run_tests = bisect_class.RunPerformanceTestAndParseResults | 362 original_run_tests = bisect_class.RunPerformanceTestAndParseResults |
328 bisect_class.RunPerformanceTestAndParseResults = _MakeMockRunTests() | 363 bisect_class.RunPerformanceTestAndParseResults = _MakeMockRunTests() |
329 | 364 |
330 try: | 365 try: |
331 dry_run_results = _GenericDryRun(_GetExtendedOptions( | 366 dry_run_results = _GenericDryRun(_GetExtendedOptions( |
(...skipping 355 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
687 (None, 0)), | 722 (None, 0)), |
688 ] | 723 ] |
689 self._SetupRunGitMock(try_cmd) | 724 self._SetupRunGitMock(try_cmd) |
690 bisect_perf_regression._StartBuilderTryJob( | 725 bisect_perf_regression._StartBuilderTryJob( |
691 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name, | 726 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name, |
692 patch) | 727 patch) |
693 | 728 |
694 | 729 |
695 if __name__ == '__main__': | 730 if __name__ == '__main__': |
696 unittest.main() | 731 unittest.main() |
OLD | NEW |