OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import os | 5 import os |
6 import re | 6 import re |
7 import shutil | 7 import shutil |
8 import sys | 8 import sys |
9 import unittest | 9 import unittest |
10 | 10 |
11 SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir) | 11 SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir) |
12 sys.path.append(os.path.join(SRC, 'third_party', 'pymock')) | 12 sys.path.append(os.path.join(SRC, 'third_party', 'pymock')) |
13 | 13 |
14 import bisect_perf_regression | 14 import bisect_perf_regression |
| 15 import bisect_results |
| 16 import bisect_state |
15 import bisect_utils | 17 import bisect_utils |
16 import fetch_build | 18 import fetch_build |
17 import mock | 19 import mock |
18 import source_control | 20 import source_control |
19 | 21 |
20 | 22 |
21 # Regression confidence: 0% | 23 # Regression confidence: 0% |
22 CLEAR_NON_REGRESSION = [ | 24 CLEAR_NON_REGRESSION = [ |
23 # Mean: 30.223 Std. Dev.: 11.383 | 25 # Mean: 30.223 Std. Dev.: 11.383 |
24 [[16.886], [16.909], [16.99], [17.723], [17.952], [18.118], [19.028], | 26 [[16.886], [16.909], [16.99], [17.723], [17.952], [18.118], [19.028], |
(...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
124 | 126 |
125 def _FakeTestResult(values, bisect_mode_is_return_code): | 127 def _FakeTestResult(values, bisect_mode_is_return_code): |
126 mean = 0.0 | 128 mean = 0.0 |
127 if bisect_mode_is_return_code: | 129 if bisect_mode_is_return_code: |
128 mean = 0 if (all(v == 0 for v in values)) else 1 | 130 mean = 0 if (all(v == 0 for v in values)) else 1 |
129 result_dict = {'mean': mean, 'std_err': 0.0, 'std_dev': 0.0, 'values': values} | 131 result_dict = {'mean': mean, 'std_err': 0.0, 'std_dev': 0.0, 'values': values} |
130 success_code = 0 | 132 success_code = 0 |
131 return (result_dict, success_code) | 133 return (result_dict, success_code) |
132 | 134 |
133 | 135 |
| 136 def _SampleBisecResult(opts): |
| 137 revisions = [ |
| 138 'ae7ef14ba2d9b5ef0d2c1c092ec98a417e44740d' |
| 139 'ab55ead638496b061c9de61685b982f7cea38ca7', |
| 140 '89aa0c99e4b977b9a4f992ac14da0d6624f7316e'] |
| 141 state = bisect_state.BisectState(depot='chromium', revisions=revisions) |
| 142 depot_registry = bisect_perf_regression.DepotDirectoryRegistry('/mock/src') |
| 143 results = bisect_results.BisectResults( |
| 144 bisect_state=state, depot_registry=depot_registry, opts=opts, |
| 145 runtime_warnings=[]) |
| 146 results.confidence = 99.9 |
| 147 results.culprit_revisions = [( |
| 148 'ab55ead638496b061c9de61685b982f7cea38ca7', |
| 149 { |
| 150 'date': 'Thu, 26 Jun 2014 14:29:49 +0000', |
| 151 'body': 'Fix', |
| 152 'author': 'author@chromium.org', |
| 153 'subject': 'Fix', |
| 154 'email': 'author@chromium.org', |
| 155 }, |
| 156 'chromium')] |
| 157 return results |
| 158 |
| 159 |
| 160 def _GetMockCallArg(function_mock, call_index): |
| 161 """Gets the list of called arguments for call at |call_index|. |
| 162 |
| 163 Args: |
| 164 function_mock: A Mock object. |
| 165 call_index: The index at which the mocked function was called. |
| 166 |
| 167 Returns: |
| 168 The called argument list. |
| 169 """ |
| 170 call_args_list = function_mock.call_args_list |
| 171 if not call_args_list or len(call_args_list) <= call_index: |
| 172 return None |
| 173 args, _ = call_args_list[call_index] |
| 174 return args |
| 175 |
| 176 |
134 def _GetBisectPerformanceMetricsInstance(options_dict): | 177 def _GetBisectPerformanceMetricsInstance(options_dict): |
135 """Returns an instance of the BisectPerformanceMetrics class.""" | 178 """Returns an instance of the BisectPerformanceMetrics class.""" |
136 opts = bisect_perf_regression.BisectOptions.FromDict(options_dict) | 179 opts = bisect_perf_regression.BisectOptions.FromDict(options_dict) |
137 return bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd()) | 180 return bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd()) |
138 | 181 |
139 | 182 |
140 def _GetExtendedOptions(improvement_dir, fake_first, ignore_confidence=True, | 183 def _GetExtendedOptions(improvement_dir, fake_first, ignore_confidence=True, |
141 **extra_opts): | 184 **extra_opts): |
142 """Returns the a copy of the default options dict plus some options.""" | 185 """Returns the a copy of the default options dict plus some options.""" |
143 result = dict(DEFAULT_OPTIONS) | 186 result = dict(DEFAULT_OPTIONS) |
(...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
312 | 355 |
313 def testBisectImprovementDirectionSucceeds(self): | 356 def testBisectImprovementDirectionSucceeds(self): |
314 """Bisects with improvement direction matching regression range.""" | 357 """Bisects with improvement direction matching regression range.""" |
315 # Test result goes from 0 to 100 where lower is better | 358 # Test result goes from 0 to 100 where lower is better |
316 results = _GenericDryRun(_GetExtendedOptions(-1, 100)) | 359 results = _GenericDryRun(_GetExtendedOptions(-1, 100)) |
317 self.assertIsNone(results.error) | 360 self.assertIsNone(results.error) |
318 # Test result goes from 0 to -100 where higher is better | 361 # Test result goes from 0 to -100 where higher is better |
319 results = _GenericDryRun(_GetExtendedOptions(1, -100)) | 362 results = _GenericDryRun(_GetExtendedOptions(1, -100)) |
320 self.assertIsNone(results.error) | 363 self.assertIsNone(results.error) |
321 | 364 |
| 365 @mock.patch('urllib2.urlopen') |
| 366 def testBisectResultsPosted(self, mock_urlopen): |
| 367 options_dict = dict(DEFAULT_OPTIONS) |
| 368 options_dict.update({ |
| 369 'bisect_mode': bisect_utils.BISECT_MODE_MEAN, |
| 370 'try_job_id': 1234, |
| 371 }) |
| 372 opts = bisect_perf_regression.BisectOptions.FromDict(options_dict) |
| 373 results = _SampleBisecResult(opts) |
| 374 bisect_perf_regression._PostBisectResults(results, opts, os.getcwd()) |
| 375 |
| 376 call_args = _GetMockCallArg(mock_urlopen, 0) |
| 377 self.assertIsNotNone(call_args) |
| 378 self.assertIn('"try_job_id": 1234', call_args[1]) |
| 379 |
322 def _CheckAbortsEarly(self, results, **extra_opts): | 380 def _CheckAbortsEarly(self, results, **extra_opts): |
323 """Returns True if the bisect job would abort early.""" | 381 """Returns True if the bisect job would abort early.""" |
324 global _MockResultsGenerator | 382 global _MockResultsGenerator |
325 _MockResultsGenerator = (r for r in results) | 383 _MockResultsGenerator = (r for r in results) |
326 bisect_class = bisect_perf_regression.BisectPerformanceMetrics | 384 bisect_class = bisect_perf_regression.BisectPerformanceMetrics |
327 original_run_tests = bisect_class.RunPerformanceTestAndParseResults | 385 original_run_tests = bisect_class.RunPerformanceTestAndParseResults |
328 bisect_class.RunPerformanceTestAndParseResults = _MakeMockRunTests() | 386 bisect_class.RunPerformanceTestAndParseResults = _MakeMockRunTests() |
329 | 387 |
330 try: | 388 try: |
331 dry_run_results = _GenericDryRun(_GetExtendedOptions( | 389 dry_run_results = _GenericDryRun(_GetExtendedOptions( |
(...skipping 355 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
687 (None, 0)), | 745 (None, 0)), |
688 ] | 746 ] |
689 self._SetupRunGitMock(try_cmd) | 747 self._SetupRunGitMock(try_cmd) |
690 bisect_perf_regression._StartBuilderTryJob( | 748 bisect_perf_regression._StartBuilderTryJob( |
691 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name, | 749 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name, |
692 patch) | 750 patch) |
693 | 751 |
694 | 752 |
695 if __name__ == '__main__': | 753 if __name__ == '__main__': |
696 unittest.main() | 754 unittest.main() |
OLD | NEW |