Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1)

Side by Side Diff: tools/auto_bisect/bisect_perf_regression_test.py

Issue 1625573004: Update legacy bisect to post results to perf dashboard. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@fix_import_path
Patch Set: update Created 4 years, 10 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tools/auto_bisect/bisect_perf_regression.py ('k') | tools/auto_bisect/bisect_results.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2014 The Chromium Authors. All rights reserved. 1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import os 5 import os
6 import re 6 import re
7 import shutil 7 import shutil
8 import sys 8 import sys
9 import unittest 9 import unittest
10 10
11 SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir) 11 SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)
12 sys.path.append(os.path.join(SRC, 'third_party', 'pymock')) 12 sys.path.append(os.path.join(SRC, 'third_party', 'pymock'))
13 13
14 import bisect_perf_regression 14 import bisect_perf_regression
15 import bisect_results
16 import bisect_state
15 import bisect_utils 17 import bisect_utils
16 import fetch_build 18 import fetch_build
17 import mock 19 import mock
18 import source_control 20 import source_control
19 21
20 22
21 # Regression confidence: 0% 23 # Regression confidence: 0%
22 CLEAR_NON_REGRESSION = [ 24 CLEAR_NON_REGRESSION = [
23 # Mean: 30.223 Std. Dev.: 11.383 25 # Mean: 30.223 Std. Dev.: 11.383
24 [[16.886], [16.909], [16.99], [17.723], [17.952], [18.118], [19.028], 26 [[16.886], [16.909], [16.99], [17.723], [17.952], [18.118], [19.028],
(...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after
124 126
125 def _FakeTestResult(values, bisect_mode_is_return_code): 127 def _FakeTestResult(values, bisect_mode_is_return_code):
126 mean = 0.0 128 mean = 0.0
127 if bisect_mode_is_return_code: 129 if bisect_mode_is_return_code:
128 mean = 0 if (all(v == 0 for v in values)) else 1 130 mean = 0 if (all(v == 0 for v in values)) else 1
129 result_dict = {'mean': mean, 'std_err': 0.0, 'std_dev': 0.0, 'values': values} 131 result_dict = {'mean': mean, 'std_err': 0.0, 'std_dev': 0.0, 'values': values}
130 success_code = 0 132 success_code = 0
131 return (result_dict, success_code) 133 return (result_dict, success_code)
132 134
133 135
136 def _SampleBisecResult(opts):
137 revisions = [
138 'ae7ef14ba2d9b5ef0d2c1c092ec98a417e44740d'
139 'ab55ead638496b061c9de61685b982f7cea38ca7',
140 '89aa0c99e4b977b9a4f992ac14da0d6624f7316e']
141 state = bisect_state.BisectState(depot='chromium', revisions=revisions)
142 depot_registry = bisect_perf_regression.DepotDirectoryRegistry('/mock/src')
143 results = bisect_results.BisectResults(
144 bisect_state=state, depot_registry=depot_registry, opts=opts,
145 runtime_warnings=[])
146 results.confidence = 99.9
147 results.culprit_revisions = [(
148 'ab55ead638496b061c9de61685b982f7cea38ca7',
149 {
150 'date': 'Thu, 26 Jun 2014 14:29:49 +0000',
151 'body': 'Fix',
152 'author': 'author@chromium.org',
153 'subject': 'Fix',
154 'email': 'author@chromium.org',
155 },
156 'chromium')]
157 return results
158
159
160 def _GetMockCallArg(function_mock, call_index):
161 """Gets the list of called arguments for call at |call_index|.
162
163 Args:
164 function_mock: A Mock object.
165 call_index: The index at which the mocked function was called.
166
167 Returns:
168 The called argument list.
169 """
170 call_args_list = function_mock.call_args_list
171 if not call_args_list or len(call_args_list) <= call_index:
172 return None
173 args, _ = call_args_list[call_index]
174 return args
175
176
134 def _GetBisectPerformanceMetricsInstance(options_dict): 177 def _GetBisectPerformanceMetricsInstance(options_dict):
135 """Returns an instance of the BisectPerformanceMetrics class.""" 178 """Returns an instance of the BisectPerformanceMetrics class."""
136 opts = bisect_perf_regression.BisectOptions.FromDict(options_dict) 179 opts = bisect_perf_regression.BisectOptions.FromDict(options_dict)
137 return bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd()) 180 return bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd())
138 181
139 182
140 def _GetExtendedOptions(improvement_dir, fake_first, ignore_confidence=True, 183 def _GetExtendedOptions(improvement_dir, fake_first, ignore_confidence=True,
141 **extra_opts): 184 **extra_opts):
142 """Returns the a copy of the default options dict plus some options.""" 185 """Returns the a copy of the default options dict plus some options."""
143 result = dict(DEFAULT_OPTIONS) 186 result = dict(DEFAULT_OPTIONS)
(...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after
312 355
313 def testBisectImprovementDirectionSucceeds(self): 356 def testBisectImprovementDirectionSucceeds(self):
314 """Bisects with improvement direction matching regression range.""" 357 """Bisects with improvement direction matching regression range."""
315 # Test result goes from 0 to 100 where lower is better 358 # Test result goes from 0 to 100 where lower is better
316 results = _GenericDryRun(_GetExtendedOptions(-1, 100)) 359 results = _GenericDryRun(_GetExtendedOptions(-1, 100))
317 self.assertIsNone(results.error) 360 self.assertIsNone(results.error)
318 # Test result goes from 0 to -100 where higher is better 361 # Test result goes from 0 to -100 where higher is better
319 results = _GenericDryRun(_GetExtendedOptions(1, -100)) 362 results = _GenericDryRun(_GetExtendedOptions(1, -100))
320 self.assertIsNone(results.error) 363 self.assertIsNone(results.error)
321 364
365 @mock.patch('urllib2.urlopen')
366 def testBisectResultsPosted(self, mock_urlopen):
367 options_dict = dict(DEFAULT_OPTIONS)
368 options_dict.update({
369 'bisect_mode': bisect_utils.BISECT_MODE_MEAN,
370 'try_job_id': 1234,
371 })
372 opts = bisect_perf_regression.BisectOptions.FromDict(options_dict)
373 results = _SampleBisecResult(opts)
374 bisect_perf_regression._PostBisectResults(results, opts, os.getcwd())
375
376 call_args = _GetMockCallArg(mock_urlopen, 0)
377 self.assertIsNotNone(call_args)
378 self.assertIn('"try_job_id": 1234', call_args[1])
379
322 def _CheckAbortsEarly(self, results, **extra_opts): 380 def _CheckAbortsEarly(self, results, **extra_opts):
323 """Returns True if the bisect job would abort early.""" 381 """Returns True if the bisect job would abort early."""
324 global _MockResultsGenerator 382 global _MockResultsGenerator
325 _MockResultsGenerator = (r for r in results) 383 _MockResultsGenerator = (r for r in results)
326 bisect_class = bisect_perf_regression.BisectPerformanceMetrics 384 bisect_class = bisect_perf_regression.BisectPerformanceMetrics
327 original_run_tests = bisect_class.RunPerformanceTestAndParseResults 385 original_run_tests = bisect_class.RunPerformanceTestAndParseResults
328 bisect_class.RunPerformanceTestAndParseResults = _MakeMockRunTests() 386 bisect_class.RunPerformanceTestAndParseResults = _MakeMockRunTests()
329 387
330 try: 388 try:
331 dry_run_results = _GenericDryRun(_GetExtendedOptions( 389 dry_run_results = _GenericDryRun(_GetExtendedOptions(
(...skipping 355 matching lines...) Expand 10 before | Expand all | Expand 10 after
687 (None, 0)), 745 (None, 0)),
688 ] 746 ]
689 self._SetupRunGitMock(try_cmd) 747 self._SetupRunGitMock(try_cmd)
690 bisect_perf_regression._StartBuilderTryJob( 748 bisect_perf_regression._StartBuilderTryJob(
691 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name, 749 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name,
692 patch) 750 patch)
693 751
694 752
695 if __name__ == '__main__': 753 if __name__ == '__main__':
696 unittest.main() 754 unittest.main()
OLDNEW
« no previous file with comments | « tools/auto_bisect/bisect_perf_regression.py ('k') | tools/auto_bisect/bisect_results.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698