Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(74)

Side by Side Diff: tools/auto_bisect/bisect_perf_regression_test.py

Issue 1625573004: Update legacy bisect to post results to perf dashboard. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@fix_import_path
Patch Set: Created 4 years, 11 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2014 The Chromium Authors. All rights reserved. 1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import argparse
5 import os 6 import os
6 import re 7 import re
7 import shutil 8 import shutil
8 import sys 9 import sys
9 import unittest 10 import unittest
10 11
11 SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir) 12 SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)
12 sys.path.append(os.path.join(SRC, 'third_party', 'pymock')) 13 sys.path.append(os.path.join(SRC, 'third_party', 'pymock'))
13 14
14 import bisect_perf_regression 15 import bisect_perf_regression
(...skipping 109 matching lines...) Expand 10 before | Expand all | Expand 10 after
124 125
125 def _FakeTestResult(values, bisect_mode_is_return_code): 126 def _FakeTestResult(values, bisect_mode_is_return_code):
126 mean = 0.0 127 mean = 0.0
127 if bisect_mode_is_return_code: 128 if bisect_mode_is_return_code:
128 mean = 0 if (all(v == 0 for v in values)) else 1 129 mean = 0 if (all(v == 0 for v in values)) else 1
129 result_dict = {'mean': mean, 'std_err': 0.0, 'std_dev': 0.0, 'values': values} 130 result_dict = {'mean': mean, 'std_err': 0.0, 'std_dev': 0.0, 'values': values}
130 success_code = 0 131 success_code = 0
131 return (result_dict, success_code) 132 return (result_dict, success_code)
132 133
133 134
135 def _GetMockCallArg(function_mock, call_index):
136 """Gets the first argument value for call at "call_index".
137
138 Args:
139 function_mock: A Mock object.
140 call_index: The index at which the mocked function was called.
141
142 Returns:
143 The first argument value.
144 """
145 call_args_list = function_mock.call_args_list
146 if not call_args_list or len(call_args_list) <= call_index:
147 return None
148 args, _ = call_args_list[call_index]
149 return args
150
151
134 def _GetBisectPerformanceMetricsInstance(options_dict): 152 def _GetBisectPerformanceMetricsInstance(options_dict):
135 """Returns an instance of the BisectPerformanceMetrics class.""" 153 """Returns an instance of the BisectPerformanceMetrics class."""
136 opts = bisect_perf_regression.BisectOptions.FromDict(options_dict) 154 opts = bisect_perf_regression.BisectOptions.FromDict(options_dict)
137 return bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd()) 155 return bisect_perf_regression.BisectPerformanceMetrics(opts, os.getcwd())
138 156
139 157
140 def _GetExtendedOptions(improvement_dir, fake_first, ignore_confidence=True, 158 def _GetExtendedOptions(improvement_dir, fake_first, ignore_confidence=True,
141 **extra_opts): 159 **extra_opts):
142 """Returns the a copy of the default options dict plus some options.""" 160 """Returns the a copy of the default options dict plus some options."""
143 result = dict(DEFAULT_OPTIONS) 161 result = dict(DEFAULT_OPTIONS)
144 result.update({ 162 result.update({
145 'improvement_direction': improvement_dir, 163 'improvement_direction': improvement_dir,
146 'debug_fake_first_test_mean': fake_first, 164 'debug_fake_first_test_mean': fake_first,
147 'debug_ignore_regression_confidence': ignore_confidence 165 'debug_ignore_regression_confidence': ignore_confidence
148 }) 166 })
149 result.update(extra_opts) 167 result.update(extra_opts)
150 return result 168 return result
151 169
152 170
153 def _GenericDryRun(options, print_results=False): 171 def _GenericDryRun(options, print_results=False):
154 """Performs a dry run of the bisector. 172 """Performs a dry run of the bisector.
155 173
156 Args: 174 Args:
157 options: Dictionary containing the options for the bisect instance. 175 options: Dictionary containing the options for the bisect instance.
158 print_results: Boolean telling whether to call FormatAndPrintResults. 176 print_results: Boolean telling whether to call FormatAndPrintResults.
159 177
160 Returns: 178 Returns:
161 The results dictionary as returned by the bisect Run method. 179 The results dictionary as returned by the bisect Run method.
162 """ 180 """
163 _AbortIfThereAreStagedChanges() 181 #_AbortIfThereAreStagedChanges()
164 # Disable rmtree to avoid deleting local trees. 182 # Disable rmtree to avoid deleting local trees.
165 old_rmtree = shutil.rmtree 183 old_rmtree = shutil.rmtree
166 shutil.rmtree = lambda path, on_error: None 184 shutil.rmtree = lambda path, on_error: None
167 # git reset HEAD may be run during the dry run, which removes staged changes. 185 # git reset HEAD may be run during the dry run, which removes staged changes.
168 try: 186 try:
169 bisect_instance = _GetBisectPerformanceMetricsInstance(options) 187 bisect_instance = _GetBisectPerformanceMetricsInstance(options)
170 results = bisect_instance.Run( 188 results = bisect_instance.Run(
171 bisect_instance.opts.command, bisect_instance.opts.bad_revision, 189 bisect_instance.opts.command, bisect_instance.opts.bad_revision,
172 bisect_instance.opts.good_revision, bisect_instance.opts.metric) 190 bisect_instance.opts.good_revision, bisect_instance.opts.metric)
173 191
(...skipping 138 matching lines...) Expand 10 before | Expand all | Expand 10 after
312 330
313 def testBisectImprovementDirectionSucceeds(self): 331 def testBisectImprovementDirectionSucceeds(self):
314 """Bisects with improvement direction matching regression range.""" 332 """Bisects with improvement direction matching regression range."""
315 # Test result goes from 0 to 100 where lower is better 333 # Test result goes from 0 to 100 where lower is better
316 results = _GenericDryRun(_GetExtendedOptions(-1, 100)) 334 results = _GenericDryRun(_GetExtendedOptions(-1, 100))
317 self.assertIsNone(results.error) 335 self.assertIsNone(results.error)
318 # Test result goes from 0 to -100 where higher is better 336 # Test result goes from 0 to -100 where higher is better
319 results = _GenericDryRun(_GetExtendedOptions(1, -100)) 337 results = _GenericDryRun(_GetExtendedOptions(1, -100))
320 self.assertIsNone(results.error) 338 self.assertIsNone(results.error)
321 339
340 @mock.patch('urllib2.urlopen')
341 def testBisectResultsPosted(self, mock_urlopen):
342 """Bisects with improvement direction matching regression range."""
343 # Test result goes from 0 to 100 where lower is better
344 options_dict = dict(DEFAULT_OPTIONS)
345 options_dict.update({
346 'bisect_mode': bisect_utils.BISECT_MODE_MEAN,
347 'try_job_id': 1234,
348 })
349 opts = bisect_perf_regression.BisectOptions.FromDict(options_dict)
350 results = _GenericDryRun(options_dict, True)
351 bisect_perf_regression._PostBisectResults(results, opts, os.getcwd())
352
353 call_args = _GetMockCallArg(mock_urlopen, 0)
354 self.assertIsNotNone(call_args)
355 self.assertIn('"try_job_id": 1234', call_args[1])
356
322 def _CheckAbortsEarly(self, results, **extra_opts): 357 def _CheckAbortsEarly(self, results, **extra_opts):
323 """Returns True if the bisect job would abort early.""" 358 """Returns True if the bisect job would abort early."""
324 global _MockResultsGenerator 359 global _MockResultsGenerator
325 _MockResultsGenerator = (r for r in results) 360 _MockResultsGenerator = (r for r in results)
326 bisect_class = bisect_perf_regression.BisectPerformanceMetrics 361 bisect_class = bisect_perf_regression.BisectPerformanceMetrics
327 original_run_tests = bisect_class.RunPerformanceTestAndParseResults 362 original_run_tests = bisect_class.RunPerformanceTestAndParseResults
328 bisect_class.RunPerformanceTestAndParseResults = _MakeMockRunTests() 363 bisect_class.RunPerformanceTestAndParseResults = _MakeMockRunTests()
329 364
330 try: 365 try:
331 dry_run_results = _GenericDryRun(_GetExtendedOptions( 366 dry_run_results = _GenericDryRun(_GetExtendedOptions(
(...skipping 355 matching lines...) Expand 10 before | Expand all | Expand 10 after
687 (None, 0)), 722 (None, 0)),
688 ] 723 ]
689 self._SetupRunGitMock(try_cmd) 724 self._SetupRunGitMock(try_cmd)
690 bisect_perf_regression._StartBuilderTryJob( 725 bisect_perf_regression._StartBuilderTryJob(
691 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name, 726 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name,
692 patch) 727 patch)
693 728
694 729
695 if __name__ == '__main__': 730 if __name__ == '__main__':
696 unittest.main() 731 unittest.main()
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698