Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(145)

Side by Side Diff: tools/auto_bisect/bisect_perf_regression_test.py

Issue 564663002: Move bisect-perf-regression.py into auto_bisect directory. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Upload a config for doing a test run. Created 6 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 # Copyright 2014 The Chromium Authors. All rights reserved. 1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import os 5 import os
6 import re 6 import re
7 import unittest 7 import unittest
8 8
9 from auto_bisect import source_control as source_control_module 9 import bisect_perf_regression
10 import source_control as source_control_module
10 11
11 # Special import necessary because filename contains dash characters.
12 bisect_perf_module = __import__('bisect-perf-regression')
13 12
14 def _GetBisectPerformanceMetricsInstance(): 13 def _GetBisectPerformanceMetricsInstance():
15 """Returns an instance of the BisectPerformanceMetrics class.""" 14 """Returns an instance of the BisectPerformanceMetrics class."""
16 options_dict = { 15 options_dict = {
17 'debug_ignore_build': True, 16 'debug_ignore_build': True,
18 'debug_ignore_sync': True, 17 'debug_ignore_sync': True,
19 'debug_ignore_perf_test': True, 18 'debug_ignore_perf_test': True,
20 'command': 'fake_command', 19 'command': 'fake_command',
21 'metric': 'fake/metric', 20 'metric': 'fake/metric',
22 'good_revision': 280000, 21 'good_revision': 280000,
23 'bad_revision': 280005, 22 'bad_revision': 280005,
24 } 23 }
25 bisect_options = bisect_perf_module.BisectOptions.FromDict(options_dict) 24 bisect_options = bisect_perf_regression.BisectOptions.FromDict(options_dict)
26 source_control = source_control_module.DetermineAndCreateSourceControl( 25 source_control = source_control_module.DetermineAndCreateSourceControl(
27 bisect_options) 26 bisect_options)
28 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( 27 bisect_instance = bisect_perf_regression.BisectPerformanceMetrics(
29 source_control, bisect_options) 28 source_control, bisect_options)
30 bisect_instance.src_cwd = os.path.abspath(
31 os.path.join(os.path.dirname(__file__), os.path.pardir))
32 return bisect_instance 29 return bisect_instance
33 30
34 31
35 class BisectPerfRegressionTest(unittest.TestCase): 32 class BisectPerfRegressionTest(unittest.TestCase):
36 """Test case for other functions and classes in bisect-perf-regression.py.""" 33 """Test case for other functions and classes in bisect-perf-regression.py."""
37 34
38 def _AssertConfidence(self, score, bad_values, good_values): 35 def _AssertConfidence(self, score, bad_values, good_values):
39 """Checks whether the given sets of values have a given confidence score. 36 """Checks whether the given sets of values have a given confidence score.
40 37
41 The score represents our confidence that the two sets of values wouldn't 38 The score represents our confidence that the two sets of values wouldn't
42 be as different as they are just by chance; that is, that some real change 39 be as different as they are just by chance; that is, that some real change
43 occurred between the two sets of values. 40 occurred between the two sets of values.
44 41
45 Args: 42 Args:
46 score: Expected confidence score. 43 score: Expected confidence score.
47 bad_values: First list of numbers. 44 bad_values: First list of numbers.
48 good_values: Second list of numbers. 45 good_values: Second list of numbers.
49 """ 46 """
50 # ConfidenceScore takes a list of lists but these lists are flattened 47 # ConfidenceScore takes a list of lists but these lists are flattened
51 # inside the function. 48 # inside the function.
52 confidence = bisect_perf_module.ConfidenceScore( 49 confidence = bisect_perf_regression.ConfidenceScore(
53 [[v] for v in bad_values], 50 [[v] for v in bad_values],
54 [[v] for v in good_values]) 51 [[v] for v in good_values])
55 self.assertEqual(score, confidence) 52 self.assertEqual(score, confidence)
56 53
57 def testConfidenceScore_ZeroConfidence(self): 54 def testConfidenceScore_ZeroConfidence(self):
58 # The good and bad sets contain the same values, so the confidence that 55 # The good and bad sets contain the same values, so the confidence that
59 # they're different should be zero. 56 # they're different should be zero.
60 self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4]) 57 self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4])
61 58
62 def testConfidenceScore_MediumConfidence(self): 59 def testConfidenceScore_MediumConfidence(self):
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after
121 118
122 # Should only expect SVN/git revisions to come through, and URLs should be 119 # Should only expect SVN/git revisions to come through, and URLs should be
123 # filtered out. 120 # filtered out.
124 expected_vars_dict = { 121 expected_vars_dict = {
125 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', 122 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
126 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', 123 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
127 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' 124 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05'
128 } 125 }
129 # Testing private function. 126 # Testing private function.
130 # pylint: disable=W0212 127 # pylint: disable=W0212
131 vars_dict = bisect_perf_module._ParseRevisionsFromDEPSFileManually( 128 vars_dict = bisect_perf_regression._ParseRevisionsFromDEPSFileManually(
132 deps_file_contents) 129 deps_file_contents)
133 self.assertEqual(vars_dict, expected_vars_dict) 130 self.assertEqual(vars_dict, expected_vars_dict)
134 131
135 def _AssertParseResult(self, expected_values, result_string): 132 def _AssertParseResult(self, expected_values, result_string):
136 """Asserts some values are parsed from a RESULT line.""" 133 """Asserts some values are parsed from a RESULT line."""
137 results_template = ('RESULT other_chart: other_trace= 123 count\n' 134 results_template = ('RESULT other_chart: other_trace= 123 count\n'
138 'RESULT my_chart: my_trace= %(value)s\n') 135 'RESULT my_chart: my_trace= %(value)s\n')
139 results = results_template % {'value': result_string} 136 results = results_template % {'value': result_string}
140 metric = ['my_chart', 'my_trace'] 137 metric = ['my_chart', 'my_trace']
141 # Testing private function. 138 # Testing private function.
142 # pylint: disable=W0212 139 # pylint: disable=W0212
143 values = bisect_perf_module._TryParseResultValuesFromOutput(metric, results) 140 values = bisect_perf_regression._TryParseResultValuesFromOutput(
141 metric, results)
144 self.assertEqual(expected_values, values) 142 self.assertEqual(expected_values, values)
145 143
146 def testTryParseResultValuesFromOutput_WithSingleValue(self): 144 def testTryParseResultValuesFromOutput_WithSingleValue(self):
147 """Tests result pattern <*>RESULT <graph>: <trace>= <value>""" 145 """Tests result pattern <*>RESULT <graph>: <trace>= <value>"""
148 self._AssertParseResult([66.88], '66.88 kb') 146 self._AssertParseResult([66.88], '66.88 kb')
149 self._AssertParseResult([66.88], '66.88 ') 147 self._AssertParseResult([66.88], '66.88 ')
150 self._AssertParseResult([-66.88], '-66.88 kb') 148 self._AssertParseResult([-66.88], '-66.88 kb')
151 self._AssertParseResult([66], '66 kb') 149 self._AssertParseResult([66], '66 kb')
152 self._AssertParseResult([0.66], '.66 kb') 150 self._AssertParseResult([0.66], '.66 kb')
153 self._AssertParseResult([], '. kb') 151 self._AssertParseResult([], '. kb')
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
185 183
186 This modification to the command is done in order to get a Telemetry 184 This modification to the command is done in order to get a Telemetry
187 command that works; before some revisions, the browser name that Telemetry 185 command that works; before some revisions, the browser name that Telemetry
188 expects is different in some cases, but we want it to work anyway. 186 expects is different in some cases, but we want it to work anyway.
189 187
190 Specifically, only for android: 188 Specifically, only for android:
191 After r276628, only android-chrome-shell works. 189 After r276628, only android-chrome-shell works.
192 Prior to r274857, only android-chromium-testshell works. 190 Prior to r274857, only android-chromium-testshell works.
193 In the range [274857, 276628], both work. 191 In the range [274857, 276628], both work.
194 """ 192 """
195 bisect_options = bisect_perf_module.BisectOptions() 193 bisect_options = bisect_perf_regression.BisectOptions()
196 bisect_options.output_buildbot_annotations = None 194 bisect_options.output_buildbot_annotations = None
197 source_control = source_control_module.DetermineAndCreateSourceControl( 195 source_control = source_control_module.DetermineAndCreateSourceControl(
198 bisect_options) 196 bisect_options)
199 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( 197 bisect_instance = bisect_perf_regression.BisectPerformanceMetrics(
200 source_control, bisect_options) 198 source_control, bisect_options)
201 bisect_instance.opts.target_platform = target_platform 199 bisect_instance.opts.target_platform = target_platform
202 git_revision = bisect_instance.source_control.ResolveToRevision( 200 git_revision = bisect_instance.source_control.ResolveToRevision(
203 revision, 'chromium', bisect_perf_module.DEPOT_DEPS_NAME, 100) 201 revision, 'chromium', bisect_perf_regression.DEPOT_DEPS_NAME, 100)
204 depot = 'chromium' 202 depot = 'chromium'
205 command = bisect_instance.GetCompatibleCommand( 203 command = bisect_instance.GetCompatibleCommand(
206 original_command, git_revision, depot) 204 original_command, git_revision, depot)
207 self.assertEqual(expected_command, command) 205 self.assertEqual(expected_command, command)
208 206
209 def testGetCompatibleCommand_ChangeToTestShell(self): 207 def testGetCompatibleCommand_ChangeToTestShell(self):
210 # For revisions <= r274857, only android-chromium-testshell is used. 208 # For revisions <= r274857, only android-chromium-testshell is used.
211 self._AssertCompatibleCommand( 209 self._AssertCompatibleCommand(
212 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo', 210 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
213 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo', 211 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after
261 self.assertEqual( 259 self.assertEqual(
262 291765, bisect_instance.source_control.GetCommitPosition(cp_git_rev)) 260 291765, bisect_instance.source_control.GetCommitPosition(cp_git_rev))
263 261
264 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58' 262 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58'
265 self.assertEqual( 263 self.assertEqual(
266 291467, bisect_instance.source_control.GetCommitPosition(svn_git_rev)) 264 291467, bisect_instance.source_control.GetCommitPosition(svn_git_rev))
267 265
268 def testGetCommitPositionForV8(self): 266 def testGetCommitPositionForV8(self):
269 bisect_instance = _GetBisectPerformanceMetricsInstance() 267 bisect_instance = _GetBisectPerformanceMetricsInstance()
270 v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb' 268 v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb'
271 depot_path = os.path.join(bisect_instance.src_cwd, 'src', 'v8') 269 depot_path = os.path.join(bisect_perf_regression.SRC_DIR, 'v8')
272 self.assertEqual( 270 self.assertEqual(
273 23634, 271 23634,
274 bisect_instance.source_control.GetCommitPosition(v8_rev, depot_path)) 272 bisect_instance.source_control.GetCommitPosition(v8_rev, depot_path))
275 273
276 def testGetCommitPositionForWebKit(self): 274 def testGetCommitPositionForWebKit(self):
277 bisect_instance = _GetBisectPerformanceMetricsInstance() 275 bisect_instance = _GetBisectPerformanceMetricsInstance()
278 wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61' 276 wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61'
279 depot_path = os.path.join(bisect_instance.src_cwd, 'src', 'third_party', 277 depot_path = os.path.join(bisect_perf_regression.SRC_DIR, 'third_party',
280 'WebKit') 278 'WebKit')
281 self.assertEqual( 279 self.assertEqual(
282 181660, 280 181660,
283 bisect_instance.source_control.GetCommitPosition(wk_rev, depot_path)) 281 bisect_instance.source_control.GetCommitPosition(wk_rev, depot_path))
284 282
285 def testUpdateDepsContent(self): 283 def testUpdateDepsContent(self):
286 bisect_instance = _GetBisectPerformanceMetricsInstance() 284 bisect_instance = _GetBisectPerformanceMetricsInstance()
287 deps_file = 'DEPS' 285 deps_file = 'DEPS'
288 # We are intentionally reading DEPS file contents instead of string literal 286 # We are intentionally reading DEPS file contents instead of string literal
289 # with few lines from DEPS because to check if the format we are expecting 287 # with few lines from DEPS because to check if the format we are expecting
290 # to search is not changed in DEPS content. 288 # to search is not changed in DEPS content.
291 # TODO (prasadv): Add a separate test to validate the DEPS contents with the 289 # TODO (prasadv): Add a separate test to validate the DEPS contents with the
292 # format that bisect script expects. 290 # format that bisect script expects.
293 deps_contents = bisect_perf_module.ReadStringFromFile(deps_file) 291 deps_contents = bisect_perf_regression.ReadStringFromFile(deps_file)
294 deps_key = 'v8_revision' 292 deps_key = 'v8_revision'
295 depot = 'v8' 293 depot = 'v8'
296 git_revision = 'a12345789a23456789a123456789a123456789' 294 git_revision = 'a12345789a23456789a123456789a123456789'
297 updated_content = bisect_instance.UpdateDepsContents( 295 updated_content = bisect_instance.UpdateDepsContents(
298 deps_contents, depot, git_revision, deps_key) 296 deps_contents, depot, git_revision, deps_key)
299 self.assertIsNotNone(updated_content) 297 self.assertIsNotNone(updated_content)
300 ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision)) 298 ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision))
301 self.assertIsNotNone(re.search(ss, updated_content)) 299 self.assertIsNotNone(re.search(ss, updated_content))
302 300
303 301
304 if __name__ == '__main__': 302 if __name__ == '__main__':
305 unittest.main() 303 unittest.main()
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698