OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import os | 5 import os |
6 import re | 6 import re |
7 import unittest | 7 import unittest |
8 | 8 |
9 from auto_bisect import source_control as source_control_module | 9 import bisect_perf_regression |
| 10 import source_control as source_control_module |
10 | 11 |
11 # Special import necessary because filename contains dash characters. | |
12 bisect_perf_module = __import__('bisect-perf-regression') | |
13 | 12 |
14 def _GetBisectPerformanceMetricsInstance(): | 13 def _GetBisectPerformanceMetricsInstance(): |
15 """Returns an instance of the BisectPerformanceMetrics class.""" | 14 """Returns an instance of the BisectPerformanceMetrics class.""" |
16 options_dict = { | 15 options_dict = { |
17 'debug_ignore_build': True, | 16 'debug_ignore_build': True, |
18 'debug_ignore_sync': True, | 17 'debug_ignore_sync': True, |
19 'debug_ignore_perf_test': True, | 18 'debug_ignore_perf_test': True, |
20 'command': 'fake_command', | 19 'command': 'fake_command', |
21 'metric': 'fake/metric', | 20 'metric': 'fake/metric', |
22 'good_revision': 280000, | 21 'good_revision': 280000, |
23 'bad_revision': 280005, | 22 'bad_revision': 280005, |
24 } | 23 } |
25 bisect_options = bisect_perf_module.BisectOptions.FromDict(options_dict) | 24 bisect_options = bisect_perf_regression.BisectOptions.FromDict(options_dict) |
26 source_control = source_control_module.DetermineAndCreateSourceControl( | 25 source_control = source_control_module.DetermineAndCreateSourceControl( |
27 bisect_options) | 26 bisect_options) |
28 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | 27 bisect_instance = bisect_perf_regression.BisectPerformanceMetrics( |
29 source_control, bisect_options) | 28 source_control, bisect_options) |
30 bisect_instance.src_cwd = os.path.abspath( | |
31 os.path.join(os.path.dirname(__file__), os.path.pardir)) | |
32 return bisect_instance | 29 return bisect_instance |
33 | 30 |
34 | 31 |
35 class BisectPerfRegressionTest(unittest.TestCase): | 32 class BisectPerfRegressionTest(unittest.TestCase): |
36 """Test case for other functions and classes in bisect-perf-regression.py.""" | 33 """Test case for other functions and classes in bisect-perf-regression.py.""" |
37 | 34 |
38 def _AssertConfidence(self, score, bad_values, good_values): | 35 def _AssertConfidence(self, score, bad_values, good_values): |
39 """Checks whether the given sets of values have a given confidence score. | 36 """Checks whether the given sets of values have a given confidence score. |
40 | 37 |
41 The score represents our confidence that the two sets of values wouldn't | 38 The score represents our confidence that the two sets of values wouldn't |
42 be as different as they are just by chance; that is, that some real change | 39 be as different as they are just by chance; that is, that some real change |
43 occurred between the two sets of values. | 40 occurred between the two sets of values. |
44 | 41 |
45 Args: | 42 Args: |
46 score: Expected confidence score. | 43 score: Expected confidence score. |
47 bad_values: First list of numbers. | 44 bad_values: First list of numbers. |
48 good_values: Second list of numbers. | 45 good_values: Second list of numbers. |
49 """ | 46 """ |
50 # ConfidenceScore takes a list of lists but these lists are flattened | 47 # ConfidenceScore takes a list of lists but these lists are flattened |
51 # inside the function. | 48 # inside the function. |
52 confidence = bisect_perf_module.ConfidenceScore( | 49 confidence = bisect_perf_regression.ConfidenceScore( |
53 [[v] for v in bad_values], | 50 [[v] for v in bad_values], |
54 [[v] for v in good_values]) | 51 [[v] for v in good_values]) |
55 self.assertEqual(score, confidence) | 52 self.assertEqual(score, confidence) |
56 | 53 |
57 def testConfidenceScore_ZeroConfidence(self): | 54 def testConfidenceScore_ZeroConfidence(self): |
58 # The good and bad sets contain the same values, so the confidence that | 55 # The good and bad sets contain the same values, so the confidence that |
59 # they're different should be zero. | 56 # they're different should be zero. |
60 self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4]) | 57 self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4]) |
61 | 58 |
62 def testConfidenceScore_MediumConfidence(self): | 59 def testConfidenceScore_MediumConfidence(self): |
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
121 | 118 |
122 # Should only expect SVN/git revisions to come through, and URLs should be | 119 # Should only expect SVN/git revisions to come through, and URLs should be |
123 # filtered out. | 120 # filtered out. |
124 expected_vars_dict = { | 121 expected_vars_dict = { |
125 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', | 122 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', |
126 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', | 123 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', |
127 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' | 124 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' |
128 } | 125 } |
129 # Testing private function. | 126 # Testing private function. |
130 # pylint: disable=W0212 | 127 # pylint: disable=W0212 |
131 vars_dict = bisect_perf_module._ParseRevisionsFromDEPSFileManually( | 128 vars_dict = bisect_perf_regression._ParseRevisionsFromDEPSFileManually( |
132 deps_file_contents) | 129 deps_file_contents) |
133 self.assertEqual(vars_dict, expected_vars_dict) | 130 self.assertEqual(vars_dict, expected_vars_dict) |
134 | 131 |
135 def _AssertParseResult(self, expected_values, result_string): | 132 def _AssertParseResult(self, expected_values, result_string): |
136 """Asserts some values are parsed from a RESULT line.""" | 133 """Asserts some values are parsed from a RESULT line.""" |
137 results_template = ('RESULT other_chart: other_trace= 123 count\n' | 134 results_template = ('RESULT other_chart: other_trace= 123 count\n' |
138 'RESULT my_chart: my_trace= %(value)s\n') | 135 'RESULT my_chart: my_trace= %(value)s\n') |
139 results = results_template % {'value': result_string} | 136 results = results_template % {'value': result_string} |
140 metric = ['my_chart', 'my_trace'] | 137 metric = ['my_chart', 'my_trace'] |
141 # Testing private function. | 138 # Testing private function. |
142 # pylint: disable=W0212 | 139 # pylint: disable=W0212 |
143 values = bisect_perf_module._TryParseResultValuesFromOutput(metric, results) | 140 values = bisect_perf_regression._TryParseResultValuesFromOutput( |
| 141 metric, results) |
144 self.assertEqual(expected_values, values) | 142 self.assertEqual(expected_values, values) |
145 | 143 |
146 def testTryParseResultValuesFromOutput_WithSingleValue(self): | 144 def testTryParseResultValuesFromOutput_WithSingleValue(self): |
147 """Tests result pattern <*>RESULT <graph>: <trace>= <value>""" | 145 """Tests result pattern <*>RESULT <graph>: <trace>= <value>""" |
148 self._AssertParseResult([66.88], '66.88 kb') | 146 self._AssertParseResult([66.88], '66.88 kb') |
149 self._AssertParseResult([66.88], '66.88 ') | 147 self._AssertParseResult([66.88], '66.88 ') |
150 self._AssertParseResult([-66.88], '-66.88 kb') | 148 self._AssertParseResult([-66.88], '-66.88 kb') |
151 self._AssertParseResult([66], '66 kb') | 149 self._AssertParseResult([66], '66 kb') |
152 self._AssertParseResult([0.66], '.66 kb') | 150 self._AssertParseResult([0.66], '.66 kb') |
153 self._AssertParseResult([], '. kb') | 151 self._AssertParseResult([], '. kb') |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
185 | 183 |
186 This modification to the command is done in order to get a Telemetry | 184 This modification to the command is done in order to get a Telemetry |
187 command that works; before some revisions, the browser name that Telemetry | 185 command that works; before some revisions, the browser name that Telemetry |
188 expects is different in some cases, but we want it to work anyway. | 186 expects is different in some cases, but we want it to work anyway. |
189 | 187 |
190 Specifically, only for android: | 188 Specifically, only for android: |
191 After r276628, only android-chrome-shell works. | 189 After r276628, only android-chrome-shell works. |
192 Prior to r274857, only android-chromium-testshell works. | 190 Prior to r274857, only android-chromium-testshell works. |
193 In the range [274857, 276628], both work. | 191 In the range [274857, 276628], both work. |
194 """ | 192 """ |
195 bisect_options = bisect_perf_module.BisectOptions() | 193 bisect_options = bisect_perf_regression.BisectOptions() |
196 bisect_options.output_buildbot_annotations = None | 194 bisect_options.output_buildbot_annotations = None |
197 source_control = source_control_module.DetermineAndCreateSourceControl( | 195 source_control = source_control_module.DetermineAndCreateSourceControl( |
198 bisect_options) | 196 bisect_options) |
199 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | 197 bisect_instance = bisect_perf_regression.BisectPerformanceMetrics( |
200 source_control, bisect_options) | 198 source_control, bisect_options) |
201 bisect_instance.opts.target_platform = target_platform | 199 bisect_instance.opts.target_platform = target_platform |
202 git_revision = bisect_instance.source_control.ResolveToRevision( | 200 git_revision = bisect_instance.source_control.ResolveToRevision( |
203 revision, 'chromium', bisect_perf_module.DEPOT_DEPS_NAME, 100) | 201 revision, 'chromium', bisect_perf_regression.DEPOT_DEPS_NAME, 100) |
204 depot = 'chromium' | 202 depot = 'chromium' |
205 command = bisect_instance.GetCompatibleCommand( | 203 command = bisect_instance.GetCompatibleCommand( |
206 original_command, git_revision, depot) | 204 original_command, git_revision, depot) |
207 self.assertEqual(expected_command, command) | 205 self.assertEqual(expected_command, command) |
208 | 206 |
209 def testGetCompatibleCommand_ChangeToTestShell(self): | 207 def testGetCompatibleCommand_ChangeToTestShell(self): |
210 # For revisions <= r274857, only android-chromium-testshell is used. | 208 # For revisions <= r274857, only android-chromium-testshell is used. |
211 self._AssertCompatibleCommand( | 209 self._AssertCompatibleCommand( |
212 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo', | 210 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo', |
213 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo', | 211 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo', |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
261 self.assertEqual( | 259 self.assertEqual( |
262 291765, bisect_instance.source_control.GetCommitPosition(cp_git_rev)) | 260 291765, bisect_instance.source_control.GetCommitPosition(cp_git_rev)) |
263 | 261 |
264 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58' | 262 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58' |
265 self.assertEqual( | 263 self.assertEqual( |
266 291467, bisect_instance.source_control.GetCommitPosition(svn_git_rev)) | 264 291467, bisect_instance.source_control.GetCommitPosition(svn_git_rev)) |
267 | 265 |
268 def testGetCommitPositionForV8(self): | 266 def testGetCommitPositionForV8(self): |
269 bisect_instance = _GetBisectPerformanceMetricsInstance() | 267 bisect_instance = _GetBisectPerformanceMetricsInstance() |
270 v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb' | 268 v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb' |
271 depot_path = os.path.join(bisect_instance.src_cwd, 'src', 'v8') | 269 depot_path = os.path.join(bisect_perf_regression.SRC_DIR, 'v8') |
272 self.assertEqual( | 270 self.assertEqual( |
273 23634, | 271 23634, |
274 bisect_instance.source_control.GetCommitPosition(v8_rev, depot_path)) | 272 bisect_instance.source_control.GetCommitPosition(v8_rev, depot_path)) |
275 | 273 |
276 def testGetCommitPositionForWebKit(self): | 274 def testGetCommitPositionForWebKit(self): |
277 bisect_instance = _GetBisectPerformanceMetricsInstance() | 275 bisect_instance = _GetBisectPerformanceMetricsInstance() |
278 wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61' | 276 wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61' |
279 depot_path = os.path.join(bisect_instance.src_cwd, 'src', 'third_party', | 277 depot_path = os.path.join(bisect_perf_regression.SRC_DIR, 'third_party', |
280 'WebKit') | 278 'WebKit') |
281 self.assertEqual( | 279 self.assertEqual( |
282 181660, | 280 181660, |
283 bisect_instance.source_control.GetCommitPosition(wk_rev, depot_path)) | 281 bisect_instance.source_control.GetCommitPosition(wk_rev, depot_path)) |
284 | 282 |
285 def testUpdateDepsContent(self): | 283 def testUpdateDepsContent(self): |
286 bisect_instance = _GetBisectPerformanceMetricsInstance() | 284 bisect_instance = _GetBisectPerformanceMetricsInstance() |
287 deps_file = 'DEPS' | 285 deps_file = 'DEPS' |
288 # We are intentionally reading DEPS file contents instead of string literal | 286 # We are intentionally reading DEPS file contents instead of string literal |
289 # with few lines from DEPS because to check if the format we are expecting | 287 # with few lines from DEPS because to check if the format we are expecting |
290 # to search is not changed in DEPS content. | 288 # to search is not changed in DEPS content. |
291 # TODO (prasadv): Add a separate test to validate the DEPS contents with the | 289 # TODO (prasadv): Add a separate test to validate the DEPS contents with the |
292 # format that bisect script expects. | 290 # format that bisect script expects. |
293 deps_contents = bisect_perf_module.ReadStringFromFile(deps_file) | 291 deps_contents = bisect_perf_regression.ReadStringFromFile(deps_file) |
294 deps_key = 'v8_revision' | 292 deps_key = 'v8_revision' |
295 depot = 'v8' | 293 depot = 'v8' |
296 git_revision = 'a12345789a23456789a123456789a123456789' | 294 git_revision = 'a12345789a23456789a123456789a123456789' |
297 updated_content = bisect_instance.UpdateDepsContents( | 295 updated_content = bisect_instance.UpdateDepsContents( |
298 deps_contents, depot, git_revision, deps_key) | 296 deps_contents, depot, git_revision, deps_key) |
299 self.assertIsNotNone(updated_content) | 297 self.assertIsNotNone(updated_content) |
300 ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision)) | 298 ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision)) |
301 self.assertIsNotNone(re.search(ss, updated_content)) | 299 self.assertIsNotNone(re.search(ss, updated_content)) |
302 | 300 |
303 | 301 |
304 if __name__ == '__main__': | 302 if __name__ == '__main__': |
305 unittest.main() | 303 unittest.main() |
OLD | NEW |