OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import os | 5 import os |
6 import re | 6 import re |
7 import unittest | 7 import unittest |
8 | 8 |
9 from auto_bisect import source_control as source_control_module | 9 import bisect_perf_regression |
10 | 10 import source_control as source_control_module |
11 # Special import necessary because filename contains dash characters. | |
12 bisect_perf_module = __import__('bisect-perf-regression') | |
13 | 11 |
14 def _GetBisectPerformanceMetricsInstance(): | 12 def _GetBisectPerformanceMetricsInstance(): |
15 """Returns an instance of the BisectPerformanceMetrics class.""" | 13 """Returns an instance of the BisectPerformanceMetrics class.""" |
16 options_dict = { | 14 options_dict = { |
17 'debug_ignore_build': True, | 15 'debug_ignore_build': True, |
18 'debug_ignore_sync': True, | 16 'debug_ignore_sync': True, |
19 'debug_ignore_perf_test': True, | 17 'debug_ignore_perf_test': True, |
20 'command': 'fake_command', | 18 'command': 'fake_command', |
21 'metric': 'fake/metric', | 19 'metric': 'fake/metric', |
22 'good_revision': 280000, | 20 'good_revision': 280000, |
23 'bad_revision': 280005, | 21 'bad_revision': 280005, |
24 } | 22 } |
25 bisect_options = bisect_perf_module.BisectOptions.FromDict(options_dict) | 23 bisect_options = bisect_perf_regression.BisectOptions.FromDict(options_dict) |
26 source_control = source_control_module.DetermineAndCreateSourceControl( | 24 source_control = source_control_module.DetermineAndCreateSourceControl( |
27 bisect_options) | 25 bisect_options) |
28 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | 26 bisect_instance = bisect_perf_regression.BisectPerformanceMetrics( |
29 source_control, bisect_options) | 27 source_control, bisect_options) |
30 bisect_instance.src_cwd = os.path.abspath( | 28 bisect_instance.src_cwd = os.path.abspath( |
31 os.path.join(os.path.dirname(__file__), os.path.pardir)) | 29 os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)) |
32 return bisect_instance | 30 return bisect_instance |
33 | 31 |
34 | 32 |
35 class BisectPerfRegressionTest(unittest.TestCase): | 33 class BisectPerfRegressionTest(unittest.TestCase): |
36 """Test case for other functions and classes in bisect-perf-regression.py.""" | 34 """Test case for other functions and classes in bisect-perf-regression.py.""" |
37 | 35 |
38 def _AssertConfidence(self, score, bad_values, good_values): | 36 def _AssertConfidence(self, score, bad_values, good_values): |
39 """Checks whether the given sets of values have a given confidence score. | 37 """Checks whether the given sets of values have a given confidence score. |
40 | 38 |
41 The score represents our confidence that the two sets of values wouldn't | 39 The score represents our confidence that the two sets of values wouldn't |
42 be as different as they are just by chance; that is, that some real change | 40 be as different as they are just by chance; that is, that some real change |
43 occurred between the two sets of values. | 41 occurred between the two sets of values. |
44 | 42 |
45 Args: | 43 Args: |
46 score: Expected confidence score. | 44 score: Expected confidence score. |
47 bad_values: First list of numbers. | 45 bad_values: First list of numbers. |
48 good_values: Second list of numbers. | 46 good_values: Second list of numbers. |
49 """ | 47 """ |
50 # ConfidenceScore takes a list of lists but these lists are flattened | 48 # ConfidenceScore takes a list of lists but these lists are flattened |
51 # inside the function. | 49 # inside the function. |
52 confidence = bisect_perf_module.ConfidenceScore( | 50 confidence = bisect_perf_regression.ConfidenceScore( |
53 [[v] for v in bad_values], | 51 [[v] for v in bad_values], |
54 [[v] for v in good_values]) | 52 [[v] for v in good_values]) |
55 self.assertEqual(score, confidence) | 53 self.assertEqual(score, confidence) |
56 | 54 |
57 def testConfidenceScore_ZeroConfidence(self): | 55 def testConfidenceScore_ZeroConfidence(self): |
58 # The good and bad sets contain the same values, so the confidence that | 56 # The good and bad sets contain the same values, so the confidence that |
59 # they're different should be zero. | 57 # they're different should be zero. |
60 self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4]) | 58 self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4]) |
61 | 59 |
62 def testConfidenceScore_MediumConfidence(self): | 60 def testConfidenceScore_MediumConfidence(self): |
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
121 | 119 |
122 # Should only expect SVN/git revisions to come through, and URLs should be | 120 # Should only expect SVN/git revisions to come through, and URLs should be |
123 # filtered out. | 121 # filtered out. |
124 expected_vars_dict = { | 122 expected_vars_dict = { |
125 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', | 123 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', |
126 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', | 124 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', |
127 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' | 125 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' |
128 } | 126 } |
129 # Testing private function. | 127 # Testing private function. |
130 # pylint: disable=W0212 | 128 # pylint: disable=W0212 |
131 vars_dict = bisect_perf_module._ParseRevisionsFromDEPSFileManually( | 129 vars_dict = bisect_perf_regression._ParseRevisionsFromDEPSFileManually( |
132 deps_file_contents) | 130 deps_file_contents) |
133 self.assertEqual(vars_dict, expected_vars_dict) | 131 self.assertEqual(vars_dict, expected_vars_dict) |
134 | 132 |
135 def _AssertParseResult(self, expected_values, result_string): | 133 def _AssertParseResult(self, expected_values, result_string): |
136 """Asserts some values are parsed from a RESULT line.""" | 134 """Asserts some values are parsed from a RESULT line.""" |
137 results_template = ('RESULT other_chart: other_trace= 123 count\n' | 135 results_template = ('RESULT other_chart: other_trace= 123 count\n' |
138 'RESULT my_chart: my_trace= %(value)s\n') | 136 'RESULT my_chart: my_trace= %(value)s\n') |
139 results = results_template % {'value': result_string} | 137 results = results_template % {'value': result_string} |
140 metric = ['my_chart', 'my_trace'] | 138 metric = ['my_chart', 'my_trace'] |
141 # Testing private function. | 139 # Testing private function. |
142 # pylint: disable=W0212 | 140 # pylint: disable=W0212 |
143 values = bisect_perf_module._TryParseResultValuesFromOutput(metric, results) | 141 values = bisect_perf_regression._TryParseResultValuesFromOutput( |
| 142 metric, results) |
144 self.assertEqual(expected_values, values) | 143 self.assertEqual(expected_values, values) |
145 | 144 |
146 def testTryParseResultValuesFromOutput_WithSingleValue(self): | 145 def testTryParseResultValuesFromOutput_WithSingleValue(self): |
147 """Tests result pattern <*>RESULT <graph>: <trace>= <value>""" | 146 """Tests result pattern <*>RESULT <graph>: <trace>= <value>""" |
148 self._AssertParseResult([66.88], '66.88 kb') | 147 self._AssertParseResult([66.88], '66.88 kb') |
149 self._AssertParseResult([66.88], '66.88 ') | 148 self._AssertParseResult([66.88], '66.88 ') |
150 self._AssertParseResult([-66.88], '-66.88 kb') | 149 self._AssertParseResult([-66.88], '-66.88 kb') |
151 self._AssertParseResult([66], '66 kb') | 150 self._AssertParseResult([66], '66 kb') |
152 self._AssertParseResult([0.66], '.66 kb') | 151 self._AssertParseResult([0.66], '.66 kb') |
153 self._AssertParseResult([], '. kb') | 152 self._AssertParseResult([], '. kb') |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
185 | 184 |
186 This modification to the command is done in order to get a Telemetry | 185 This modification to the command is done in order to get a Telemetry |
187 command that works; before some revisions, the browser name that Telemetry | 186 command that works; before some revisions, the browser name that Telemetry |
188 expects is different in some cases, but we want it to work anyway. | 187 expects is different in some cases, but we want it to work anyway. |
189 | 188 |
190 Specifically, only for android: | 189 Specifically, only for android: |
191 After r276628, only android-chrome-shell works. | 190 After r276628, only android-chrome-shell works. |
192 Prior to r274857, only android-chromium-testshell works. | 191 Prior to r274857, only android-chromium-testshell works. |
193 In the range [274857, 276628], both work. | 192 In the range [274857, 276628], both work. |
194 """ | 193 """ |
195 bisect_options = bisect_perf_module.BisectOptions() | 194 bisect_options = bisect_perf_regression.BisectOptions() |
196 bisect_options.output_buildbot_annotations = None | 195 bisect_options.output_buildbot_annotations = None |
197 source_control = source_control_module.DetermineAndCreateSourceControl( | 196 source_control = source_control_module.DetermineAndCreateSourceControl( |
198 bisect_options) | 197 bisect_options) |
199 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | 198 bisect_instance = bisect_perf_regression.BisectPerformanceMetrics( |
200 source_control, bisect_options) | 199 source_control, bisect_options) |
201 bisect_instance.opts.target_platform = target_platform | 200 bisect_instance.opts.target_platform = target_platform |
202 git_revision = bisect_instance.source_control.ResolveToRevision( | 201 git_revision = bisect_instance.source_control.ResolveToRevision( |
203 revision, 'chromium', bisect_perf_module.DEPOT_DEPS_NAME, 100) | 202 revision, 'chromium', bisect_perf_regression.DEPOT_DEPS_NAME, 100) |
204 depot = 'chromium' | 203 depot = 'chromium' |
205 command = bisect_instance.GetCompatibleCommand( | 204 command = bisect_instance.GetCompatibleCommand( |
206 original_command, git_revision, depot) | 205 original_command, git_revision, depot) |
207 self.assertEqual(expected_command, command) | 206 self.assertEqual(expected_command, command) |
208 | 207 |
209 def testGetCompatibleCommand_ChangeToTestShell(self): | 208 def testGetCompatibleCommand_ChangeToTestShell(self): |
210 # For revisions <= r274857, only android-chromium-testshell is used. | 209 # For revisions <= r274857, only android-chromium-testshell is used. |
211 self._AssertCompatibleCommand( | 210 self._AssertCompatibleCommand( |
212 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo', | 211 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo', |
213 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo', | 212 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo', |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
261 self.assertEqual( | 260 self.assertEqual( |
262 291765, bisect_instance.source_control.GetCommitPosition(cp_git_rev)) | 261 291765, bisect_instance.source_control.GetCommitPosition(cp_git_rev)) |
263 | 262 |
264 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58' | 263 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58' |
265 self.assertEqual( | 264 self.assertEqual( |
266 291467, bisect_instance.source_control.GetCommitPosition(svn_git_rev)) | 265 291467, bisect_instance.source_control.GetCommitPosition(svn_git_rev)) |
267 | 266 |
268 def testGetCommitPositionForV8(self): | 267 def testGetCommitPositionForV8(self): |
269 bisect_instance = _GetBisectPerformanceMetricsInstance() | 268 bisect_instance = _GetBisectPerformanceMetricsInstance() |
270 v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb' | 269 v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb' |
271 depot_path = os.path.join(bisect_instance.src_cwd, 'src', 'v8') | 270 depot_path = os.path.join(bisect_instance.src_cwd, 'v8') |
272 self.assertEqual( | 271 self.assertEqual( |
273 23634, | 272 23634, |
274 bisect_instance.source_control.GetCommitPosition(v8_rev, depot_path)) | 273 bisect_instance.source_control.GetCommitPosition(v8_rev, depot_path)) |
275 | 274 |
276 def testGetCommitPositionForWebKit(self): | 275 def testGetCommitPositionForWebKit(self): |
277 bisect_instance = _GetBisectPerformanceMetricsInstance() | 276 bisect_instance = _GetBisectPerformanceMetricsInstance() |
278 wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61' | 277 wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61' |
279 depot_path = os.path.join(bisect_instance.src_cwd, 'src', 'third_party', | 278 depot_path = os.path.join(bisect_instance.src_cwd, 'third_party', 'WebKit') |
280 'WebKit') | |
281 self.assertEqual( | 279 self.assertEqual( |
282 181660, | 280 181660, |
283 bisect_instance.source_control.GetCommitPosition(wk_rev, depot_path)) | 281 bisect_instance.source_control.GetCommitPosition(wk_rev, depot_path)) |
284 | 282 |
285 def testUpdateDepsContent(self): | 283 def testUpdateDepsContent(self): |
286 bisect_instance = _GetBisectPerformanceMetricsInstance() | 284 bisect_instance = _GetBisectPerformanceMetricsInstance() |
287 deps_file = 'DEPS' | 285 deps_file = 'DEPS' |
288 # We are intentionally reading DEPS file contents instead of string literal | 286 # We are intentionally reading DEPS file contents instead of string literal |
289 # with few lines from DEPS because to check if the format we are expecting | 287 # with few lines from DEPS because to check if the format we are expecting |
290 # to search is not changed in DEPS content. | 288 # to search is not changed in DEPS content. |
291 # TODO (prasadv): Add a separate test to validate the DEPS contents with the | 289 # TODO (prasadv): Add a separate test to validate the DEPS contents with the |
292 # format that bisect script expects. | 290 # format that bisect script expects. |
293 deps_contents = bisect_perf_module.ReadStringFromFile(deps_file) | 291 deps_contents = bisect_perf_regression.ReadStringFromFile(deps_file) |
294 deps_key = 'v8_revision' | 292 deps_key = 'v8_revision' |
295 depot = 'v8' | 293 depot = 'v8' |
296 git_revision = 'a12345789a23456789a123456789a123456789' | 294 git_revision = 'a12345789a23456789a123456789a123456789' |
297 updated_content = bisect_instance.UpdateDepsContents( | 295 updated_content = bisect_instance.UpdateDepsContents( |
298 deps_contents, depot, git_revision, deps_key) | 296 deps_contents, depot, git_revision, deps_key) |
299 self.assertIsNotNone(updated_content) | 297 self.assertIsNotNone(updated_content) |
300 ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision)) | 298 ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision)) |
301 self.assertIsNotNone(re.search(ss, updated_content)) | 299 self.assertIsNotNone(re.search(ss, updated_content)) |
302 | 300 |
303 | 301 |
304 if __name__ == '__main__': | 302 if __name__ == '__main__': |
305 unittest.main() | 303 unittest.main() |
OLD | NEW |