| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import os | 5 import os |
| 6 import re | 6 import re |
| 7 import unittest | 7 import unittest |
| 8 | 8 |
| 9 from auto_bisect import source_control as source_control_module | 9 import bisect_perf_regression |
| 10 import source_control as source_control_module |
| 10 | 11 |
| 11 # Special import necessary because filename contains dash characters. | |
| 12 bisect_perf_module = __import__('bisect-perf-regression') | |
| 13 | 12 |
| 14 def _GetBisectPerformanceMetricsInstance(): | 13 def _GetBisectPerformanceMetricsInstance(): |
| 15 """Returns an instance of the BisectPerformanceMetrics class.""" | 14 """Returns an instance of the BisectPerformanceMetrics class.""" |
| 16 options_dict = { | 15 options_dict = { |
| 17 'debug_ignore_build': True, | 16 'debug_ignore_build': True, |
| 18 'debug_ignore_sync': True, | 17 'debug_ignore_sync': True, |
| 19 'debug_ignore_perf_test': True, | 18 'debug_ignore_perf_test': True, |
| 20 'command': 'fake_command', | 19 'command': 'fake_command', |
| 21 'metric': 'fake/metric', | 20 'metric': 'fake/metric', |
| 22 'good_revision': 280000, | 21 'good_revision': 280000, |
| 23 'bad_revision': 280005, | 22 'bad_revision': 280005, |
| 24 } | 23 } |
| 25 bisect_options = bisect_perf_module.BisectOptions.FromDict(options_dict) | 24 bisect_options = bisect_perf_regression.BisectOptions.FromDict(options_dict) |
| 26 source_control = source_control_module.DetermineAndCreateSourceControl( | 25 source_control = source_control_module.DetermineAndCreateSourceControl( |
| 27 bisect_options) | 26 bisect_options) |
| 28 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | 27 bisect_instance = bisect_perf_regression.BisectPerformanceMetrics( |
| 29 source_control, bisect_options) | 28 source_control, bisect_options) |
| 30 bisect_instance.src_cwd = os.path.abspath( | |
| 31 os.path.join(os.path.dirname(__file__), os.path.pardir)) | |
| 32 return bisect_instance | 29 return bisect_instance |
| 33 | 30 |
| 34 | 31 |
| 35 class BisectPerfRegressionTest(unittest.TestCase): | 32 class BisectPerfRegressionTest(unittest.TestCase): |
| 36 """Test case for other functions and classes in bisect-perf-regression.py.""" | 33 """Test case for other functions and classes in bisect-perf-regression.py.""" |
| 37 | 34 |
| 38 def _AssertConfidence(self, score, bad_values, good_values): | 35 def _AssertConfidence(self, score, bad_values, good_values): |
| 39 """Checks whether the given sets of values have a given confidence score. | 36 """Checks whether the given sets of values have a given confidence score. |
| 40 | 37 |
| 41 The score represents our confidence that the two sets of values wouldn't | 38 The score represents our confidence that the two sets of values wouldn't |
| 42 be as different as they are just by chance; that is, that some real change | 39 be as different as they are just by chance; that is, that some real change |
| 43 occurred between the two sets of values. | 40 occurred between the two sets of values. |
| 44 | 41 |
| 45 Args: | 42 Args: |
| 46 score: Expected confidence score. | 43 score: Expected confidence score. |
| 47 bad_values: First list of numbers. | 44 bad_values: First list of numbers. |
| 48 good_values: Second list of numbers. | 45 good_values: Second list of numbers. |
| 49 """ | 46 """ |
| 50 # ConfidenceScore takes a list of lists but these lists are flattened | 47 # ConfidenceScore takes a list of lists but these lists are flattened |
| 51 # inside the function. | 48 # inside the function. |
| 52 confidence = bisect_perf_module.ConfidenceScore( | 49 confidence = bisect_perf_regression.ConfidenceScore( |
| 53 [[v] for v in bad_values], | 50 [[v] for v in bad_values], |
| 54 [[v] for v in good_values]) | 51 [[v] for v in good_values]) |
| 55 self.assertEqual(score, confidence) | 52 self.assertEqual(score, confidence) |
| 56 | 53 |
| 57 def testConfidenceScore_ZeroConfidence(self): | 54 def testConfidenceScore_ZeroConfidence(self): |
| 58 # The good and bad sets contain the same values, so the confidence that | 55 # The good and bad sets contain the same values, so the confidence that |
| 59 # they're different should be zero. | 56 # they're different should be zero. |
| 60 self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4]) | 57 self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4]) |
| 61 | 58 |
| 62 def testConfidenceScore_MediumConfidence(self): | 59 def testConfidenceScore_MediumConfidence(self): |
| (...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 121 | 118 |
| 122 # Should only expect SVN/git revisions to come through, and URLs should be | 119 # Should only expect SVN/git revisions to come through, and URLs should be |
| 123 # filtered out. | 120 # filtered out. |
| 124 expected_vars_dict = { | 121 expected_vars_dict = { |
| 125 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', | 122 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', |
| 126 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', | 123 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', |
| 127 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' | 124 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' |
| 128 } | 125 } |
| 129 # Testing private function. | 126 # Testing private function. |
| 130 # pylint: disable=W0212 | 127 # pylint: disable=W0212 |
| 131 vars_dict = bisect_perf_module._ParseRevisionsFromDEPSFileManually( | 128 vars_dict = bisect_perf_regression._ParseRevisionsFromDEPSFileManually( |
| 132 deps_file_contents) | 129 deps_file_contents) |
| 133 self.assertEqual(vars_dict, expected_vars_dict) | 130 self.assertEqual(vars_dict, expected_vars_dict) |
| 134 | 131 |
| 135 def _AssertParseResult(self, expected_values, result_string): | 132 def _AssertParseResult(self, expected_values, result_string): |
| 136 """Asserts some values are parsed from a RESULT line.""" | 133 """Asserts some values are parsed from a RESULT line.""" |
| 137 results_template = ('RESULT other_chart: other_trace= 123 count\n' | 134 results_template = ('RESULT other_chart: other_trace= 123 count\n' |
| 138 'RESULT my_chart: my_trace= %(value)s\n') | 135 'RESULT my_chart: my_trace= %(value)s\n') |
| 139 results = results_template % {'value': result_string} | 136 results = results_template % {'value': result_string} |
| 140 metric = ['my_chart', 'my_trace'] | 137 metric = ['my_chart', 'my_trace'] |
| 141 # Testing private function. | 138 # Testing private function. |
| 142 # pylint: disable=W0212 | 139 # pylint: disable=W0212 |
| 143 values = bisect_perf_module._TryParseResultValuesFromOutput(metric, results) | 140 values = bisect_perf_regression._TryParseResultValuesFromOutput( |
| 141 metric, results) |
| 144 self.assertEqual(expected_values, values) | 142 self.assertEqual(expected_values, values) |
| 145 | 143 |
| 146 def testTryParseResultValuesFromOutput_WithSingleValue(self): | 144 def testTryParseResultValuesFromOutput_WithSingleValue(self): |
| 147 """Tests result pattern <*>RESULT <graph>: <trace>= <value>""" | 145 """Tests result pattern <*>RESULT <graph>: <trace>= <value>""" |
| 148 self._AssertParseResult([66.88], '66.88 kb') | 146 self._AssertParseResult([66.88], '66.88 kb') |
| 149 self._AssertParseResult([66.88], '66.88 ') | 147 self._AssertParseResult([66.88], '66.88 ') |
| 150 self._AssertParseResult([-66.88], '-66.88 kb') | 148 self._AssertParseResult([-66.88], '-66.88 kb') |
| 151 self._AssertParseResult([66], '66 kb') | 149 self._AssertParseResult([66], '66 kb') |
| 152 self._AssertParseResult([0.66], '.66 kb') | 150 self._AssertParseResult([0.66], '.66 kb') |
| 153 self._AssertParseResult([], '. kb') | 151 self._AssertParseResult([], '. kb') |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 185 | 183 |
| 186 This modification to the command is done in order to get a Telemetry | 184 This modification to the command is done in order to get a Telemetry |
| 187 command that works; before some revisions, the browser name that Telemetry | 185 command that works; before some revisions, the browser name that Telemetry |
| 188 expects is different in some cases, but we want it to work anyway. | 186 expects is different in some cases, but we want it to work anyway. |
| 189 | 187 |
| 190 Specifically, only for android: | 188 Specifically, only for android: |
| 191 After r276628, only android-chrome-shell works. | 189 After r276628, only android-chrome-shell works. |
| 192 Prior to r274857, only android-chromium-testshell works. | 190 Prior to r274857, only android-chromium-testshell works. |
| 193 In the range [274857, 276628], both work. | 191 In the range [274857, 276628], both work. |
| 194 """ | 192 """ |
| 195 bisect_options = bisect_perf_module.BisectOptions() | 193 bisect_options = bisect_perf_regression.BisectOptions() |
| 196 bisect_options.output_buildbot_annotations = None | 194 bisect_options.output_buildbot_annotations = None |
| 197 source_control = source_control_module.DetermineAndCreateSourceControl( | 195 source_control = source_control_module.DetermineAndCreateSourceControl( |
| 198 bisect_options) | 196 bisect_options) |
| 199 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | 197 bisect_instance = bisect_perf_regression.BisectPerformanceMetrics( |
| 200 source_control, bisect_options) | 198 source_control, bisect_options) |
| 201 bisect_instance.opts.target_platform = target_platform | 199 bisect_instance.opts.target_platform = target_platform |
| 202 git_revision = bisect_instance.source_control.ResolveToRevision( | 200 git_revision = bisect_instance.source_control.ResolveToRevision( |
| 203 revision, 'chromium', bisect_perf_module.DEPOT_DEPS_NAME, 100) | 201 revision, 'chromium', bisect_perf_regression.DEPOT_DEPS_NAME, 100) |
| 204 depot = 'chromium' | 202 depot = 'chromium' |
| 205 command = bisect_instance.GetCompatibleCommand( | 203 command = bisect_instance.GetCompatibleCommand( |
| 206 original_command, git_revision, depot) | 204 original_command, git_revision, depot) |
| 207 self.assertEqual(expected_command, command) | 205 self.assertEqual(expected_command, command) |
| 208 | 206 |
| 209 def testGetCompatibleCommand_ChangeToTestShell(self): | 207 def testGetCompatibleCommand_ChangeToTestShell(self): |
| 210 # For revisions <= r274857, only android-chromium-testshell is used. | 208 # For revisions <= r274857, only android-chromium-testshell is used. |
| 211 self._AssertCompatibleCommand( | 209 self._AssertCompatibleCommand( |
| 212 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo', | 210 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo', |
| 213 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo', | 211 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo', |
| (...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 261 self.assertEqual( | 259 self.assertEqual( |
| 262 291765, bisect_instance.source_control.GetCommitPosition(cp_git_rev)) | 260 291765, bisect_instance.source_control.GetCommitPosition(cp_git_rev)) |
| 263 | 261 |
| 264 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58' | 262 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58' |
| 265 self.assertEqual( | 263 self.assertEqual( |
| 266 291467, bisect_instance.source_control.GetCommitPosition(svn_git_rev)) | 264 291467, bisect_instance.source_control.GetCommitPosition(svn_git_rev)) |
| 267 | 265 |
| 268 def testGetCommitPositionForV8(self): | 266 def testGetCommitPositionForV8(self): |
| 269 bisect_instance = _GetBisectPerformanceMetricsInstance() | 267 bisect_instance = _GetBisectPerformanceMetricsInstance() |
| 270 v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb' | 268 v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb' |
| 271 depot_path = os.path.join(bisect_instance.src_cwd, 'src', 'v8') | 269 depot_path = os.path.join(bisect_perf_regression.SRC_DIR, 'v8') |
| 272 self.assertEqual( | 270 self.assertEqual( |
| 273 23634, | 271 23634, |
| 274 bisect_instance.source_control.GetCommitPosition(v8_rev, depot_path)) | 272 bisect_instance.source_control.GetCommitPosition(v8_rev, depot_path)) |
| 275 | 273 |
| 276 def testGetCommitPositionForWebKit(self): | 274 def testGetCommitPositionForWebKit(self): |
| 277 bisect_instance = _GetBisectPerformanceMetricsInstance() | 275 bisect_instance = _GetBisectPerformanceMetricsInstance() |
| 278 wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61' | 276 wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61' |
| 279 depot_path = os.path.join(bisect_instance.src_cwd, 'src', 'third_party', | 277 depot_path = os.path.join(bisect_perf_regression.SRC_DIR, 'third_party', |
| 280 'WebKit') | 278 'WebKit') |
| 281 self.assertEqual( | 279 self.assertEqual( |
| 282 181660, | 280 181660, |
| 283 bisect_instance.source_control.GetCommitPosition(wk_rev, depot_path)) | 281 bisect_instance.source_control.GetCommitPosition(wk_rev, depot_path)) |
| 284 | 282 |
| 285 def testUpdateDepsContent(self): | 283 def testUpdateDepsContent(self): |
| 286 bisect_instance = _GetBisectPerformanceMetricsInstance() | 284 bisect_instance = _GetBisectPerformanceMetricsInstance() |
| 287 deps_file = 'DEPS' | 285 deps_file = 'DEPS' |
| 288 # We are intentionally reading DEPS file contents instead of string literal | 286 # We are intentionally reading DEPS file contents instead of string literal |
| 289 # with few lines from DEPS because to check if the format we are expecting | 287 # with few lines from DEPS because to check if the format we are expecting |
| 290 # to search is not changed in DEPS content. | 288 # to search is not changed in DEPS content. |
| 291 # TODO (prasadv): Add a separate test to validate the DEPS contents with the | 289 # TODO (prasadv): Add a separate test to validate the DEPS contents with the |
| 292 # format that bisect script expects. | 290 # format that bisect script expects. |
| 293 deps_contents = bisect_perf_module.ReadStringFromFile(deps_file) | 291 deps_contents = bisect_perf_regression.ReadStringFromFile(deps_file) |
| 294 deps_key = 'v8_revision' | 292 deps_key = 'v8_revision' |
| 295 depot = 'v8' | 293 depot = 'v8' |
| 296 git_revision = 'a12345789a23456789a123456789a123456789' | 294 git_revision = 'a12345789a23456789a123456789a123456789' |
| 297 updated_content = bisect_instance.UpdateDepsContents( | 295 updated_content = bisect_instance.UpdateDepsContents( |
| 298 deps_contents, depot, git_revision, deps_key) | 296 deps_contents, depot, git_revision, deps_key) |
| 299 self.assertIsNotNone(updated_content) | 297 self.assertIsNotNone(updated_content) |
| 300 ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision)) | 298 ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision)) |
| 301 self.assertIsNotNone(re.search(ss, updated_content)) | 299 self.assertIsNotNone(re.search(ss, updated_content)) |
| 302 | 300 |
| 303 | 301 |
| 304 if __name__ == '__main__': | 302 if __name__ == '__main__': |
| 305 unittest.main() | 303 unittest.main() |
| OLD | NEW |