| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import os | 5 import os |
| 6 import re | 6 import re |
| 7 import unittest | 7 import unittest |
| 8 | 8 |
| 9 from auto_bisect import source_control as source_control_module | 9 import bisect_perf_regression |
| 10 | 10 import source_control as source_control_module |
| 11 # Special import necessary because filename contains dash characters. | |
| 12 bisect_perf_module = __import__('bisect-perf-regression') | |
| 13 | 11 |
| 14 def _GetBisectPerformanceMetricsInstance(): | 12 def _GetBisectPerformanceMetricsInstance(): |
| 15 """Returns an instance of the BisectPerformanceMetrics class.""" | 13 """Returns an instance of the BisectPerformanceMetrics class.""" |
| 16 options_dict = { | 14 options_dict = { |
| 17 'debug_ignore_build': True, | 15 'debug_ignore_build': True, |
| 18 'debug_ignore_sync': True, | 16 'debug_ignore_sync': True, |
| 19 'debug_ignore_perf_test': True, | 17 'debug_ignore_perf_test': True, |
| 20 'command': 'fake_command', | 18 'command': 'fake_command', |
| 21 'metric': 'fake/metric', | 19 'metric': 'fake/metric', |
| 22 'good_revision': 280000, | 20 'good_revision': 280000, |
| 23 'bad_revision': 280005, | 21 'bad_revision': 280005, |
| 24 } | 22 } |
| 25 bisect_options = bisect_perf_module.BisectOptions.FromDict(options_dict) | 23 bisect_options = bisect_perf_regression.BisectOptions.FromDict(options_dict) |
| 26 source_control = source_control_module.DetermineAndCreateSourceControl( | 24 source_control = source_control_module.DetermineAndCreateSourceControl( |
| 27 bisect_options) | 25 bisect_options) |
| 28 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | 26 bisect_instance = bisect_perf_regression.BisectPerformanceMetrics( |
| 29 source_control, bisect_options) | 27 source_control, bisect_options) |
| 30 bisect_instance.src_cwd = os.path.abspath( | 28 bisect_instance.src_cwd = os.path.abspath( |
| 31 os.path.join(os.path.dirname(__file__), os.path.pardir)) | 29 os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)) |
| 32 return bisect_instance | 30 return bisect_instance |
| 33 | 31 |
| 34 | 32 |
| 35 class BisectPerfRegressionTest(unittest.TestCase): | 33 class BisectPerfRegressionTest(unittest.TestCase): |
| 36 """Test case for other functions and classes in bisect-perf-regression.py.""" | 34 """Test case for other functions and classes in bisect-perf-regression.py.""" |
| 37 | 35 |
| 38 def _AssertConfidence(self, score, bad_values, good_values): | 36 def _AssertConfidence(self, score, bad_values, good_values): |
| 39 """Checks whether the given sets of values have a given confidence score. | 37 """Checks whether the given sets of values have a given confidence score. |
| 40 | 38 |
| 41 The score represents our confidence that the two sets of values wouldn't | 39 The score represents our confidence that the two sets of values wouldn't |
| 42 be as different as they are just by chance; that is, that some real change | 40 be as different as they are just by chance; that is, that some real change |
| 43 occurred between the two sets of values. | 41 occurred between the two sets of values. |
| 44 | 42 |
| 45 Args: | 43 Args: |
| 46 score: Expected confidence score. | 44 score: Expected confidence score. |
| 47 bad_values: First list of numbers. | 45 bad_values: First list of numbers. |
| 48 good_values: Second list of numbers. | 46 good_values: Second list of numbers. |
| 49 """ | 47 """ |
| 50 # ConfidenceScore takes a list of lists but these lists are flattened | 48 # ConfidenceScore takes a list of lists but these lists are flattened |
| 51 # inside the function. | 49 # inside the function. |
| 52 confidence = bisect_perf_module.ConfidenceScore( | 50 confidence = bisect_perf_regression.ConfidenceScore( |
| 53 [[v] for v in bad_values], | 51 [[v] for v in bad_values], |
| 54 [[v] for v in good_values]) | 52 [[v] for v in good_values]) |
| 55 self.assertEqual(score, confidence) | 53 self.assertEqual(score, confidence) |
| 56 | 54 |
| 57 def testConfidenceScore_ZeroConfidence(self): | 55 def testConfidenceScore_ZeroConfidence(self): |
| 58 # The good and bad sets contain the same values, so the confidence that | 56 # The good and bad sets contain the same values, so the confidence that |
| 59 # they're different should be zero. | 57 # they're different should be zero. |
| 60 self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4]) | 58 self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4]) |
| 61 | 59 |
| 62 def testConfidenceScore_MediumConfidence(self): | 60 def testConfidenceScore_MediumConfidence(self): |
| (...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 121 | 119 |
| 122 # Should only expect SVN/git revisions to come through, and URLs should be | 120 # Should only expect SVN/git revisions to come through, and URLs should be |
| 123 # filtered out. | 121 # filtered out. |
| 124 expected_vars_dict = { | 122 expected_vars_dict = { |
| 125 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', | 123 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', |
| 126 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', | 124 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', |
| 127 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' | 125 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' |
| 128 } | 126 } |
| 129 # Testing private function. | 127 # Testing private function. |
| 130 # pylint: disable=W0212 | 128 # pylint: disable=W0212 |
| 131 vars_dict = bisect_perf_module._ParseRevisionsFromDEPSFileManually( | 129 vars_dict = bisect_perf_regression._ParseRevisionsFromDEPSFileManually( |
| 132 deps_file_contents) | 130 deps_file_contents) |
| 133 self.assertEqual(vars_dict, expected_vars_dict) | 131 self.assertEqual(vars_dict, expected_vars_dict) |
| 134 | 132 |
| 135 def _AssertParseResult(self, expected_values, result_string): | 133 def _AssertParseResult(self, expected_values, result_string): |
| 136 """Asserts some values are parsed from a RESULT line.""" | 134 """Asserts some values are parsed from a RESULT line.""" |
| 137 results_template = ('RESULT other_chart: other_trace= 123 count\n' | 135 results_template = ('RESULT other_chart: other_trace= 123 count\n' |
| 138 'RESULT my_chart: my_trace= %(value)s\n') | 136 'RESULT my_chart: my_trace= %(value)s\n') |
| 139 results = results_template % {'value': result_string} | 137 results = results_template % {'value': result_string} |
| 140 metric = ['my_chart', 'my_trace'] | 138 metric = ['my_chart', 'my_trace'] |
| 141 # Testing private function. | 139 # Testing private function. |
| 142 # pylint: disable=W0212 | 140 # pylint: disable=W0212 |
| 143 values = bisect_perf_module._TryParseResultValuesFromOutput(metric, results) | 141 values = bisect_perf_regression._TryParseResultValuesFromOutput( |
| 142 metric, results) |
| 144 self.assertEqual(expected_values, values) | 143 self.assertEqual(expected_values, values) |
| 145 | 144 |
| 146 def testTryParseResultValuesFromOutput_WithSingleValue(self): | 145 def testTryParseResultValuesFromOutput_WithSingleValue(self): |
| 147 """Tests result pattern <*>RESULT <graph>: <trace>= <value>""" | 146 """Tests result pattern <*>RESULT <graph>: <trace>= <value>""" |
| 148 self._AssertParseResult([66.88], '66.88 kb') | 147 self._AssertParseResult([66.88], '66.88 kb') |
| 149 self._AssertParseResult([66.88], '66.88 ') | 148 self._AssertParseResult([66.88], '66.88 ') |
| 150 self._AssertParseResult([-66.88], '-66.88 kb') | 149 self._AssertParseResult([-66.88], '-66.88 kb') |
| 151 self._AssertParseResult([66], '66 kb') | 150 self._AssertParseResult([66], '66 kb') |
| 152 self._AssertParseResult([0.66], '.66 kb') | 151 self._AssertParseResult([0.66], '.66 kb') |
| 153 self._AssertParseResult([], '. kb') | 152 self._AssertParseResult([], '. kb') |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 185 | 184 |
| 186 This modification to the command is done in order to get a Telemetry | 185 This modification to the command is done in order to get a Telemetry |
| 187 command that works; before some revisions, the browser name that Telemetry | 186 command that works; before some revisions, the browser name that Telemetry |
| 188 expects is different in some cases, but we want it to work anyway. | 187 expects is different in some cases, but we want it to work anyway. |
| 189 | 188 |
| 190 Specifically, only for android: | 189 Specifically, only for android: |
| 191 After r276628, only android-chrome-shell works. | 190 After r276628, only android-chrome-shell works. |
| 192 Prior to r274857, only android-chromium-testshell works. | 191 Prior to r274857, only android-chromium-testshell works. |
| 193 In the range [274857, 276628], both work. | 192 In the range [274857, 276628], both work. |
| 194 """ | 193 """ |
| 195 bisect_options = bisect_perf_module.BisectOptions() | 194 bisect_options = bisect_perf_regression.BisectOptions() |
| 196 bisect_options.output_buildbot_annotations = None | 195 bisect_options.output_buildbot_annotations = None |
| 197 source_control = source_control_module.DetermineAndCreateSourceControl( | 196 source_control = source_control_module.DetermineAndCreateSourceControl( |
| 198 bisect_options) | 197 bisect_options) |
| 199 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | 198 bisect_instance = bisect_perf_regression.BisectPerformanceMetrics( |
| 200 source_control, bisect_options) | 199 source_control, bisect_options) |
| 201 bisect_instance.opts.target_platform = target_platform | 200 bisect_instance.opts.target_platform = target_platform |
| 202 git_revision = bisect_instance.source_control.ResolveToRevision( | 201 git_revision = bisect_instance.source_control.ResolveToRevision( |
| 203 revision, 'chromium', bisect_perf_module.DEPOT_DEPS_NAME, 100) | 202 revision, 'chromium', bisect_perf_regression.DEPOT_DEPS_NAME, 100) |
| 204 depot = 'chromium' | 203 depot = 'chromium' |
| 205 command = bisect_instance.GetCompatibleCommand( | 204 command = bisect_instance.GetCompatibleCommand( |
| 206 original_command, git_revision, depot) | 205 original_command, git_revision, depot) |
| 207 self.assertEqual(expected_command, command) | 206 self.assertEqual(expected_command, command) |
| 208 | 207 |
| 209 def testGetCompatibleCommand_ChangeToTestShell(self): | 208 def testGetCompatibleCommand_ChangeToTestShell(self): |
| 210 # For revisions <= r274857, only android-chromium-testshell is used. | 209 # For revisions <= r274857, only android-chromium-testshell is used. |
| 211 self._AssertCompatibleCommand( | 210 self._AssertCompatibleCommand( |
| 212 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo', | 211 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo', |
| 213 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo', | 212 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo', |
| (...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 261 self.assertEqual( | 260 self.assertEqual( |
| 262 291765, bisect_instance.source_control.GetCommitPosition(cp_git_rev)) | 261 291765, bisect_instance.source_control.GetCommitPosition(cp_git_rev)) |
| 263 | 262 |
| 264 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58' | 263 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58' |
| 265 self.assertEqual( | 264 self.assertEqual( |
| 266 291467, bisect_instance.source_control.GetCommitPosition(svn_git_rev)) | 265 291467, bisect_instance.source_control.GetCommitPosition(svn_git_rev)) |
| 267 | 266 |
| 268 def testGetCommitPositionForV8(self): | 267 def testGetCommitPositionForV8(self): |
| 269 bisect_instance = _GetBisectPerformanceMetricsInstance() | 268 bisect_instance = _GetBisectPerformanceMetricsInstance() |
| 270 v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb' | 269 v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb' |
| 271 depot_path = os.path.join(bisect_instance.src_cwd, 'src', 'v8') | 270 depot_path = os.path.join(bisect_instance.src_cwd, 'v8') |
| 272 self.assertEqual( | 271 self.assertEqual( |
| 273 23634, | 272 23634, |
| 274 bisect_instance.source_control.GetCommitPosition(v8_rev, depot_path)) | 273 bisect_instance.source_control.GetCommitPosition(v8_rev, depot_path)) |
| 275 | 274 |
| 276 def testGetCommitPositionForWebKit(self): | 275 def testGetCommitPositionForWebKit(self): |
| 277 bisect_instance = _GetBisectPerformanceMetricsInstance() | 276 bisect_instance = _GetBisectPerformanceMetricsInstance() |
| 278 wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61' | 277 wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61' |
| 279 depot_path = os.path.join(bisect_instance.src_cwd, 'src', 'third_party', | 278 depot_path = os.path.join(bisect_instance.src_cwd, 'third_party', 'WebKit') |
| 280 'WebKit') | |
| 281 self.assertEqual( | 279 self.assertEqual( |
| 282 181660, | 280 181660, |
| 283 bisect_instance.source_control.GetCommitPosition(wk_rev, depot_path)) | 281 bisect_instance.source_control.GetCommitPosition(wk_rev, depot_path)) |
| 284 | 282 |
| 285 def testUpdateDepsContent(self): | 283 def testUpdateDepsContent(self): |
| 286 bisect_instance = _GetBisectPerformanceMetricsInstance() | 284 bisect_instance = _GetBisectPerformanceMetricsInstance() |
| 287 deps_file = 'DEPS' | 285 deps_file = 'DEPS' |
| 288 # We are intentionally reading DEPS file contents instead of string literal | 286 # We are intentionally reading DEPS file contents instead of string literal |
| 289 # with few lines from DEPS because to check if the format we are expecting | 287 # with few lines from DEPS because to check if the format we are expecting |
| 290 # to search is not changed in DEPS content. | 288 # to search is not changed in DEPS content. |
| 291 # TODO (prasadv): Add a separate test to validate the DEPS contents with the | 289 # TODO (prasadv): Add a separate test to validate the DEPS contents with the |
| 292 # format that bisect script expects. | 290 # format that bisect script expects. |
| 293 deps_contents = bisect_perf_module.ReadStringFromFile(deps_file) | 291 deps_contents = bisect_perf_regression.ReadStringFromFile(deps_file) |
| 294 deps_key = 'v8_revision' | 292 deps_key = 'v8_revision' |
| 295 depot = 'v8' | 293 depot = 'v8' |
| 296 git_revision = 'a12345789a23456789a123456789a123456789' | 294 git_revision = 'a12345789a23456789a123456789a123456789' |
| 297 updated_content = bisect_instance.UpdateDepsContents( | 295 updated_content = bisect_instance.UpdateDepsContents( |
| 298 deps_contents, depot, git_revision, deps_key) | 296 deps_contents, depot, git_revision, deps_key) |
| 299 self.assertIsNotNone(updated_content) | 297 self.assertIsNotNone(updated_content) |
| 300 ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision)) | 298 ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision)) |
| 301 self.assertIsNotNone(re.search(ss, updated_content)) | 299 self.assertIsNotNone(re.search(ss, updated_content)) |
| 302 | 300 |
| 303 | 301 |
| 304 if __name__ == '__main__': | 302 if __name__ == '__main__': |
| 305 unittest.main() | 303 unittest.main() |
| OLD | NEW |