| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import math | |
| 6 import unittest | 5 import unittest |
| 7 | 6 |
| 8 from auto_bisect import source_control as source_control_module | 7 from auto_bisect import source_control as source_control_module |
| 9 | 8 |
| 10 # Special import necessary because filename contains dash characters. | 9 # Special import necessary because filename contains dash characters. |
| 11 bisect_perf_module = __import__('bisect-perf-regression') | 10 bisect_perf_module = __import__('bisect-perf-regression') |
| 12 | 11 |
| 13 # Sample output for a performance test used in the results parsing tests below. | 12 # Sample output for a performance test used in the results parsing tests below. |
| 14 RESULTS_OUTPUT = """RESULT write_operations: write_operations= 23089 count | 13 RESULTS_OUTPUT = """RESULT write_operations: write_operations= 23089 count |
| 15 RESULT read_bytes_gpu: read_bytes_gpu= 35201 kb | 14 RESULT read_bytes_gpu: read_bytes_gpu= 35201 kb |
| 16 RESULT write_bytes_gpu: write_bytes_gpu= 542 kb | 15 RESULT write_bytes_gpu: write_bytes_gpu= 542 kb |
| 17 RESULT telemetry_page_measurement_results: num_failed= 0 count | 16 RESULT telemetry_page_measurement_results: num_failed= 0 count |
| 18 RESULT telemetry_page_measurement_results: num_errored= 0 count | 17 RESULT telemetry_page_measurement_results: num_errored= 0 count |
| 19 *RESULT Total: Total_ref= %(value)s | 18 *RESULT Total: Total_ref= %(value)s |
| 20 """ | 19 """ |
| 21 | 20 |
| 22 | 21 |
| 22 # Some private methods of the bisect-perf-regression module are tested below. |
| 23 # pylint: disable=W0212 |
| 23 class BisectPerfRegressionTest(unittest.TestCase): | 24 class BisectPerfRegressionTest(unittest.TestCase): |
| 24 """Test case for top-level functions in the bisect-perf-regrssion module.""" | 25 """Test case for top-level functions in the bisect-perf-regrssion module.""" |
| 25 | 26 |
| 26 def setUp(self): | 27 def setUp(self): |
| 27 """Sets up the test environment before each test method.""" | 28 """Sets up the test environment before each test method.""" |
| 28 pass | 29 pass |
| 29 | 30 |
| 30 def tearDown(self): | 31 def tearDown(self): |
| 31 """Cleans up the test environment after each test method.""" | 32 """Cleans up the test environment after each test method.""" |
| 32 pass | 33 pass |
| (...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 86 | 87 |
| 87 def testConfidenceScoreFunctionalTestResultsConsistent(self): | 88 def testConfidenceScoreFunctionalTestResultsConsistent(self): |
| 88 """Tests the confidence calculation when the numbers are 0 and 1.""" | 89 """Tests the confidence calculation when the numbers are 0 and 1.""" |
| 89 bad_values = [[1], [1], [1], [1], [1], [1], [1], [1]] | 90 bad_values = [[1], [1], [1], [1], [1], [1], [1], [1]] |
| 90 good_values = [[0], [0], [0], [0], [0], [0]] | 91 good_values = [[0], [0], [0], [0], [0], [0]] |
| 91 confidence = bisect_perf_module.ConfidenceScore(bad_values, good_values) | 92 confidence = bisect_perf_module.ConfidenceScore(bad_values, good_values) |
| 92 self.assertEqual(99.9, confidence) | 93 self.assertEqual(99.9, confidence) |
| 93 | 94 |
| 94 def testParseDEPSStringManually(self): | 95 def testParseDEPSStringManually(self): |
| 95 """Tests DEPS parsing.""" | 96 """Tests DEPS parsing.""" |
| 96 bisect_options = bisect_perf_module.BisectOptions() | 97 deps_file_contents = """ |
| 97 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | 98 vars = { |
| 98 None, bisect_options) | 99 'ffmpeg_hash': |
| 100 '@ac4a9f31fe2610bd146857bbd55d7a260003a888', |
| 101 'webkit_url': |
| 102 'https://chromium.googlesource.com/chromium/blink.git', |
| 103 'git_url': |
| 104 'https://chromium.googlesource.com', |
| 105 'webkit_rev': |
| 106 '@e01ac0a267d1017288bc67fa3c366b10469d8a24', |
| 107 'angle_revision': |
| 108 '74697cf2064c0a2c0d7e1b1b28db439286766a05' |
| 109 }""" |
| 99 | 110 |
| 100 deps_file_contents = """ | 111 # Should only expect svn/git revisions to come through, and urls should be |
| 101 vars = { | |
| 102 'ffmpeg_hash': | |
| 103 '@ac4a9f31fe2610bd146857bbd55d7a260003a888', | |
| 104 'webkit_url': | |
| 105 'https://chromium.googlesource.com/chromium/blink.git', | |
| 106 'git_url': | |
| 107 'https://chromium.googlesource.com', | |
| 108 'webkit_rev': | |
| 109 '@e01ac0a267d1017288bc67fa3c366b10469d8a24', | |
| 110 'angle_revision': | |
| 111 '74697cf2064c0a2c0d7e1b1b28db439286766a05' | |
| 112 }""" | |
| 113 | |
| 114 # Should only expect svn/git revisions to come through, and urls to be | |
| 115 # filtered out. | 112 # filtered out. |
| 116 expected_vars_dict = { | 113 expected_vars_dict = { |
| 117 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', | 114 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', |
| 118 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', | 115 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', |
| 119 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' | 116 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' |
| 120 } | 117 } |
| 121 vars_dict = bisect_instance._ParseRevisionsFromDEPSFileManually( | 118 vars_dict = bisect_perf_module._ParseRevisionsFromDEPSFileManually( |
| 122 deps_file_contents) | 119 deps_file_contents) |
| 123 self.assertEqual(vars_dict, expected_vars_dict) | 120 self.assertEqual(vars_dict, expected_vars_dict) |
| 124 | 121 |
| 125 def testTryParseResultValuesFromOutputWithSingleValue(self): | 122 def testTryParseResultValuesFromOutputWithSingleValue(self): |
| 126 """Tests result pattern <*>RESULT <graph>: <trace>= <value>""" | 123 """Tests result pattern <*>RESULT <graph>: <trace>= <value>""" |
| 127 bisect_options = bisect_perf_module.BisectOptions() | |
| 128 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | |
| 129 None, bisect_options) | |
| 130 metrics = ['Total', 'Total_ref'] | 124 metrics = ['Total', 'Total_ref'] |
| 131 self.assertEqual( | 125 self.assertEqual( |
| 132 [66.88], bisect_instance.TryParseResultValuesFromOutput( | 126 [66.88], bisect_perf_module._TryParseResultValuesFromOutput( |
| 133 metrics, RESULTS_OUTPUT % {'value': '66.88 kb'})) | 127 metrics, RESULTS_OUTPUT % {'value': '66.88 kb'})) |
| 134 self.assertEqual( | 128 self.assertEqual( |
| 135 [66.88], bisect_instance.TryParseResultValuesFromOutput( | 129 [66.88], bisect_perf_module._TryParseResultValuesFromOutput( |
| 136 metrics, RESULTS_OUTPUT % {'value': '66.88kb'})) | 130 metrics, RESULTS_OUTPUT % {'value': '66.88kb'})) |
| 137 self.assertEqual( | 131 self.assertEqual( |
| 138 [66.88], bisect_instance.TryParseResultValuesFromOutput( | 132 [66.88], bisect_perf_module._TryParseResultValuesFromOutput( |
| 139 metrics, RESULTS_OUTPUT % {'value': ' 66.88 '})) | 133 metrics, RESULTS_OUTPUT % {'value': ' 66.88 '})) |
| 140 self.assertEqual( | 134 self.assertEqual( |
| 141 [-66.88], bisect_instance.TryParseResultValuesFromOutput( | 135 [-66.88], bisect_perf_module._TryParseResultValuesFromOutput( |
| 142 metrics, RESULTS_OUTPUT % {'value': ' -66.88 kb'})) | 136 metrics, RESULTS_OUTPUT % {'value': ' -66.88 kb'})) |
| 143 self.assertEqual( | 137 self.assertEqual( |
| 144 [66], bisect_instance.TryParseResultValuesFromOutput( | 138 [66], bisect_perf_module._TryParseResultValuesFromOutput( |
| 145 metrics, RESULTS_OUTPUT % {'value': '66 kb'})) | 139 metrics, RESULTS_OUTPUT % {'value': '66 kb'})) |
| 146 self.assertEqual( | 140 self.assertEqual( |
| 147 [.66], bisect_instance.TryParseResultValuesFromOutput( | 141 [.66], bisect_perf_module._TryParseResultValuesFromOutput( |
| 148 metrics, RESULTS_OUTPUT % {'value': '.66 kb'})) | 142 metrics, RESULTS_OUTPUT % {'value': '.66 kb'})) |
| 149 self.assertEqual( | 143 self.assertEqual( |
| 150 [], bisect_instance.TryParseResultValuesFromOutput( | 144 [], bisect_perf_module._TryParseResultValuesFromOutput( |
| 151 metrics, RESULTS_OUTPUT % {'value': '. kb'})) | 145 metrics, RESULTS_OUTPUT % {'value': '. kb'})) |
| 152 self.assertEqual( | 146 self.assertEqual( |
| 153 [], bisect_instance.TryParseResultValuesFromOutput( | 147 [], bisect_perf_module._TryParseResultValuesFromOutput( |
| 154 metrics, RESULTS_OUTPUT % {'value': 'aaa kb'})) | 148 metrics, RESULTS_OUTPUT % {'value': 'aaa kb'})) |
| 155 | 149 |
| 156 def testTryParseResultValuesFromOutputWithMulitValue(self): | 150 def testTryParseResultValuesFromOutputWithMulitValue(self): |
| 157 """Tests result pattern <*>RESULT <graph>: <trace>= [<value>,<value>, ..]""" | 151 """Tests result pattern <*>RESULT <graph>: <trace>= [<value>,<value>, ..]""" |
| 158 bisect_options = bisect_perf_module.BisectOptions() | |
| 159 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | |
| 160 None, bisect_options) | |
| 161 metrics = ['Total', 'Total_ref'] | 152 metrics = ['Total', 'Total_ref'] |
| 162 self.assertEqual( | 153 self.assertEqual( |
| 163 [66.88], bisect_instance.TryParseResultValuesFromOutput( | 154 [66.88], bisect_perf_module._TryParseResultValuesFromOutput( |
| 164 metrics, RESULTS_OUTPUT % {'value': '[66.88] kb'})) | 155 metrics, RESULTS_OUTPUT % {'value': '[66.88] kb'})) |
| 165 self.assertEqual( | 156 self.assertEqual( |
| 166 [66.88, 99.44], bisect_instance.TryParseResultValuesFromOutput( | 157 [66.88, 99.44], bisect_perf_module._TryParseResultValuesFromOutput( |
| 167 metrics, RESULTS_OUTPUT % {'value': '[66.88, 99.44]kb'})) | 158 metrics, RESULTS_OUTPUT % {'value': '[66.88, 99.44]kb'})) |
| 168 self.assertEqual( | 159 self.assertEqual( |
| 169 [66.88, 99.44], bisect_instance.TryParseResultValuesFromOutput( | 160 [66.88, 99.44], bisect_perf_module._TryParseResultValuesFromOutput( |
| 170 metrics, RESULTS_OUTPUT % {'value': '[ 66.88, 99.44 ]'})) | 161 metrics, RESULTS_OUTPUT % {'value': '[ 66.88, 99.44 ]'})) |
| 171 self.assertEqual( | 162 self.assertEqual( |
| 172 [-66.88, 99.44], bisect_instance.TryParseResultValuesFromOutput( | 163 [-66.88, 99.44], bisect_perf_module._TryParseResultValuesFromOutput( |
| 173 metrics, RESULTS_OUTPUT % {'value': '[-66.88,99.44] kb'})) | 164 metrics, RESULTS_OUTPUT % {'value': '[-66.88,99.44] kb'})) |
| 174 self.assertEqual( | 165 self.assertEqual( |
| 175 [-66, 99], bisect_instance.TryParseResultValuesFromOutput( | 166 [-66, 99], bisect_perf_module._TryParseResultValuesFromOutput( |
| 176 metrics, RESULTS_OUTPUT % {'value': '[-66,99] kb'})) | 167 metrics, RESULTS_OUTPUT % {'value': '[-66,99] kb'})) |
| 177 self.assertEqual( | 168 self.assertEqual( |
| 178 [-66, 99], bisect_instance.TryParseResultValuesFromOutput( | 169 [-66, 99], bisect_perf_module._TryParseResultValuesFromOutput( |
| 179 metrics, RESULTS_OUTPUT % {'value': '[-66,99,] kb'})) | 170 metrics, RESULTS_OUTPUT % {'value': '[-66,99,] kb'})) |
| 180 self.assertEqual( | 171 self.assertEqual( |
| 181 [.66, .99], bisect_instance.TryParseResultValuesFromOutput( | 172 [.66, .99], bisect_perf_module._TryParseResultValuesFromOutput( |
| 182 metrics, RESULTS_OUTPUT % {'value': '[.66,.99] kb'})) | 173 metrics, RESULTS_OUTPUT % {'value': '[.66,.99] kb'})) |
| 183 self.assertEqual( | 174 self.assertEqual( |
| 184 [], bisect_instance.TryParseResultValuesFromOutput( | 175 [], bisect_perf_module._TryParseResultValuesFromOutput( |
| 185 metrics, RESULTS_OUTPUT % {'value': '[] kb'})) | 176 metrics, RESULTS_OUTPUT % {'value': '[] kb'})) |
| 186 self.assertEqual( | 177 self.assertEqual( |
| 187 [], bisect_instance.TryParseResultValuesFromOutput( | 178 [], bisect_perf_module._TryParseResultValuesFromOutput( |
| 188 metrics, RESULTS_OUTPUT % {'value': '[-66,abc] kb'})) | 179 metrics, RESULTS_OUTPUT % {'value': '[-66,abc] kb'})) |
| 189 | 180 |
| 190 def testTryParseResultValuesFromOutputWithMeanStd(self): | 181 def testTryParseResultValuesFromOutputWithMeanStd(self): |
| 191 """Tests result pattern <*>RESULT <graph>: <trace>= {<mean, std}""" | 182 """Tests result pattern <*>RESULT <graph>: <trace>= {<mean, std}""" |
| 192 bisect_options = bisect_perf_module.BisectOptions() | |
| 193 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | |
| 194 None, bisect_options) | |
| 195 metrics = ['Total', 'Total_ref'] | 183 metrics = ['Total', 'Total_ref'] |
| 196 self.assertEqual( | 184 self.assertEqual( |
| 197 [33.22], bisect_instance.TryParseResultValuesFromOutput( | 185 [33.22], bisect_perf_module._TryParseResultValuesFromOutput( |
| 198 metrics, RESULTS_OUTPUT % {'value': '{33.22, 3.6} kb'})) | 186 metrics, RESULTS_OUTPUT % {'value': '{33.22, 3.6} kb'})) |
| 199 self.assertEqual( | 187 self.assertEqual( |
| 200 [33.22], bisect_instance.TryParseResultValuesFromOutput( | 188 [33.22], bisect_perf_module._TryParseResultValuesFromOutput( |
| 201 metrics, RESULTS_OUTPUT % {'value': '{33.22,3.6}kb'})) | 189 metrics, RESULTS_OUTPUT % {'value': '{33.22,3.6}kb'})) |
| 202 self.assertEqual( | 190 self.assertEqual( |
| 203 [33.22], bisect_instance.TryParseResultValuesFromOutput( | 191 [33.22], bisect_perf_module._TryParseResultValuesFromOutput( |
| 204 metrics, RESULTS_OUTPUT % {'value': '{33.22,3.6} kb'})) | 192 metrics, RESULTS_OUTPUT % {'value': '{33.22,3.6} kb'})) |
| 205 self.assertEqual( | 193 self.assertEqual( |
| 206 [33.22], bisect_instance.TryParseResultValuesFromOutput( | 194 [33.22], bisect_perf_module._TryParseResultValuesFromOutput( |
| 207 metrics, RESULTS_OUTPUT % {'value': '{ 33.22,3.6 }kb'})) | 195 metrics, RESULTS_OUTPUT % {'value': '{ 33.22,3.6 }kb'})) |
| 208 self.assertEqual( | 196 self.assertEqual( |
| 209 [-33.22], bisect_instance.TryParseResultValuesFromOutput( | 197 [-33.22], bisect_perf_module._TryParseResultValuesFromOutput( |
| 210 metrics, RESULTS_OUTPUT % {'value': '{-33.22,3.6}kb'})) | 198 metrics, RESULTS_OUTPUT % {'value': '{-33.22,3.6}kb'})) |
| 211 self.assertEqual( | 199 self.assertEqual( |
| 212 [22], bisect_instance.TryParseResultValuesFromOutput( | 200 [22], bisect_perf_module._TryParseResultValuesFromOutput( |
| 213 metrics, RESULTS_OUTPUT % {'value': '{22,6}kb'})) | 201 metrics, RESULTS_OUTPUT % {'value': '{22,6}kb'})) |
| 214 self.assertEqual( | 202 self.assertEqual( |
| 215 [.22], bisect_instance.TryParseResultValuesFromOutput( | 203 [.22], bisect_perf_module._TryParseResultValuesFromOutput( |
| 216 metrics, RESULTS_OUTPUT % {'value': '{.22,6}kb'})) | 204 metrics, RESULTS_OUTPUT % {'value': '{.22,6}kb'})) |
| 217 self.assertEqual( | 205 self.assertEqual( |
| 218 [], bisect_instance.TryParseResultValuesFromOutput( | 206 [], bisect_perf_module._TryParseResultValuesFromOutput( |
| 219 metrics, RESULTS_OUTPUT % {'value': '{.22,6, 44}kb'})) | 207 metrics, RESULTS_OUTPUT % {'value': '{.22,6, 44}kb'})) |
| 220 self.assertEqual( | 208 self.assertEqual( |
| 221 [], bisect_instance.TryParseResultValuesFromOutput( | 209 [], bisect_perf_module._TryParseResultValuesFromOutput( |
| 222 metrics, RESULTS_OUTPUT % {'value': '{}kb'})) | 210 metrics, RESULTS_OUTPUT % {'value': '{}kb'})) |
| 223 self.assertEqual( | 211 self.assertEqual( |
| 224 [], bisect_instance.TryParseResultValuesFromOutput( | 212 [], bisect_perf_module._TryParseResultValuesFromOutput( |
| 225 metrics, RESULTS_OUTPUT % {'value': '{XYZ}kb'})) | 213 metrics, RESULTS_OUTPUT % {'value': '{XYZ}kb'})) |
| 226 | 214 |
| 227 def testGetCompatibleCommand(self): | 215 def testGetCompatibleCommand(self): |
| 228 bisect_options = bisect_perf_module.BisectOptions() | 216 bisect_options = bisect_perf_module.BisectOptions() |
| 229 bisect_options.output_buildbot_annotations = None | 217 bisect_options.output_buildbot_annotations = None |
| 230 source_control = source_control_module.DetermineAndCreateSourceControl( | 218 source_control = source_control_module.DetermineAndCreateSourceControl( |
| 231 bisect_options) | 219 bisect_options) |
| 232 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | 220 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( |
| 233 source_control, bisect_options) | 221 source_control, bisect_options) |
| 234 bisect_instance.opts.target_platform = 'android' | 222 bisect_instance.opts.target_platform = 'android' |
| (...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 282 '--browser=release page_cycler.intl_ja_zh') | 270 '--browser=release page_cycler.intl_ja_zh') |
| 283 expected_command = ('tools/perf/run_benchmark -v --browser=' | 271 expected_command = ('tools/perf/run_benchmark -v --browser=' |
| 284 'release page_cycler.intl_ja_zh') | 272 'release page_cycler.intl_ja_zh') |
| 285 self.assertEqual( | 273 self.assertEqual( |
| 286 bisect_instance.GetCompatibleCommand(command, git_revision, depot), | 274 bisect_instance.GetCompatibleCommand(command, git_revision, depot), |
| 287 expected_command) | 275 expected_command) |
| 288 | 276 |
| 289 | 277 |
| 290 if __name__ == '__main__': | 278 if __name__ == '__main__': |
| 291 unittest.main() | 279 unittest.main() |
| OLD | NEW |