OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import math | |
6 import unittest | 5 import unittest |
7 | 6 |
8 from auto_bisect import source_control as source_control_module | 7 from auto_bisect import source_control as source_control_module |
9 | 8 |
10 # Special import necessary because filename contains dash characters. | 9 # Special import necessary because filename contains dash characters. |
11 bisect_perf_module = __import__('bisect-perf-regression') | 10 bisect_perf_module = __import__('bisect-perf-regression') |
12 | 11 |
13 # Sample output for a performance test used in the results parsing tests below. | 12 # Sample output for a performance test used in the results parsing tests below. |
14 RESULTS_OUTPUT = """RESULT write_operations: write_operations= 23089 count | 13 RESULTS_OUTPUT = """RESULT write_operations: write_operations= 23089 count |
15 RESULT read_bytes_gpu: read_bytes_gpu= 35201 kb | 14 RESULT read_bytes_gpu: read_bytes_gpu= 35201 kb |
16 RESULT write_bytes_gpu: write_bytes_gpu= 542 kb | 15 RESULT write_bytes_gpu: write_bytes_gpu= 542 kb |
17 RESULT telemetry_page_measurement_results: num_failed= 0 count | 16 RESULT telemetry_page_measurement_results: num_failed= 0 count |
18 RESULT telemetry_page_measurement_results: num_errored= 0 count | 17 RESULT telemetry_page_measurement_results: num_errored= 0 count |
19 *RESULT Total: Total_ref= %(value)s | 18 *RESULT Total: Total_ref= %(value)s |
20 """ | 19 """ |
21 | 20 |
22 | 21 |
| 22 # Some private methods of the bisect-perf-regression module are tested below. |
| 23 # pylint: disable=W0212 |
23 class BisectPerfRegressionTest(unittest.TestCase): | 24 class BisectPerfRegressionTest(unittest.TestCase): |
24 """Test case for top-level functions in the bisect-perf-regrssion module.""" | 25 """Test case for top-level functions in the bisect-perf-regrssion module.""" |
25 | 26 |
26 def setUp(self): | 27 def setUp(self): |
27 """Sets up the test environment before each test method.""" | 28 """Sets up the test environment before each test method.""" |
28 pass | 29 pass |
29 | 30 |
30 def tearDown(self): | 31 def tearDown(self): |
31 """Cleans up the test environment after each test method.""" | 32 """Cleans up the test environment after each test method.""" |
32 pass | 33 pass |
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
86 | 87 |
87 def testConfidenceScoreFunctionalTestResultsConsistent(self): | 88 def testConfidenceScoreFunctionalTestResultsConsistent(self): |
88 """Tests the confidence calculation when the numbers are 0 and 1.""" | 89 """Tests the confidence calculation when the numbers are 0 and 1.""" |
89 bad_values = [[1], [1], [1], [1], [1], [1], [1], [1]] | 90 bad_values = [[1], [1], [1], [1], [1], [1], [1], [1]] |
90 good_values = [[0], [0], [0], [0], [0], [0]] | 91 good_values = [[0], [0], [0], [0], [0], [0]] |
91 confidence = bisect_perf_module.ConfidenceScore(bad_values, good_values) | 92 confidence = bisect_perf_module.ConfidenceScore(bad_values, good_values) |
92 self.assertEqual(99.9, confidence) | 93 self.assertEqual(99.9, confidence) |
93 | 94 |
94 def testParseDEPSStringManually(self): | 95 def testParseDEPSStringManually(self): |
95 """Tests DEPS parsing.""" | 96 """Tests DEPS parsing.""" |
96 bisect_options = bisect_perf_module.BisectOptions() | 97 deps_file_contents = """ |
97 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | 98 vars = { |
98 None, bisect_options) | 99 'ffmpeg_hash': |
| 100 '@ac4a9f31fe2610bd146857bbd55d7a260003a888', |
| 101 'webkit_url': |
| 102 'https://chromium.googlesource.com/chromium/blink.git', |
| 103 'git_url': |
| 104 'https://chromium.googlesource.com', |
| 105 'webkit_rev': |
| 106 '@e01ac0a267d1017288bc67fa3c366b10469d8a24', |
| 107 'angle_revision': |
| 108 '74697cf2064c0a2c0d7e1b1b28db439286766a05' |
| 109 }""" |
99 | 110 |
100 deps_file_contents = """ | 111 # Should only expect svn/git revisions to come through, and urls should be |
101 vars = { | |
102 'ffmpeg_hash': | |
103 '@ac4a9f31fe2610bd146857bbd55d7a260003a888', | |
104 'webkit_url': | |
105 'https://chromium.googlesource.com/chromium/blink.git', | |
106 'git_url': | |
107 'https://chromium.googlesource.com', | |
108 'webkit_rev': | |
109 '@e01ac0a267d1017288bc67fa3c366b10469d8a24', | |
110 'angle_revision': | |
111 '74697cf2064c0a2c0d7e1b1b28db439286766a05' | |
112 }""" | |
113 | |
114 # Should only expect svn/git revisions to come through, and urls to be | |
115 # filtered out. | 112 # filtered out. |
116 expected_vars_dict = { | 113 expected_vars_dict = { |
117 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', | 114 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', |
118 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', | 115 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', |
119 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' | 116 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' |
120 } | 117 } |
121 vars_dict = bisect_instance._ParseRevisionsFromDEPSFileManually( | 118 vars_dict = bisect_perf_module._ParseRevisionsFromDEPSFileManually( |
122 deps_file_contents) | 119 deps_file_contents) |
123 self.assertEqual(vars_dict, expected_vars_dict) | 120 self.assertEqual(vars_dict, expected_vars_dict) |
124 | 121 |
125 def testTryParseResultValuesFromOutputWithSingleValue(self): | 122 def testTryParseResultValuesFromOutputWithSingleValue(self): |
126 """Tests result pattern <*>RESULT <graph>: <trace>= <value>""" | 123 """Tests result pattern <*>RESULT <graph>: <trace>= <value>""" |
127 bisect_options = bisect_perf_module.BisectOptions() | |
128 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | |
129 None, bisect_options) | |
130 metrics = ['Total', 'Total_ref'] | 124 metrics = ['Total', 'Total_ref'] |
131 self.assertEqual( | 125 self.assertEqual( |
132 [66.88], bisect_instance.TryParseResultValuesFromOutput( | 126 [66.88], bisect_perf_module._TryParseResultValuesFromOutput( |
133 metrics, RESULTS_OUTPUT % {'value': '66.88 kb'})) | 127 metrics, RESULTS_OUTPUT % {'value': '66.88 kb'})) |
134 self.assertEqual( | 128 self.assertEqual( |
135 [66.88], bisect_instance.TryParseResultValuesFromOutput( | 129 [66.88], bisect_perf_module._TryParseResultValuesFromOutput( |
136 metrics, RESULTS_OUTPUT % {'value': '66.88kb'})) | 130 metrics, RESULTS_OUTPUT % {'value': '66.88kb'})) |
137 self.assertEqual( | 131 self.assertEqual( |
138 [66.88], bisect_instance.TryParseResultValuesFromOutput( | 132 [66.88], bisect_perf_module._TryParseResultValuesFromOutput( |
139 metrics, RESULTS_OUTPUT % {'value': ' 66.88 '})) | 133 metrics, RESULTS_OUTPUT % {'value': ' 66.88 '})) |
140 self.assertEqual( | 134 self.assertEqual( |
141 [-66.88], bisect_instance.TryParseResultValuesFromOutput( | 135 [-66.88], bisect_perf_module._TryParseResultValuesFromOutput( |
142 metrics, RESULTS_OUTPUT % {'value': ' -66.88 kb'})) | 136 metrics, RESULTS_OUTPUT % {'value': ' -66.88 kb'})) |
143 self.assertEqual( | 137 self.assertEqual( |
144 [66], bisect_instance.TryParseResultValuesFromOutput( | 138 [66], bisect_perf_module._TryParseResultValuesFromOutput( |
145 metrics, RESULTS_OUTPUT % {'value': '66 kb'})) | 139 metrics, RESULTS_OUTPUT % {'value': '66 kb'})) |
146 self.assertEqual( | 140 self.assertEqual( |
147 [.66], bisect_instance.TryParseResultValuesFromOutput( | 141 [.66], bisect_perf_module._TryParseResultValuesFromOutput( |
148 metrics, RESULTS_OUTPUT % {'value': '.66 kb'})) | 142 metrics, RESULTS_OUTPUT % {'value': '.66 kb'})) |
149 self.assertEqual( | 143 self.assertEqual( |
150 [], bisect_instance.TryParseResultValuesFromOutput( | 144 [], bisect_perf_module._TryParseResultValuesFromOutput( |
151 metrics, RESULTS_OUTPUT % {'value': '. kb'})) | 145 metrics, RESULTS_OUTPUT % {'value': '. kb'})) |
152 self.assertEqual( | 146 self.assertEqual( |
153 [], bisect_instance.TryParseResultValuesFromOutput( | 147 [], bisect_perf_module._TryParseResultValuesFromOutput( |
154 metrics, RESULTS_OUTPUT % {'value': 'aaa kb'})) | 148 metrics, RESULTS_OUTPUT % {'value': 'aaa kb'})) |
155 | 149 |
156 def testTryParseResultValuesFromOutputWithMulitValue(self): | 150 def testTryParseResultValuesFromOutputWithMulitValue(self): |
157 """Tests result pattern <*>RESULT <graph>: <trace>= [<value>,<value>, ..]""" | 151 """Tests result pattern <*>RESULT <graph>: <trace>= [<value>,<value>, ..]""" |
158 bisect_options = bisect_perf_module.BisectOptions() | |
159 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | |
160 None, bisect_options) | |
161 metrics = ['Total', 'Total_ref'] | 152 metrics = ['Total', 'Total_ref'] |
162 self.assertEqual( | 153 self.assertEqual( |
163 [66.88], bisect_instance.TryParseResultValuesFromOutput( | 154 [66.88], bisect_perf_module._TryParseResultValuesFromOutput( |
164 metrics, RESULTS_OUTPUT % {'value': '[66.88] kb'})) | 155 metrics, RESULTS_OUTPUT % {'value': '[66.88] kb'})) |
165 self.assertEqual( | 156 self.assertEqual( |
166 [66.88, 99.44], bisect_instance.TryParseResultValuesFromOutput( | 157 [66.88, 99.44], bisect_perf_module._TryParseResultValuesFromOutput( |
167 metrics, RESULTS_OUTPUT % {'value': '[66.88, 99.44]kb'})) | 158 metrics, RESULTS_OUTPUT % {'value': '[66.88, 99.44]kb'})) |
168 self.assertEqual( | 159 self.assertEqual( |
169 [66.88, 99.44], bisect_instance.TryParseResultValuesFromOutput( | 160 [66.88, 99.44], bisect_perf_module._TryParseResultValuesFromOutput( |
170 metrics, RESULTS_OUTPUT % {'value': '[ 66.88, 99.44 ]'})) | 161 metrics, RESULTS_OUTPUT % {'value': '[ 66.88, 99.44 ]'})) |
171 self.assertEqual( | 162 self.assertEqual( |
172 [-66.88, 99.44], bisect_instance.TryParseResultValuesFromOutput( | 163 [-66.88, 99.44], bisect_perf_module._TryParseResultValuesFromOutput( |
173 metrics, RESULTS_OUTPUT % {'value': '[-66.88,99.44] kb'})) | 164 metrics, RESULTS_OUTPUT % {'value': '[-66.88,99.44] kb'})) |
174 self.assertEqual( | 165 self.assertEqual( |
175 [-66, 99], bisect_instance.TryParseResultValuesFromOutput( | 166 [-66, 99], bisect_perf_module._TryParseResultValuesFromOutput( |
176 metrics, RESULTS_OUTPUT % {'value': '[-66,99] kb'})) | 167 metrics, RESULTS_OUTPUT % {'value': '[-66,99] kb'})) |
177 self.assertEqual( | 168 self.assertEqual( |
178 [-66, 99], bisect_instance.TryParseResultValuesFromOutput( | 169 [-66, 99], bisect_perf_module._TryParseResultValuesFromOutput( |
179 metrics, RESULTS_OUTPUT % {'value': '[-66,99,] kb'})) | 170 metrics, RESULTS_OUTPUT % {'value': '[-66,99,] kb'})) |
180 self.assertEqual( | 171 self.assertEqual( |
181 [.66, .99], bisect_instance.TryParseResultValuesFromOutput( | 172 [.66, .99], bisect_perf_module._TryParseResultValuesFromOutput( |
182 metrics, RESULTS_OUTPUT % {'value': '[.66,.99] kb'})) | 173 metrics, RESULTS_OUTPUT % {'value': '[.66,.99] kb'})) |
183 self.assertEqual( | 174 self.assertEqual( |
184 [], bisect_instance.TryParseResultValuesFromOutput( | 175 [], bisect_perf_module._TryParseResultValuesFromOutput( |
185 metrics, RESULTS_OUTPUT % {'value': '[] kb'})) | 176 metrics, RESULTS_OUTPUT % {'value': '[] kb'})) |
186 self.assertEqual( | 177 self.assertEqual( |
187 [], bisect_instance.TryParseResultValuesFromOutput( | 178 [], bisect_perf_module._TryParseResultValuesFromOutput( |
188 metrics, RESULTS_OUTPUT % {'value': '[-66,abc] kb'})) | 179 metrics, RESULTS_OUTPUT % {'value': '[-66,abc] kb'})) |
189 | 180 |
190 def testTryParseResultValuesFromOutputWithMeanStd(self): | 181 def testTryParseResultValuesFromOutputWithMeanStd(self): |
191 """Tests result pattern <*>RESULT <graph>: <trace>= {<mean, std}""" | 182 """Tests result pattern <*>RESULT <graph>: <trace>= {<mean, std}""" |
192 bisect_options = bisect_perf_module.BisectOptions() | |
193 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | |
194 None, bisect_options) | |
195 metrics = ['Total', 'Total_ref'] | 183 metrics = ['Total', 'Total_ref'] |
196 self.assertEqual( | 184 self.assertEqual( |
197 [33.22], bisect_instance.TryParseResultValuesFromOutput( | 185 [33.22], bisect_perf_module._TryParseResultValuesFromOutput( |
198 metrics, RESULTS_OUTPUT % {'value': '{33.22, 3.6} kb'})) | 186 metrics, RESULTS_OUTPUT % {'value': '{33.22, 3.6} kb'})) |
199 self.assertEqual( | 187 self.assertEqual( |
200 [33.22], bisect_instance.TryParseResultValuesFromOutput( | 188 [33.22], bisect_perf_module._TryParseResultValuesFromOutput( |
201 metrics, RESULTS_OUTPUT % {'value': '{33.22,3.6}kb'})) | 189 metrics, RESULTS_OUTPUT % {'value': '{33.22,3.6}kb'})) |
202 self.assertEqual( | 190 self.assertEqual( |
203 [33.22], bisect_instance.TryParseResultValuesFromOutput( | 191 [33.22], bisect_perf_module._TryParseResultValuesFromOutput( |
204 metrics, RESULTS_OUTPUT % {'value': '{33.22,3.6} kb'})) | 192 metrics, RESULTS_OUTPUT % {'value': '{33.22,3.6} kb'})) |
205 self.assertEqual( | 193 self.assertEqual( |
206 [33.22], bisect_instance.TryParseResultValuesFromOutput( | 194 [33.22], bisect_perf_module._TryParseResultValuesFromOutput( |
207 metrics, RESULTS_OUTPUT % {'value': '{ 33.22,3.6 }kb'})) | 195 metrics, RESULTS_OUTPUT % {'value': '{ 33.22,3.6 }kb'})) |
208 self.assertEqual( | 196 self.assertEqual( |
209 [-33.22], bisect_instance.TryParseResultValuesFromOutput( | 197 [-33.22], bisect_perf_module._TryParseResultValuesFromOutput( |
210 metrics, RESULTS_OUTPUT % {'value': '{-33.22,3.6}kb'})) | 198 metrics, RESULTS_OUTPUT % {'value': '{-33.22,3.6}kb'})) |
211 self.assertEqual( | 199 self.assertEqual( |
212 [22], bisect_instance.TryParseResultValuesFromOutput( | 200 [22], bisect_perf_module._TryParseResultValuesFromOutput( |
213 metrics, RESULTS_OUTPUT % {'value': '{22,6}kb'})) | 201 metrics, RESULTS_OUTPUT % {'value': '{22,6}kb'})) |
214 self.assertEqual( | 202 self.assertEqual( |
215 [.22], bisect_instance.TryParseResultValuesFromOutput( | 203 [.22], bisect_perf_module._TryParseResultValuesFromOutput( |
216 metrics, RESULTS_OUTPUT % {'value': '{.22,6}kb'})) | 204 metrics, RESULTS_OUTPUT % {'value': '{.22,6}kb'})) |
217 self.assertEqual( | 205 self.assertEqual( |
218 [], bisect_instance.TryParseResultValuesFromOutput( | 206 [], bisect_perf_module._TryParseResultValuesFromOutput( |
219 metrics, RESULTS_OUTPUT % {'value': '{.22,6, 44}kb'})) | 207 metrics, RESULTS_OUTPUT % {'value': '{.22,6, 44}kb'})) |
220 self.assertEqual( | 208 self.assertEqual( |
221 [], bisect_instance.TryParseResultValuesFromOutput( | 209 [], bisect_perf_module._TryParseResultValuesFromOutput( |
222 metrics, RESULTS_OUTPUT % {'value': '{}kb'})) | 210 metrics, RESULTS_OUTPUT % {'value': '{}kb'})) |
223 self.assertEqual( | 211 self.assertEqual( |
224 [], bisect_instance.TryParseResultValuesFromOutput( | 212 [], bisect_perf_module._TryParseResultValuesFromOutput( |
225 metrics, RESULTS_OUTPUT % {'value': '{XYZ}kb'})) | 213 metrics, RESULTS_OUTPUT % {'value': '{XYZ}kb'})) |
226 | 214 |
227 def testGetCompatibleCommand(self): | 215 def testGetCompatibleCommand(self): |
228 bisect_options = bisect_perf_module.BisectOptions() | 216 bisect_options = bisect_perf_module.BisectOptions() |
229 bisect_options.output_buildbot_annotations = None | 217 bisect_options.output_buildbot_annotations = None |
230 source_control = source_control_module.DetermineAndCreateSourceControl( | 218 source_control = source_control_module.DetermineAndCreateSourceControl( |
231 bisect_options) | 219 bisect_options) |
232 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( | 220 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( |
233 source_control, bisect_options) | 221 source_control, bisect_options) |
234 bisect_instance.opts.target_platform = 'android' | 222 bisect_instance.opts.target_platform = 'android' |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
282 '--browser=release page_cycler.intl_ja_zh') | 270 '--browser=release page_cycler.intl_ja_zh') |
283 expected_command = ('tools/perf/run_benchmark -v --browser=' | 271 expected_command = ('tools/perf/run_benchmark -v --browser=' |
284 'release page_cycler.intl_ja_zh') | 272 'release page_cycler.intl_ja_zh') |
285 self.assertEqual( | 273 self.assertEqual( |
286 bisect_instance.GetCompatibleCommand(command, git_revision, depot), | 274 bisect_instance.GetCompatibleCommand(command, git_revision, depot), |
287 expected_command) | 275 expected_command) |
288 | 276 |
289 | 277 |
290 if __name__ == '__main__': | 278 if __name__ == '__main__': |
291 unittest.main() | 279 unittest.main() |
OLD | NEW |