Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(285)

Side by Side Diff: tools/auto_bisect/bisect_perf_regression_test.py

Issue 564663002: Move bisect-perf-regression.py into auto_bisect directory. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Add telemetry to PYTHONPATH for PRESUBMIT.py. Created 6 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tools/auto_bisect/bisect_perf_regression.py ('k') | tools/auto_bisect/bisect_utils.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2014 The Chromium Authors. All rights reserved. 1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import os 5 import os
6 import re 6 import re
7 import unittest 7 import unittest
8 8
9 from auto_bisect import source_control as source_control_module 9 import bisect_perf_regression
10 10 import source_control as source_control_module
11 # Special import necessary because filename contains dash characters.
12 bisect_perf_module = __import__('bisect-perf-regression')
13 11
14 def _GetBisectPerformanceMetricsInstance(): 12 def _GetBisectPerformanceMetricsInstance():
15 """Returns an instance of the BisectPerformanceMetrics class.""" 13 """Returns an instance of the BisectPerformanceMetrics class."""
16 options_dict = { 14 options_dict = {
17 'debug_ignore_build': True, 15 'debug_ignore_build': True,
18 'debug_ignore_sync': True, 16 'debug_ignore_sync': True,
19 'debug_ignore_perf_test': True, 17 'debug_ignore_perf_test': True,
20 'command': 'fake_command', 18 'command': 'fake_command',
21 'metric': 'fake/metric', 19 'metric': 'fake/metric',
22 'good_revision': 280000, 20 'good_revision': 280000,
23 'bad_revision': 280005, 21 'bad_revision': 280005,
24 } 22 }
25 bisect_options = bisect_perf_module.BisectOptions.FromDict(options_dict) 23 bisect_options = bisect_perf_regression.BisectOptions.FromDict(options_dict)
26 source_control = source_control_module.DetermineAndCreateSourceControl( 24 source_control = source_control_module.DetermineAndCreateSourceControl(
27 bisect_options) 25 bisect_options)
28 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( 26 bisect_instance = bisect_perf_regression.BisectPerformanceMetrics(
29 source_control, bisect_options) 27 source_control, bisect_options)
30 bisect_instance.src_cwd = os.path.abspath( 28 bisect_instance.src_cwd = os.path.abspath(
31 os.path.join(os.path.dirname(__file__), os.path.pardir)) 29 os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir))
32 return bisect_instance 30 return bisect_instance
33 31
34 32
35 class BisectPerfRegressionTest(unittest.TestCase): 33 class BisectPerfRegressionTest(unittest.TestCase):
36 """Test case for other functions and classes in bisect-perf-regression.py.""" 34 """Test case for other functions and classes in bisect-perf-regression.py."""
37 35
38 def _AssertConfidence(self, score, bad_values, good_values): 36 def _AssertConfidence(self, score, bad_values, good_values):
39 """Checks whether the given sets of values have a given confidence score. 37 """Checks whether the given sets of values have a given confidence score.
40 38
41 The score represents our confidence that the two sets of values wouldn't 39 The score represents our confidence that the two sets of values wouldn't
42 be as different as they are just by chance; that is, that some real change 40 be as different as they are just by chance; that is, that some real change
43 occurred between the two sets of values. 41 occurred between the two sets of values.
44 42
45 Args: 43 Args:
46 score: Expected confidence score. 44 score: Expected confidence score.
47 bad_values: First list of numbers. 45 bad_values: First list of numbers.
48 good_values: Second list of numbers. 46 good_values: Second list of numbers.
49 """ 47 """
50 # ConfidenceScore takes a list of lists but these lists are flattened 48 # ConfidenceScore takes a list of lists but these lists are flattened
51 # inside the function. 49 # inside the function.
52 confidence = bisect_perf_module.ConfidenceScore( 50 confidence = bisect_perf_regression.ConfidenceScore(
53 [[v] for v in bad_values], 51 [[v] for v in bad_values],
54 [[v] for v in good_values]) 52 [[v] for v in good_values])
55 self.assertEqual(score, confidence) 53 self.assertEqual(score, confidence)
56 54
57 def testConfidenceScore_ZeroConfidence(self): 55 def testConfidenceScore_ZeroConfidence(self):
58 # The good and bad sets contain the same values, so the confidence that 56 # The good and bad sets contain the same values, so the confidence that
59 # they're different should be zero. 57 # they're different should be zero.
60 self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4]) 58 self._AssertConfidence(0.0, [4, 5, 7, 6, 8, 7], [8, 7, 6, 7, 5, 4])
61 59
62 def testConfidenceScore_MediumConfidence(self): 60 def testConfidenceScore_MediumConfidence(self):
(...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after
121 119
122 # Should only expect SVN/git revisions to come through, and URLs should be 120 # Should only expect SVN/git revisions to come through, and URLs should be
123 # filtered out. 121 # filtered out.
124 expected_vars_dict = { 122 expected_vars_dict = {
125 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888', 123 'ffmpeg_hash': '@ac4a9f31fe2610bd146857bbd55d7a260003a888',
126 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24', 124 'webkit_rev': '@e01ac0a267d1017288bc67fa3c366b10469d8a24',
127 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05' 125 'angle_revision': '74697cf2064c0a2c0d7e1b1b28db439286766a05'
128 } 126 }
129 # Testing private function. 127 # Testing private function.
130 # pylint: disable=W0212 128 # pylint: disable=W0212
131 vars_dict = bisect_perf_module._ParseRevisionsFromDEPSFileManually( 129 vars_dict = bisect_perf_regression._ParseRevisionsFromDEPSFileManually(
132 deps_file_contents) 130 deps_file_contents)
133 self.assertEqual(vars_dict, expected_vars_dict) 131 self.assertEqual(vars_dict, expected_vars_dict)
134 132
135 def _AssertParseResult(self, expected_values, result_string): 133 def _AssertParseResult(self, expected_values, result_string):
136 """Asserts some values are parsed from a RESULT line.""" 134 """Asserts some values are parsed from a RESULT line."""
137 results_template = ('RESULT other_chart: other_trace= 123 count\n' 135 results_template = ('RESULT other_chart: other_trace= 123 count\n'
138 'RESULT my_chart: my_trace= %(value)s\n') 136 'RESULT my_chart: my_trace= %(value)s\n')
139 results = results_template % {'value': result_string} 137 results = results_template % {'value': result_string}
140 metric = ['my_chart', 'my_trace'] 138 metric = ['my_chart', 'my_trace']
141 # Testing private function. 139 # Testing private function.
142 # pylint: disable=W0212 140 # pylint: disable=W0212
143 values = bisect_perf_module._TryParseResultValuesFromOutput(metric, results) 141 values = bisect_perf_regression._TryParseResultValuesFromOutput(
142 metric, results)
144 self.assertEqual(expected_values, values) 143 self.assertEqual(expected_values, values)
145 144
146 def testTryParseResultValuesFromOutput_WithSingleValue(self): 145 def testTryParseResultValuesFromOutput_WithSingleValue(self):
147 """Tests result pattern <*>RESULT <graph>: <trace>= <value>""" 146 """Tests result pattern <*>RESULT <graph>: <trace>= <value>"""
148 self._AssertParseResult([66.88], '66.88 kb') 147 self._AssertParseResult([66.88], '66.88 kb')
149 self._AssertParseResult([66.88], '66.88 ') 148 self._AssertParseResult([66.88], '66.88 ')
150 self._AssertParseResult([-66.88], '-66.88 kb') 149 self._AssertParseResult([-66.88], '-66.88 kb')
151 self._AssertParseResult([66], '66 kb') 150 self._AssertParseResult([66], '66 kb')
152 self._AssertParseResult([0.66], '.66 kb') 151 self._AssertParseResult([0.66], '.66 kb')
153 self._AssertParseResult([], '. kb') 152 self._AssertParseResult([], '. kb')
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
185 184
186 This modification to the command is done in order to get a Telemetry 185 This modification to the command is done in order to get a Telemetry
187 command that works; before some revisions, the browser name that Telemetry 186 command that works; before some revisions, the browser name that Telemetry
188 expects is different in some cases, but we want it to work anyway. 187 expects is different in some cases, but we want it to work anyway.
189 188
190 Specifically, only for android: 189 Specifically, only for android:
191 After r276628, only android-chrome-shell works. 190 After r276628, only android-chrome-shell works.
192 Prior to r274857, only android-chromium-testshell works. 191 Prior to r274857, only android-chromium-testshell works.
193 In the range [274857, 276628], both work. 192 In the range [274857, 276628], both work.
194 """ 193 """
195 bisect_options = bisect_perf_module.BisectOptions() 194 bisect_options = bisect_perf_regression.BisectOptions()
196 bisect_options.output_buildbot_annotations = None 195 bisect_options.output_buildbot_annotations = None
197 source_control = source_control_module.DetermineAndCreateSourceControl( 196 source_control = source_control_module.DetermineAndCreateSourceControl(
198 bisect_options) 197 bisect_options)
199 bisect_instance = bisect_perf_module.BisectPerformanceMetrics( 198 bisect_instance = bisect_perf_regression.BisectPerformanceMetrics(
200 source_control, bisect_options) 199 source_control, bisect_options)
201 bisect_instance.opts.target_platform = target_platform 200 bisect_instance.opts.target_platform = target_platform
202 git_revision = bisect_instance.source_control.ResolveToRevision( 201 git_revision = bisect_instance.source_control.ResolveToRevision(
203 revision, 'chromium', bisect_perf_module.DEPOT_DEPS_NAME, 100) 202 revision, 'chromium', bisect_perf_regression.DEPOT_DEPS_NAME, 100)
204 depot = 'chromium' 203 depot = 'chromium'
205 command = bisect_instance.GetCompatibleCommand( 204 command = bisect_instance.GetCompatibleCommand(
206 original_command, git_revision, depot) 205 original_command, git_revision, depot)
207 self.assertEqual(expected_command, command) 206 self.assertEqual(expected_command, command)
208 207
209 def testGetCompatibleCommand_ChangeToTestShell(self): 208 def testGetCompatibleCommand_ChangeToTestShell(self):
210 # For revisions <= r274857, only android-chromium-testshell is used. 209 # For revisions <= r274857, only android-chromium-testshell is used.
211 self._AssertCompatibleCommand( 210 self._AssertCompatibleCommand(
212 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo', 211 'tools/perf/run_benchmark -v --browser=android-chromium-testshell foo',
213 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo', 212 'tools/perf/run_benchmark -v --browser=android-chrome-shell foo',
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after
261 self.assertEqual( 260 self.assertEqual(
262 291765, bisect_instance.source_control.GetCommitPosition(cp_git_rev)) 261 291765, bisect_instance.source_control.GetCommitPosition(cp_git_rev))
263 262
264 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58' 263 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58'
265 self.assertEqual( 264 self.assertEqual(
266 291467, bisect_instance.source_control.GetCommitPosition(svn_git_rev)) 265 291467, bisect_instance.source_control.GetCommitPosition(svn_git_rev))
267 266
268 def testGetCommitPositionForV8(self): 267 def testGetCommitPositionForV8(self):
269 bisect_instance = _GetBisectPerformanceMetricsInstance() 268 bisect_instance = _GetBisectPerformanceMetricsInstance()
270 v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb' 269 v8_rev = '21d700eedcdd6570eff22ece724b63a5eefe78cb'
271 depot_path = os.path.join(bisect_instance.src_cwd, 'src', 'v8') 270 depot_path = os.path.join(bisect_instance.src_cwd, 'v8')
272 self.assertEqual( 271 self.assertEqual(
273 23634, 272 23634,
274 bisect_instance.source_control.GetCommitPosition(v8_rev, depot_path)) 273 bisect_instance.source_control.GetCommitPosition(v8_rev, depot_path))
275 274
276 def testGetCommitPositionForWebKit(self): 275 def testGetCommitPositionForWebKit(self):
277 bisect_instance = _GetBisectPerformanceMetricsInstance() 276 bisect_instance = _GetBisectPerformanceMetricsInstance()
278 wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61' 277 wk_rev = 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61'
279 depot_path = os.path.join(bisect_instance.src_cwd, 'src', 'third_party', 278 depot_path = os.path.join(bisect_instance.src_cwd, 'third_party', 'WebKit')
280 'WebKit')
281 self.assertEqual( 279 self.assertEqual(
282 181660, 280 181660,
283 bisect_instance.source_control.GetCommitPosition(wk_rev, depot_path)) 281 bisect_instance.source_control.GetCommitPosition(wk_rev, depot_path))
284 282
285 def testUpdateDepsContent(self): 283 def testUpdateDepsContent(self):
286 bisect_instance = _GetBisectPerformanceMetricsInstance() 284 bisect_instance = _GetBisectPerformanceMetricsInstance()
287 deps_file = 'DEPS' 285 deps_file = 'DEPS'
288 # We are intentionally reading DEPS file contents instead of string literal 286 # We are intentionally reading DEPS file contents instead of string literal
289 # with few lines from DEPS because to check if the format we are expecting 287 # with few lines from DEPS because to check if the format we are expecting
290 # to search is not changed in DEPS content. 288 # to search is not changed in DEPS content.
291 # TODO (prasadv): Add a separate test to validate the DEPS contents with the 289 # TODO (prasadv): Add a separate test to validate the DEPS contents with the
292 # format that bisect script expects. 290 # format that bisect script expects.
293 deps_contents = bisect_perf_module.ReadStringFromFile(deps_file) 291 deps_contents = bisect_perf_regression.ReadStringFromFile(deps_file)
294 deps_key = 'v8_revision' 292 deps_key = 'v8_revision'
295 depot = 'v8' 293 depot = 'v8'
296 git_revision = 'a12345789a23456789a123456789a123456789' 294 git_revision = 'a12345789a23456789a123456789a123456789'
297 updated_content = bisect_instance.UpdateDepsContents( 295 updated_content = bisect_instance.UpdateDepsContents(
298 deps_contents, depot, git_revision, deps_key) 296 deps_contents, depot, git_revision, deps_key)
299 self.assertIsNotNone(updated_content) 297 self.assertIsNotNone(updated_content)
300 ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision)) 298 ss = re.compile('["\']%s["\']: ["\']%s["\']' % (deps_key, git_revision))
301 self.assertIsNotNone(re.search(ss, updated_content)) 299 self.assertIsNotNone(re.search(ss, updated_content))
302 300
303 301
304 if __name__ == '__main__': 302 if __name__ == '__main__':
305 unittest.main() 303 unittest.main()
OLDNEW
« no previous file with comments | « tools/auto_bisect/bisect_perf_regression.py ('k') | tools/auto_bisect/bisect_utils.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698