Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(45)

Side by Side Diff: tools/auto_bisect/bisect_perf_regression_test.py

Issue 1001033004: Fix style in tools/auto_bisect. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Rebased Created 5 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tools/auto_bisect/bisect_perf_regression.py ('k') | tools/auto_bisect/bisect_printer.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2014 The Chromium Authors. All rights reserved. 1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import os 5 import os
6 import re 6 import re
7 import shutil 7 import shutil
8 import sys 8 import sys
9 import unittest 9 import unittest
10 10
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after
63 [[29.547], [29.713], [29.835], [30.132], [30.132], [30.33], [30.406], 63 [[29.547], [29.713], [29.835], [30.132], [30.132], [30.33], [30.406],
64 [30.592], [30.72], [34.486], [35.247], [35.253], [35.335], [35.378], 64 [30.592], [30.72], [34.486], [35.247], [35.253], [35.335], [35.378],
65 [35.934], [36.233], [36.41], [36.947], [37.982]] 65 [35.934], [36.233], [36.41], [36.947], [37.982]]
66 ] 66 ]
67 67
68 # Regression confidence > 95%, taken from: crbug.com/434318 68 # Regression confidence > 95%, taken from: crbug.com/434318
69 # Specifically from Builder android_nexus10_perf_bisect Build #1198 69 # Specifically from Builder android_nexus10_perf_bisect Build #1198
70 MULTIPLE_VALUES = [ 70 MULTIPLE_VALUES = [
71 [ 71 [
72 [18.916, 22.371, 8.527, 5.877, 5.407, 9.476, 8.100, 5.334, 72 [18.916, 22.371, 8.527, 5.877, 5.407, 9.476, 8.100, 5.334,
73 4.507, 4.842, 8.485, 8.308, 27.490, 4.560, 4.804, 23.068, 17.577, 73 4.507, 4.842, 8.485, 8.308, 27.490, 4.560, 4.804, 23.068, 17.577,
74 17.346, 26.738, 60.330, 32.307, 5.468, 27.803, 27.373, 17.823, 74 17.346, 26.738, 60.330, 32.307, 5.468, 27.803, 27.373, 17.823,
75 5.158, 27.439, 5.236, 11.413], 75 5.158, 27.439, 5.236, 11.413],
76 [18.999, 22.642, 8.158, 5.995, 5.495, 9.499, 8.092, 5.324, 76 [18.999, 22.642, 8.158, 5.995, 5.495, 9.499, 8.092, 5.324,
77 4.468, 4.788, 8.248, 7.853, 27.533, 4.410, 4.622, 22.341, 22.313, 77 4.468, 4.788, 8.248, 7.853, 27.533, 4.410, 4.622, 22.341, 22.313,
78 17.072, 26.731, 57.513, 33.001, 5.500, 28.297, 27.277, 26.462, 78 17.072, 26.731, 57.513, 33.001, 5.500, 28.297, 27.277, 26.462,
79 5.009, 27.361, 5.130, 10.955] 79 5.009, 27.361, 5.130, 10.955]
80 ], 80 ],
81 [ 81 [
82 [18.238, 22.365, 8.555, 5.939, 5.437, 9.463, 7.047, 5.345, 4.517, 82 [18.238, 22.365, 8.555, 5.939, 5.437, 9.463, 7.047, 5.345, 4.517,
83 4.796, 8.593, 7.901, 27.499, 4.378, 5.040, 4.904, 4.816, 4.828, 83 4.796, 8.593, 7.901, 27.499, 4.378, 5.040, 4.904, 4.816, 4.828,
84 4.853, 57.363, 34.184, 5.482, 28.190, 27.290, 26.694, 5.099, 84 4.853, 57.363, 34.184, 5.482, 28.190, 27.290, 26.694, 5.099,
85 4.905, 5.290, 4.813], 85 4.905, 5.290, 4.813],
86 [18.301, 22.522, 8.035, 6.021, 5.565, 9.037, 6.998, 5.321, 4.485, 86 [18.301, 22.522, 8.035, 6.021, 5.565, 9.037, 6.998, 5.321, 4.485,
87 4.768, 8.397, 7.865, 27.636, 4.640, 5.015, 4.962, 4.933, 4.977, 87 4.768, 8.397, 7.865, 27.636, 4.640, 5.015, 4.962, 4.933, 4.977,
88 4.961, 60.648, 34.593, 5.538, 28.454, 27.297, 26.490, 5.099, 5, 88 4.961, 60.648, 34.593, 5.538, 28.454, 27.297, 26.490, 5.099, 5,
89 5.247, 4.945], 89 5.247, 4.945],
90 [18.907, 23.368, 8.100, 6.169, 5.621, 9.971, 8.161, 5.331, 4.513, 90 [18.907, 23.368, 8.100, 6.169, 5.621, 9.971, 8.161, 5.331, 4.513,
91 4.837, 8.255, 7.852, 26.209, 4.388, 5.045, 5.029, 5.032, 4.946, 91 4.837, 8.255, 7.852, 26.209, 4.388, 5.045, 5.029, 5.032, 4.946,
92 4.973, 60.334, 33.377, 5.499, 28.275, 27.550, 26.103, 5.108, 92 4.973, 60.334, 33.377, 5.499, 28.275, 27.550, 26.103, 5.108,
93 4.951, 5.285, 4.910], 93 4.951, 5.285, 4.910],
94 [18.715, 23.748, 8.128, 6.148, 5.691, 9.361, 8.106, 5.334, 4.528, 94 [18.715, 23.748, 8.128, 6.148, 5.691, 9.361, 8.106, 5.334, 4.528,
95 4.965, 8.261, 7.851, 27.282, 4.391, 4.949, 4.981, 4.964, 4.935, 95 4.965, 8.261, 7.851, 27.282, 4.391, 4.949, 4.981, 4.964, 4.935,
96 4.933, 60.231, 33.361, 5.489, 28.106, 27.457, 26.648, 5.108, 96 4.933, 60.231, 33.361, 5.489, 28.106, 27.457, 26.648, 5.108,
97 4.963, 5.272, 4.954] 97 4.963, 5.272, 4.954]
98 ] 98 ]
99 ] 99 ]
100 100
101 # Default options for the dry run 101 # Default options for the dry run
102 DEFAULT_OPTIONS = { 102 DEFAULT_OPTIONS = {
103 'debug_ignore_build': True, 103 'debug_ignore_build': True,
104 'debug_ignore_sync': True, 104 'debug_ignore_sync': True,
105 'debug_ignore_perf_test': True, 105 'debug_ignore_perf_test': True,
106 'debug_ignore_regression_confidence': True, 106 'debug_ignore_regression_confidence': True,
107 'command': 'fake_command', 107 'command': 'fake_command',
108 'metric': 'fake/metric', 108 'metric': 'fake/metric',
109 'good_revision': 280000, 109 'good_revision': 280000,
110 'bad_revision': 280005, 110 'bad_revision': 280005,
111 } 111 }
112 112
113 # This global is a placeholder for a generator to be defined by the test cases 113 # This global is a placeholder for a generator to be defined by the test cases
114 # that use _MockRunTests. 114 # that use _MockRunTests.
115 _MockResultsGenerator = (x for x in []) 115 _MockResultsGenerator = (x for x in [])
116 116
117 117
118 def _MockRunTests(*args, **kwargs): 118 def _MockRunTests(*args, **kwargs): # pylint: disable=unused-argument
119 _, _ = args, kwargs
120 return _FakeTestResult(_MockResultsGenerator.next()) 119 return _FakeTestResult(_MockResultsGenerator.next())
121 120
122 121
123 def _FakeTestResult(values): 122 def _FakeTestResult(values):
124 result_dict = {'mean': 0.0, 'std_err': 0.0, 'std_dev': 0.0, 'values': values} 123 result_dict = {'mean': 0.0, 'std_err': 0.0, 'std_dev': 0.0, 'values': values}
125 success_code = 0 124 success_code = 0
126 return (result_dict, success_code) 125 return (result_dict, success_code)
127 126
128 127
129 def _GetBisectPerformanceMetricsInstance(options_dict): 128 def _GetBisectPerformanceMetricsInstance(options_dict):
(...skipping 242 matching lines...) Expand 10 before | Expand all | Expand 10 after
372 371
373 def _CheckAbortsEarly(self, results): 372 def _CheckAbortsEarly(self, results):
374 """Returns True if the bisect job would abort early.""" 373 """Returns True if the bisect job would abort early."""
375 global _MockResultsGenerator 374 global _MockResultsGenerator
376 _MockResultsGenerator = (r for r in results) 375 _MockResultsGenerator = (r for r in results)
377 bisect_class = bisect_perf_regression.BisectPerformanceMetrics 376 bisect_class = bisect_perf_regression.BisectPerformanceMetrics
378 original_run_tests = bisect_class.RunPerformanceTestAndParseResults 377 original_run_tests = bisect_class.RunPerformanceTestAndParseResults
379 bisect_class.RunPerformanceTestAndParseResults = _MockRunTests 378 bisect_class.RunPerformanceTestAndParseResults = _MockRunTests
380 379
381 try: 380 try:
382 _GenericDryRun(_GetExtendedOptions(0, 0, False)) 381 dry_run_results = _GenericDryRun(_GetExtendedOptions(0, 0, False))
383 except StopIteration: 382 except StopIteration:
384 # If StopIteration was raised, that means that the next value after 383 # If StopIteration was raised, that means that the next value after
385 # the first two values was requested, so the job was not aborted. 384 # the first two values was requested, so the job was not aborted.
386 return False 385 return False
387 finally: 386 finally:
388 bisect_class.RunPerformanceTestAndParseResults = original_run_tests 387 bisect_class.RunPerformanceTestAndParseResults = original_run_tests
389 388
390 # If the job was aborted, there should be a warning about it. 389 # If the job was aborted, there should be a warning about it.
391 assert [w for w in results.warnings 390 assert [w for w in dry_run_results.warnings
392 if 'could not reproduce the regression' in w] 391 if 'could not reproduce the regression' in w]
393 return True 392 return True
394 393
395 def testBisectStopsOnClearUnclearRegression(self): 394 def testBisectAbortedOnClearNonRegression(self):
396 self.assertTrue(self._CheckAbortsEarly(CLEAR_NON_REGRESSION)) 395 self.assertTrue(self._CheckAbortsEarly(CLEAR_NON_REGRESSION))
397 396
398 def testBisectStopsOnClearUnclearRegression(self): 397 def testBisectNotAborted_AlmostRegression(self):
399 self.assertFalse(self._CheckAbortsEarly(ALMOST_REGRESSION)) 398 self.assertFalse(self._CheckAbortsEarly(ALMOST_REGRESSION))
400 399
401 def testBisectStopsOnClearUnclearRegression(self): 400 def testBisectNotAborted_ClearRegression(self):
402 self.assertFalse(self._CheckAbortsEarly(CLEAR_REGRESSION)) 401 self.assertFalse(self._CheckAbortsEarly(CLEAR_REGRESSION))
403 402
404 def testBisectStopsOnClearUnclearRegression(self): 403 def testBisectNotAborted_BarelyRegression(self):
405 self.assertFalse(self._CheckAbortsEarly(BARELY_REGRESSION)) 404 self.assertFalse(self._CheckAbortsEarly(BARELY_REGRESSION))
406 405
407 def testBisectStopsOnClearUnclearRegression(self): 406 def testBisectNotAborted_MultipleValues(self):
408 self.assertFalse(self._CheckAbortsEarly(MULTIPLE_VALUES)) 407 self.assertFalse(self._CheckAbortsEarly(MULTIPLE_VALUES))
409 408
410 def testGetCommitPosition(self): 409 def testGetCommitPosition(self):
411 cp_git_rev = '7017a81991de983e12ab50dfc071c70e06979531' 410 cp_git_rev = '7017a81991de983e12ab50dfc071c70e06979531'
412 self.assertEqual(291765, source_control.GetCommitPosition(cp_git_rev)) 411 self.assertEqual(291765, source_control.GetCommitPosition(cp_git_rev))
413 412
414 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58' 413 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58'
415 self.assertEqual(291467, source_control.GetCommitPosition(svn_git_rev)) 414 self.assertEqual(291467, source_control.GetCommitPosition(svn_git_rev))
416 415
417 def testGetCommitPositionForV8(self): 416 def testGetCommitPositionForV8(self):
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after
460 mock_RunGClient.return_value = 0 459 mock_RunGClient.return_value = 0
461 bisect_instance._SyncRevision( 460 bisect_instance._SyncRevision(
462 'chromium', 'e6db23a037cad47299a94b155b95eebd1ee61a58', 'gclient') 461 'chromium', 'e6db23a037cad47299a94b155b95eebd1ee61a58', 'gclient')
463 expected_params = [ 462 expected_params = [
464 'sync', 463 'sync',
465 '--verbose', 464 '--verbose',
466 '--nohooks', 465 '--nohooks',
467 '--force', 466 '--force',
468 '--delete_unversioned_trees', 467 '--delete_unversioned_trees',
469 '--revision', 468 '--revision',
470 'src@e6db23a037cad47299a94b155b95eebd1ee61a58' 469 'src@e6db23a037cad47299a94b155b95eebd1ee61a58',
471 ] 470 ]
472 471
473 mock_RunGClient.assert_called_with(expected_params, cwd=None) 472 mock_RunGClient.assert_called_with(expected_params, cwd=None)
474 473
475 @mock.patch('bisect_utils.RunGit') 474 @mock.patch('bisect_utils.RunGit')
476 def testSyncToRevisionForWebKit(self, mock_RunGit): 475 def testSyncToRevisionForWebKit(self, mock_RunGit):
477 bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS) 476 bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS)
478 mock_RunGit.return_value = None, None 477 mock_RunGit.return_value = None, None
479 bisect_instance._SyncRevision( 478 bisect_instance._SyncRevision(
480 'webkit', 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61' , None) 479 'webkit', 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61', None)
481 expected_params = ['checkout', 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61'] 480 expected_params = ['checkout', 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61']
482 mock_RunGit.assert_called_with(expected_params) 481 mock_RunGit.assert_called_with(expected_params)
483 482
484 def testTryJobSvnRepo_PerfBuilderType_ReturnsRepoUrl(self): 483 def testTryJobSvnRepo_PerfBuilderType_ReturnsRepoUrl(self):
485 self.assertEqual(bisect_perf_regression.PERF_SVN_REPO_URL, 484 self.assertEqual(
485 bisect_perf_regression.PERF_SVN_REPO_URL,
486 bisect_perf_regression._TryJobSvnRepo(fetch_build.PERF_BUILDER)) 486 bisect_perf_regression._TryJobSvnRepo(fetch_build.PERF_BUILDER))
487 487
488 def testTryJobSvnRepo_FullBuilderType_ReturnsRepoUrl(self): 488 def testTryJobSvnRepo_FullBuilderType_ReturnsRepoUrl(self):
489 self.assertEqual(bisect_perf_regression.FULL_SVN_REPO_URL, 489 self.assertEqual(
490 bisect_perf_regression.FULL_SVN_REPO_URL,
490 bisect_perf_regression._TryJobSvnRepo(fetch_build.FULL_BUILDER)) 491 bisect_perf_regression._TryJobSvnRepo(fetch_build.FULL_BUILDER))
491 492
492 def testTryJobSvnRepo_WithUnknownBuilderType_ThrowsError(self): 493 def testTryJobSvnRepo_WithUnknownBuilderType_ThrowsError(self):
493 with self.assertRaises(NotImplementedError): 494 with self.assertRaises(NotImplementedError):
494 bisect_perf_regression._TryJobSvnRepo('foo') 495 bisect_perf_regression._TryJobSvnRepo('foo')
495 496
496 def _CheckIsDownloadable(self, depot, target_platform='chromium', 497 def _CheckIsDownloadable(self, depot, target_platform='chromium',
497 builder_type='perf'): 498 builder_type='perf'):
498 opts = dict(DEFAULT_OPTIONS) 499 opts = dict(DEFAULT_OPTIONS)
499 opts.update({'target_platform': target_platform, 500 opts.update({'target_platform': target_platform,
(...skipping 165 matching lines...) Expand 10 before | Expand all | Expand 10 after
665 (['update-index', '--refresh', '-q'], (None, 0)), 666 (['update-index', '--refresh', '-q'], (None, 0)),
666 (['diff-index', 'HEAD'], (None, 0)), 667 (['diff-index', 'HEAD'], (None, 0)),
667 (['checkout', '-b', new_branch], ('None', 0)), 668 (['checkout', '-b', new_branch], ('None', 0)),
668 (['branch', '--set-upstream-to', parent_branch], 669 (['branch', '--set-upstream-to', parent_branch],
669 ('Setuptream fails', 0)), 670 ('Setuptream fails', 0)),
670 (['try', 671 (['try',
671 '--bot=%s' % bot_name, 672 '--bot=%s' % bot_name,
672 '--revision=%s' % git_revision, 673 '--revision=%s' % git_revision,
673 '--name=%s' % bisect_job_name, 674 '--name=%s' % bisect_job_name,
674 '--svn_repo=%s' % bisect_perf_regression.PERF_SVN_REPO_URL, 675 '--svn_repo=%s' % bisect_perf_regression.PERF_SVN_REPO_URL,
675 '--diff=%s' % patch_content 676 '--diff=%s' % patch_content],
676 ], (None, 1)), 677 (None, 1)),
677 ] 678 ]
678 self._AssertRunGitExceptions( 679 self._AssertRunGitExceptions(
679 try_cmd, bisect_perf_regression._StartBuilderTryJob, 680 try_cmd, bisect_perf_regression._StartBuilderTryJob,
680 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name, 681 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name,
681 patch) 682 patch)
682 683
683 def testBuilderTryJob(self): 684 def testBuilderTryJob(self):
684 git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888' 685 git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888'
685 bot_name = 'linux_perf_bisect_builder' 686 bot_name = 'linux_perf_bisect_builder'
686 bisect_job_name = 'testBisectJobname' 687 bisect_job_name = 'testBisectJobname'
687 patch = None 688 patch = None
688 patch_content = '/dev/null' 689 patch_content = '/dev/null'
689 new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH 690 new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
690 parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH 691 parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
691 try_cmd = [ 692 try_cmd = [
692 (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)), 693 (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
693 (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)), 694 (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)),
694 (['branch', '-D', new_branch], ('None', 0)), 695 (['branch', '-D', new_branch], ('None', 0)),
695 (['update-index', '--refresh', '-q'], (None, 0)), 696 (['update-index', '--refresh', '-q'], (None, 0)),
696 (['diff-index', 'HEAD'], (None, 0)), 697 (['diff-index', 'HEAD'], (None, 0)),
697 (['checkout', '-b', new_branch], ('None', 0)), 698 (['checkout', '-b', new_branch], ('None', 0)),
698 (['branch', '--set-upstream-to', parent_branch], 699 (['branch', '--set-upstream-to', parent_branch],
699 ('Setuptream fails', 0)), 700 ('Setuptream fails', 0)),
700 (['try', 701 (['try',
701 '--bot=%s' % bot_name, 702 '--bot=%s' % bot_name,
702 '--revision=%s' % git_revision, 703 '--revision=%s' % git_revision,
703 '--name=%s' % bisect_job_name, 704 '--name=%s' % bisect_job_name,
704 '--svn_repo=%s' % bisect_perf_regression.PERF_SVN_REPO_URL, 705 '--svn_repo=%s' % bisect_perf_regression.PERF_SVN_REPO_URL,
705 '--diff=%s' % patch_content 706 '--diff=%s' % patch_content],
706 ], (None, 0)), 707 (None, 0)),
707 ] 708 ]
708 self._SetupRunGitMock(try_cmd) 709 self._SetupRunGitMock(try_cmd)
709 bisect_perf_regression._StartBuilderTryJob( 710 bisect_perf_regression._StartBuilderTryJob(
710 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name, 711 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name,
711 patch) 712 patch)
712 713
713 714
714 if __name__ == '__main__': 715 if __name__ == '__main__':
715 unittest.main() 716 unittest.main()
OLDNEW
« no previous file with comments | « tools/auto_bisect/bisect_perf_regression.py ('k') | tools/auto_bisect/bisect_printer.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698