OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import os | 5 import os |
6 import re | 6 import re |
7 import shutil | 7 import shutil |
8 import sys | 8 import sys |
9 import unittest | 9 import unittest |
10 | 10 |
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
63 [[29.547], [29.713], [29.835], [30.132], [30.132], [30.33], [30.406], | 63 [[29.547], [29.713], [29.835], [30.132], [30.132], [30.33], [30.406], |
64 [30.592], [30.72], [34.486], [35.247], [35.253], [35.335], [35.378], | 64 [30.592], [30.72], [34.486], [35.247], [35.253], [35.335], [35.378], |
65 [35.934], [36.233], [36.41], [36.947], [37.982]] | 65 [35.934], [36.233], [36.41], [36.947], [37.982]] |
66 ] | 66 ] |
67 | 67 |
68 # Regression confidence > 95%, taken from: crbug.com/434318 | 68 # Regression confidence > 95%, taken from: crbug.com/434318 |
69 # Specifically from Builder android_nexus10_perf_bisect Build #1198 | 69 # Specifically from Builder android_nexus10_perf_bisect Build #1198 |
70 MULTIPLE_VALUES = [ | 70 MULTIPLE_VALUES = [ |
71 [ | 71 [ |
72 [18.916, 22.371, 8.527, 5.877, 5.407, 9.476, 8.100, 5.334, | 72 [18.916, 22.371, 8.527, 5.877, 5.407, 9.476, 8.100, 5.334, |
73 4.507, 4.842, 8.485, 8.308, 27.490, 4.560, 4.804, 23.068, 17.577, | 73 4.507, 4.842, 8.485, 8.308, 27.490, 4.560, 4.804, 23.068, 17.577, |
74 17.346, 26.738, 60.330, 32.307, 5.468, 27.803, 27.373, 17.823, | 74 17.346, 26.738, 60.330, 32.307, 5.468, 27.803, 27.373, 17.823, |
75 5.158, 27.439, 5.236, 11.413], | 75 5.158, 27.439, 5.236, 11.413], |
76 [18.999, 22.642, 8.158, 5.995, 5.495, 9.499, 8.092, 5.324, | 76 [18.999, 22.642, 8.158, 5.995, 5.495, 9.499, 8.092, 5.324, |
77 4.468, 4.788, 8.248, 7.853, 27.533, 4.410, 4.622, 22.341, 22.313, | 77 4.468, 4.788, 8.248, 7.853, 27.533, 4.410, 4.622, 22.341, 22.313, |
78 17.072, 26.731, 57.513, 33.001, 5.500, 28.297, 27.277, 26.462, | 78 17.072, 26.731, 57.513, 33.001, 5.500, 28.297, 27.277, 26.462, |
79 5.009, 27.361, 5.130, 10.955] | 79 5.009, 27.361, 5.130, 10.955] |
80 ], | 80 ], |
81 [ | 81 [ |
82 [18.238, 22.365, 8.555, 5.939, 5.437, 9.463, 7.047, 5.345, 4.517, | 82 [18.238, 22.365, 8.555, 5.939, 5.437, 9.463, 7.047, 5.345, 4.517, |
83 4.796, 8.593, 7.901, 27.499, 4.378, 5.040, 4.904, 4.816, 4.828, | 83 4.796, 8.593, 7.901, 27.499, 4.378, 5.040, 4.904, 4.816, 4.828, |
84 4.853, 57.363, 34.184, 5.482, 28.190, 27.290, 26.694, 5.099, | 84 4.853, 57.363, 34.184, 5.482, 28.190, 27.290, 26.694, 5.099, |
85 4.905, 5.290, 4.813], | 85 4.905, 5.290, 4.813], |
86 [18.301, 22.522, 8.035, 6.021, 5.565, 9.037, 6.998, 5.321, 4.485, | 86 [18.301, 22.522, 8.035, 6.021, 5.565, 9.037, 6.998, 5.321, 4.485, |
87 4.768, 8.397, 7.865, 27.636, 4.640, 5.015, 4.962, 4.933, 4.977, | 87 4.768, 8.397, 7.865, 27.636, 4.640, 5.015, 4.962, 4.933, 4.977, |
88 4.961, 60.648, 34.593, 5.538, 28.454, 27.297, 26.490, 5.099, 5, | 88 4.961, 60.648, 34.593, 5.538, 28.454, 27.297, 26.490, 5.099, 5, |
89 5.247, 4.945], | 89 5.247, 4.945], |
90 [18.907, 23.368, 8.100, 6.169, 5.621, 9.971, 8.161, 5.331, 4.513, | 90 [18.907, 23.368, 8.100, 6.169, 5.621, 9.971, 8.161, 5.331, 4.513, |
91 4.837, 8.255, 7.852, 26.209, 4.388, 5.045, 5.029, 5.032, 4.946, | 91 4.837, 8.255, 7.852, 26.209, 4.388, 5.045, 5.029, 5.032, 4.946, |
92 4.973, 60.334, 33.377, 5.499, 28.275, 27.550, 26.103, 5.108, | 92 4.973, 60.334, 33.377, 5.499, 28.275, 27.550, 26.103, 5.108, |
93 4.951, 5.285, 4.910], | 93 4.951, 5.285, 4.910], |
94 [18.715, 23.748, 8.128, 6.148, 5.691, 9.361, 8.106, 5.334, 4.528, | 94 [18.715, 23.748, 8.128, 6.148, 5.691, 9.361, 8.106, 5.334, 4.528, |
95 4.965, 8.261, 7.851, 27.282, 4.391, 4.949, 4.981, 4.964, 4.935, | 95 4.965, 8.261, 7.851, 27.282, 4.391, 4.949, 4.981, 4.964, 4.935, |
96 4.933, 60.231, 33.361, 5.489, 28.106, 27.457, 26.648, 5.108, | 96 4.933, 60.231, 33.361, 5.489, 28.106, 27.457, 26.648, 5.108, |
97 4.963, 5.272, 4.954] | 97 4.963, 5.272, 4.954] |
98 ] | 98 ] |
99 ] | 99 ] |
100 | 100 |
101 # Default options for the dry run | 101 # Default options for the dry run |
102 DEFAULT_OPTIONS = { | 102 DEFAULT_OPTIONS = { |
103 'debug_ignore_build': True, | 103 'debug_ignore_build': True, |
104 'debug_ignore_sync': True, | 104 'debug_ignore_sync': True, |
105 'debug_ignore_perf_test': True, | 105 'debug_ignore_perf_test': True, |
106 'debug_ignore_regression_confidence': True, | 106 'debug_ignore_regression_confidence': True, |
107 'command': 'fake_command', | 107 'command': 'fake_command', |
108 'metric': 'fake/metric', | 108 'metric': 'fake/metric', |
109 'good_revision': 280000, | 109 'good_revision': 280000, |
110 'bad_revision': 280005, | 110 'bad_revision': 280005, |
111 } | 111 } |
112 | 112 |
113 # This global is a placeholder for a generator to be defined by the test cases | 113 # This global is a placeholder for a generator to be defined by the test cases |
114 # that use _MockRunTests. | 114 # that use _MockRunTests. |
115 _MockResultsGenerator = (x for x in []) | 115 _MockResultsGenerator = (x for x in []) |
116 | 116 |
117 | 117 |
118 def _MockRunTests(*args, **kwargs): | 118 def _MockRunTests(*args, **kwargs): # pylint: disable=unused-argument |
119 _, _ = args, kwargs | |
120 return _FakeTestResult(_MockResultsGenerator.next()) | 119 return _FakeTestResult(_MockResultsGenerator.next()) |
121 | 120 |
122 | 121 |
123 def _FakeTestResult(values): | 122 def _FakeTestResult(values): |
124 result_dict = {'mean': 0.0, 'std_err': 0.0, 'std_dev': 0.0, 'values': values} | 123 result_dict = {'mean': 0.0, 'std_err': 0.0, 'std_dev': 0.0, 'values': values} |
125 success_code = 0 | 124 success_code = 0 |
126 return (result_dict, success_code) | 125 return (result_dict, success_code) |
127 | 126 |
128 | 127 |
129 def _GetBisectPerformanceMetricsInstance(options_dict): | 128 def _GetBisectPerformanceMetricsInstance(options_dict): |
(...skipping 242 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
372 | 371 |
373 def _CheckAbortsEarly(self, results): | 372 def _CheckAbortsEarly(self, results): |
374 """Returns True if the bisect job would abort early.""" | 373 """Returns True if the bisect job would abort early.""" |
375 global _MockResultsGenerator | 374 global _MockResultsGenerator |
376 _MockResultsGenerator = (r for r in results) | 375 _MockResultsGenerator = (r for r in results) |
377 bisect_class = bisect_perf_regression.BisectPerformanceMetrics | 376 bisect_class = bisect_perf_regression.BisectPerformanceMetrics |
378 original_run_tests = bisect_class.RunPerformanceTestAndParseResults | 377 original_run_tests = bisect_class.RunPerformanceTestAndParseResults |
379 bisect_class.RunPerformanceTestAndParseResults = _MockRunTests | 378 bisect_class.RunPerformanceTestAndParseResults = _MockRunTests |
380 | 379 |
381 try: | 380 try: |
382 _GenericDryRun(_GetExtendedOptions(0, 0, False)) | 381 dry_run_results = _GenericDryRun(_GetExtendedOptions(0, 0, False)) |
383 except StopIteration: | 382 except StopIteration: |
384 # If StopIteration was raised, that means that the next value after | 383 # If StopIteration was raised, that means that the next value after |
385 # the first two values was requested, so the job was not aborted. | 384 # the first two values was requested, so the job was not aborted. |
386 return False | 385 return False |
387 finally: | 386 finally: |
388 bisect_class.RunPerformanceTestAndParseResults = original_run_tests | 387 bisect_class.RunPerformanceTestAndParseResults = original_run_tests |
389 | 388 |
390 # If the job was aborted, there should be a warning about it. | 389 # If the job was aborted, there should be a warning about it. |
391 assert [w for w in results.warnings | 390 assert [w for w in dry_run_results.warnings |
392 if 'could not reproduce the regression' in w] | 391 if 'could not reproduce the regression' in w] |
393 return True | 392 return True |
394 | 393 |
395 def testBisectStopsOnClearUnclearRegression(self): | 394 def testBisectAbortedOnClearNonRegression(self): |
396 self.assertTrue(self._CheckAbortsEarly(CLEAR_NON_REGRESSION)) | 395 self.assertTrue(self._CheckAbortsEarly(CLEAR_NON_REGRESSION)) |
397 | 396 |
398 def testBisectStopsOnClearUnclearRegression(self): | 397 def testBisectNotAborted_AlmostRegression(self): |
399 self.assertFalse(self._CheckAbortsEarly(ALMOST_REGRESSION)) | 398 self.assertFalse(self._CheckAbortsEarly(ALMOST_REGRESSION)) |
400 | 399 |
401 def testBisectStopsOnClearUnclearRegression(self): | 400 def testBisectNotAborted_ClearRegression(self): |
402 self.assertFalse(self._CheckAbortsEarly(CLEAR_REGRESSION)) | 401 self.assertFalse(self._CheckAbortsEarly(CLEAR_REGRESSION)) |
403 | 402 |
404 def testBisectStopsOnClearUnclearRegression(self): | 403 def testBisectNotAborted_BarelyRegression(self): |
405 self.assertFalse(self._CheckAbortsEarly(BARELY_REGRESSION)) | 404 self.assertFalse(self._CheckAbortsEarly(BARELY_REGRESSION)) |
406 | 405 |
407 def testBisectStopsOnClearUnclearRegression(self): | 406 def testBisectNotAborted_MultipleValues(self): |
408 self.assertFalse(self._CheckAbortsEarly(MULTIPLE_VALUES)) | 407 self.assertFalse(self._CheckAbortsEarly(MULTIPLE_VALUES)) |
409 | 408 |
410 def testGetCommitPosition(self): | 409 def testGetCommitPosition(self): |
411 cp_git_rev = '7017a81991de983e12ab50dfc071c70e06979531' | 410 cp_git_rev = '7017a81991de983e12ab50dfc071c70e06979531' |
412 self.assertEqual(291765, source_control.GetCommitPosition(cp_git_rev)) | 411 self.assertEqual(291765, source_control.GetCommitPosition(cp_git_rev)) |
413 | 412 |
414 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58' | 413 svn_git_rev = 'e6db23a037cad47299a94b155b95eebd1ee61a58' |
415 self.assertEqual(291467, source_control.GetCommitPosition(svn_git_rev)) | 414 self.assertEqual(291467, source_control.GetCommitPosition(svn_git_rev)) |
416 | 415 |
417 def testGetCommitPositionForV8(self): | 416 def testGetCommitPositionForV8(self): |
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
460 mock_RunGClient.return_value = 0 | 459 mock_RunGClient.return_value = 0 |
461 bisect_instance._SyncRevision( | 460 bisect_instance._SyncRevision( |
462 'chromium', 'e6db23a037cad47299a94b155b95eebd1ee61a58', 'gclient') | 461 'chromium', 'e6db23a037cad47299a94b155b95eebd1ee61a58', 'gclient') |
463 expected_params = [ | 462 expected_params = [ |
464 'sync', | 463 'sync', |
465 '--verbose', | 464 '--verbose', |
466 '--nohooks', | 465 '--nohooks', |
467 '--force', | 466 '--force', |
468 '--delete_unversioned_trees', | 467 '--delete_unversioned_trees', |
469 '--revision', | 468 '--revision', |
470 'src@e6db23a037cad47299a94b155b95eebd1ee61a58' | 469 'src@e6db23a037cad47299a94b155b95eebd1ee61a58', |
471 ] | 470 ] |
472 | 471 |
473 mock_RunGClient.assert_called_with(expected_params, cwd=None) | 472 mock_RunGClient.assert_called_with(expected_params, cwd=None) |
474 | 473 |
475 @mock.patch('bisect_utils.RunGit') | 474 @mock.patch('bisect_utils.RunGit') |
476 def testSyncToRevisionForWebKit(self, mock_RunGit): | 475 def testSyncToRevisionForWebKit(self, mock_RunGit): |
477 bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS) | 476 bisect_instance = _GetBisectPerformanceMetricsInstance(DEFAULT_OPTIONS) |
478 mock_RunGit.return_value = None, None | 477 mock_RunGit.return_value = None, None |
479 bisect_instance._SyncRevision( | 478 bisect_instance._SyncRevision( |
480 'webkit', 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61' , None) | 479 'webkit', 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61', None) |
481 expected_params = ['checkout', 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61'] | 480 expected_params = ['checkout', 'a94d028e0f2c77f159b3dac95eb90c3b4cf48c61'] |
482 mock_RunGit.assert_called_with(expected_params) | 481 mock_RunGit.assert_called_with(expected_params) |
483 | 482 |
484 def testTryJobSvnRepo_PerfBuilderType_ReturnsRepoUrl(self): | 483 def testTryJobSvnRepo_PerfBuilderType_ReturnsRepoUrl(self): |
485 self.assertEqual(bisect_perf_regression.PERF_SVN_REPO_URL, | 484 self.assertEqual( |
| 485 bisect_perf_regression.PERF_SVN_REPO_URL, |
486 bisect_perf_regression._TryJobSvnRepo(fetch_build.PERF_BUILDER)) | 486 bisect_perf_regression._TryJobSvnRepo(fetch_build.PERF_BUILDER)) |
487 | 487 |
488 def testTryJobSvnRepo_FullBuilderType_ReturnsRepoUrl(self): | 488 def testTryJobSvnRepo_FullBuilderType_ReturnsRepoUrl(self): |
489 self.assertEqual(bisect_perf_regression.FULL_SVN_REPO_URL, | 489 self.assertEqual( |
| 490 bisect_perf_regression.FULL_SVN_REPO_URL, |
490 bisect_perf_regression._TryJobSvnRepo(fetch_build.FULL_BUILDER)) | 491 bisect_perf_regression._TryJobSvnRepo(fetch_build.FULL_BUILDER)) |
491 | 492 |
492 def testTryJobSvnRepo_WithUnknownBuilderType_ThrowsError(self): | 493 def testTryJobSvnRepo_WithUnknownBuilderType_ThrowsError(self): |
493 with self.assertRaises(NotImplementedError): | 494 with self.assertRaises(NotImplementedError): |
494 bisect_perf_regression._TryJobSvnRepo('foo') | 495 bisect_perf_regression._TryJobSvnRepo('foo') |
495 | 496 |
496 def _CheckIsDownloadable(self, depot, target_platform='chromium', | 497 def _CheckIsDownloadable(self, depot, target_platform='chromium', |
497 builder_type='perf'): | 498 builder_type='perf'): |
498 opts = dict(DEFAULT_OPTIONS) | 499 opts = dict(DEFAULT_OPTIONS) |
499 opts.update({'target_platform': target_platform, | 500 opts.update({'target_platform': target_platform, |
(...skipping 165 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
665 (['update-index', '--refresh', '-q'], (None, 0)), | 666 (['update-index', '--refresh', '-q'], (None, 0)), |
666 (['diff-index', 'HEAD'], (None, 0)), | 667 (['diff-index', 'HEAD'], (None, 0)), |
667 (['checkout', '-b', new_branch], ('None', 0)), | 668 (['checkout', '-b', new_branch], ('None', 0)), |
668 (['branch', '--set-upstream-to', parent_branch], | 669 (['branch', '--set-upstream-to', parent_branch], |
669 ('Setuptream fails', 0)), | 670 ('Setuptream fails', 0)), |
670 (['try', | 671 (['try', |
671 '--bot=%s' % bot_name, | 672 '--bot=%s' % bot_name, |
672 '--revision=%s' % git_revision, | 673 '--revision=%s' % git_revision, |
673 '--name=%s' % bisect_job_name, | 674 '--name=%s' % bisect_job_name, |
674 '--svn_repo=%s' % bisect_perf_regression.PERF_SVN_REPO_URL, | 675 '--svn_repo=%s' % bisect_perf_regression.PERF_SVN_REPO_URL, |
675 '--diff=%s' % patch_content | 676 '--diff=%s' % patch_content], |
676 ], (None, 1)), | 677 (None, 1)), |
677 ] | 678 ] |
678 self._AssertRunGitExceptions( | 679 self._AssertRunGitExceptions( |
679 try_cmd, bisect_perf_regression._StartBuilderTryJob, | 680 try_cmd, bisect_perf_regression._StartBuilderTryJob, |
680 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name, | 681 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name, |
681 patch) | 682 patch) |
682 | 683 |
683 def testBuilderTryJob(self): | 684 def testBuilderTryJob(self): |
684 git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888' | 685 git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888' |
685 bot_name = 'linux_perf_bisect_builder' | 686 bot_name = 'linux_perf_bisect_builder' |
686 bisect_job_name = 'testBisectJobname' | 687 bisect_job_name = 'testBisectJobname' |
687 patch = None | 688 patch = None |
688 patch_content = '/dev/null' | 689 patch_content = '/dev/null' |
689 new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH | 690 new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH |
690 parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH | 691 parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH |
691 try_cmd = [ | 692 try_cmd = [ |
692 (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)), | 693 (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)), |
693 (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)), | 694 (['branch', '--list'], ('bisect-tryjob\n*master\nsomebranch', 0)), |
694 (['branch', '-D', new_branch], ('None', 0)), | 695 (['branch', '-D', new_branch], ('None', 0)), |
695 (['update-index', '--refresh', '-q'], (None, 0)), | 696 (['update-index', '--refresh', '-q'], (None, 0)), |
696 (['diff-index', 'HEAD'], (None, 0)), | 697 (['diff-index', 'HEAD'], (None, 0)), |
697 (['checkout', '-b', new_branch], ('None', 0)), | 698 (['checkout', '-b', new_branch], ('None', 0)), |
698 (['branch', '--set-upstream-to', parent_branch], | 699 (['branch', '--set-upstream-to', parent_branch], |
699 ('Setuptream fails', 0)), | 700 ('Setuptream fails', 0)), |
700 (['try', | 701 (['try', |
701 '--bot=%s' % bot_name, | 702 '--bot=%s' % bot_name, |
702 '--revision=%s' % git_revision, | 703 '--revision=%s' % git_revision, |
703 '--name=%s' % bisect_job_name, | 704 '--name=%s' % bisect_job_name, |
704 '--svn_repo=%s' % bisect_perf_regression.PERF_SVN_REPO_URL, | 705 '--svn_repo=%s' % bisect_perf_regression.PERF_SVN_REPO_URL, |
705 '--diff=%s' % patch_content | 706 '--diff=%s' % patch_content], |
706 ], (None, 0)), | 707 (None, 0)), |
707 ] | 708 ] |
708 self._SetupRunGitMock(try_cmd) | 709 self._SetupRunGitMock(try_cmd) |
709 bisect_perf_regression._StartBuilderTryJob( | 710 bisect_perf_regression._StartBuilderTryJob( |
710 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name, | 711 fetch_build.PERF_BUILDER, git_revision, bot_name, bisect_job_name, |
711 patch) | 712 patch) |
712 | 713 |
713 | 714 |
714 if __name__ == '__main__': | 715 if __name__ == '__main__': |
715 unittest.main() | 716 unittest.main() |
OLD | NEW |