OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Performance Test Bisect Tool | 6 """Performance Test Bisect Tool |
7 | 7 |
8 This script bisects a series of changelists using binary search. It starts at | 8 This script bisects a series of changelists using binary search. It starts at |
9 a bad revision where a performance metric has regressed, and asks for a last | 9 a bad revision where a performance metric has regressed, and asks for a last |
10 known-good revision. It will then binary search across this revision range by | 10 known-good revision. It will then binary search across this revision range by |
(...skipping 268 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
279 | 279 |
280 Returns: | 280 Returns: |
281 A number in the range [0, 100]. | 281 A number in the range [0, 100]. |
282 """ | 282 """ |
283 if not good_results_lists or not bad_results_lists: | 283 if not good_results_lists or not bad_results_lists: |
284 return 0.0 | 284 return 0.0 |
285 | 285 |
286 # Flatten the lists of results lists. | 286 # Flatten the lists of results lists. |
287 sample1 = sum(good_results_lists, []) | 287 sample1 = sum(good_results_lists, []) |
288 sample2 = sum(bad_results_lists, []) | 288 sample2 = sum(bad_results_lists, []) |
| 289 if not sample1 or not sample2: |
| 290 return 0.0 |
289 | 291 |
290 # The p-value is approximately the probability of obtaining the given set | 292 # The p-value is approximately the probability of obtaining the given set |
291 # of good and bad values just by chance. | 293 # of good and bad values just by chance. |
292 _, _, p_value = ttest.WelchsTTest(sample1, sample2) | 294 _, _, p_value = ttest.WelchsTTest(sample1, sample2) |
293 return 100.0 * (1.0 - p_value) | 295 return 100.0 * (1.0 - p_value) |
294 | 296 |
295 | 297 |
296 def GetSHA1HexDigest(contents): | 298 def GetSHA1HexDigest(contents): |
297 """Returns SHA1 hex digest of the given string.""" | 299 """Returns SHA1 hex digest of the given string.""" |
298 return hashlib.sha1(contents).hexdigest() | 300 return hashlib.sha1(contents).hexdigest() |
(...skipping 2748 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3047 previous_data['depot'], previous_link) | 3049 previous_data['depot'], previous_link) |
3048 print | 3050 print |
3049 | 3051 |
3050 def _GetResultsDict(self, revision_data, revision_data_sorted): | 3052 def _GetResultsDict(self, revision_data, revision_data_sorted): |
3051 # Find range where it possibly broke. | 3053 # Find range where it possibly broke. |
3052 first_working_revision = None | 3054 first_working_revision = None |
3053 first_working_revision_index = -1 | 3055 first_working_revision_index = -1 |
3054 last_broken_revision = None | 3056 last_broken_revision = None |
3055 last_broken_revision_index = -1 | 3057 last_broken_revision_index = -1 |
3056 | 3058 |
| 3059 culprit_revisions = [] |
| 3060 other_regressions = [] |
| 3061 regression_size = 0.0 |
| 3062 regression_std_err = 0.0 |
| 3063 confidence = 0.0 |
| 3064 |
3057 for i in xrange(len(revision_data_sorted)): | 3065 for i in xrange(len(revision_data_sorted)): |
3058 k, v = revision_data_sorted[i] | 3066 k, v = revision_data_sorted[i] |
3059 if v['passed'] == 1: | 3067 if v['passed'] == 1: |
3060 if not first_working_revision: | 3068 if not first_working_revision: |
3061 first_working_revision = k | 3069 first_working_revision = k |
3062 first_working_revision_index = i | 3070 first_working_revision_index = i |
3063 | 3071 |
3064 if not v['passed']: | 3072 if not v['passed']: |
3065 last_broken_revision = k | 3073 last_broken_revision = k |
3066 last_broken_revision_index = i | 3074 last_broken_revision_index = i |
(...skipping 585 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3652 # bugs. If you change this, please update the perf dashboard as well. | 3660 # bugs. If you change this, please update the perf dashboard as well. |
3653 bisect_utils.OutputAnnotationStepStart('Results') | 3661 bisect_utils.OutputAnnotationStepStart('Results') |
3654 print 'Error: %s' % e.message | 3662 print 'Error: %s' % e.message |
3655 if opts.output_buildbot_annotations: | 3663 if opts.output_buildbot_annotations: |
3656 bisect_utils.OutputAnnotationStepClosed() | 3664 bisect_utils.OutputAnnotationStepClosed() |
3657 return 1 | 3665 return 1 |
3658 | 3666 |
3659 | 3667 |
3660 if __name__ == '__main__': | 3668 if __name__ == '__main__': |
3661 sys.exit(main()) | 3669 sys.exit(main()) |
OLD | NEW |