OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Performance Test Bisect Tool | 6 """Performance Test Bisect Tool |
7 | 7 |
8 This script bisects a series of changelists using binary search. It starts at | 8 This script bisects a series of changelists using binary search. It starts at |
9 a bad revision where a performance metric has regressed, and asks for a last | 9 a bad revision where a performance metric has regressed, and asks for a last |
10 known-good revision. It will then binary search across this revision range by | 10 known-good revision. It will then binary search across this revision range by |
(...skipping 1357 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1368 RmTreeAndMkDir(output_dir, skip_makedir=True) | 1368 RmTreeAndMkDir(output_dir, skip_makedir=True) |
1369 ExtractZip(archive_file_dest, abs_build_dir) | 1369 ExtractZip(archive_file_dest, abs_build_dir) |
1370 if os.path.exists(output_dir): | 1370 if os.path.exists(output_dir): |
1371 self.BackupOrRestoreOutputdirectory(restore=False) | 1371 self.BackupOrRestoreOutputdirectory(restore=False) |
1372 print 'Moving build from %s to %s' % ( | 1372 print 'Moving build from %s to %s' % ( |
1373 output_dir, target_build_output_dir) | 1373 output_dir, target_build_output_dir) |
1374 shutil.move(output_dir, target_build_output_dir) | 1374 shutil.move(output_dir, target_build_output_dir) |
1375 return True | 1375 return True |
1376 raise IOError('Missing extracted folder %s ' % output_dir) | 1376 raise IOError('Missing extracted folder %s ' % output_dir) |
1377 except e: | 1377 except e: |
1378 print 'Something went wrong while extracting archive file: %s' % e | 1378 print 'Somewthing went wrong while extracting archive file: %s' % e |
1379 self.BackupOrRestoreOutputdirectory(restore=True) | 1379 self.BackupOrRestoreOutputdirectory(restore=True) |
1380 # Cleanup any leftovers from unzipping. | 1380 # Cleanup any leftovers from unzipping. |
1381 if os.path.exists(output_dir): | 1381 if os.path.exists(output_dir): |
1382 RmTreeAndMkDir(output_dir, skip_makedir=True) | 1382 RmTreeAndMkDir(output_dir, skip_makedir=True) |
1383 finally: | 1383 finally: |
1384 # Delete downloaded archive | 1384 # Delete downloaded archive |
1385 if os.path.exists(archive_file_dest): | 1385 if os.path.exists(archive_file_dest): |
1386 os.remove(archive_file_dest) | 1386 os.remove(archive_file_dest) |
1387 return False | 1387 return False |
1388 | 1388 |
(...skipping 1392 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2781 bad_greater_than_good else not prev_less_than_current) | 2781 bad_greater_than_good else not prev_less_than_current) |
2782 | 2782 |
2783 # Only report potential regressions with high confidence. | 2783 # Only report potential regressions with high confidence. |
2784 if is_same_direction and confidence > 50: | 2784 if is_same_direction and confidence > 50: |
2785 other_regressions.append([current_id, previous_id, confidence]) | 2785 other_regressions.append([current_id, previous_id, confidence]) |
2786 previous_values.append(current_values) | 2786 previous_values.append(current_values) |
2787 previous_id = current_id | 2787 previous_id = current_id |
2788 return other_regressions | 2788 return other_regressions |
2789 | 2789 |
2790 def _CalculateConfidence(self, working_means, broken_means): | 2790 def _CalculateConfidence(self, working_means, broken_means): |
2791 """Calculates the confidence percentage. | 2791 bounds_working = [] |
2792 | 2792 bounds_broken = [] |
2793 This is calculated based on how distinct the values are before and after | 2793 for m in working_means: |
2794 the last broken revision, and how noisy the results are. | 2794 current_mean = CalculateTruncatedMean(m, 0) |
2795 | 2795 if bounds_working: |
2796 Args: | 2796 bounds_working[0] = min(current_mean, bounds_working[0]) |
2797 working_means: A list of lists of "good" result numbers. | 2797 bounds_working[1] = max(current_mean, bounds_working[0]) |
2798 broken means: A list of lists of "bad" result numbers. | 2798 else: |
2799 | 2799 bounds_working = [current_mean, current_mean] |
2800 Returns: | 2800 for m in broken_means: |
2801 A number between in the range [0, 100]. | 2801 current_mean = CalculateTruncatedMean(m, 0) |
2802 """ | 2802 if bounds_broken: |
2803 bounds_working = self._CalculateBounds(working_means) | 2803 bounds_broken[0] = min(current_mean, bounds_broken[0]) |
2804 bounds_broken = self._CalculateBounds(broken_means) | 2804 bounds_broken[1] = max(current_mean, bounds_broken[0]) |
| 2805 else: |
| 2806 bounds_broken = [current_mean, current_mean] |
2805 dist_between_groups = min(math.fabs(bounds_broken[1] - bounds_working[0]), | 2807 dist_between_groups = min(math.fabs(bounds_broken[1] - bounds_working[0]), |
2806 math.fabs(bounds_broken[0] - bounds_working[1])) | 2808 math.fabs(bounds_broken[0] - bounds_working[1])) |
2807 working_mean = sum(working_means, []) | 2809 working_mean = sum(working_means, []) |
2808 broken_mean = sum(broken_means, []) | 2810 broken_mean = sum(broken_means, []) |
2809 len_working_group = CalculateStandardDeviation(working_mean) | 2811 len_working_group = CalculateStandardDeviation(working_mean) |
2810 len_broken_group = CalculateStandardDeviation(broken_mean) | 2812 len_broken_group = CalculateStandardDeviation(broken_mean) |
2811 confidence = (dist_between_groups / | 2813 |
2812 (max(0.0001, (len_broken_group + len_working_group)))) | 2814 confidence = (dist_between_groups / ( |
| 2815 max(0.0001, (len_broken_group + len_working_group )))) |
2813 confidence = int(min(1.0, max(confidence, 0.0)) * 100.0) | 2816 confidence = int(min(1.0, max(confidence, 0.0)) * 100.0) |
2814 return confidence | 2817 return confidence |
2815 | 2818 |
2816 def _CalculateBounds(self, values_list): | |
2817 """Returns the lower/upper bounds for the means of the given value lists. | |
2818 | |
2819 Args: | |
2820 values_list: A non-empty list of lists of numbers. | |
2821 | |
2822 Returns: | |
2823 A (lower, upper) pair of bounds. | |
2824 """ | |
2825 bounds = None | |
2826 for values in values_list: | |
2827 mean = CalculateTruncatedMean(values, 0) | |
2828 if bounds: | |
2829 bounds[0] = min(mean, bounds[0]) | |
2830 bounds[1] = max(mean, bounds[1]) | |
2831 else: | |
2832 bounds = (mean, mean) | |
2833 return bounds | |
2834 | |
2835 def _GetResultsDict(self, revision_data, revision_data_sorted): | 2819 def _GetResultsDict(self, revision_data, revision_data_sorted): |
2836 """Makes and returns a dictionary of overall results from the bisect. | |
2837 | |
2838 Args: | |
2839 revision_data: The revisions_data dict as returned by the Run method. | |
2840 revision_data_sorted: A list of pairs from the above dictionary, sorted | |
2841 in order of commits. | |
2842 | |
2843 Returns: | |
2844 A dictionary containing results from the bisect. | |
2845 """ | |
2846 # Find range where it possibly broke. | 2820 # Find range where it possibly broke. |
2847 first_working_revision = None | 2821 first_working_revision = None |
2848 first_working_revision_index = -1 | 2822 first_working_revision_index = -1 |
2849 last_broken_revision = None | 2823 last_broken_revision = None |
2850 last_broken_revision_index = -1 | 2824 last_broken_revision_index = -1 |
2851 | 2825 |
2852 for i in xrange(len(revision_data_sorted)): | 2826 for i in xrange(len(revision_data_sorted)): |
2853 k, v = revision_data_sorted[i] | 2827 k, v = revision_data_sorted[i] |
2854 if v['passed'] == 1: | 2828 if v['passed'] == 1: |
2855 if not first_working_revision: | 2829 if not first_working_revision: |
(...skipping 552 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
3408 # The perf dashboard scrapes the "results" step in order to comment on | 3382 # The perf dashboard scrapes the "results" step in order to comment on |
3409 # bugs. If you change this, please update the perf dashboard as well. | 3383 # bugs. If you change this, please update the perf dashboard as well. |
3410 bisect_utils.OutputAnnotationStepStart('Results') | 3384 bisect_utils.OutputAnnotationStepStart('Results') |
3411 print 'Error: %s' % e.message | 3385 print 'Error: %s' % e.message |
3412 if opts.output_buildbot_annotations: | 3386 if opts.output_buildbot_annotations: |
3413 bisect_utils.OutputAnnotationStepClosed() | 3387 bisect_utils.OutputAnnotationStepClosed() |
3414 return 1 | 3388 return 1 |
3415 | 3389 |
3416 if __name__ == '__main__': | 3390 if __name__ == '__main__': |
3417 sys.exit(main()) | 3391 sys.exit(main()) |
OLD | NEW |