Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(102)

Side by Side Diff: tools/bisect-perf-regression.py

Issue 218613012: Revert of Refactor perf bisect script _CalculateConfidence method. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 6 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Performance Test Bisect Tool 6 """Performance Test Bisect Tool
7 7
8 This script bisects a series of changelists using binary search. It starts at 8 This script bisects a series of changelists using binary search. It starts at
9 a bad revision where a performance metric has regressed, and asks for a last 9 a bad revision where a performance metric has regressed, and asks for a last
10 known-good revision. It will then binary search across this revision range by 10 known-good revision. It will then binary search across this revision range by
(...skipping 1357 matching lines...) Expand 10 before | Expand all | Expand 10 after
1368 RmTreeAndMkDir(output_dir, skip_makedir=True) 1368 RmTreeAndMkDir(output_dir, skip_makedir=True)
1369 ExtractZip(archive_file_dest, abs_build_dir) 1369 ExtractZip(archive_file_dest, abs_build_dir)
1370 if os.path.exists(output_dir): 1370 if os.path.exists(output_dir):
1371 self.BackupOrRestoreOutputdirectory(restore=False) 1371 self.BackupOrRestoreOutputdirectory(restore=False)
1372 print 'Moving build from %s to %s' % ( 1372 print 'Moving build from %s to %s' % (
1373 output_dir, target_build_output_dir) 1373 output_dir, target_build_output_dir)
1374 shutil.move(output_dir, target_build_output_dir) 1374 shutil.move(output_dir, target_build_output_dir)
1375 return True 1375 return True
1376 raise IOError('Missing extracted folder %s ' % output_dir) 1376 raise IOError('Missing extracted folder %s ' % output_dir)
1377 except e: 1377 except e:
1378 print 'Something went wrong while extracting archive file: %s' % e 1378 print 'Somewthing went wrong while extracting archive file: %s' % e
1379 self.BackupOrRestoreOutputdirectory(restore=True) 1379 self.BackupOrRestoreOutputdirectory(restore=True)
1380 # Cleanup any leftovers from unzipping. 1380 # Cleanup any leftovers from unzipping.
1381 if os.path.exists(output_dir): 1381 if os.path.exists(output_dir):
1382 RmTreeAndMkDir(output_dir, skip_makedir=True) 1382 RmTreeAndMkDir(output_dir, skip_makedir=True)
1383 finally: 1383 finally:
1384 # Delete downloaded archive 1384 # Delete downloaded archive
1385 if os.path.exists(archive_file_dest): 1385 if os.path.exists(archive_file_dest):
1386 os.remove(archive_file_dest) 1386 os.remove(archive_file_dest)
1387 return False 1387 return False
1388 1388
(...skipping 1392 matching lines...) Expand 10 before | Expand all | Expand 10 after
2781 bad_greater_than_good else not prev_less_than_current) 2781 bad_greater_than_good else not prev_less_than_current)
2782 2782
2783 # Only report potential regressions with high confidence. 2783 # Only report potential regressions with high confidence.
2784 if is_same_direction and confidence > 50: 2784 if is_same_direction and confidence > 50:
2785 other_regressions.append([current_id, previous_id, confidence]) 2785 other_regressions.append([current_id, previous_id, confidence])
2786 previous_values.append(current_values) 2786 previous_values.append(current_values)
2787 previous_id = current_id 2787 previous_id = current_id
2788 return other_regressions 2788 return other_regressions
2789 2789
2790 def _CalculateConfidence(self, working_means, broken_means): 2790 def _CalculateConfidence(self, working_means, broken_means):
2791 """Calculates the confidence percentage. 2791 bounds_working = []
2792 2792 bounds_broken = []
2793 This is calculated based on how distinct the values are before and after 2793 for m in working_means:
2794 the last broken revision, and how noisy the results are. 2794 current_mean = CalculateTruncatedMean(m, 0)
2795 2795 if bounds_working:
2796 Args: 2796 bounds_working[0] = min(current_mean, bounds_working[0])
2797 working_means: A list of lists of "good" result numbers. 2797 bounds_working[1] = max(current_mean, bounds_working[0])
2798 broken means: A list of lists of "bad" result numbers. 2798 else:
2799 2799 bounds_working = [current_mean, current_mean]
2800 Returns: 2800 for m in broken_means:
2801 A number between in the range [0, 100]. 2801 current_mean = CalculateTruncatedMean(m, 0)
2802 """ 2802 if bounds_broken:
2803 bounds_working = self._CalculateBounds(working_means) 2803 bounds_broken[0] = min(current_mean, bounds_broken[0])
2804 bounds_broken = self._CalculateBounds(broken_means) 2804 bounds_broken[1] = max(current_mean, bounds_broken[0])
2805 else:
2806 bounds_broken = [current_mean, current_mean]
2805 dist_between_groups = min(math.fabs(bounds_broken[1] - bounds_working[0]), 2807 dist_between_groups = min(math.fabs(bounds_broken[1] - bounds_working[0]),
2806 math.fabs(bounds_broken[0] - bounds_working[1])) 2808 math.fabs(bounds_broken[0] - bounds_working[1]))
2807 working_mean = sum(working_means, []) 2809 working_mean = sum(working_means, [])
2808 broken_mean = sum(broken_means, []) 2810 broken_mean = sum(broken_means, [])
2809 len_working_group = CalculateStandardDeviation(working_mean) 2811 len_working_group = CalculateStandardDeviation(working_mean)
2810 len_broken_group = CalculateStandardDeviation(broken_mean) 2812 len_broken_group = CalculateStandardDeviation(broken_mean)
2811 confidence = (dist_between_groups / 2813
2812 (max(0.0001, (len_broken_group + len_working_group)))) 2814 confidence = (dist_between_groups / (
2815 max(0.0001, (len_broken_group + len_working_group ))))
2813 confidence = int(min(1.0, max(confidence, 0.0)) * 100.0) 2816 confidence = int(min(1.0, max(confidence, 0.0)) * 100.0)
2814 return confidence 2817 return confidence
2815 2818
2816 def _CalculateBounds(self, values_list):
2817 """Returns the lower/upper bounds for the means of the given value lists.
2818
2819 Args:
2820 values_list: A non-empty list of lists of numbers.
2821
2822 Returns:
2823 A (lower, upper) pair of bounds.
2824 """
2825 bounds = None
2826 for values in values_list:
2827 mean = CalculateTruncatedMean(values, 0)
2828 if bounds:
2829 bounds[0] = min(mean, bounds[0])
2830 bounds[1] = max(mean, bounds[1])
2831 else:
2832 bounds = (mean, mean)
2833 return bounds
2834
2835 def _GetResultsDict(self, revision_data, revision_data_sorted): 2819 def _GetResultsDict(self, revision_data, revision_data_sorted):
2836 """Makes and returns a dictionary of overall results from the bisect.
2837
2838 Args:
2839 revision_data: The revisions_data dict as returned by the Run method.
2840 revision_data_sorted: A list of pairs from the above dictionary, sorted
2841 in order of commits.
2842
2843 Returns:
2844 A dictionary containing results from the bisect.
2845 """
2846 # Find range where it possibly broke. 2820 # Find range where it possibly broke.
2847 first_working_revision = None 2821 first_working_revision = None
2848 first_working_revision_index = -1 2822 first_working_revision_index = -1
2849 last_broken_revision = None 2823 last_broken_revision = None
2850 last_broken_revision_index = -1 2824 last_broken_revision_index = -1
2851 2825
2852 for i in xrange(len(revision_data_sorted)): 2826 for i in xrange(len(revision_data_sorted)):
2853 k, v = revision_data_sorted[i] 2827 k, v = revision_data_sorted[i]
2854 if v['passed'] == 1: 2828 if v['passed'] == 1:
2855 if not first_working_revision: 2829 if not first_working_revision:
(...skipping 552 matching lines...) Expand 10 before | Expand all | Expand 10 after
3408 # The perf dashboard scrapes the "results" step in order to comment on 3382 # The perf dashboard scrapes the "results" step in order to comment on
3409 # bugs. If you change this, please update the perf dashboard as well. 3383 # bugs. If you change this, please update the perf dashboard as well.
3410 bisect_utils.OutputAnnotationStepStart('Results') 3384 bisect_utils.OutputAnnotationStepStart('Results')
3411 print 'Error: %s' % e.message 3385 print 'Error: %s' % e.message
3412 if opts.output_buildbot_annotations: 3386 if opts.output_buildbot_annotations:
3413 bisect_utils.OutputAnnotationStepClosed() 3387 bisect_utils.OutputAnnotationStepClosed()
3414 return 1 3388 return 1
3415 3389
3416 if __name__ == '__main__': 3390 if __name__ == '__main__':
3417 sys.exit(main()) 3391 sys.exit(main())
OLDNEW
« no previous file with comments | « no previous file | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698