| OLD | NEW |
| 1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import math | 5 import math |
| 6 import os | 6 import os |
| 7 | 7 |
| 8 import bisect_utils | 8 import bisect_utils |
| 9 import math_utils | 9 import math_utils |
| 10 import source_control | 10 import source_control |
| (...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 51 second option creates an error result. | 51 second option creates an error result. |
| 52 | 52 |
| 53 Args: | 53 Args: |
| 54 bisect_state: BisectState object representing latest bisect state. | 54 bisect_state: BisectState object representing latest bisect state. |
| 55 depot_registry: DepotDirectoryRegistry object with information on each | 55 depot_registry: DepotDirectoryRegistry object with information on each |
| 56 repository in the bisect_state. | 56 repository in the bisect_state. |
| 57 opts: Options passed to the bisect run. | 57 opts: Options passed to the bisect run. |
| 58 runtime_warnings: A list of warnings from the bisect run. | 58 runtime_warnings: A list of warnings from the bisect run. |
| 59 error: Error message. When error is not None, other arguments are ignored. | 59 error: Error message. When error is not None, other arguments are ignored. |
| 60 """ | 60 """ |
| 61 | |
| 62 self.error = error | 61 self.error = error |
| 63 self.abort_reason = abort_reason | 62 self.abort_reason = abort_reason |
| 64 if error is not None or abort_reason is not None: | 63 if error is not None or abort_reason is not None: |
| 65 return | 64 return |
| 66 | 65 |
| 67 assert (bisect_state is not None and depot_registry is not None and | 66 assert (bisect_state is not None and depot_registry is not None and |
| 68 opts is not None and runtime_warnings is not None), ( | 67 opts is not None and runtime_warnings is not None), ( |
| 69 'Incorrect use of the BisectResults constructor. When error is ' | 68 'Incorrect use of the BisectResults constructor. ' |
| 70 'None, all other arguments are required') | 69 'When error is None, all other arguments are required.') |
| 71 | 70 |
| 72 self.state = bisect_state | 71 self.state = bisect_state |
| 73 | 72 |
| 74 rev_states = bisect_state.GetRevisionStates() | 73 rev_states = bisect_state.GetRevisionStates() |
| 75 first_working_rev, last_broken_rev = self.FindBreakingRevRange(rev_states) | 74 first_working_rev, last_broken_rev = self.FindBreakingRevRange(rev_states) |
| 76 self.first_working_revision = first_working_rev | 75 self.first_working_revision = first_working_rev |
| 77 self.last_broken_revision = last_broken_rev | 76 self.last_broken_revision = last_broken_rev |
| 78 | 77 |
| 79 self.warnings = runtime_warnings | 78 self.warnings = runtime_warnings |
| 80 | 79 |
| (...skipping 25 matching lines...) Expand all Loading... |
| 106 self.culprit_revisions = [] | 105 self.culprit_revisions = [] |
| 107 self.other_regressions = [] | 106 self.other_regressions = [] |
| 108 | 107 |
| 109 def AddRetestResults(self, results_tot, results_reverted): | 108 def AddRetestResults(self, results_tot, results_reverted): |
| 110 if not results_tot or not results_reverted: | 109 if not results_tot or not results_reverted: |
| 111 self.warnings.append( | 110 self.warnings.append( |
| 112 'Failed to re-test reverted culprit CL against ToT.') | 111 'Failed to re-test reverted culprit CL against ToT.') |
| 113 return | 112 return |
| 114 | 113 |
| 115 confidence_params = (results_reverted[0]['values'], | 114 confidence_params = (results_reverted[0]['values'], |
| 116 results_tot[0]['values']) | 115 results_tot[0]['values']) |
| 117 confidence = BisectResults.ConfidenceScore(*confidence_params) | 116 confidence = BisectResults.ConfidenceScore(*confidence_params) |
| 118 | 117 |
| 119 self.retest_results_tot = RevisionState('ToT', 'n/a', 0) | 118 self.retest_results_tot = RevisionState('ToT', 'n/a', 0) |
| 120 self.retest_results_tot.value = results_tot[0] | 119 self.retest_results_tot.value = results_tot[0] |
| 121 | 120 |
| 122 self.retest_results_reverted = RevisionState('Reverted', 'n/a', 0) | 121 self.retest_results_reverted = RevisionState('Reverted', 'n/a', 0) |
| 123 self.retest_results_reverted.value = results_reverted[0] | 122 self.retest_results_reverted.value = results_reverted[0] |
| 124 | 123 |
| 125 if confidence <= bisect_utils.HIGH_CONFIDENCE: | 124 if confidence <= bisect_utils.HIGH_CONFIDENCE: |
| 126 self.warnings.append( | 125 self.warnings.append( |
| 127 'Confidence of re-test with reverted CL is not high.' | 126 'Confidence of re-test with reverted CL is not high.' |
| 128 ' Check that the regression hasn\'t already recovered. ' | 127 ' Check that the regression hasn\'t already recovered. ' |
| 129 ' There\'s still a chance this is a regression, as performance of' | 128 ' There\'s still a chance this is a regression, as performance of' |
| 130 ' local builds may not match official builds.' ) | 129 ' local builds may not match official builds.') |
| 131 | 130 |
| 132 @staticmethod | 131 @staticmethod |
| 133 def _GetResultBasedWarnings(culprit_revisions, opts, confidence): | 132 def _GetResultBasedWarnings(culprit_revisions, opts, confidence): |
| 134 warnings = [] | 133 warnings = [] |
| 135 if len(culprit_revisions) > 1: | 134 if len(culprit_revisions) > 1: |
| 136 warnings.append('Due to build errors, regression range could ' | 135 warnings.append('Due to build errors, regression range could ' |
| 137 'not be narrowed down to a single commit.') | 136 'not be narrowed down to a single commit.') |
| 138 if opts.repeat_test_count == 1: | 137 if opts.repeat_test_count == 1: |
| 139 warnings.append('Tests were only set to run once. This may ' | 138 warnings.append('Tests were only set to run once. This may ' |
| 140 'be insufficient to get meaningful results.') | 139 'be insufficient to get meaningful results.') |
| (...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 211 confidence = cls.ConfidenceScore(*confidence_params, | 210 confidence = cls.ConfidenceScore(*confidence_params, |
| 212 accept_single_bad_or_good=True) | 211 accept_single_bad_or_good=True) |
| 213 mean_of_prev_runs = math_utils.Mean(sum(previous_values, [])) | 212 mean_of_prev_runs = math_utils.Mean(sum(previous_values, [])) |
| 214 mean_of_current_runs = math_utils.Mean(current_values) | 213 mean_of_current_runs = math_utils.Mean(current_values) |
| 215 | 214 |
| 216 # Check that the potential regression is in the same direction as | 215 # Check that the potential regression is in the same direction as |
| 217 # the overall regression. If the mean of the previous runs < the | 216 # the overall regression. If the mean of the previous runs < the |
| 218 # mean of the current runs, this local regression is in same | 217 # mean of the current runs, this local regression is in same |
| 219 # direction. | 218 # direction. |
| 220 prev_greater_than_current = mean_of_prev_runs > mean_of_current_runs | 219 prev_greater_than_current = mean_of_prev_runs > mean_of_current_runs |
| 221 is_same_direction = (prev_greater_than_current if | 220 if bad_greater_than_good: |
| 222 bad_greater_than_good else not prev_greater_than_current) | 221 is_same_direction = prev_greater_than_current |
| 222 else: |
| 223 is_same_direction = not prev_greater_than_current |
| 223 | 224 |
| 224 # Only report potential regressions with high confidence. | 225 # Only report potential regressions with high confidence. |
| 225 if is_same_direction and confidence > 50: | 226 if is_same_direction and confidence > 50: |
| 226 other_regressions.append([revision_state, prev_state, confidence]) | 227 other_regressions.append([revision_state, prev_state, confidence]) |
| 227 previous_values.append(current_values) | 228 previous_values.append(current_values) |
| 228 prev_state = revision_state | 229 prev_state = revision_state |
| 229 return other_regressions | 230 return other_regressions |
| 230 | 231 |
| 231 @staticmethod | 232 @staticmethod |
| 232 def FindBreakingRevRange(revision_states): | 233 def FindBreakingRevRange(revision_states): |
| (...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 309 sum([last_broken_rev.value['values']], []) | 310 sum([last_broken_rev.value['values']], []) |
| 310 ) | 311 ) |
| 311 confidence = cls.ConfidenceScore(*confidence_params) | 312 confidence = cls.ConfidenceScore(*confidence_params) |
| 312 | 313 |
| 313 bad_greater_than_good = mean_of_bad_runs > mean_of_good_runs | 314 bad_greater_than_good = mean_of_bad_runs > mean_of_good_runs |
| 314 | 315 |
| 315 return {'regression_size': regression_size, | 316 return {'regression_size': regression_size, |
| 316 'regression_std_err': regression_std_err, | 317 'regression_std_err': regression_std_err, |
| 317 'confidence': confidence, | 318 'confidence': confidence, |
| 318 'bad_greater_than_good': bad_greater_than_good} | 319 'bad_greater_than_good': bad_greater_than_good} |
| OLD | NEW |