OLD | NEW |
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import math | 5 import math |
6 import os | 6 import os |
7 | 7 |
8 import bisect_utils | 8 import bisect_utils |
9 import math_utils | 9 import math_utils |
10 import source_control | 10 import source_control |
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
51 second option creates an error result. | 51 second option creates an error result. |
52 | 52 |
53 Args: | 53 Args: |
54 bisect_state: BisectState object representing latest bisect state. | 54 bisect_state: BisectState object representing latest bisect state. |
55 depot_registry: DepotDirectoryRegistry object with information on each | 55 depot_registry: DepotDirectoryRegistry object with information on each |
56 repository in the bisect_state. | 56 repository in the bisect_state. |
57 opts: Options passed to the bisect run. | 57 opts: Options passed to the bisect run. |
58 runtime_warnings: A list of warnings from the bisect run. | 58 runtime_warnings: A list of warnings from the bisect run. |
59 error: Error message. When error is not None, other arguments are ignored. | 59 error: Error message. When error is not None, other arguments are ignored. |
60 """ | 60 """ |
61 | |
62 self.error = error | 61 self.error = error |
63 self.abort_reason = abort_reason | 62 self.abort_reason = abort_reason |
64 if error is not None or abort_reason is not None: | 63 if error is not None or abort_reason is not None: |
65 return | 64 return |
66 | 65 |
67 assert (bisect_state is not None and depot_registry is not None and | 66 assert (bisect_state is not None and depot_registry is not None and |
68 opts is not None and runtime_warnings is not None), ( | 67 opts is not None and runtime_warnings is not None), ( |
69 'Incorrect use of the BisectResults constructor. When error is ' | 68 'Incorrect use of the BisectResults constructor. ' |
70 'None, all other arguments are required') | 69 'When error is None, all other arguments are required.') |
71 | 70 |
72 self.state = bisect_state | 71 self.state = bisect_state |
73 | 72 |
74 rev_states = bisect_state.GetRevisionStates() | 73 rev_states = bisect_state.GetRevisionStates() |
75 first_working_rev, last_broken_rev = self.FindBreakingRevRange(rev_states) | 74 first_working_rev, last_broken_rev = self.FindBreakingRevRange(rev_states) |
76 self.first_working_revision = first_working_rev | 75 self.first_working_revision = first_working_rev |
77 self.last_broken_revision = last_broken_rev | 76 self.last_broken_revision = last_broken_rev |
78 | 77 |
79 self.warnings = runtime_warnings | 78 self.warnings = runtime_warnings |
80 | 79 |
(...skipping 25 matching lines...) Expand all Loading... |
106 self.culprit_revisions = [] | 105 self.culprit_revisions = [] |
107 self.other_regressions = [] | 106 self.other_regressions = [] |
108 | 107 |
109 def AddRetestResults(self, results_tot, results_reverted): | 108 def AddRetestResults(self, results_tot, results_reverted): |
110 if not results_tot or not results_reverted: | 109 if not results_tot or not results_reverted: |
111 self.warnings.append( | 110 self.warnings.append( |
112 'Failed to re-test reverted culprit CL against ToT.') | 111 'Failed to re-test reverted culprit CL against ToT.') |
113 return | 112 return |
114 | 113 |
115 confidence_params = (results_reverted[0]['values'], | 114 confidence_params = (results_reverted[0]['values'], |
116 results_tot[0]['values']) | 115 results_tot[0]['values']) |
117 confidence = BisectResults.ConfidenceScore(*confidence_params) | 116 confidence = BisectResults.ConfidenceScore(*confidence_params) |
118 | 117 |
119 self.retest_results_tot = RevisionState('ToT', 'n/a', 0) | 118 self.retest_results_tot = RevisionState('ToT', 'n/a', 0) |
120 self.retest_results_tot.value = results_tot[0] | 119 self.retest_results_tot.value = results_tot[0] |
121 | 120 |
122 self.retest_results_reverted = RevisionState('Reverted', 'n/a', 0) | 121 self.retest_results_reverted = RevisionState('Reverted', 'n/a', 0) |
123 self.retest_results_reverted.value = results_reverted[0] | 122 self.retest_results_reverted.value = results_reverted[0] |
124 | 123 |
125 if confidence <= bisect_utils.HIGH_CONFIDENCE: | 124 if confidence <= bisect_utils.HIGH_CONFIDENCE: |
126 self.warnings.append( | 125 self.warnings.append( |
127 'Confidence of re-test with reverted CL is not high.' | 126 'Confidence of re-test with reverted CL is not high.' |
128 ' Check that the regression hasn\'t already recovered. ' | 127 ' Check that the regression hasn\'t already recovered. ' |
129 ' There\'s still a chance this is a regression, as performance of' | 128 ' There\'s still a chance this is a regression, as performance of' |
130 ' local builds may not match official builds.' ) | 129 ' local builds may not match official builds.') |
131 | 130 |
132 @staticmethod | 131 @staticmethod |
133 def _GetResultBasedWarnings(culprit_revisions, opts, confidence): | 132 def _GetResultBasedWarnings(culprit_revisions, opts, confidence): |
134 warnings = [] | 133 warnings = [] |
135 if len(culprit_revisions) > 1: | 134 if len(culprit_revisions) > 1: |
136 warnings.append('Due to build errors, regression range could ' | 135 warnings.append('Due to build errors, regression range could ' |
137 'not be narrowed down to a single commit.') | 136 'not be narrowed down to a single commit.') |
138 if opts.repeat_test_count == 1: | 137 if opts.repeat_test_count == 1: |
139 warnings.append('Tests were only set to run once. This may ' | 138 warnings.append('Tests were only set to run once. This may ' |
140 'be insufficient to get meaningful results.') | 139 'be insufficient to get meaningful results.') |
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
211 confidence = cls.ConfidenceScore(*confidence_params, | 210 confidence = cls.ConfidenceScore(*confidence_params, |
212 accept_single_bad_or_good=True) | 211 accept_single_bad_or_good=True) |
213 mean_of_prev_runs = math_utils.Mean(sum(previous_values, [])) | 212 mean_of_prev_runs = math_utils.Mean(sum(previous_values, [])) |
214 mean_of_current_runs = math_utils.Mean(current_values) | 213 mean_of_current_runs = math_utils.Mean(current_values) |
215 | 214 |
216 # Check that the potential regression is in the same direction as | 215 # Check that the potential regression is in the same direction as |
217 # the overall regression. If the mean of the previous runs < the | 216 # the overall regression. If the mean of the previous runs < the |
218 # mean of the current runs, this local regression is in same | 217 # mean of the current runs, this local regression is in same |
219 # direction. | 218 # direction. |
220 prev_greater_than_current = mean_of_prev_runs > mean_of_current_runs | 219 prev_greater_than_current = mean_of_prev_runs > mean_of_current_runs |
221 is_same_direction = (prev_greater_than_current if | 220 if bad_greater_than_good: |
222 bad_greater_than_good else not prev_greater_than_current) | 221 is_same_direction = prev_greater_than_current |
| 222 else: |
| 223 is_same_direction = not prev_greater_than_current |
223 | 224 |
224 # Only report potential regressions with high confidence. | 225 # Only report potential regressions with high confidence. |
225 if is_same_direction and confidence > 50: | 226 if is_same_direction and confidence > 50: |
226 other_regressions.append([revision_state, prev_state, confidence]) | 227 other_regressions.append([revision_state, prev_state, confidence]) |
227 previous_values.append(current_values) | 228 previous_values.append(current_values) |
228 prev_state = revision_state | 229 prev_state = revision_state |
229 return other_regressions | 230 return other_regressions |
230 | 231 |
231 @staticmethod | 232 @staticmethod |
232 def FindBreakingRevRange(revision_states): | 233 def FindBreakingRevRange(revision_states): |
(...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
309 sum([last_broken_rev.value['values']], []) | 310 sum([last_broken_rev.value['values']], []) |
310 ) | 311 ) |
311 confidence = cls.ConfidenceScore(*confidence_params) | 312 confidence = cls.ConfidenceScore(*confidence_params) |
312 | 313 |
313 bad_greater_than_good = mean_of_bad_runs > mean_of_good_runs | 314 bad_greater_than_good = mean_of_bad_runs > mean_of_good_runs |
314 | 315 |
315 return {'regression_size': regression_size, | 316 return {'regression_size': regression_size, |
316 'regression_std_err': regression_std_err, | 317 'regression_std_err': regression_std_err, |
317 'confidence': confidence, | 318 'confidence': confidence, |
318 'bad_greater_than_good': bad_greater_than_good} | 319 'bad_greater_than_good': bad_greater_than_good} |
OLD | NEW |