OLD | NEW |
1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /// Compares the test log of a build step with previous builds. | 5 /// Compares the test log of a build step with previous builds. |
6 /// | 6 /// |
7 /// Use this to detect flakiness of failures, especially timeouts. | 7 /// Use this to detect flakiness of failures, especially timeouts. |
8 | 8 |
9 import 'dart:async'; | 9 import 'dart:async'; |
10 | 10 |
(...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
57 } else { | 57 } else { |
58 nonEmptySummaries.add(summary); | 58 nonEmptySummaries.add(summary); |
59 } | 59 } |
60 }); | 60 }); |
61 StringBuffer sb = new StringBuffer(); | 61 StringBuffer sb = new StringBuffer(); |
62 if (nonEmptySummaries.isEmpty) { | 62 if (nonEmptySummaries.isEmpty) { |
63 if (emptySummaries.isNotEmpty) { | 63 if (emptySummaries.isNotEmpty) { |
64 if (LOG || emptySummaries.length < 3) { | 64 if (LOG || emptySummaries.length < 3) { |
65 if (emptySummaries.length == 1) { | 65 if (emptySummaries.length == 1) { |
66 sb.writeln('No errors found for build bot:'); | 66 sb.writeln('No errors found for build bot:'); |
67 sb.write(emptySummaries.single.buildUri); | 67 sb.write(emptySummaries.single.name); |
68 } else { | 68 } else { |
69 sb.writeln('No errors found for any of these build bots:'); | 69 sb.writeln('No errors found for any of these build bots:'); |
70 for (Summary summary in emptySummaries) { | 70 for (Summary summary in emptySummaries) { |
71 sb.writeln('${summary.buildUri}'); | 71 sb.writeln('${summary.name}'); |
72 } | 72 } |
73 } | 73 } |
74 } else { | 74 } else { |
75 sb.write('No errors found for any of the ' | 75 sb.write('No errors found for any of the ' |
76 '${emptySummaries.length} bots.'); | 76 '${emptySummaries.length} bots.'); |
77 } | 77 } |
78 } else { | 78 } else { |
79 sb.write('No build bot results found for args: ${args}'); | 79 sb.write('No build bot results found for args: ${args}'); |
80 } | 80 } |
81 } else { | 81 } else { |
82 for (Summary summary in nonEmptySummaries) { | 82 for (Summary summary in nonEmptySummaries) { |
83 summary.printOn(sb); | 83 summary.printOn(sb); |
84 } | 84 } |
85 if (emptySummaries.isNotEmpty) { | 85 if (emptySummaries.isNotEmpty) { |
86 if (LOG || emptySummaries.length < 3) { | 86 if (LOG || emptySummaries.length < 3) { |
87 sb.writeln('No errors found for the remaining build bots:'); | 87 sb.writeln('No errors found for the remaining build bots:'); |
88 for (Summary summary in emptySummaries) { | 88 for (Summary summary in emptySummaries) { |
89 sb.writeln('${summary.buildUri}'); | 89 sb.writeln('${summary.name}'); |
90 } | 90 } |
91 } else { | 91 } else { |
92 sb.write( | 92 sb.write( |
93 'No errors found for the ${emptySummaries.length} remaining bots.'); | 93 'No errors found for the ${emptySummaries.length} remaining bots.'); |
94 } | 94 } |
95 } | 95 } |
96 } | 96 } |
97 print(sb); | 97 print(sb); |
98 } | 98 } |
99 | 99 |
100 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the | 100 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the |
101 /// [BuildResult]s for the previous [runCount] builds. | 101 /// [BuildResult]s for the previous [runCount] builds. |
102 Future<List<BuildResult>> readBuildResults( | 102 Future<List<BuildResult>> readBuildResults( |
103 BuildbotClient client, BuildUri buildUri, int runCount) async { | 103 BuildbotClient client, BuildUri buildUri, int runCount) async { |
104 List<BuildResult> summaries = <BuildResult>[]; | 104 List<BuildResult> summaries = <BuildResult>[]; |
105 BuildResult summary = await client.readResult(buildUri); | 105 BuildResult summary = await client.readResult(buildUri); |
| 106 if (summary == null) { |
| 107 print('No result found for $buildUri'); |
| 108 return summaries; |
| 109 } |
106 summaries.add(summary); | 110 summaries.add(summary); |
107 if (summary.hasFailures) { | 111 if (summary.hasFailures) { |
108 for (int i = 0; i < runCount; i++) { | 112 for (int i = 0; i < runCount; i++) { |
109 buildUri = summary.buildUri.prev(); | 113 buildUri = summary.buildUri.prev(); |
110 summary = await client.readResult(buildUri); | 114 summary = await client.readResult(buildUri); |
111 summaries.add(summary); | 115 summaries.add(summary); |
112 } | 116 } |
113 } | 117 } |
114 return summaries; | 118 return summaries; |
115 } | 119 } |
(...skipping 22 matching lines...) Expand all Loading... |
138 Set<String> stepNames = new Set<String>(); | 142 Set<String> stepNames = new Set<String>(); |
139 for (BuildResult result in results) { | 143 for (BuildResult result in results) { |
140 for (Timing timing in result.timings) { | 144 for (Timing timing in result.timings) { |
141 Map<int, Map<String, Timing>> builds = map.putIfAbsent( | 145 Map<int, Map<String, Timing>> builds = map.putIfAbsent( |
142 timing.step.id, () => <int, Map<String, Timing>>{}); | 146 timing.step.id, () => <int, Map<String, Timing>>{}); |
143 stepNames.add(timing.step.stepName); | 147 stepNames.add(timing.step.stepName); |
144 builds.putIfAbsent(timing.uri.buildNumber, () => <String, Timing>{})[ | 148 builds.putIfAbsent(timing.uri.buildNumber, () => <String, Timing>{})[ |
145 timing.step.stepName] = timing; | 149 timing.step.stepName] = timing; |
146 } | 150 } |
147 } | 151 } |
148 sb.write('Timeouts for ${buildUri} :\n'); | 152 sb.write('Timeouts for ${name} :\n'); |
149 map.forEach( | 153 map.forEach( |
150 (TestConfiguration id, Map<int, Map<String, Timing>> timings) { | 154 (TestConfiguration id, Map<int, Map<String, Timing>> timings) { |
151 if (!timeoutIds.contains(id)) return; | 155 if (!timeoutIds.contains(id)) return; |
152 sb.write('$id\n'); | 156 sb.write('$id\n'); |
153 sb.write( | 157 sb.write( |
154 '${' ' * 8} ${stepNames.map((t) => padRight(t, 14)).join(' ')}\n'); | 158 '${' ' * 8} ${stepNames.map((t) => padRight(t, 14)).join(' ')}\n'); |
155 for (BuildResult result in results) { | 159 for (BuildResult result in results) { |
156 int buildNumber = result.buildUri.buildNumber; | 160 int buildNumber = result.buildUri.buildNumber; |
157 Map<String, Timing> steps = timings[buildNumber] ?? const {}; | 161 Map<String, Timing> steps = timings[buildNumber] ?? const {}; |
158 sb.write(padRight(' ${buildNumber}: ', 8)); | 162 sb.write(padRight(' ${buildNumber}: ', 8)); |
(...skipping 12 matching lines...) Expand all Loading... |
171 } | 175 } |
172 if (errorIds.isNotEmpty) { | 176 if (errorIds.isNotEmpty) { |
173 Map<TestConfiguration, Map<int, TestFailure>> map = | 177 Map<TestConfiguration, Map<int, TestFailure>> map = |
174 <TestConfiguration, Map<int, TestFailure>>{}; | 178 <TestConfiguration, Map<int, TestFailure>>{}; |
175 for (BuildResult result in results) { | 179 for (BuildResult result in results) { |
176 for (TestFailure failure in result.errors) { | 180 for (TestFailure failure in result.errors) { |
177 map.putIfAbsent(failure.id, () => <int, TestFailure>{})[ | 181 map.putIfAbsent(failure.id, () => <int, TestFailure>{})[ |
178 failure.uri.buildNumber] = failure; | 182 failure.uri.buildNumber] = failure; |
179 } | 183 } |
180 } | 184 } |
181 sb.write('Errors for ${buildUri} :\n'); | 185 sb.write('Errors for ${name} :\n'); |
182 // TODO(johnniwinther): Improve comparison of non-timeouts. | 186 // TODO(johnniwinther): Improve comparison of non-timeouts. |
183 map.forEach((TestConfiguration id, Map<int, TestFailure> failures) { | 187 map.forEach((TestConfiguration id, Map<int, TestFailure> failures) { |
184 if (!errorIds.contains(id)) return; | 188 if (!errorIds.contains(id)) return; |
185 sb.write('$id\n'); | 189 sb.write('$id\n'); |
186 for (BuildResult result in results) { | 190 for (BuildResult result in results) { |
187 int buildNumber = result.buildUri.buildNumber; | 191 int buildNumber = result.buildUri.buildNumber; |
188 TestFailure failure = failures[buildNumber]; | 192 TestFailure failure = failures[buildNumber]; |
189 sb.write(padRight(' ${buildNumber}: ', 8)); | 193 sb.write(padRight(' ${buildNumber}: ', 8)); |
190 if (failure != null) { | 194 if (failure != null) { |
191 sb.write(padRight(failure.expected, 10)); | 195 sb.write(padRight(failure.expected, 10)); |
192 sb.write(' / '); | 196 sb.write(' / '); |
193 sb.write(padRight(failure.actual, 10)); | 197 sb.write(padRight(failure.actual, 10)); |
194 } else { | 198 } else { |
195 sb.write(' ' * 10); | 199 sb.write(' ' * 10); |
196 sb.write(' / '); | 200 sb.write(' / '); |
197 sb.write(padRight('-- OK --', 10)); | 201 sb.write(padRight('-- OK --', 10)); |
198 } | 202 } |
199 sb.write('\n'); | 203 sb.write('\n'); |
200 } | 204 } |
201 sb.write('\n'); | 205 sb.write('\n'); |
202 }); | 206 }); |
203 } | 207 } |
204 if (timeoutIds.isEmpty && errorIds.isEmpty) { | 208 if (timeoutIds.isEmpty && errorIds.isEmpty) { |
205 sb.write('No errors found for ${buildUri}'); | 209 sb.write('No errors found for ${name}'); |
206 } | 210 } |
207 } | 211 } |
| 212 |
| 213 String get name => results.isNotEmpty |
| 214 // Use the first result as name since it most likely has an absolute build |
| 215 // number. |
| 216 ? results.first.buildUri.toString() |
| 217 : buildUri.toString(); |
208 } | 218 } |
OLD | NEW |