| OLD | NEW |
| 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /// Compares the test log of a build step with previous builds. | 5 /// Compares the test log of a build step with previous builds. |
| 6 /// | 6 /// |
| 7 /// Use this to detect flakiness of failures, especially timeouts. | 7 /// Use this to detect flakiness of failures, especially timeouts. |
| 8 | 8 |
| 9 import 'dart:async'; | 9 import 'dart:async'; |
| 10 | 10 |
| (...skipping 46 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 57 } else { | 57 } else { |
| 58 nonEmptySummaries.add(summary); | 58 nonEmptySummaries.add(summary); |
| 59 } | 59 } |
| 60 }); | 60 }); |
| 61 StringBuffer sb = new StringBuffer(); | 61 StringBuffer sb = new StringBuffer(); |
| 62 if (nonEmptySummaries.isEmpty) { | 62 if (nonEmptySummaries.isEmpty) { |
| 63 if (emptySummaries.isNotEmpty) { | 63 if (emptySummaries.isNotEmpty) { |
| 64 if (LOG || emptySummaries.length < 3) { | 64 if (LOG || emptySummaries.length < 3) { |
| 65 if (emptySummaries.length == 1) { | 65 if (emptySummaries.length == 1) { |
| 66 sb.writeln('No errors found for build bot:'); | 66 sb.writeln('No errors found for build bot:'); |
| 67 sb.write(emptySummaries.single.buildUri); | 67 sb.write(emptySummaries.single.name); |
| 68 } else { | 68 } else { |
| 69 sb.writeln('No errors found for any of these build bots:'); | 69 sb.writeln('No errors found for any of these build bots:'); |
| 70 for (Summary summary in emptySummaries) { | 70 for (Summary summary in emptySummaries) { |
| 71 sb.writeln('${summary.buildUri}'); | 71 sb.writeln('${summary.name}'); |
| 72 } | 72 } |
| 73 } | 73 } |
| 74 } else { | 74 } else { |
| 75 sb.write('No errors found for any of the ' | 75 sb.write('No errors found for any of the ' |
| 76 '${emptySummaries.length} bots.'); | 76 '${emptySummaries.length} bots.'); |
| 77 } | 77 } |
| 78 } else { | 78 } else { |
| 79 sb.write('No build bot results found for args: ${args}'); | 79 sb.write('No build bot results found for args: ${args}'); |
| 80 } | 80 } |
| 81 } else { | 81 } else { |
| 82 for (Summary summary in nonEmptySummaries) { | 82 for (Summary summary in nonEmptySummaries) { |
| 83 summary.printOn(sb); | 83 summary.printOn(sb); |
| 84 } | 84 } |
| 85 if (emptySummaries.isNotEmpty) { | 85 if (emptySummaries.isNotEmpty) { |
| 86 if (LOG || emptySummaries.length < 3) { | 86 if (LOG || emptySummaries.length < 3) { |
| 87 sb.writeln('No errors found for the remaining build bots:'); | 87 sb.writeln('No errors found for the remaining build bots:'); |
| 88 for (Summary summary in emptySummaries) { | 88 for (Summary summary in emptySummaries) { |
| 89 sb.writeln('${summary.buildUri}'); | 89 sb.writeln('${summary.name}'); |
| 90 } | 90 } |
| 91 } else { | 91 } else { |
| 92 sb.write( | 92 sb.write( |
| 93 'No errors found for the ${emptySummaries.length} remaining bots.'); | 93 'No errors found for the ${emptySummaries.length} remaining bots.'); |
| 94 } | 94 } |
| 95 } | 95 } |
| 96 } | 96 } |
| 97 print(sb); | 97 print(sb); |
| 98 } | 98 } |
| 99 | 99 |
| 100 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the | 100 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the |
| 101 /// [BuildResult]s for the previous [runCount] builds. | 101 /// [BuildResult]s for the previous [runCount] builds. |
| 102 Future<List<BuildResult>> readBuildResults( | 102 Future<List<BuildResult>> readBuildResults( |
| 103 BuildbotClient client, BuildUri buildUri, int runCount) async { | 103 BuildbotClient client, BuildUri buildUri, int runCount) async { |
| 104 List<BuildResult> summaries = <BuildResult>[]; | 104 List<BuildResult> summaries = <BuildResult>[]; |
| 105 BuildResult summary = await client.readResult(buildUri); | 105 BuildResult summary = await client.readResult(buildUri); |
| 106 if (summary == null) { |
| 107 print('No result found for $buildUri'); |
| 108 return summaries; |
| 109 } |
| 106 summaries.add(summary); | 110 summaries.add(summary); |
| 107 if (summary.hasFailures) { | 111 if (summary.hasFailures) { |
| 108 for (int i = 0; i < runCount; i++) { | 112 for (int i = 0; i < runCount; i++) { |
| 109 buildUri = summary.buildUri.prev(); | 113 buildUri = summary.buildUri.prev(); |
| 110 summary = await client.readResult(buildUri); | 114 summary = await client.readResult(buildUri); |
| 111 summaries.add(summary); | 115 summaries.add(summary); |
| 112 } | 116 } |
| 113 } | 117 } |
| 114 return summaries; | 118 return summaries; |
| 115 } | 119 } |
| (...skipping 22 matching lines...) Expand all Loading... |
| 138 Set<String> stepNames = new Set<String>(); | 142 Set<String> stepNames = new Set<String>(); |
| 139 for (BuildResult result in results) { | 143 for (BuildResult result in results) { |
| 140 for (Timing timing in result.timings) { | 144 for (Timing timing in result.timings) { |
| 141 Map<int, Map<String, Timing>> builds = map.putIfAbsent( | 145 Map<int, Map<String, Timing>> builds = map.putIfAbsent( |
| 142 timing.step.id, () => <int, Map<String, Timing>>{}); | 146 timing.step.id, () => <int, Map<String, Timing>>{}); |
| 143 stepNames.add(timing.step.stepName); | 147 stepNames.add(timing.step.stepName); |
| 144 builds.putIfAbsent(timing.uri.buildNumber, () => <String, Timing>{})[ | 148 builds.putIfAbsent(timing.uri.buildNumber, () => <String, Timing>{})[ |
| 145 timing.step.stepName] = timing; | 149 timing.step.stepName] = timing; |
| 146 } | 150 } |
| 147 } | 151 } |
| 148 sb.write('Timeouts for ${buildUri} :\n'); | 152 sb.write('Timeouts for ${name} :\n'); |
| 149 map.forEach( | 153 map.forEach( |
| 150 (TestConfiguration id, Map<int, Map<String, Timing>> timings) { | 154 (TestConfiguration id, Map<int, Map<String, Timing>> timings) { |
| 151 if (!timeoutIds.contains(id)) return; | 155 if (!timeoutIds.contains(id)) return; |
| 152 sb.write('$id\n'); | 156 sb.write('$id\n'); |
| 153 sb.write( | 157 sb.write( |
| 154 '${' ' * 8} ${stepNames.map((t) => padRight(t, 14)).join(' ')}\n'); | 158 '${' ' * 8} ${stepNames.map((t) => padRight(t, 14)).join(' ')}\n'); |
| 155 for (BuildResult result in results) { | 159 for (BuildResult result in results) { |
| 156 int buildNumber = result.buildUri.buildNumber; | 160 int buildNumber = result.buildUri.buildNumber; |
| 157 Map<String, Timing> steps = timings[buildNumber] ?? const {}; | 161 Map<String, Timing> steps = timings[buildNumber] ?? const {}; |
| 158 sb.write(padRight(' ${buildNumber}: ', 8)); | 162 sb.write(padRight(' ${buildNumber}: ', 8)); |
| (...skipping 12 matching lines...) Expand all Loading... |
| 171 } | 175 } |
| 172 if (errorIds.isNotEmpty) { | 176 if (errorIds.isNotEmpty) { |
| 173 Map<TestConfiguration, Map<int, TestFailure>> map = | 177 Map<TestConfiguration, Map<int, TestFailure>> map = |
| 174 <TestConfiguration, Map<int, TestFailure>>{}; | 178 <TestConfiguration, Map<int, TestFailure>>{}; |
| 175 for (BuildResult result in results) { | 179 for (BuildResult result in results) { |
| 176 for (TestFailure failure in result.errors) { | 180 for (TestFailure failure in result.errors) { |
| 177 map.putIfAbsent(failure.id, () => <int, TestFailure>{})[ | 181 map.putIfAbsent(failure.id, () => <int, TestFailure>{})[ |
| 178 failure.uri.buildNumber] = failure; | 182 failure.uri.buildNumber] = failure; |
| 179 } | 183 } |
| 180 } | 184 } |
| 181 sb.write('Errors for ${buildUri} :\n'); | 185 sb.write('Errors for ${name} :\n'); |
| 182 // TODO(johnniwinther): Improve comparison of non-timeouts. | 186 // TODO(johnniwinther): Improve comparison of non-timeouts. |
| 183 map.forEach((TestConfiguration id, Map<int, TestFailure> failures) { | 187 map.forEach((TestConfiguration id, Map<int, TestFailure> failures) { |
| 184 if (!errorIds.contains(id)) return; | 188 if (!errorIds.contains(id)) return; |
| 185 sb.write('$id\n'); | 189 sb.write('$id\n'); |
| 186 for (BuildResult result in results) { | 190 for (BuildResult result in results) { |
| 187 int buildNumber = result.buildUri.buildNumber; | 191 int buildNumber = result.buildUri.buildNumber; |
| 188 TestFailure failure = failures[buildNumber]; | 192 TestFailure failure = failures[buildNumber]; |
| 189 sb.write(padRight(' ${buildNumber}: ', 8)); | 193 sb.write(padRight(' ${buildNumber}: ', 8)); |
| 190 if (failure != null) { | 194 if (failure != null) { |
| 191 sb.write(padRight(failure.expected, 10)); | 195 sb.write(padRight(failure.expected, 10)); |
| 192 sb.write(' / '); | 196 sb.write(' / '); |
| 193 sb.write(padRight(failure.actual, 10)); | 197 sb.write(padRight(failure.actual, 10)); |
| 194 } else { | 198 } else { |
| 195 sb.write(' ' * 10); | 199 sb.write(' ' * 10); |
| 196 sb.write(' / '); | 200 sb.write(' / '); |
| 197 sb.write(padRight('-- OK --', 10)); | 201 sb.write(padRight('-- OK --', 10)); |
| 198 } | 202 } |
| 199 sb.write('\n'); | 203 sb.write('\n'); |
| 200 } | 204 } |
| 201 sb.write('\n'); | 205 sb.write('\n'); |
| 202 }); | 206 }); |
| 203 } | 207 } |
| 204 if (timeoutIds.isEmpty && errorIds.isEmpty) { | 208 if (timeoutIds.isEmpty && errorIds.isEmpty) { |
| 205 sb.write('No errors found for ${buildUri}'); | 209 sb.write('No errors found for ${name}'); |
| 206 } | 210 } |
| 207 } | 211 } |
| 212 |
| 213 String get name => results.isNotEmpty |
| 214 // Use the first result as name since it most likely has an absolute build |
| 215 // number. |
| 216 ? results.first.buildUri.toString() |
| 217 : buildUri.toString(); |
| 208 } | 218 } |
| OLD | NEW |