| OLD | NEW |
| 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /// Compares the test log of a build step with previous builds. | 5 /// Compares the test log of a build step with previous builds. |
| 6 /// | 6 /// |
| 7 /// Use this to detect flakiness of failures, especially timeouts. | 7 /// Use this to detect flakiness of failures, especially timeouts. |
| 8 | 8 |
| 9 import 'dart:async'; | 9 import 'dart:async'; |
| 10 import 'dart:io'; | 10 import 'dart:io'; |
| 11 | 11 |
| 12 import 'package:args/args.dart'; | 12 import 'package:args/args.dart'; |
| 13 import 'package:gardening/src/buildbot_structures.dart'; | 13 import 'package:gardening/src/buildbot_structures.dart'; |
| 14 import 'package:gardening/src/buildbot_loading.dart'; | 14 import 'package:gardening/src/client.dart'; |
| 15 import 'package:gardening/src/util.dart'; | 15 import 'package:gardening/src/util.dart'; |
| 16 | 16 |
| 17 main(List<String> args) async { | 17 main(List<String> args) async { |
| 18 ArgParser argParser = createArgParser(); | 18 ArgParser argParser = createArgParser(); |
| 19 ArgResults argResults = argParser.parse(args); | 19 ArgResults argResults = argParser.parse(args); |
| 20 processArgResults(argResults); | 20 processArgResults(argResults); |
| 21 |
| 22 BuildbotClient client = argResults['logdog'] |
| 23 ? new LogdogBuildbotClient() |
| 24 : new HttpBuildbotClient(); |
| 25 |
| 21 if (argResults.rest.length != 1) { | 26 if (argResults.rest.length != 1) { |
| 22 print('Usage: compare_failures [options] <log-uri>'); | 27 print('Usage: compare_failures [options] <log-uri>'); |
| 23 print('where <log-uri> is the uri the stdio output of a failing test step'); | 28 print('where <log-uri> is the uri the stdio output of a failing test step'); |
| 24 print('and options are:'); | 29 print('and options are:'); |
| 25 print(argParser.usage); | 30 print(argParser.usage); |
| 26 exit(1); | 31 exit(1); |
| 27 } | 32 } |
| 28 String url = argResults.rest.first; | 33 String url = argResults.rest.first; |
| 29 if (!url.endsWith('/text')) { | 34 if (!url.endsWith('/text')) { |
| 30 // Use the text version of the stdio log. | 35 // Use the text version of the stdio log. |
| 31 url += '/text'; | 36 url += '/text'; |
| 32 } | 37 } |
| 33 Uri uri = Uri.parse(url); | 38 Uri uri = Uri.parse(url); |
| 34 HttpClient client = new HttpClient(); | |
| 35 BuildUri buildUri = new BuildUri(uri); | 39 BuildUri buildUri = new BuildUri(uri); |
| 36 List<BuildResult> results = await readBuildResults(client, buildUri); | 40 List<BuildResult> results = await readBuildResults(client, buildUri); |
| 37 print(generateBuildResultsSummary(buildUri, results)); | 41 print(generateBuildResultsSummary(buildUri, results)); |
| 38 client.close(); | 42 client.close(); |
| 39 } | 43 } |
| 40 | 44 |
| 41 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the | 45 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the |
| 42 /// [BuildResult]s for the previous 5 builds. | 46 /// [BuildResult]s for the previous 5 builds. |
| 43 Future<List<BuildResult>> readBuildResults( | 47 Future<List<BuildResult>> readBuildResults( |
| 44 HttpClient client, BuildUri buildUri) async { | 48 BuildbotClient client, BuildUri buildUri) async { |
| 45 List<BuildResult> summaries = <BuildResult>[]; | 49 List<BuildResult> summaries = <BuildResult>[]; |
| 46 BuildResult firstSummary = await readBuildResult(client, buildUri); | 50 BuildResult summary = await client.readResult(buildUri); |
| 47 summaries.add(firstSummary); | 51 summaries.add(summary); |
| 48 if (firstSummary.hasFailures) { | 52 if (summary.hasFailures) { |
| 49 for (int i = 0; i < 10; i++) { | 53 for (int i = 0; i < 10; i++) { |
| 50 buildUri = buildUri.prev(); | 54 buildUri = summary.buildUri.prev(); |
| 51 summaries.add(await readBuildResult(client, buildUri)); | 55 summary = await client.readResult(buildUri); |
| 56 summaries.add(summary); |
| 52 } | 57 } |
| 53 } | 58 } |
| 54 return summaries; | 59 return summaries; |
| 55 } | 60 } |
| 56 | 61 |
| 57 /// Generate a summary of the timeouts and other failures in [results]. | 62 /// Generate a summary of the timeouts and other failures in [results]. |
| 58 String generateBuildResultsSummary( | 63 String generateBuildResultsSummary( |
| 59 BuildUri buildUri, List<BuildResult> results) { | 64 BuildUri buildUri, List<BuildResult> results) { |
| 60 StringBuffer sb = new StringBuffer(); | 65 StringBuffer sb = new StringBuffer(); |
| 61 sb.write('Results for $buildUri:\n'); | 66 sb.write('Results for $buildUri:\n'); |
| 62 Set<TestConfiguration> timeoutIds = new Set<TestConfiguration>(); | 67 Set<TestConfiguration> timeoutIds = new Set<TestConfiguration>(); |
| 63 for (BuildResult result in results) { | 68 for (BuildResult result in results) { |
| 64 timeoutIds.addAll(result.timeouts.map((TestFailure failure) => failure.id)); | 69 timeoutIds.addAll(result.timeouts.map((TestFailure failure) => failure.id)); |
| 65 } | 70 } |
| 66 if (timeoutIds.isNotEmpty) { | 71 if (timeoutIds.isNotEmpty) { |
| 67 int firstBuildNumber = results.first.buildUri.buildNumber; | |
| 68 int lastBuildNumber = results.last.buildUri.buildNumber; | |
| 69 Map<TestConfiguration, Map<int, Map<String, Timing>>> map = | 72 Map<TestConfiguration, Map<int, Map<String, Timing>>> map = |
| 70 <TestConfiguration, Map<int, Map<String, Timing>>>{}; | 73 <TestConfiguration, Map<int, Map<String, Timing>>>{}; |
| 71 Set<String> stepNames = new Set<String>(); | 74 Set<String> stepNames = new Set<String>(); |
| 72 for (BuildResult result in results) { | 75 for (BuildResult result in results) { |
| 73 for (Timing timing in result.timings) { | 76 for (Timing timing in result.timings) { |
| 74 Map<int, Map<String, Timing>> builds = | 77 Map<int, Map<String, Timing>> builds = |
| 75 map.putIfAbsent(timing.step.id, () => <int, Map<String, Timing>>{}); | 78 map.putIfAbsent(timing.step.id, () => <int, Map<String, Timing>>{}); |
| 76 stepNames.add(timing.step.stepName); | 79 stepNames.add(timing.step.stepName); |
| 77 builds.putIfAbsent(timing.uri.buildNumber, () => <String, Timing>{})[ | 80 builds.putIfAbsent(timing.uri.buildNumber, () => <String, Timing>{})[ |
| 78 timing.step.stepName] = timing; | 81 timing.step.stepName] = timing; |
| 79 } | 82 } |
| 80 } | 83 } |
| 81 sb.write('Timeouts for ${buildUri} :\n'); | 84 sb.write('Timeouts for ${buildUri} :\n'); |
| 82 map.forEach((TestConfiguration id, Map<int, Map<String, Timing>> timings) { | 85 map.forEach((TestConfiguration id, Map<int, Map<String, Timing>> timings) { |
| 83 if (!timeoutIds.contains(id)) return; | 86 if (!timeoutIds.contains(id)) return; |
| 84 sb.write('$id\n'); | 87 sb.write('$id\n'); |
| 85 sb.write( | 88 sb.write( |
| 86 '${' ' * 8} ${stepNames.map((t) => padRight(t, 14)).join(' ')}\n'); | 89 '${' ' * 8} ${stepNames.map((t) => padRight(t, 14)).join(' ')}\n'); |
| 87 for (int buildNumber = firstBuildNumber; | 90 for (BuildResult result in results) { |
| 88 buildNumber >= lastBuildNumber; | 91 int buildNumber = result.buildUri.buildNumber; |
| 89 buildNumber--) { | |
| 90 Map<String, Timing> steps = timings[buildNumber] ?? const {}; | 92 Map<String, Timing> steps = timings[buildNumber] ?? const {}; |
| 91 sb.write(padRight(' ${buildNumber}: ', 8)); | 93 sb.write(padRight(' ${buildNumber}: ', 8)); |
| 92 for (String stepName in stepNames) { | 94 for (String stepName in stepNames) { |
| 93 Timing timing = steps[stepName]; | 95 Timing timing = steps[stepName]; |
| 94 if (timing != null) { | 96 if (timing != null) { |
| 95 sb.write(' ${timing.time}'); | 97 sb.write(' ${timing.time}'); |
| 96 } else { | 98 } else { |
| 97 sb.write(' --------------'); | 99 sb.write(' --------------'); |
| 98 } | 100 } |
| 99 } | 101 } |
| 100 sb.write('\n'); | 102 sb.write('\n'); |
| 101 } | 103 } |
| 102 sb.write('\n'); | 104 sb.write('\n'); |
| 103 }); | 105 }); |
| 104 } | 106 } |
| 105 Set<TestConfiguration> errorIds = new Set<TestConfiguration>(); | 107 Set<TestConfiguration> errorIds = new Set<TestConfiguration>(); |
| 106 for (BuildResult result in results) { | 108 for (BuildResult result in results) { |
| 107 errorIds.addAll(result.errors.map((TestFailure failure) => failure.id)); | 109 errorIds.addAll(result.errors.map((TestFailure failure) => failure.id)); |
| 108 } | 110 } |
| 109 if (errorIds.isNotEmpty) { | 111 if (errorIds.isNotEmpty) { |
| 110 int firstBuildNumber = results.first.buildUri.buildNumber; | |
| 111 int lastBuildNumber = results.last.buildUri.buildNumber; | |
| 112 Map<TestConfiguration, Map<int, TestFailure>> map = | 112 Map<TestConfiguration, Map<int, TestFailure>> map = |
| 113 <TestConfiguration, Map<int, TestFailure>>{}; | 113 <TestConfiguration, Map<int, TestFailure>>{}; |
| 114 for (BuildResult result in results) { | 114 for (BuildResult result in results) { |
| 115 for (TestFailure failure in result.errors) { | 115 for (TestFailure failure in result.errors) { |
| 116 map.putIfAbsent(failure.id, () => <int, TestFailure>{})[ | 116 map.putIfAbsent(failure.id, () => <int, TestFailure>{})[ |
| 117 failure.uri.buildNumber] = failure; | 117 failure.uri.buildNumber] = failure; |
| 118 } | 118 } |
| 119 } | 119 } |
| 120 sb.write('Errors for ${buildUri} :\n'); | 120 sb.write('Errors for ${buildUri} :\n'); |
| 121 // TODO(johnniwinther): Improve comparison of non-timeouts. | 121 // TODO(johnniwinther): Improve comparison of non-timeouts. |
| 122 map.forEach((TestConfiguration id, Map<int, TestFailure> failures) { | 122 map.forEach((TestConfiguration id, Map<int, TestFailure> failures) { |
| 123 if (!errorIds.contains(id)) return; | 123 if (!errorIds.contains(id)) return; |
| 124 sb.write('$id\n'); | 124 sb.write('$id\n'); |
| 125 for (int buildNumber = firstBuildNumber; | 125 for (BuildResult result in results) { |
| 126 buildNumber >= lastBuildNumber; | 126 int buildNumber = result.buildUri.buildNumber; |
| 127 buildNumber--) { | |
| 128 TestFailure failure = failures[buildNumber]; | 127 TestFailure failure = failures[buildNumber]; |
| 129 sb.write(padRight(' ${buildNumber}: ', 8)); | 128 sb.write(padRight(' ${buildNumber}: ', 8)); |
| 130 if (failure != null) { | 129 if (failure != null) { |
| 131 sb.write(padRight(failure.expected, 10)); | 130 sb.write(padRight(failure.expected, 10)); |
| 132 sb.write(' / '); | 131 sb.write(' / '); |
| 133 sb.write(padRight(failure.actual, 10)); | 132 sb.write(padRight(failure.actual, 10)); |
| 134 } else { | 133 } else { |
| 135 sb.write(' ' * 10); | 134 sb.write(' ' * 10); |
| 136 sb.write(' / '); | 135 sb.write(' / '); |
| 137 sb.write(padRight('-- OK --', 10)); | 136 sb.write(padRight('-- OK --', 10)); |
| 138 } | 137 } |
| 139 sb.write('\n'); | 138 sb.write('\n'); |
| 140 } | 139 } |
| 141 sb.write('\n'); | 140 sb.write('\n'); |
| 142 }); | 141 }); |
| 143 } | 142 } |
| 144 return sb.toString(); | 143 return sb.toString(); |
| 145 } | 144 } |
| OLD | NEW |