OLD | NEW |
1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /// Compares the test log of a build step with previous builds. | 5 /// Compares the test log of a build step with previous builds. |
6 /// | 6 /// |
7 /// Use this to detect flakiness of failures, especially timeouts. | 7 /// Use this to detect flakiness of failures, especially timeouts. |
8 | 8 |
9 import 'dart:async'; | |
10 import 'dart:io'; | 9 import 'dart:io'; |
11 | 10 |
12 import 'package:args/args.dart'; | 11 import 'package:args/args.dart'; |
13 import 'package:gardening/src/buildbot_structures.dart'; | |
14 import 'package:gardening/src/client.dart'; | 12 import 'package:gardening/src/client.dart'; |
| 13 import 'package:gardening/src/compare_failures_impl.dart'; |
15 import 'package:gardening/src/util.dart'; | 14 import 'package:gardening/src/util.dart'; |
16 | 15 |
17 void help(ArgParser argParser) { | 16 void help(ArgParser argParser) { |
18 print('Given a <log-uri> finds all failing tests in that stdout. Then '); | 17 print('Given a <log-uri> finds all failing tests in that stdout. Then '); |
19 print('fetches earlier runs of the same bot and compares the results.'); | 18 print('fetches earlier runs of the same bot and compares the results.'); |
20 print('This tool is particularly useful to detect flakes and their '); | 19 print('This tool is particularly useful to detect flakes and their '); |
21 print('frequency.'); | 20 print('frequency.'); |
22 print('Usage: compare_failures [options] <log-uri>'); | 21 print('Usage: compare_failures [options] '); |
23 print('where <log-uri> is the uri the stdio output of a failing test step'); | 22 print(' (<log-uri> [<log-uri> ...] | <build-group> [<build-group> ...])'); |
24 print('and options are:'); | 23 print('where <log-uri> is the uri the stdio output of a failing test step '); |
| 24 print('and <build-group> is the name of a buildbot group, for instance '); |
| 25 print('`vm-kernel`, and options are:'); |
25 print(argParser.usage); | 26 print(argParser.usage); |
26 } | 27 } |
27 | 28 |
28 main(List<String> args) async { | 29 main(List<String> args) async { |
29 ArgParser argParser = createArgParser(); | 30 ArgParser argParser = createArgParser(); |
30 argParser.addOption("run-count", | 31 argParser.addOption("run-count", |
31 defaultsTo: "10", help: "How many previous runs should be fetched"); | 32 defaultsTo: "10", help: "How many previous runs should be fetched"); |
32 ArgResults argResults = argParser.parse(args); | 33 ArgResults argResults = argParser.parse(args); |
33 processArgResults(argResults); | 34 processArgResults(argResults); |
34 | 35 |
35 BuildbotClient client = argResults['logdog'] | |
36 ? new LogdogBuildbotClient() | |
37 : new HttpBuildbotClient(); | |
38 | |
39 var runCount = int.parse(argResults['run-count'], onError: (_) => null); | 36 var runCount = int.parse(argResults['run-count'], onError: (_) => null); |
40 | 37 |
41 if (argResults.rest.length != 1 || argResults['help'] || runCount == null) { | 38 if (argResults.rest.length != 1 || argResults['help'] || runCount == null) { |
42 help(argParser); | 39 help(argParser); |
43 if (argResults['help']) return; | 40 if (argResults['help']) return; |
44 exit(1); | 41 exit(1); |
45 } | 42 } |
46 String url = argResults.rest.first; | 43 |
47 if (!url.endsWith('/text')) { | 44 BuildbotClient client = argResults['logdog'] |
48 // Use the text version of the stdio log. | 45 ? new LogdogBuildbotClient() |
49 url += '/text'; | 46 : new HttpBuildbotClient(); |
50 } | 47 await mainInternal(client, argResults.rest, runCount: runCount); |
51 Uri uri = Uri.parse(url); | |
52 BuildUri buildUri = new BuildUri(uri); | |
53 List<BuildResult> results = | |
54 await readBuildResults(client, buildUri, runCount); | |
55 print(generateBuildResultsSummary(buildUri, results)); | |
56 client.close(); | 48 client.close(); |
57 } | 49 } |
58 | |
59 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the | |
60 /// [BuildResult]s for the previous [runCount] builds. | |
61 Future<List<BuildResult>> readBuildResults( | |
62 BuildbotClient client, BuildUri buildUri, int runCount) async { | |
63 List<BuildResult> summaries = <BuildResult>[]; | |
64 BuildResult summary = await client.readResult(buildUri); | |
65 summaries.add(summary); | |
66 if (summary.hasFailures) { | |
67 for (int i = 0; i < runCount; i++) { | |
68 buildUri = summary.buildUri.prev(); | |
69 summary = await client.readResult(buildUri); | |
70 summaries.add(summary); | |
71 } | |
72 } | |
73 return summaries; | |
74 } | |
75 | |
76 /// Generate a summary of the timeouts and other failures in [results]. | |
77 String generateBuildResultsSummary( | |
78 BuildUri buildUri, List<BuildResult> results) { | |
79 StringBuffer sb = new StringBuffer(); | |
80 sb.write('Results for $buildUri:\n'); | |
81 Set<TestConfiguration> timeoutIds = new Set<TestConfiguration>(); | |
82 for (BuildResult result in results) { | |
83 timeoutIds.addAll(result.timeouts.map((TestFailure failure) => failure.id)); | |
84 } | |
85 if (timeoutIds.isNotEmpty) { | |
86 Map<TestConfiguration, Map<int, Map<String, Timing>>> map = | |
87 <TestConfiguration, Map<int, Map<String, Timing>>>{}; | |
88 Set<String> stepNames = new Set<String>(); | |
89 for (BuildResult result in results) { | |
90 for (Timing timing in result.timings) { | |
91 Map<int, Map<String, Timing>> builds = | |
92 map.putIfAbsent(timing.step.id, () => <int, Map<String, Timing>>{}); | |
93 stepNames.add(timing.step.stepName); | |
94 builds.putIfAbsent(timing.uri.buildNumber, () => <String, Timing>{})[ | |
95 timing.step.stepName] = timing; | |
96 } | |
97 } | |
98 sb.write('Timeouts for ${buildUri} :\n'); | |
99 map.forEach((TestConfiguration id, Map<int, Map<String, Timing>> timings) { | |
100 if (!timeoutIds.contains(id)) return; | |
101 sb.write('$id\n'); | |
102 sb.write( | |
103 '${' ' * 8} ${stepNames.map((t) => padRight(t, 14)).join(' ')}\n'); | |
104 for (BuildResult result in results) { | |
105 int buildNumber = result.buildUri.buildNumber; | |
106 Map<String, Timing> steps = timings[buildNumber] ?? const {}; | |
107 sb.write(padRight(' ${buildNumber}: ', 8)); | |
108 for (String stepName in stepNames) { | |
109 Timing timing = steps[stepName]; | |
110 if (timing != null) { | |
111 sb.write(' ${timing.time}'); | |
112 } else { | |
113 sb.write(' --------------'); | |
114 } | |
115 } | |
116 sb.write('\n'); | |
117 } | |
118 sb.write('\n'); | |
119 }); | |
120 } | |
121 Set<TestConfiguration> errorIds = new Set<TestConfiguration>(); | |
122 for (BuildResult result in results) { | |
123 errorIds.addAll(result.errors.map((TestFailure failure) => failure.id)); | |
124 } | |
125 if (errorIds.isNotEmpty) { | |
126 Map<TestConfiguration, Map<int, TestFailure>> map = | |
127 <TestConfiguration, Map<int, TestFailure>>{}; | |
128 for (BuildResult result in results) { | |
129 for (TestFailure failure in result.errors) { | |
130 map.putIfAbsent(failure.id, () => <int, TestFailure>{})[ | |
131 failure.uri.buildNumber] = failure; | |
132 } | |
133 } | |
134 sb.write('Errors for ${buildUri} :\n'); | |
135 // TODO(johnniwinther): Improve comparison of non-timeouts. | |
136 map.forEach((TestConfiguration id, Map<int, TestFailure> failures) { | |
137 if (!errorIds.contains(id)) return; | |
138 sb.write('$id\n'); | |
139 for (BuildResult result in results) { | |
140 int buildNumber = result.buildUri.buildNumber; | |
141 TestFailure failure = failures[buildNumber]; | |
142 sb.write(padRight(' ${buildNumber}: ', 8)); | |
143 if (failure != null) { | |
144 sb.write(padRight(failure.expected, 10)); | |
145 sb.write(' / '); | |
146 sb.write(padRight(failure.actual, 10)); | |
147 } else { | |
148 sb.write(' ' * 10); | |
149 sb.write(' / '); | |
150 sb.write(padRight('-- OK --', 10)); | |
151 } | |
152 sb.write('\n'); | |
153 } | |
154 sb.write('\n'); | |
155 }); | |
156 } | |
157 return sb.toString(); | |
158 } | |
OLD | NEW |