OLD | NEW |
1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /// Compares the test log of a build step with previous builds. | 5 /// Compares the test log of a build step with previous builds. |
6 /// | 6 /// |
7 /// Use this to detect flakiness of failures, especially timeouts. | 7 /// Use this to detect flakiness of failures, especially timeouts. |
8 | 8 |
9 import 'dart:async'; | 9 import 'dart:async'; |
10 import 'dart:io'; | 10 import 'dart:io'; |
11 | 11 |
12 import 'package:args/args.dart'; | 12 import 'package:args/args.dart'; |
13 import 'package:gardening/src/buildbot_structures.dart'; | 13 import 'package:gardening/src/buildbot_structures.dart'; |
14 import 'package:gardening/src/buildbot_loading.dart'; | 14 import 'package:gardening/src/client.dart'; |
15 import 'package:gardening/src/util.dart'; | 15 import 'package:gardening/src/util.dart'; |
16 | 16 |
17 main(List<String> args) async { | 17 main(List<String> args) async { |
18 ArgParser argParser = createArgParser(); | 18 ArgParser argParser = createArgParser(); |
19 ArgResults argResults = argParser.parse(args); | 19 ArgResults argResults = argParser.parse(args); |
20 processArgResults(argResults); | 20 processArgResults(argResults); |
| 21 |
| 22 BuildbotClient client = argResults['logdog'] |
| 23 ? new LogdogBuildbotClient() |
| 24 : new HttpBuildbotClient(); |
| 25 |
21 if (argResults.rest.length != 1) { | 26 if (argResults.rest.length != 1) { |
22 print('Usage: compare_failures [options] <log-uri>'); | 27 print('Usage: compare_failures [options] <log-uri>'); |
23 print('where <log-uri> is the uri the stdio output of a failing test step'); | 28 print('where <log-uri> is the uri the stdio output of a failing test step'); |
24 print('and options are:'); | 29 print('and options are:'); |
25 print(argParser.usage); | 30 print(argParser.usage); |
26 exit(1); | 31 exit(1); |
27 } | 32 } |
28 String url = argResults.rest.first; | 33 String url = argResults.rest.first; |
29 if (!url.endsWith('/text')) { | 34 if (!url.endsWith('/text')) { |
30 // Use the text version of the stdio log. | 35 // Use the text version of the stdio log. |
31 url += '/text'; | 36 url += '/text'; |
32 } | 37 } |
33 Uri uri = Uri.parse(url); | 38 Uri uri = Uri.parse(url); |
34 HttpClient client = new HttpClient(); | |
35 BuildUri buildUri = new BuildUri(uri); | 39 BuildUri buildUri = new BuildUri(uri); |
36 List<BuildResult> results = await readBuildResults(client, buildUri); | 40 List<BuildResult> results = await readBuildResults(client, buildUri); |
37 print(generateBuildResultsSummary(buildUri, results)); | 41 print(generateBuildResultsSummary(buildUri, results)); |
38 client.close(); | 42 client.close(); |
39 } | 43 } |
40 | 44 |
41 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the | 45 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the |
42 /// [BuildResult]s for the previous 5 builds. | 46 /// [BuildResult]s for the previous 5 builds. |
43 Future<List<BuildResult>> readBuildResults( | 47 Future<List<BuildResult>> readBuildResults( |
44 HttpClient client, BuildUri buildUri) async { | 48 BuildbotClient client, BuildUri buildUri) async { |
45 List<BuildResult> summaries = <BuildResult>[]; | 49 List<BuildResult> summaries = <BuildResult>[]; |
46 BuildResult firstSummary = await readBuildResult(client, buildUri); | 50 BuildResult summary = await client.readResult(buildUri); |
47 summaries.add(firstSummary); | 51 summaries.add(summary); |
48 if (firstSummary.hasFailures) { | 52 if (summary.hasFailures) { |
49 for (int i = 0; i < 10; i++) { | 53 for (int i = 0; i < 10; i++) { |
50 buildUri = buildUri.prev(); | 54 buildUri = summary.buildUri.prev(); |
51 summaries.add(await readBuildResult(client, buildUri)); | 55 summary = await client.readResult(buildUri); |
| 56 summaries.add(summary); |
52 } | 57 } |
53 } | 58 } |
54 return summaries; | 59 return summaries; |
55 } | 60 } |
56 | 61 |
57 /// Generate a summary of the timeouts and other failures in [results]. | 62 /// Generate a summary of the timeouts and other failures in [results]. |
58 String generateBuildResultsSummary( | 63 String generateBuildResultsSummary( |
59 BuildUri buildUri, List<BuildResult> results) { | 64 BuildUri buildUri, List<BuildResult> results) { |
60 StringBuffer sb = new StringBuffer(); | 65 StringBuffer sb = new StringBuffer(); |
61 sb.write('Results for $buildUri:\n'); | 66 sb.write('Results for $buildUri:\n'); |
62 Set<TestConfiguration> timeoutIds = new Set<TestConfiguration>(); | 67 Set<TestConfiguration> timeoutIds = new Set<TestConfiguration>(); |
63 for (BuildResult result in results) { | 68 for (BuildResult result in results) { |
64 timeoutIds.addAll(result.timeouts.map((TestFailure failure) => failure.id)); | 69 timeoutIds.addAll(result.timeouts.map((TestFailure failure) => failure.id)); |
65 } | 70 } |
66 if (timeoutIds.isNotEmpty) { | 71 if (timeoutIds.isNotEmpty) { |
67 int firstBuildNumber = results.first.buildUri.buildNumber; | |
68 int lastBuildNumber = results.last.buildUri.buildNumber; | |
69 Map<TestConfiguration, Map<int, Map<String, Timing>>> map = | 72 Map<TestConfiguration, Map<int, Map<String, Timing>>> map = |
70 <TestConfiguration, Map<int, Map<String, Timing>>>{}; | 73 <TestConfiguration, Map<int, Map<String, Timing>>>{}; |
71 Set<String> stepNames = new Set<String>(); | 74 Set<String> stepNames = new Set<String>(); |
72 for (BuildResult result in results) { | 75 for (BuildResult result in results) { |
73 for (Timing timing in result.timings) { | 76 for (Timing timing in result.timings) { |
74 Map<int, Map<String, Timing>> builds = | 77 Map<int, Map<String, Timing>> builds = |
75 map.putIfAbsent(timing.step.id, () => <int, Map<String, Timing>>{}); | 78 map.putIfAbsent(timing.step.id, () => <int, Map<String, Timing>>{}); |
76 stepNames.add(timing.step.stepName); | 79 stepNames.add(timing.step.stepName); |
77 builds.putIfAbsent(timing.uri.buildNumber, () => <String, Timing>{})[ | 80 builds.putIfAbsent(timing.uri.buildNumber, () => <String, Timing>{})[ |
78 timing.step.stepName] = timing; | 81 timing.step.stepName] = timing; |
79 } | 82 } |
80 } | 83 } |
81 sb.write('Timeouts for ${buildUri} :\n'); | 84 sb.write('Timeouts for ${buildUri} :\n'); |
82 map.forEach((TestConfiguration id, Map<int, Map<String, Timing>> timings) { | 85 map.forEach((TestConfiguration id, Map<int, Map<String, Timing>> timings) { |
83 if (!timeoutIds.contains(id)) return; | 86 if (!timeoutIds.contains(id)) return; |
84 sb.write('$id\n'); | 87 sb.write('$id\n'); |
85 sb.write( | 88 sb.write( |
86 '${' ' * 8} ${stepNames.map((t) => padRight(t, 14)).join(' ')}\n'); | 89 '${' ' * 8} ${stepNames.map((t) => padRight(t, 14)).join(' ')}\n'); |
87 for (int buildNumber = firstBuildNumber; | 90 for (BuildResult result in results) { |
88 buildNumber >= lastBuildNumber; | 91 int buildNumber = result.buildUri.buildNumber; |
89 buildNumber--) { | |
90 Map<String, Timing> steps = timings[buildNumber] ?? const {}; | 92 Map<String, Timing> steps = timings[buildNumber] ?? const {}; |
91 sb.write(padRight(' ${buildNumber}: ', 8)); | 93 sb.write(padRight(' ${buildNumber}: ', 8)); |
92 for (String stepName in stepNames) { | 94 for (String stepName in stepNames) { |
93 Timing timing = steps[stepName]; | 95 Timing timing = steps[stepName]; |
94 if (timing != null) { | 96 if (timing != null) { |
95 sb.write(' ${timing.time}'); | 97 sb.write(' ${timing.time}'); |
96 } else { | 98 } else { |
97 sb.write(' --------------'); | 99 sb.write(' --------------'); |
98 } | 100 } |
99 } | 101 } |
100 sb.write('\n'); | 102 sb.write('\n'); |
101 } | 103 } |
102 sb.write('\n'); | 104 sb.write('\n'); |
103 }); | 105 }); |
104 } | 106 } |
105 Set<TestConfiguration> errorIds = new Set<TestConfiguration>(); | 107 Set<TestConfiguration> errorIds = new Set<TestConfiguration>(); |
106 for (BuildResult result in results) { | 108 for (BuildResult result in results) { |
107 errorIds.addAll(result.errors.map((TestFailure failure) => failure.id)); | 109 errorIds.addAll(result.errors.map((TestFailure failure) => failure.id)); |
108 } | 110 } |
109 if (errorIds.isNotEmpty) { | 111 if (errorIds.isNotEmpty) { |
110 int firstBuildNumber = results.first.buildUri.buildNumber; | |
111 int lastBuildNumber = results.last.buildUri.buildNumber; | |
112 Map<TestConfiguration, Map<int, TestFailure>> map = | 112 Map<TestConfiguration, Map<int, TestFailure>> map = |
113 <TestConfiguration, Map<int, TestFailure>>{}; | 113 <TestConfiguration, Map<int, TestFailure>>{}; |
114 for (BuildResult result in results) { | 114 for (BuildResult result in results) { |
115 for (TestFailure failure in result.errors) { | 115 for (TestFailure failure in result.errors) { |
116 map.putIfAbsent(failure.id, () => <int, TestFailure>{})[ | 116 map.putIfAbsent(failure.id, () => <int, TestFailure>{})[ |
117 failure.uri.buildNumber] = failure; | 117 failure.uri.buildNumber] = failure; |
118 } | 118 } |
119 } | 119 } |
120 sb.write('Errors for ${buildUri} :\n'); | 120 sb.write('Errors for ${buildUri} :\n'); |
121 // TODO(johnniwinther): Improve comparison of non-timeouts. | 121 // TODO(johnniwinther): Improve comparison of non-timeouts. |
122 map.forEach((TestConfiguration id, Map<int, TestFailure> failures) { | 122 map.forEach((TestConfiguration id, Map<int, TestFailure> failures) { |
123 if (!errorIds.contains(id)) return; | 123 if (!errorIds.contains(id)) return; |
124 sb.write('$id\n'); | 124 sb.write('$id\n'); |
125 for (int buildNumber = firstBuildNumber; | 125 for (BuildResult result in results) { |
126 buildNumber >= lastBuildNumber; | 126 int buildNumber = result.buildUri.buildNumber; |
127 buildNumber--) { | |
128 TestFailure failure = failures[buildNumber]; | 127 TestFailure failure = failures[buildNumber]; |
129 sb.write(padRight(' ${buildNumber}: ', 8)); | 128 sb.write(padRight(' ${buildNumber}: ', 8)); |
130 if (failure != null) { | 129 if (failure != null) { |
131 sb.write(padRight(failure.expected, 10)); | 130 sb.write(padRight(failure.expected, 10)); |
132 sb.write(' / '); | 131 sb.write(' / '); |
133 sb.write(padRight(failure.actual, 10)); | 132 sb.write(padRight(failure.actual, 10)); |
134 } else { | 133 } else { |
135 sb.write(' ' * 10); | 134 sb.write(' ' * 10); |
136 sb.write(' / '); | 135 sb.write(' / '); |
137 sb.write(padRight('-- OK --', 10)); | 136 sb.write(padRight('-- OK --', 10)); |
138 } | 137 } |
139 sb.write('\n'); | 138 sb.write('\n'); |
140 } | 139 } |
141 sb.write('\n'); | 140 sb.write('\n'); |
142 }); | 141 }); |
143 } | 142 } |
144 return sb.toString(); | 143 return sb.toString(); |
145 } | 144 } |
OLD | NEW |