OLD | NEW |
1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /// Compares the test log of a build step with previous builds. | 5 /// Compares the test log of a build step with previous builds. |
6 /// | 6 /// |
7 /// Use this to detect flakiness of failures, especially timeouts. | 7 /// Use this to detect flakiness of failures, especially timeouts. |
8 | 8 |
9 import 'dart:async'; | 9 import 'dart:async'; |
10 import 'dart:io'; | 10 import 'dart:io'; |
11 | 11 |
12 import 'package:args/args.dart'; | 12 import 'package:args/args.dart'; |
13 import 'package:gardening/src/buildbot_structures.dart'; | 13 import 'package:gardening/src/buildbot_structures.dart'; |
14 import 'package:gardening/src/buildbot_loading.dart'; | 14 import 'package:gardening/src/buildbot_loading.dart'; |
15 import 'package:gardening/src/util.dart'; | 15 import 'package:gardening/src/util.dart'; |
16 | 16 |
| 17 void help(ArgParser argParser) { |
| 18 print('Given a <log-uri> finds all failing tests in that stdout. Then '); |
| 19 print('fetches earlier runs of the same bot and compares the results.'); |
| 20 print('This tool is particularly useful to detect flakes and their '); |
| 21 print('frequency.'); |
| 22 print('Usage: compare_failures [options] <log-uri>'); |
| 23 print('where <log-uri> is the uri the stdio output of a failing test step'); |
| 24 print('and options are:'); |
| 25 print(argParser.usage); |
| 26 } |
| 27 |
17 main(List<String> args) async { | 28 main(List<String> args) async { |
18 ArgParser argParser = createArgParser(); | 29 ArgParser argParser = createArgParser(); |
| 30 argParser.addOption("run-count", |
| 31 defaultsTo: "10", help: "How many previous runs should be fetched"); |
19 ArgResults argResults = argParser.parse(args); | 32 ArgResults argResults = argParser.parse(args); |
20 processArgResults(argResults); | 33 processArgResults(argResults); |
21 if (argResults.rest.length != 1) { | 34 var runCount = int.parse(argResults['run-count'], onError: (_) => null); |
22 print('Usage: compare_failures [options] <log-uri>'); | 35 |
23 print('where <log-uri> is the uri the stdio output of a failing test step'); | 36 if (argResults.rest.length != 1 || argResults['help'] || runCount == null) { |
24 print('and options are:'); | 37 help(argParser); |
25 print(argParser.usage); | 38 if (argResults['help']) return; |
26 exit(1); | 39 exit(1); |
27 } | 40 } |
28 String url = argResults.rest.first; | 41 String url = argResults.rest.first; |
29 if (!url.endsWith('/text')) { | 42 if (!url.endsWith('/text')) { |
30 // Use the text version of the stdio log. | 43 // Use the text version of the stdio log. |
31 url += '/text'; | 44 url += '/text'; |
32 } | 45 } |
33 Uri uri = Uri.parse(url); | 46 Uri uri = Uri.parse(url); |
34 HttpClient client = new HttpClient(); | 47 HttpClient client = new HttpClient(); |
35 BuildUri buildUri = new BuildUri(uri); | 48 BuildUri buildUri = new BuildUri(uri); |
36 List<BuildResult> results = await readBuildResults(client, buildUri); | 49 List<BuildResult> results = |
| 50 await readBuildResults(client, buildUri, runCount); |
37 print(generateBuildResultsSummary(buildUri, results)); | 51 print(generateBuildResultsSummary(buildUri, results)); |
38 client.close(); | 52 client.close(); |
39 } | 53 } |
40 | 54 |
41 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the | 55 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the |
42 /// [BuildResult]s for the previous 5 builds. | 56 /// [BuildResult]s for the previous 5 builds. |
43 Future<List<BuildResult>> readBuildResults( | 57 Future<List<BuildResult>> readBuildResults( |
44 HttpClient client, BuildUri buildUri) async { | 58 HttpClient client, BuildUri buildUri, int runCount) async { |
45 List<BuildResult> summaries = <BuildResult>[]; | 59 List<BuildResult> summaries = <BuildResult>[]; |
46 BuildResult firstSummary = await readBuildResult(client, buildUri); | 60 BuildResult firstSummary = await readBuildResult(client, buildUri); |
47 summaries.add(firstSummary); | 61 summaries.add(firstSummary); |
48 if (firstSummary.hasFailures) { | 62 if (firstSummary.hasFailures) { |
49 for (int i = 0; i < 10; i++) { | 63 for (int i = 0; i < runCount; i++) { |
50 buildUri = buildUri.prev(); | 64 buildUri = buildUri.prev(); |
51 summaries.add(await readBuildResult(client, buildUri)); | 65 summaries.add(await readBuildResult(client, buildUri)); |
52 } | 66 } |
53 } | 67 } |
54 return summaries; | 68 return summaries; |
55 } | 69 } |
56 | 70 |
57 /// Generate a summary of the timeouts and other failures in [results]. | 71 /// Generate a summary of the timeouts and other failures in [results]. |
58 String generateBuildResultsSummary( | 72 String generateBuildResultsSummary( |
59 BuildUri buildUri, List<BuildResult> results) { | 73 BuildUri buildUri, List<BuildResult> results) { |
(...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
136 sb.write(' / '); | 150 sb.write(' / '); |
137 sb.write(padRight('-- OK --', 10)); | 151 sb.write(padRight('-- OK --', 10)); |
138 } | 152 } |
139 sb.write('\n'); | 153 sb.write('\n'); |
140 } | 154 } |
141 sb.write('\n'); | 155 sb.write('\n'); |
142 }); | 156 }); |
143 } | 157 } |
144 return sb.toString(); | 158 return sb.toString(); |
145 } | 159 } |
OLD | NEW |