| OLD | NEW |
| 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /// Compares the test log of a build step with previous builds. | 5 /// Compares the test log of a build step with previous builds. |
| 6 /// | 6 /// |
| 7 /// Use this to detect flakiness of failures, especially timeouts. | 7 /// Use this to detect flakiness of failures, especially timeouts. |
| 8 | 8 |
| 9 import 'dart:async'; | 9 import 'dart:async'; |
| 10 import 'dart:io'; | 10 import 'dart:io'; |
| 11 | 11 |
| 12 import 'package:args/args.dart'; | 12 import 'package:args/args.dart'; |
| 13 import 'package:gardening/src/buildbot_structures.dart'; | 13 import 'package:gardening/src/buildbot_structures.dart'; |
| 14 import 'package:gardening/src/buildbot_loading.dart'; | 14 import 'package:gardening/src/buildbot_loading.dart'; |
| 15 import 'package:gardening/src/util.dart'; | 15 import 'package:gardening/src/util.dart'; |
| 16 | 16 |
| 17 void help(ArgParser argParser) { |
| 18 print('Given a <log-uri> finds all failing tests in that stdout. Then '); |
| 19 print('fetches earlier runs of the same bot and compares the results.'); |
| 20 print('This tool is particularly useful to detect flakes and their '); |
| 21 print('frequency.'); |
| 22 print('Usage: compare_failures [options] <log-uri>'); |
| 23 print('where <log-uri> is the uri the stdio output of a failing test step'); |
| 24 print('and options are:'); |
| 25 print(argParser.usage); |
| 26 } |
| 27 |
| 17 main(List<String> args) async { | 28 main(List<String> args) async { |
| 18 ArgParser argParser = createArgParser(); | 29 ArgParser argParser = createArgParser(); |
| 30 argParser.addOption("run-count", |
| 31 defaultsTo: "10", help: "How many previous runs should be fetched"); |
| 19 ArgResults argResults = argParser.parse(args); | 32 ArgResults argResults = argParser.parse(args); |
| 20 processArgResults(argResults); | 33 processArgResults(argResults); |
| 21 if (argResults.rest.length != 1) { | 34 var runCount = int.parse(argResults['run-count'], onError: (_) => null); |
| 22 print('Usage: compare_failures [options] <log-uri>'); | 35 |
| 23 print('where <log-uri> is the uri the stdio output of a failing test step'); | 36 if (argResults.rest.length != 1 || argResults['help'] || runCount == null) { |
| 24 print('and options are:'); | 37 help(argParser); |
| 25 print(argParser.usage); | 38 if (argResults['help']) return; |
| 26 exit(1); | 39 exit(1); |
| 27 } | 40 } |
| 28 String url = argResults.rest.first; | 41 String url = argResults.rest.first; |
| 29 if (!url.endsWith('/text')) { | 42 if (!url.endsWith('/text')) { |
| 30 // Use the text version of the stdio log. | 43 // Use the text version of the stdio log. |
| 31 url += '/text'; | 44 url += '/text'; |
| 32 } | 45 } |
| 33 Uri uri = Uri.parse(url); | 46 Uri uri = Uri.parse(url); |
| 34 HttpClient client = new HttpClient(); | 47 HttpClient client = new HttpClient(); |
| 35 BuildUri buildUri = new BuildUri(uri); | 48 BuildUri buildUri = new BuildUri(uri); |
| 36 List<BuildResult> results = await readBuildResults(client, buildUri); | 49 List<BuildResult> results = |
| 50 await readBuildResults(client, buildUri, runCount); |
| 37 print(generateBuildResultsSummary(buildUri, results)); | 51 print(generateBuildResultsSummary(buildUri, results)); |
| 38 client.close(); | 52 client.close(); |
| 39 } | 53 } |
| 40 | 54 |
| 41 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the | 55 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the |
| 42 /// [BuildResult]s for the previous 5 builds. | 56 /// [BuildResult]s for the previous 5 builds. |
| 43 Future<List<BuildResult>> readBuildResults( | 57 Future<List<BuildResult>> readBuildResults( |
| 44 HttpClient client, BuildUri buildUri) async { | 58 HttpClient client, BuildUri buildUri, int runCount) async { |
| 45 List<BuildResult> summaries = <BuildResult>[]; | 59 List<BuildResult> summaries = <BuildResult>[]; |
| 46 BuildResult firstSummary = await readBuildResult(client, buildUri); | 60 BuildResult firstSummary = await readBuildResult(client, buildUri); |
| 47 summaries.add(firstSummary); | 61 summaries.add(firstSummary); |
| 48 if (firstSummary.hasFailures) { | 62 if (firstSummary.hasFailures) { |
| 49 for (int i = 0; i < 10; i++) { | 63 for (int i = 0; i < runCount; i++) { |
| 50 buildUri = buildUri.prev(); | 64 buildUri = buildUri.prev(); |
| 51 summaries.add(await readBuildResult(client, buildUri)); | 65 summaries.add(await readBuildResult(client, buildUri)); |
| 52 } | 66 } |
| 53 } | 67 } |
| 54 return summaries; | 68 return summaries; |
| 55 } | 69 } |
| 56 | 70 |
| 57 /// Generate a summary of the timeouts and other failures in [results]. | 71 /// Generate a summary of the timeouts and other failures in [results]. |
| 58 String generateBuildResultsSummary( | 72 String generateBuildResultsSummary( |
| 59 BuildUri buildUri, List<BuildResult> results) { | 73 BuildUri buildUri, List<BuildResult> results) { |
| (...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 136 sb.write(' / '); | 150 sb.write(' / '); |
| 137 sb.write(padRight('-- OK --', 10)); | 151 sb.write(padRight('-- OK --', 10)); |
| 138 } | 152 } |
| 139 sb.write('\n'); | 153 sb.write('\n'); |
| 140 } | 154 } |
| 141 sb.write('\n'); | 155 sb.write('\n'); |
| 142 }); | 156 }); |
| 143 } | 157 } |
| 144 return sb.toString(); | 158 return sb.toString(); |
| 145 } | 159 } |
| OLD | NEW |