Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(291)

Side by Side Diff: tools/gardening/bin/compare_failures.dart

Issue 2987253002: Add test to compare-failures (Closed)
Patch Set: Created 3 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | tools/gardening/bin/find_shard.dart » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 /// Compares the test log of a build step with previous builds. 5 /// Compares the test log of a build step with previous builds.
6 /// 6 ///
7 /// Use this to detect flakiness of failures, especially timeouts. 7 /// Use this to detect flakiness of failures, especially timeouts.
8 8
9 import 'dart:async';
10 import 'dart:io'; 9 import 'dart:io';
11 10
12 import 'package:args/args.dart'; 11 import 'package:args/args.dart';
13 import 'package:gardening/src/buildbot_structures.dart';
14 import 'package:gardening/src/client.dart'; 12 import 'package:gardening/src/client.dart';
13 import 'package:gardening/src/compare_failures_impl.dart';
15 import 'package:gardening/src/util.dart'; 14 import 'package:gardening/src/util.dart';
16 15
17 void help(ArgParser argParser) { 16 void help(ArgParser argParser) {
18 print('Given a <log-uri> finds all failing tests in that stdout. Then '); 17 print('Given a <log-uri> finds all failing tests in that stdout. Then ');
19 print('fetches earlier runs of the same bot and compares the results.'); 18 print('fetches earlier runs of the same bot and compares the results.');
20 print('This tool is particularly useful to detect flakes and their '); 19 print('This tool is particularly useful to detect flakes and their ');
21 print('frequency.'); 20 print('frequency.');
22 print('Usage: compare_failures [options] <log-uri>'); 21 print('Usage: compare_failures [options] ');
23 print('where <log-uri> is the uri the stdio output of a failing test step'); 22 print(' (<log-uri> [<log-uri> ...] | <build-group> [<build-group> ...])');
24 print('and options are:'); 23 print('where <log-uri> is the uri the stdio output of a failing test step ');
24 print('and <build-group> is the name of a buildbot group, for instance ');
25 print('`vm-kernel`, and options are:');
25 print(argParser.usage); 26 print(argParser.usage);
26 } 27 }
27 28
28 main(List<String> args) async { 29 main(List<String> args) async {
29 ArgParser argParser = createArgParser(); 30 ArgParser argParser = createArgParser();
30 argParser.addOption("run-count", 31 argParser.addOption("run-count",
31 defaultsTo: "10", help: "How many previous runs should be fetched"); 32 defaultsTo: "10", help: "How many previous runs should be fetched");
32 ArgResults argResults = argParser.parse(args); 33 ArgResults argResults = argParser.parse(args);
33 processArgResults(argResults); 34 processArgResults(argResults);
34 35
35 BuildbotClient client = argResults['logdog']
36 ? new LogdogBuildbotClient()
37 : new HttpBuildbotClient();
38
39 var runCount = int.parse(argResults['run-count'], onError: (_) => null); 36 var runCount = int.parse(argResults['run-count'], onError: (_) => null);
40 37
41 if (argResults.rest.length != 1 || argResults['help'] || runCount == null) { 38 if (argResults.rest.length != 1 || argResults['help'] || runCount == null) {
42 help(argParser); 39 help(argParser);
43 if (argResults['help']) return; 40 if (argResults['help']) return;
44 exit(1); 41 exit(1);
45 } 42 }
46 String url = argResults.rest.first; 43
47 if (!url.endsWith('/text')) { 44 BuildbotClient client = argResults['logdog']
48 // Use the text version of the stdio log. 45 ? new LogdogBuildbotClient()
49 url += '/text'; 46 : new HttpBuildbotClient();
50 } 47 await mainInternal(client, argResults.rest, runCount: runCount);
51 Uri uri = Uri.parse(url);
52 BuildUri buildUri = new BuildUri(uri);
53 List<BuildResult> results =
54 await readBuildResults(client, buildUri, runCount);
55 print(generateBuildResultsSummary(buildUri, results));
56 client.close(); 48 client.close();
57 } 49 }
58
59 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the
60 /// [BuildResult]s for the previous [runCount] builds.
61 Future<List<BuildResult>> readBuildResults(
62 BuildbotClient client, BuildUri buildUri, int runCount) async {
63 List<BuildResult> summaries = <BuildResult>[];
64 BuildResult summary = await client.readResult(buildUri);
65 summaries.add(summary);
66 if (summary.hasFailures) {
67 for (int i = 0; i < runCount; i++) {
68 buildUri = summary.buildUri.prev();
69 summary = await client.readResult(buildUri);
70 summaries.add(summary);
71 }
72 }
73 return summaries;
74 }
75
76 /// Generate a summary of the timeouts and other failures in [results].
77 String generateBuildResultsSummary(
78 BuildUri buildUri, List<BuildResult> results) {
79 StringBuffer sb = new StringBuffer();
80 sb.write('Results for $buildUri:\n');
81 Set<TestConfiguration> timeoutIds = new Set<TestConfiguration>();
82 for (BuildResult result in results) {
83 timeoutIds.addAll(result.timeouts.map((TestFailure failure) => failure.id));
84 }
85 if (timeoutIds.isNotEmpty) {
86 Map<TestConfiguration, Map<int, Map<String, Timing>>> map =
87 <TestConfiguration, Map<int, Map<String, Timing>>>{};
88 Set<String> stepNames = new Set<String>();
89 for (BuildResult result in results) {
90 for (Timing timing in result.timings) {
91 Map<int, Map<String, Timing>> builds =
92 map.putIfAbsent(timing.step.id, () => <int, Map<String, Timing>>{});
93 stepNames.add(timing.step.stepName);
94 builds.putIfAbsent(timing.uri.buildNumber, () => <String, Timing>{})[
95 timing.step.stepName] = timing;
96 }
97 }
98 sb.write('Timeouts for ${buildUri} :\n');
99 map.forEach((TestConfiguration id, Map<int, Map<String, Timing>> timings) {
100 if (!timeoutIds.contains(id)) return;
101 sb.write('$id\n');
102 sb.write(
103 '${' ' * 8} ${stepNames.map((t) => padRight(t, 14)).join(' ')}\n');
104 for (BuildResult result in results) {
105 int buildNumber = result.buildUri.buildNumber;
106 Map<String, Timing> steps = timings[buildNumber] ?? const {};
107 sb.write(padRight(' ${buildNumber}: ', 8));
108 for (String stepName in stepNames) {
109 Timing timing = steps[stepName];
110 if (timing != null) {
111 sb.write(' ${timing.time}');
112 } else {
113 sb.write(' --------------');
114 }
115 }
116 sb.write('\n');
117 }
118 sb.write('\n');
119 });
120 }
121 Set<TestConfiguration> errorIds = new Set<TestConfiguration>();
122 for (BuildResult result in results) {
123 errorIds.addAll(result.errors.map((TestFailure failure) => failure.id));
124 }
125 if (errorIds.isNotEmpty) {
126 Map<TestConfiguration, Map<int, TestFailure>> map =
127 <TestConfiguration, Map<int, TestFailure>>{};
128 for (BuildResult result in results) {
129 for (TestFailure failure in result.errors) {
130 map.putIfAbsent(failure.id, () => <int, TestFailure>{})[
131 failure.uri.buildNumber] = failure;
132 }
133 }
134 sb.write('Errors for ${buildUri} :\n');
135 // TODO(johnniwinther): Improve comparison of non-timeouts.
136 map.forEach((TestConfiguration id, Map<int, TestFailure> failures) {
137 if (!errorIds.contains(id)) return;
138 sb.write('$id\n');
139 for (BuildResult result in results) {
140 int buildNumber = result.buildUri.buildNumber;
141 TestFailure failure = failures[buildNumber];
142 sb.write(padRight(' ${buildNumber}: ', 8));
143 if (failure != null) {
144 sb.write(padRight(failure.expected, 10));
145 sb.write(' / ');
146 sb.write(padRight(failure.actual, 10));
147 } else {
148 sb.write(' ' * 10);
149 sb.write(' / ');
150 sb.write(padRight('-- OK --', 10));
151 }
152 sb.write('\n');
153 }
154 sb.write('\n');
155 });
156 }
157 return sb.toString();
158 }
OLDNEW
« no previous file with comments | « no previous file | tools/gardening/bin/find_shard.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698