Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(165)

Side by Side Diff: tools/gardening/bin/compare_failures.dart

Issue 2912343002: Add BuildBotClient to abstract http/logdog access (Closed)
Patch Set: Created 3 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | tools/gardening/bin/current_summary.dart » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 /// Compares the test log of a build step with previous builds. 5 /// Compares the test log of a build step with previous builds.
6 /// 6 ///
7 /// Use this to detect flakiness of failures, especially timeouts. 7 /// Use this to detect flakiness of failures, especially timeouts.
8 8
9 import 'dart:async'; 9 import 'dart:async';
10 import 'dart:io'; 10 import 'dart:io';
11 11
12 import 'package:args/args.dart'; 12 import 'package:args/args.dart';
13 import 'package:gardening/src/buildbot_structures.dart'; 13 import 'package:gardening/src/buildbot_structures.dart';
14 import 'package:gardening/src/buildbot_loading.dart'; 14 import 'package:gardening/src/client.dart';
15 import 'package:gardening/src/util.dart'; 15 import 'package:gardening/src/util.dart';
16 16
17 main(List<String> args) async { 17 main(List<String> args) async {
18 ArgParser argParser = createArgParser(); 18 ArgParser argParser = createArgParser();
19 ArgResults argResults = argParser.parse(args); 19 ArgResults argResults = argParser.parse(args);
20 processArgResults(argResults); 20 processArgResults(argResults);
21
22 BuildbotClient client = argResults['logdog']
23 ? new LogdogBuildbotClient()
24 : new HttpBuildbotClient();
25
21 if (argResults.rest.length != 1) { 26 if (argResults.rest.length != 1) {
22 print('Usage: compare_failures [options] <log-uri>'); 27 print('Usage: compare_failures [options] <log-uri>');
23 print('where <log-uri> is the uri the stdio output of a failing test step'); 28 print('where <log-uri> is the uri the stdio output of a failing test step');
24 print('and options are:'); 29 print('and options are:');
25 print(argParser.usage); 30 print(argParser.usage);
26 exit(1); 31 exit(1);
27 } 32 }
28 String url = argResults.rest.first; 33 String url = argResults.rest.first;
29 if (!url.endsWith('/text')) { 34 if (!url.endsWith('/text')) {
30 // Use the text version of the stdio log. 35 // Use the text version of the stdio log.
31 url += '/text'; 36 url += '/text';
32 } 37 }
33 Uri uri = Uri.parse(url); 38 Uri uri = Uri.parse(url);
34 HttpClient client = new HttpClient();
35 BuildUri buildUri = new BuildUri(uri); 39 BuildUri buildUri = new BuildUri(uri);
36 List<BuildResult> results = await readBuildResults(client, buildUri); 40 List<BuildResult> results = await readBuildResults(client, buildUri);
37 print(generateBuildResultsSummary(buildUri, results)); 41 print(generateBuildResultsSummary(buildUri, results));
38 client.close(); 42 client.close();
39 } 43 }
40 44
41 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the 45 /// Creates a [BuildResult] for [buildUri] and, if it contains failures, the
42 /// [BuildResult]s for the previous 5 builds. 46 /// [BuildResult]s for the previous 5 builds.
43 Future<List<BuildResult>> readBuildResults( 47 Future<List<BuildResult>> readBuildResults(
44 HttpClient client, BuildUri buildUri) async { 48 BuildbotClient client, BuildUri buildUri) async {
45 List<BuildResult> summaries = <BuildResult>[]; 49 List<BuildResult> summaries = <BuildResult>[];
46 BuildResult firstSummary = await readBuildResult(client, buildUri); 50 BuildResult summary = await client.readResult(buildUri);
47 summaries.add(firstSummary); 51 summaries.add(summary);
48 if (firstSummary.hasFailures) { 52 if (summary.hasFailures) {
49 for (int i = 0; i < 10; i++) { 53 for (int i = 0; i < 10; i++) {
50 buildUri = buildUri.prev(); 54 buildUri = summary.buildUri.prev();
51 summaries.add(await readBuildResult(client, buildUri)); 55 summary = await client.readResult(buildUri);
56 summaries.add(summary);
52 } 57 }
53 } 58 }
54 return summaries; 59 return summaries;
55 } 60 }
56 61
57 /// Generate a summary of the timeouts and other failures in [results]. 62 /// Generate a summary of the timeouts and other failures in [results].
58 String generateBuildResultsSummary( 63 String generateBuildResultsSummary(
59 BuildUri buildUri, List<BuildResult> results) { 64 BuildUri buildUri, List<BuildResult> results) {
60 StringBuffer sb = new StringBuffer(); 65 StringBuffer sb = new StringBuffer();
61 sb.write('Results for $buildUri:\n'); 66 sb.write('Results for $buildUri:\n');
62 Set<TestConfiguration> timeoutIds = new Set<TestConfiguration>(); 67 Set<TestConfiguration> timeoutIds = new Set<TestConfiguration>();
63 for (BuildResult result in results) { 68 for (BuildResult result in results) {
64 timeoutIds.addAll(result.timeouts.map((TestFailure failure) => failure.id)); 69 timeoutIds.addAll(result.timeouts.map((TestFailure failure) => failure.id));
65 } 70 }
66 if (timeoutIds.isNotEmpty) { 71 if (timeoutIds.isNotEmpty) {
67 int firstBuildNumber = results.first.buildUri.buildNumber;
68 int lastBuildNumber = results.last.buildUri.buildNumber;
69 Map<TestConfiguration, Map<int, Map<String, Timing>>> map = 72 Map<TestConfiguration, Map<int, Map<String, Timing>>> map =
70 <TestConfiguration, Map<int, Map<String, Timing>>>{}; 73 <TestConfiguration, Map<int, Map<String, Timing>>>{};
71 Set<String> stepNames = new Set<String>(); 74 Set<String> stepNames = new Set<String>();
72 for (BuildResult result in results) { 75 for (BuildResult result in results) {
73 for (Timing timing in result.timings) { 76 for (Timing timing in result.timings) {
74 Map<int, Map<String, Timing>> builds = 77 Map<int, Map<String, Timing>> builds =
75 map.putIfAbsent(timing.step.id, () => <int, Map<String, Timing>>{}); 78 map.putIfAbsent(timing.step.id, () => <int, Map<String, Timing>>{});
76 stepNames.add(timing.step.stepName); 79 stepNames.add(timing.step.stepName);
77 builds.putIfAbsent(timing.uri.buildNumber, () => <String, Timing>{})[ 80 builds.putIfAbsent(timing.uri.buildNumber, () => <String, Timing>{})[
78 timing.step.stepName] = timing; 81 timing.step.stepName] = timing;
79 } 82 }
80 } 83 }
81 sb.write('Timeouts for ${buildUri} :\n'); 84 sb.write('Timeouts for ${buildUri} :\n');
82 map.forEach((TestConfiguration id, Map<int, Map<String, Timing>> timings) { 85 map.forEach((TestConfiguration id, Map<int, Map<String, Timing>> timings) {
83 if (!timeoutIds.contains(id)) return; 86 if (!timeoutIds.contains(id)) return;
84 sb.write('$id\n'); 87 sb.write('$id\n');
85 sb.write( 88 sb.write(
86 '${' ' * 8} ${stepNames.map((t) => padRight(t, 14)).join(' ')}\n'); 89 '${' ' * 8} ${stepNames.map((t) => padRight(t, 14)).join(' ')}\n');
87 for (int buildNumber = firstBuildNumber; 90 for (BuildResult result in results) {
88 buildNumber >= lastBuildNumber; 91 int buildNumber = result.buildUri.buildNumber;
89 buildNumber--) {
90 Map<String, Timing> steps = timings[buildNumber] ?? const {}; 92 Map<String, Timing> steps = timings[buildNumber] ?? const {};
91 sb.write(padRight(' ${buildNumber}: ', 8)); 93 sb.write(padRight(' ${buildNumber}: ', 8));
92 for (String stepName in stepNames) { 94 for (String stepName in stepNames) {
93 Timing timing = steps[stepName]; 95 Timing timing = steps[stepName];
94 if (timing != null) { 96 if (timing != null) {
95 sb.write(' ${timing.time}'); 97 sb.write(' ${timing.time}');
96 } else { 98 } else {
97 sb.write(' --------------'); 99 sb.write(' --------------');
98 } 100 }
99 } 101 }
100 sb.write('\n'); 102 sb.write('\n');
101 } 103 }
102 sb.write('\n'); 104 sb.write('\n');
103 }); 105 });
104 } 106 }
105 Set<TestConfiguration> errorIds = new Set<TestConfiguration>(); 107 Set<TestConfiguration> errorIds = new Set<TestConfiguration>();
106 for (BuildResult result in results) { 108 for (BuildResult result in results) {
107 errorIds.addAll(result.errors.map((TestFailure failure) => failure.id)); 109 errorIds.addAll(result.errors.map((TestFailure failure) => failure.id));
108 } 110 }
109 if (errorIds.isNotEmpty) { 111 if (errorIds.isNotEmpty) {
110 int firstBuildNumber = results.first.buildUri.buildNumber;
111 int lastBuildNumber = results.last.buildUri.buildNumber;
112 Map<TestConfiguration, Map<int, TestFailure>> map = 112 Map<TestConfiguration, Map<int, TestFailure>> map =
113 <TestConfiguration, Map<int, TestFailure>>{}; 113 <TestConfiguration, Map<int, TestFailure>>{};
114 for (BuildResult result in results) { 114 for (BuildResult result in results) {
115 for (TestFailure failure in result.errors) { 115 for (TestFailure failure in result.errors) {
116 map.putIfAbsent(failure.id, () => <int, TestFailure>{})[ 116 map.putIfAbsent(failure.id, () => <int, TestFailure>{})[
117 failure.uri.buildNumber] = failure; 117 failure.uri.buildNumber] = failure;
118 } 118 }
119 } 119 }
120 sb.write('Errors for ${buildUri} :\n'); 120 sb.write('Errors for ${buildUri} :\n');
121 // TODO(johnniwinther): Improve comparison of non-timeouts. 121 // TODO(johnniwinther): Improve comparison of non-timeouts.
122 map.forEach((TestConfiguration id, Map<int, TestFailure> failures) { 122 map.forEach((TestConfiguration id, Map<int, TestFailure> failures) {
123 if (!errorIds.contains(id)) return; 123 if (!errorIds.contains(id)) return;
124 sb.write('$id\n'); 124 sb.write('$id\n');
125 for (int buildNumber = firstBuildNumber; 125 for (BuildResult result in results) {
126 buildNumber >= lastBuildNumber; 126 int buildNumber = result.buildUri.buildNumber;
127 buildNumber--) {
128 TestFailure failure = failures[buildNumber]; 127 TestFailure failure = failures[buildNumber];
129 sb.write(padRight(' ${buildNumber}: ', 8)); 128 sb.write(padRight(' ${buildNumber}: ', 8));
130 if (failure != null) { 129 if (failure != null) {
131 sb.write(padRight(failure.expected, 10)); 130 sb.write(padRight(failure.expected, 10));
132 sb.write(' / '); 131 sb.write(' / ');
133 sb.write(padRight(failure.actual, 10)); 132 sb.write(padRight(failure.actual, 10));
134 } else { 133 } else {
135 sb.write(' ' * 10); 134 sb.write(' ' * 10);
136 sb.write(' / '); 135 sb.write(' / ');
137 sb.write(padRight('-- OK --', 10)); 136 sb.write(padRight('-- OK --', 10));
138 } 137 }
139 sb.write('\n'); 138 sb.write('\n');
140 } 139 }
141 sb.write('\n'); 140 sb.write('\n');
142 }); 141 });
143 } 142 }
144 return sb.toString(); 143 return sb.toString();
145 } 144 }
OLDNEW
« no previous file with comments | « no previous file | tools/gardening/bin/current_summary.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698