Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(2)

Side by Side Diff: tools/gardening/test/compare_failures_test.dart

Issue 3000953002: Use Bot in compare_failures (Closed)
Patch Set: Created 3 years, 4 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « tools/gardening/lib/src/compare_failures_impl.dart ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 /// Compares the test log of a build step with previous builds. 5 /// Compares the test log of a build step with previous builds.
6 /// 6 ///
7 /// Use this to detect flakiness of failures, especially timeouts. 7 /// Use this to detect flakiness of failures, especially timeouts.
8 8
9 import 'dart:async'; 9 import 'dart:async';
10 10
11 import 'package:args/args.dart'; 11 import 'package:args/args.dart';
12 import 'package:expect/expect.dart'; 12 import 'package:expect/expect.dart';
13 import 'package:gardening/src/bot.dart';
13 import 'package:gardening/src/buildbot_structures.dart'; 14 import 'package:gardening/src/buildbot_structures.dart';
14 import 'package:gardening/src/client.dart'; 15 import 'package:gardening/src/client.dart';
15 import 'package:gardening/src/compare_failures_impl.dart'; 16 import 'package:gardening/src/compare_failures_impl.dart';
16 import 'package:gardening/src/util.dart'; 17 import 'package:gardening/src/util.dart';
17 18
18 import 'test_client.dart'; 19 import 'test_client.dart';
19 20
20 // TODO(johnniwinther): Use 'package:testing' to run all tests. 21 // TODO(johnniwinther): Use 'package:testing' to run all tests.
21 main(List<String> args) async { 22 main(List<String> args) async {
22 ArgParser argParser = createArgParser(); 23 ArgParser argParser = createArgParser();
23 argParser.addFlag('force', abbr: 'f'); 24 argParser.addFlag('force', abbr: 'f');
24 ArgResults argResults = argParser.parse(args); 25 ArgResults argResults = argParser.parse(args);
25 processArgResults(argResults); 26 processArgResults(argResults);
26 27
27 await runSingleTests(argResults); 28 await runSingleTests(argResults);
28 await runGroupTests(argResults); 29 await runGroupTests(argResults);
29 } 30 }
30 31
31 Future runGroupTests(ArgResults argResults) async { 32 Future runGroupTests(ArgResults argResults) async {
32 BuildbotClient client = argResults['logdog'] 33 BuildbotClient client = argResults['logdog']
33 ? new LogdogBuildbotClient() 34 ? new LogdogBuildbotClient()
34 : new HttpBuildbotClient(); 35 : new HttpBuildbotClient();
35 await runGroupTest(client, 'vm-kernel'); 36 Bot bot = new Bot.internal(client);
36 client.close(); 37 await runGroupTest(bot, 'vm-kernel');
38 bot.close();
37 } 39 }
38 40
39 Future runGroupTest(BuildbotClient client, String testGroup) async { 41 Future runGroupTest(Bot bot, String testGroup) async {
40 print('Testing group compare-failures: $testGroup runCount=1'); 42 print('Testing group compare-failures: $testGroup runCount=1');
41 Map<BuildUri, List<BuildResult>> buildResults = 43 Map<BuildUri, List<BuildResult>> buildResults =
42 await loadBuildResults(client, [testGroup], runCount: 1); 44 await loadBuildResults(bot, [testGroup], runCount: 1);
43 print('- checking results for ${buildResults.keys}'); 45 print('- checking results for ${buildResults.keys}');
44 buildResults.forEach((BuildUri buildUri, List<BuildResult> results) { 46 buildResults.forEach((BuildUri buildUri, List<BuildResult> results) {
45 Expect.isTrue(buildUri.buildNumber < 0, 47 Expect.isTrue(buildUri.buildNumber < 0,
46 "Expected relative build number for $buildUri"); 48 "Expected relative build number for $buildUri");
47 for (BuildResult result in results) { 49 for (BuildResult result in results) {
48 Expect.isTrue( 50 Expect.isTrue(
49 result.buildUri.buildNumber > 0, 51 result.buildUri.buildNumber > 0,
50 "Expected absolute build number for build result " 52 "Expected absolute build number for build result "
51 "${result.buildUri}."); 53 "${result.buildUri}.");
52 for (TestFailure failure in result.failures) { 54 for (TestFailure failure in result.failures) {
53 Expect.equals( 55 Expect.equals(
54 result.buildUri.buildNumber, 56 result.buildUri.buildNumber,
55 failure.uri.buildNumber, 57 failure.uri.buildNumber,
56 "Expected absolute build number for test failure " 58 "Expected absolute build number for test failure "
57 "'${failure.id}' from ${failure.uri}."); 59 "'${failure.id}' from ${failure.uri}.");
58 } 60 }
59 } 61 }
60 }); 62 });
61 } 63 }
62 64
63 Future runSingleTests(ArgResults argResults) async { 65 Future runSingleTests(ArgResults argResults) async {
64 bool force = argResults['force']; 66 bool force = argResults['force'];
65 67
66 BuildbotClient client = argResults['logdog'] 68 BuildbotClient client = argResults['logdog']
67 ? new LogdogBuildbotClient() 69 ? new LogdogBuildbotClient()
68 : new TestClient(force: force); 70 : new TestClient(force: force);
71 Bot bot = new Bot.internal(client);
69 72
70 await runSingleTest( 73 await runSingleTest(
71 client, 74 bot,
72 'https://build.chromium.org/p/client.dart/builders/' 75 'https://build.chromium.org/p/client.dart/builders/'
73 'vm-kernel-linux-debug-x64-be/builds/1884/steps/' 76 'vm-kernel-linux-debug-x64-be/builds/1884/steps/'
74 'vm%20tests/logs/stdio', 77 'vm%20tests/logs/stdio',
75 1, 78 1,
76 { 79 {
77 1884: { 80 1884: {
78 'dartk-vm debug_x64 corelib_2/map_keys2_test': 'RuntimeError/Pass', 81 'dartk-vm debug_x64 corelib_2/map_keys2_test': 'RuntimeError/Pass',
79 }, 82 },
80 1883: { 83 1883: {
81 'dartk-vm debug_x64 corelib_2/map_keys2_test': 'RuntimeError/Pass', 84 'dartk-vm debug_x64 corelib_2/map_keys2_test': 'RuntimeError/Pass',
82 'dartk-vm debug_x64 corelib_2/package_resource_test': 85 'dartk-vm debug_x64 corelib_2/package_resource_test':
83 'Pass/CompileTimeError', 86 'Pass/CompileTimeError',
84 }, 87 },
85 }); 88 });
86 89
87 client.close(); 90 bot.close();
88 } 91 }
89 92
90 testSingleResults( 93 testSingleResults(
91 Map<int, Map<String, String>> expected, List<BuildResult> results) { 94 Map<int, Map<String, String>> expected, List<BuildResult> results) {
92 Expect.equals(expected.length, results.length); 95 Expect.equals(expected.length, results.length);
93 int i = 0; 96 int i = 0;
94 expected.forEach((int buildNumber, Map<String, String> failures) { 97 expected.forEach((int buildNumber, Map<String, String> failures) {
95 BuildResult result = results[i++]; 98 BuildResult result = results[i++];
96 Expect.equals(failures.length, result.errors.length); 99 Expect.equals(failures.length, result.errors.length);
97 failures.forEach((String testName, String resultText) { 100 failures.forEach((String testName, String resultText) {
98 List<String> nameParts = split(testName, [' ', ' ']); 101 List<String> nameParts = split(testName, [' ', ' ']);
99 TestConfiguration id = 102 TestConfiguration id =
100 new TestConfiguration(nameParts[0], nameParts[1], nameParts[2]); 103 new TestConfiguration(nameParts[0], nameParts[1], nameParts[2]);
101 List<String> resultParts = split(resultText, ['/']); 104 List<String> resultParts = split(resultText, ['/']);
102 TestFailure failure = result.errors.singleWhere((t) => t.id == id); 105 TestFailure failure = result.errors.singleWhere((t) => t.id == id);
103 Expect.equals(failure.expected, resultParts[0]); 106 Expect.equals(failure.expected, resultParts[0]);
104 Expect.equals(failure.actual, resultParts[1]); 107 Expect.equals(failure.actual, resultParts[1]);
105 }); 108 });
106 }); 109 });
107 } 110 }
108 111
109 Future runSingleTest(BuildbotClient client, String testUri, int runCount, 112 Future runSingleTest(Bot bot, String testUri, int runCount,
110 Map<int, Map<String, String>> expectedResult) async { 113 Map<int, Map<String, String>> expectedResult) async {
111 print('Testing single compare-failures: $testUri runCount=$runCount'); 114 print('Testing single compare-failures: $testUri runCount=$runCount');
112 Map<BuildUri, List<BuildResult>> buildResults = 115 Map<BuildUri, List<BuildResult>> buildResults =
113 await loadBuildResults(client, [testUri], runCount: runCount); 116 await loadBuildResults(bot, [testUri], runCount: runCount);
114 print('- checking results for ${buildResults.keys}'); 117 print('- checking results for ${buildResults.keys}');
115 if (LOG) { 118 if (LOG) {
116 printBuildResultsSummary(buildResults, [testUri]); 119 printBuildResultsSummary(buildResults, [testUri]);
117 } 120 }
118 Expect.equals(1, buildResults.length); 121 Expect.equals(1, buildResults.length);
119 testSingleResults(expectedResult, buildResults.values.first); 122 testSingleResults(expectedResult, buildResults.values.first);
120 } 123 }
OLDNEW
« no previous file with comments | « tools/gardening/lib/src/compare_failures_impl.dart ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698