OLD | NEW |
1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /// Compares the test log of a build step with previous builds. | 5 /// Compares the test log of a build step with previous builds. |
6 /// | 6 /// |
7 /// Use this to detect flakiness of failures, especially timeouts. | 7 /// Use this to detect flakiness of failures, especially timeouts. |
8 | 8 |
9 import 'dart:async'; | 9 import 'dart:async'; |
10 | 10 |
11 import 'package:args/args.dart'; | 11 import 'package:args/args.dart'; |
12 import 'package:expect/expect.dart'; | 12 import 'package:expect/expect.dart'; |
| 13 import 'package:gardening/src/bot.dart'; |
13 import 'package:gardening/src/buildbot_structures.dart'; | 14 import 'package:gardening/src/buildbot_structures.dart'; |
14 import 'package:gardening/src/client.dart'; | 15 import 'package:gardening/src/client.dart'; |
15 import 'package:gardening/src/compare_failures_impl.dart'; | 16 import 'package:gardening/src/compare_failures_impl.dart'; |
16 import 'package:gardening/src/util.dart'; | 17 import 'package:gardening/src/util.dart'; |
17 | 18 |
18 import 'test_client.dart'; | 19 import 'test_client.dart'; |
19 | 20 |
20 // TODO(johnniwinther): Use 'package:testing' to run all tests. | 21 // TODO(johnniwinther): Use 'package:testing' to run all tests. |
21 main(List<String> args) async { | 22 main(List<String> args) async { |
22 ArgParser argParser = createArgParser(); | 23 ArgParser argParser = createArgParser(); |
23 argParser.addFlag('force', abbr: 'f'); | 24 argParser.addFlag('force', abbr: 'f'); |
24 ArgResults argResults = argParser.parse(args); | 25 ArgResults argResults = argParser.parse(args); |
25 processArgResults(argResults); | 26 processArgResults(argResults); |
26 | 27 |
27 await runSingleTests(argResults); | 28 await runSingleTests(argResults); |
28 await runGroupTests(argResults); | 29 await runGroupTests(argResults); |
29 } | 30 } |
30 | 31 |
31 Future runGroupTests(ArgResults argResults) async { | 32 Future runGroupTests(ArgResults argResults) async { |
32 BuildbotClient client = argResults['logdog'] | 33 BuildbotClient client = argResults['logdog'] |
33 ? new LogdogBuildbotClient() | 34 ? new LogdogBuildbotClient() |
34 : new HttpBuildbotClient(); | 35 : new HttpBuildbotClient(); |
35 await runGroupTest(client, 'vm-kernel'); | 36 Bot bot = new Bot.internal(client); |
36 client.close(); | 37 await runGroupTest(bot, 'vm-kernel'); |
| 38 bot.close(); |
37 } | 39 } |
38 | 40 |
39 Future runGroupTest(BuildbotClient client, String testGroup) async { | 41 Future runGroupTest(Bot bot, String testGroup) async { |
40 print('Testing group compare-failures: $testGroup runCount=1'); | 42 print('Testing group compare-failures: $testGroup runCount=1'); |
41 Map<BuildUri, List<BuildResult>> buildResults = | 43 Map<BuildUri, List<BuildResult>> buildResults = |
42 await loadBuildResults(client, [testGroup], runCount: 1); | 44 await loadBuildResults(bot, [testGroup], runCount: 1); |
43 print('- checking results for ${buildResults.keys}'); | 45 print('- checking results for ${buildResults.keys}'); |
44 buildResults.forEach((BuildUri buildUri, List<BuildResult> results) { | 46 buildResults.forEach((BuildUri buildUri, List<BuildResult> results) { |
45 Expect.isTrue(buildUri.buildNumber < 0, | 47 Expect.isTrue(buildUri.buildNumber < 0, |
46 "Expected relative build number for $buildUri"); | 48 "Expected relative build number for $buildUri"); |
47 for (BuildResult result in results) { | 49 for (BuildResult result in results) { |
48 Expect.isTrue( | 50 Expect.isTrue( |
49 result.buildUri.buildNumber > 0, | 51 result.buildUri.buildNumber > 0, |
50 "Expected absolute build number for build result " | 52 "Expected absolute build number for build result " |
51 "${result.buildUri}."); | 53 "${result.buildUri}."); |
52 for (TestFailure failure in result.failures) { | 54 for (TestFailure failure in result.failures) { |
53 Expect.equals( | 55 Expect.equals( |
54 result.buildUri.buildNumber, | 56 result.buildUri.buildNumber, |
55 failure.uri.buildNumber, | 57 failure.uri.buildNumber, |
56 "Expected absolute build number for test failure " | 58 "Expected absolute build number for test failure " |
57 "'${failure.id}' from ${failure.uri}."); | 59 "'${failure.id}' from ${failure.uri}."); |
58 } | 60 } |
59 } | 61 } |
60 }); | 62 }); |
61 } | 63 } |
62 | 64 |
63 Future runSingleTests(ArgResults argResults) async { | 65 Future runSingleTests(ArgResults argResults) async { |
64 bool force = argResults['force']; | 66 bool force = argResults['force']; |
65 | 67 |
66 BuildbotClient client = argResults['logdog'] | 68 BuildbotClient client = argResults['logdog'] |
67 ? new LogdogBuildbotClient() | 69 ? new LogdogBuildbotClient() |
68 : new TestClient(force: force); | 70 : new TestClient(force: force); |
| 71 Bot bot = new Bot.internal(client); |
69 | 72 |
70 await runSingleTest( | 73 await runSingleTest( |
71 client, | 74 bot, |
72 'https://build.chromium.org/p/client.dart/builders/' | 75 'https://build.chromium.org/p/client.dart/builders/' |
73 'vm-kernel-linux-debug-x64-be/builds/1884/steps/' | 76 'vm-kernel-linux-debug-x64-be/builds/1884/steps/' |
74 'vm%20tests/logs/stdio', | 77 'vm%20tests/logs/stdio', |
75 1, | 78 1, |
76 { | 79 { |
77 1884: { | 80 1884: { |
78 'dartk-vm debug_x64 corelib_2/map_keys2_test': 'RuntimeError/Pass', | 81 'dartk-vm debug_x64 corelib_2/map_keys2_test': 'RuntimeError/Pass', |
79 }, | 82 }, |
80 1883: { | 83 1883: { |
81 'dartk-vm debug_x64 corelib_2/map_keys2_test': 'RuntimeError/Pass', | 84 'dartk-vm debug_x64 corelib_2/map_keys2_test': 'RuntimeError/Pass', |
82 'dartk-vm debug_x64 corelib_2/package_resource_test': | 85 'dartk-vm debug_x64 corelib_2/package_resource_test': |
83 'Pass/CompileTimeError', | 86 'Pass/CompileTimeError', |
84 }, | 87 }, |
85 }); | 88 }); |
86 | 89 |
87 client.close(); | 90 bot.close(); |
88 } | 91 } |
89 | 92 |
90 testSingleResults( | 93 testSingleResults( |
91 Map<int, Map<String, String>> expected, List<BuildResult> results) { | 94 Map<int, Map<String, String>> expected, List<BuildResult> results) { |
92 Expect.equals(expected.length, results.length); | 95 Expect.equals(expected.length, results.length); |
93 int i = 0; | 96 int i = 0; |
94 expected.forEach((int buildNumber, Map<String, String> failures) { | 97 expected.forEach((int buildNumber, Map<String, String> failures) { |
95 BuildResult result = results[i++]; | 98 BuildResult result = results[i++]; |
96 Expect.equals(failures.length, result.errors.length); | 99 Expect.equals(failures.length, result.errors.length); |
97 failures.forEach((String testName, String resultText) { | 100 failures.forEach((String testName, String resultText) { |
98 List<String> nameParts = split(testName, [' ', ' ']); | 101 List<String> nameParts = split(testName, [' ', ' ']); |
99 TestConfiguration id = | 102 TestConfiguration id = |
100 new TestConfiguration(nameParts[0], nameParts[1], nameParts[2]); | 103 new TestConfiguration(nameParts[0], nameParts[1], nameParts[2]); |
101 List<String> resultParts = split(resultText, ['/']); | 104 List<String> resultParts = split(resultText, ['/']); |
102 TestFailure failure = result.errors.singleWhere((t) => t.id == id); | 105 TestFailure failure = result.errors.singleWhere((t) => t.id == id); |
103 Expect.equals(failure.expected, resultParts[0]); | 106 Expect.equals(failure.expected, resultParts[0]); |
104 Expect.equals(failure.actual, resultParts[1]); | 107 Expect.equals(failure.actual, resultParts[1]); |
105 }); | 108 }); |
106 }); | 109 }); |
107 } | 110 } |
108 | 111 |
109 Future runSingleTest(BuildbotClient client, String testUri, int runCount, | 112 Future runSingleTest(Bot bot, String testUri, int runCount, |
110 Map<int, Map<String, String>> expectedResult) async { | 113 Map<int, Map<String, String>> expectedResult) async { |
111 print('Testing single compare-failures: $testUri runCount=$runCount'); | 114 print('Testing single compare-failures: $testUri runCount=$runCount'); |
112 Map<BuildUri, List<BuildResult>> buildResults = | 115 Map<BuildUri, List<BuildResult>> buildResults = |
113 await loadBuildResults(client, [testUri], runCount: runCount); | 116 await loadBuildResults(bot, [testUri], runCount: runCount); |
114 print('- checking results for ${buildResults.keys}'); | 117 print('- checking results for ${buildResults.keys}'); |
115 if (LOG) { | 118 if (LOG) { |
116 printBuildResultsSummary(buildResults, [testUri]); | 119 printBuildResultsSummary(buildResults, [testUri]); |
117 } | 120 } |
118 Expect.equals(1, buildResults.length); | 121 Expect.equals(1, buildResults.length); |
119 testSingleResults(expectedResult, buildResults.values.first); | 122 testSingleResults(expectedResult, buildResults.values.first); |
120 } | 123 } |
OLD | NEW |