| OLD | NEW |
| 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /// Compares the test log of a build step with previous builds. | 5 /// Compares the test log of a build step with previous builds. |
| 6 /// | 6 /// |
| 7 /// Use this to detect flakiness of failures, especially timeouts. | 7 /// Use this to detect flakiness of failures, especially timeouts. |
| 8 | 8 |
| 9 import 'dart:async'; | 9 import 'dart:async'; |
| 10 | 10 |
| 11 import 'package:args/args.dart'; | 11 import 'package:args/args.dart'; |
| 12 import 'package:expect/expect.dart'; | 12 import 'package:expect/expect.dart'; |
| 13 import 'package:gardening/src/bot.dart'; |
| 13 import 'package:gardening/src/buildbot_structures.dart'; | 14 import 'package:gardening/src/buildbot_structures.dart'; |
| 14 import 'package:gardening/src/client.dart'; | 15 import 'package:gardening/src/client.dart'; |
| 15 import 'package:gardening/src/compare_failures_impl.dart'; | 16 import 'package:gardening/src/compare_failures_impl.dart'; |
| 16 import 'package:gardening/src/util.dart'; | 17 import 'package:gardening/src/util.dart'; |
| 17 | 18 |
| 18 import 'test_client.dart'; | 19 import 'test_client.dart'; |
| 19 | 20 |
| 20 // TODO(johnniwinther): Use 'package:testing' to run all tests. | 21 // TODO(johnniwinther): Use 'package:testing' to run all tests. |
| 21 main(List<String> args) async { | 22 main(List<String> args) async { |
| 22 ArgParser argParser = createArgParser(); | 23 ArgParser argParser = createArgParser(); |
| 23 argParser.addFlag('force', abbr: 'f'); | 24 argParser.addFlag('force', abbr: 'f'); |
| 24 ArgResults argResults = argParser.parse(args); | 25 ArgResults argResults = argParser.parse(args); |
| 25 processArgResults(argResults); | 26 processArgResults(argResults); |
| 26 | 27 |
| 27 await runSingleTests(argResults); | 28 await runSingleTests(argResults); |
| 28 await runGroupTests(argResults); | 29 await runGroupTests(argResults); |
| 29 } | 30 } |
| 30 | 31 |
| 31 Future runGroupTests(ArgResults argResults) async { | 32 Future runGroupTests(ArgResults argResults) async { |
| 32 BuildbotClient client = argResults['logdog'] | 33 BuildbotClient client = argResults['logdog'] |
| 33 ? new LogdogBuildbotClient() | 34 ? new LogdogBuildbotClient() |
| 34 : new HttpBuildbotClient(); | 35 : new HttpBuildbotClient(); |
| 35 await runGroupTest(client, 'vm-kernel'); | 36 Bot bot = new Bot.internal(client); |
| 36 client.close(); | 37 await runGroupTest(bot, 'vm-kernel'); |
| 38 bot.close(); |
| 37 } | 39 } |
| 38 | 40 |
| 39 Future runGroupTest(BuildbotClient client, String testGroup) async { | 41 Future runGroupTest(Bot bot, String testGroup) async { |
| 40 print('Testing group compare-failures: $testGroup runCount=1'); | 42 print('Testing group compare-failures: $testGroup runCount=1'); |
| 41 Map<BuildUri, List<BuildResult>> buildResults = | 43 Map<BuildUri, List<BuildResult>> buildResults = |
| 42 await loadBuildResults(client, [testGroup], runCount: 1); | 44 await loadBuildResults(bot, [testGroup], runCount: 1); |
| 43 print('- checking results for ${buildResults.keys}'); | 45 print('- checking results for ${buildResults.keys}'); |
| 44 buildResults.forEach((BuildUri buildUri, List<BuildResult> results) { | 46 buildResults.forEach((BuildUri buildUri, List<BuildResult> results) { |
| 45 Expect.isTrue(buildUri.buildNumber < 0, | 47 Expect.isTrue(buildUri.buildNumber < 0, |
| 46 "Expected relative build number for $buildUri"); | 48 "Expected relative build number for $buildUri"); |
| 47 for (BuildResult result in results) { | 49 for (BuildResult result in results) { |
| 48 Expect.isTrue( | 50 Expect.isTrue( |
| 49 result.buildUri.buildNumber > 0, | 51 result.buildUri.buildNumber > 0, |
| 50 "Expected absolute build number for build result " | 52 "Expected absolute build number for build result " |
| 51 "${result.buildUri}."); | 53 "${result.buildUri}."); |
| 52 for (TestFailure failure in result.failures) { | 54 for (TestFailure failure in result.failures) { |
| 53 Expect.equals( | 55 Expect.equals( |
| 54 result.buildUri.buildNumber, | 56 result.buildUri.buildNumber, |
| 55 failure.uri.buildNumber, | 57 failure.uri.buildNumber, |
| 56 "Expected absolute build number for test failure " | 58 "Expected absolute build number for test failure " |
| 57 "'${failure.id}' from ${failure.uri}."); | 59 "'${failure.id}' from ${failure.uri}."); |
| 58 } | 60 } |
| 59 } | 61 } |
| 60 }); | 62 }); |
| 61 } | 63 } |
| 62 | 64 |
| 63 Future runSingleTests(ArgResults argResults) async { | 65 Future runSingleTests(ArgResults argResults) async { |
| 64 bool force = argResults['force']; | 66 bool force = argResults['force']; |
| 65 | 67 |
| 66 BuildbotClient client = argResults['logdog'] | 68 BuildbotClient client = argResults['logdog'] |
| 67 ? new LogdogBuildbotClient() | 69 ? new LogdogBuildbotClient() |
| 68 : new TestClient(force: force); | 70 : new TestClient(force: force); |
| 71 Bot bot = new Bot.internal(client); |
| 69 | 72 |
| 70 await runSingleTest( | 73 await runSingleTest( |
| 71 client, | 74 bot, |
| 72 'https://build.chromium.org/p/client.dart/builders/' | 75 'https://build.chromium.org/p/client.dart/builders/' |
| 73 'vm-kernel-linux-debug-x64-be/builds/1884/steps/' | 76 'vm-kernel-linux-debug-x64-be/builds/1884/steps/' |
| 74 'vm%20tests/logs/stdio', | 77 'vm%20tests/logs/stdio', |
| 75 1, | 78 1, |
| 76 { | 79 { |
| 77 1884: { | 80 1884: { |
| 78 'dartk-vm debug_x64 corelib_2/map_keys2_test': 'RuntimeError/Pass', | 81 'dartk-vm debug_x64 corelib_2/map_keys2_test': 'RuntimeError/Pass', |
| 79 }, | 82 }, |
| 80 1883: { | 83 1883: { |
| 81 'dartk-vm debug_x64 corelib_2/map_keys2_test': 'RuntimeError/Pass', | 84 'dartk-vm debug_x64 corelib_2/map_keys2_test': 'RuntimeError/Pass', |
| 82 'dartk-vm debug_x64 corelib_2/package_resource_test': | 85 'dartk-vm debug_x64 corelib_2/package_resource_test': |
| 83 'Pass/CompileTimeError', | 86 'Pass/CompileTimeError', |
| 84 }, | 87 }, |
| 85 }); | 88 }); |
| 86 | 89 |
| 87 client.close(); | 90 bot.close(); |
| 88 } | 91 } |
| 89 | 92 |
| 90 testSingleResults( | 93 testSingleResults( |
| 91 Map<int, Map<String, String>> expected, List<BuildResult> results) { | 94 Map<int, Map<String, String>> expected, List<BuildResult> results) { |
| 92 Expect.equals(expected.length, results.length); | 95 Expect.equals(expected.length, results.length); |
| 93 int i = 0; | 96 int i = 0; |
| 94 expected.forEach((int buildNumber, Map<String, String> failures) { | 97 expected.forEach((int buildNumber, Map<String, String> failures) { |
| 95 BuildResult result = results[i++]; | 98 BuildResult result = results[i++]; |
| 96 Expect.equals(failures.length, result.errors.length); | 99 Expect.equals(failures.length, result.errors.length); |
| 97 failures.forEach((String testName, String resultText) { | 100 failures.forEach((String testName, String resultText) { |
| 98 List<String> nameParts = split(testName, [' ', ' ']); | 101 List<String> nameParts = split(testName, [' ', ' ']); |
| 99 TestConfiguration id = | 102 TestConfiguration id = |
| 100 new TestConfiguration(nameParts[0], nameParts[1], nameParts[2]); | 103 new TestConfiguration(nameParts[0], nameParts[1], nameParts[2]); |
| 101 List<String> resultParts = split(resultText, ['/']); | 104 List<String> resultParts = split(resultText, ['/']); |
| 102 TestFailure failure = result.errors.singleWhere((t) => t.id == id); | 105 TestFailure failure = result.errors.singleWhere((t) => t.id == id); |
| 103 Expect.equals(failure.expected, resultParts[0]); | 106 Expect.equals(failure.expected, resultParts[0]); |
| 104 Expect.equals(failure.actual, resultParts[1]); | 107 Expect.equals(failure.actual, resultParts[1]); |
| 105 }); | 108 }); |
| 106 }); | 109 }); |
| 107 } | 110 } |
| 108 | 111 |
| 109 Future runSingleTest(BuildbotClient client, String testUri, int runCount, | 112 Future runSingleTest(Bot bot, String testUri, int runCount, |
| 110 Map<int, Map<String, String>> expectedResult) async { | 113 Map<int, Map<String, String>> expectedResult) async { |
| 111 print('Testing single compare-failures: $testUri runCount=$runCount'); | 114 print('Testing single compare-failures: $testUri runCount=$runCount'); |
| 112 Map<BuildUri, List<BuildResult>> buildResults = | 115 Map<BuildUri, List<BuildResult>> buildResults = |
| 113 await loadBuildResults(client, [testUri], runCount: runCount); | 116 await loadBuildResults(bot, [testUri], runCount: runCount); |
| 114 print('- checking results for ${buildResults.keys}'); | 117 print('- checking results for ${buildResults.keys}'); |
| 115 if (LOG) { | 118 if (LOG) { |
| 116 printBuildResultsSummary(buildResults, [testUri]); | 119 printBuildResultsSummary(buildResults, [testUri]); |
| 117 } | 120 } |
| 118 Expect.equals(1, buildResults.length); | 121 Expect.equals(1, buildResults.length); |
| 119 testSingleResults(expectedResult, buildResults.values.first); | 122 testSingleResults(expectedResult, buildResults.values.first); |
| 120 } | 123 } |
| OLD | NEW |