OLD | NEW |
1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /// Runs the tests in a batch on the various configurations used on the bots. | 5 /// Runs the tests in a batch on the various configurations used on the bots. |
6 | 6 |
7 import 'dart:async'; | 7 import 'dart:async'; |
8 import 'dart:io'; | 8 import 'dart:io'; |
9 | 9 |
10 import 'package:args/args.dart'; | 10 import 'package:args/args.dart'; |
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
51 "vm-kernel": [dartk, releaseMode, vm], | 51 "vm-kernel": [dartk, releaseMode, vm], |
52 "vm-precomp": [precompiler, precompiled], | 52 "vm-precomp": [precompiler, precompiled], |
53 "vm-product": [productMode, noCompiler, vm], | 53 "vm-product": [productMode, noCompiler, vm], |
54 // TODO(rnystrom): Add dart2js-d8-hostchecked, dart2js-d8-minified, or | 54 // TODO(rnystrom): Add dart2js-d8-hostchecked, dart2js-d8-minified, or |
55 // dart2js-jsshell? | 55 // dart2js-jsshell? |
56 "analyzer": [analyzer, noRuntime, useSdk], | 56 "analyzer": [analyzer, noRuntime, useSdk], |
57 "analyzer-checked": [analyzer, noRuntime, checked, useSdk], | 57 "analyzer-checked": [analyzer, noRuntime, checked, useSdk], |
58 "analyzer-checked-strong": [analyzer, noRuntime, checked, strong, useSdk], | 58 "analyzer-checked-strong": [analyzer, noRuntime, checked, strong, useSdk], |
59 "analyzer-strong": [analyzer, noRuntime, strong, useSdk], | 59 "analyzer-strong": [analyzer, noRuntime, strong, useSdk], |
60 "dart2js": [dart2js, chrome, useSdk, dart2jsBatch], | 60 "dart2js": [dart2js, chrome, useSdk, dart2jsBatch], |
61 "dart2js-d8-checked": [dart2js, d8, checked, fastStartup, useSdk, dart2jsBatch
], | 61 "dart2js-d8-checked": [ |
| 62 dart2js, |
| 63 d8, |
| 64 checked, |
| 65 fastStartup, |
| 66 useSdk, |
| 67 dart2jsBatch |
| 68 ], |
62 "dart2js-jsshell": [dart2js, jsshell, fastStartup, useSdk, dart2jsBatch], | 69 "dart2js-jsshell": [dart2js, jsshell, fastStartup, useSdk, dart2jsBatch], |
63 // TODO(rnystrom): Is it worth running dart2js on Firefox too? | 70 // TODO(rnystrom): Is it worth running dart2js on Firefox too? |
64 "dartdevc": [dartdevc, chrome, useSdk, strong], | 71 "dartdevc": [dartdevc, chrome, useSdk, strong], |
65 }; | 72 }; |
66 | 73 |
| 74 /// A subset of the configurations that run quickly and give a decent amount of |
| 75 /// coverage for the platforms that do not implement Dart 2.0 yet. |
| 76 final oneConfigs = const [ |
| 77 "vm", |
| 78 "vm-checked", |
| 79 "analyzer", |
| 80 "analyzer-checked", |
| 81 "dart2js", |
| 82 "dart2js-d8-checked", |
| 83 "dartdevc", |
| 84 ]; |
| 85 |
| 86 /// The configurations that should correctly implement Dart 2.0 (more or less) |
| 87 /// already. |
| 88 final twoConfigs = const [ |
| 89 "analyzer-checked-strong", |
| 90 "analyzer-strong", |
| 91 "dartdevc" |
| 92 ]; |
| 93 |
67 final buildSteps = [ | 94 final buildSteps = [ |
68 // The SDK, which also builds the VM. | 95 // The SDK, which also builds the VM. |
69 ["--mode=release", "create_sdk"], | 96 ["--mode=release", "create_sdk"], |
70 // The kernel service. | 97 // The kernel service. |
71 ["--mode=release", "kernel-service"], | 98 ["--mode=release", "kernel-service"], |
72 // Precompiled runtime for release | 99 // Precompiled runtime for release |
73 ["--mode=release", "runtime_precompiled"], | 100 ["--mode=release", "runtime_precompiled"], |
74 // Product version of the runtime and precompiled runtime. | 101 // Product version of the runtime and precompiled runtime. |
75 ["--mode=product", "runtime", "runtime_precompiled"], | 102 ["--mode=product", "runtime", "runtime_precompiled"], |
76 // Dartdevc and its dependencies. | 103 // Dartdevc and its dependencies. |
77 ["--mode=release", "dartdevc_test"], | 104 ["--mode=release", "dartdevc_test"], |
78 ]; | 105 ]; |
79 | 106 |
80 Future<Null> main(List<String> arguments) async { | 107 Future<Null> main(List<String> arguments) async { |
81 var argParser = new ArgParser(allowTrailingOptions: true); | 108 var argParser = new ArgParser(allowTrailingOptions: true); |
82 argParser.addFlag("build", help: "Build runtimes before running tests."); | 109 argParser.addFlag("build", help: "Build runtimes before running tests."); |
83 argParser.addOption("config", | 110 argParser.addOption("config", |
84 abbr: "c", allowMultiple: true, help: "Which configurations to run."); | 111 abbr: "c", allowMultiple: true, help: "Which configurations to run."); |
85 argParser.addFlag("help"); | 112 argParser.addFlag("help"); |
| 113 argParser.addFlag("1", |
| 114 abbr: "1", help: "Run some of the 1.0-supporting configurations."); |
| 115 argParser.addFlag("2", |
| 116 abbr: "2", help: "Run the 2.0-supporting configurations."); |
86 | 117 |
87 var argResults = argParser.parse(arguments); | 118 var argResults = argParser.parse(arguments); |
88 if (argResults["help"] as bool) { | 119 if (argResults["help"] as bool) { |
89 usage(argParser); | 120 usage(argParser); |
90 return; | 121 return; |
91 } | 122 } |
92 | 123 |
93 var remainingArgs = []..addAll(argResults.rest); | 124 String start; |
| 125 String end; |
94 | 126 |
95 if (remainingArgs.length == 1) { | 127 if (argResults.rest.length == 1) { |
96 remainingArgs.add(remainingArgs.first); | 128 // Just run a single test. |
97 } | 129 start = argResults.rest[0]; |
98 | 130 end = start; |
99 if (remainingArgs.length != 2) { | 131 } else if (argResults.rest.length == 2) { |
| 132 start = argResults.rest[0]; |
| 133 end = argResults.rest[1]; |
| 134 } else { |
100 usage(argParser); | 135 usage(argParser); |
101 exit(1); | 136 exit(1); |
102 } | 137 } |
103 | 138 |
104 var build = argResults["build"] as bool; | 139 var build = argResults["build"] as bool; |
105 var configs = argResults["config"] as List<String>; | 140 var configs = argResults["config"] as List<String>; |
106 if (configs.isEmpty) configs = allConfigs.keys.toList(); | 141 |
| 142 if (argResults["1"] as bool) { |
| 143 configs.addAll(oneConfigs); |
| 144 } |
| 145 |
| 146 if (argResults["2"] as bool) { |
| 147 configs.addAll(twoConfigs); |
| 148 } |
| 149 |
| 150 if (configs.isEmpty) configs.addAll(allConfigs.keys); |
107 | 151 |
108 var tests = scanTests(); | 152 var tests = scanTests(); |
109 | 153 |
110 var startIndex = findFork(tests, remainingArgs[0]); | 154 var startIndex = findFork(tests, start); |
111 var endIndex = findFork(tests, remainingArgs[1]); | 155 var endIndex = findFork(tests, end); |
112 | 156 |
113 if (startIndex == null || endIndex == null) exit(1); | 157 if (startIndex == null || endIndex == null) exit(1); |
114 | 158 |
115 tests = tests.sublist(startIndex, endIndex + 1); | 159 tests = tests.sublist(startIndex, endIndex + 1); |
116 | 160 |
117 if (tests.isEmpty) { | 161 if (tests.isEmpty) { |
118 print("No tests in range."); | 162 print("No tests in range."); |
119 return; | 163 return; |
120 } | 164 } |
121 | 165 |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
158 args.add("$selector/${paths.join('|')}"); | 202 args.add("$selector/${paths.join('|')}"); |
159 }); | 203 }); |
160 | 204 |
161 var exitCode = await run("tools/test.py", args); | 205 var exitCode = await run("tools/test.py", args); |
162 if (exitCode != 0) { | 206 if (exitCode != 0) { |
163 print(red("Configuration failed: $name")); | 207 print(red("Configuration failed: $name")); |
164 failed.add(name); | 208 failed.add(name); |
165 } else { | 209 } else { |
166 passed.add(name); | 210 passed.add(name); |
167 } | 211 } |
| 212 |
| 213 print(""); |
168 } | 214 } |
169 | 215 |
170 if (failed.length == 0) { | 216 if (failed.length == 0) { |
171 var s = passed.length == 1 ? "" : "s"; | 217 var s = passed.length == 1 ? "" : "s"; |
172 print("${green('PASSED')} all ${bold(passed.length)} configuration$s!"); | 218 print("${green('PASSED')} all ${bold(passed.length)} configuration$s!"); |
173 } else { | 219 } else { |
174 if (passed.length > 0) { | 220 if (passed.length > 0) { |
175 var s = passed == 1 ? "" : "s"; | 221 var s = passed == 1 ? "" : "s"; |
176 print("${green('PASSED')} ${bold(passed.length)} configuration$s:"); | 222 print("${green('PASSED')} ${bold(passed.length)} configuration$s:"); |
177 for (var config in passed) { | 223 for (var config in passed) { |
178 print("- ${bold(config)}"); | 224 print("- ${bold(config)}"); |
179 } | 225 } |
180 } | 226 } |
181 | 227 |
182 var s = failed == 1 ? "" : "s"; | 228 var s = failed == 1 ? "" : "s"; |
183 print("${red("FAILED")} ${bold(failed.length)} configuration$s:"); | 229 print("${red("FAILED")} ${bold(failed.length)} configuration$s:"); |
184 for (var config in failed) { | 230 for (var config in failed) { |
185 print("- ${bold(config)}"); | 231 print("- ${bold(config)}"); |
186 } | 232 } |
187 } | 233 } |
188 } | 234 } |
189 | 235 |
190 void usage(ArgParser parser) { | 236 void usage(ArgParser parser) { |
191 print("Usage: dart run_tests.dart [--build] [--configs=...] " | 237 print("Usage: dart run_tests.dart [--build] [-2] [-1] [--configs=...]" |
192 "<first file> [last file]"); | 238 "<first file> [last file]"); |
193 print("\n"); | 239 print("\n"); |
194 print("Example:"); | 240 print("Example:"); |
195 print("\n"); | 241 print("\n"); |
196 print(" \$ dart run_tests.dart map_to_string queue"); | 242 print(" \$ dart run_tests.dart map_to_string queue"); |
197 print("\n"); | 243 print("\n"); |
198 print(parser.usage); | 244 print(parser.usage); |
199 } | 245 } |
200 | 246 |
201 Future<int> run(String executable, List<String> arguments) async { | 247 Future<int> run(String executable, List<String> arguments) async { |
202 var process = await Process.start(executable, arguments); | 248 var process = await Process.start(executable, arguments); |
203 process.stdout.listen((bytes) { | 249 process.stdout.listen((bytes) { |
204 stdout.add(bytes); | 250 stdout.add(bytes); |
205 }); | 251 }); |
206 | 252 |
207 process.stderr.listen((bytes) { | 253 process.stderr.listen((bytes) { |
208 stderr.add(bytes); | 254 stderr.add(bytes); |
209 }); | 255 }); |
210 | 256 |
211 return await process.exitCode; | 257 return await process.exitCode; |
212 } | 258 } |
OLD | NEW |