OLD | NEW |
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 import 'dart:io'; | 5 import 'dart:io'; |
6 | |
7 import 'configuration.dart'; | |
8 import 'drt_updater.dart'; | 6 import 'drt_updater.dart'; |
| 7 import 'test_suite.dart'; |
9 import 'path.dart'; | 8 import 'path.dart'; |
10 import 'test_suite.dart'; | 9 import 'compiler_configuration.dart' show CompilerConfiguration; |
| 10 import 'runtime_configuration.dart' show RuntimeConfiguration; |
11 | 11 |
12 const _defaultTestSelectors = const [ | 12 const _defaultTestSelectors = const [ |
13 'samples', | 13 'samples', |
14 'standalone', | 14 'standalone', |
15 'corelib', | 15 'corelib', |
16 'co19', | 16 'co19', |
17 'language', | 17 'language', |
18 'isolate', | 18 'isolate', |
19 'vm', | 19 'vm', |
20 'html', | 20 'html', |
21 'benchmark_smoke', | 21 'benchmark_smoke', |
22 'utils', | 22 'utils', |
23 'lib', | 23 'lib', |
24 'analyze_library', | 24 'analyze_library', |
25 'service', | 25 'service', |
26 'kernel', | 26 'kernel', |
27 'observatory_ui' | 27 'observatory_ui' |
28 ]; | 28 ]; |
29 | 29 |
30 /// Specifies a single command line option. | 30 /// Specifies a single command line option. |
31 /// | 31 /// |
32 /// The name of the specification is used as the key for the option in the Map | 32 /// The name of the specification is used as the key for the option in the Map |
33 /// returned from the [TestOptionParser] parse method. | 33 /// returned from the [TestOptionParser] parse method. |
34 class _Option { | 34 class _Option { |
35 // TODO(rnystrom): Some string options use "" to mean "no value" and others | 35 // TODO(rnystrom): Some string options use "" to mean "no value" and others |
36 // use null. Clean that up. | 36 // use null. Clean that up. |
37 _Option(this.name, this.description, | 37 _Option(this.name, this.description, |
38 {String abbr, List<String> values, String defaultsTo}) | 38 {String abbr, List<String> values, String defaultsTo = ""}) |
39 : abbreviation = abbr, | 39 : abbreviation = abbr, |
40 values = values ?? [], | 40 values = values ?? [], |
41 defaultValue = defaultsTo, | 41 defaultValue = defaultsTo, |
42 type = _OptionValueType.string; | 42 type = _OptionValueType.string; |
43 | 43 |
44 _Option.bool(this.name, this.description, [this.abbreviation]) | 44 _Option.bool(this.name, this.description, [this.abbreviation]) |
45 : values = [], | 45 : values = [], |
46 defaultValue = false, | 46 defaultValue = false, |
47 type = _OptionValueType.bool; | 47 type = _OptionValueType.bool; |
48 | 48 |
(...skipping 18 matching lines...) Expand all Loading... |
67 String get command => "--${name.replaceAll('_', '-')}"; | 67 String get command => "--${name.replaceAll('_', '-')}"; |
68 } | 68 } |
69 | 69 |
70 enum _OptionValueType { bool, int, string } | 70 enum _OptionValueType { bool, int, string } |
71 | 71 |
72 /// Parses command line arguments and produces a test runner configuration. | 72 /// Parses command line arguments and produces a test runner configuration. |
73 class OptionsParser { | 73 class OptionsParser { |
74 static final List<_Option> _options = [ | 74 static final List<_Option> _options = [ |
75 new _Option('mode', 'Mode in which to run the tests.', | 75 new _Option('mode', 'Mode in which to run the tests.', |
76 abbr: 'm', | 76 abbr: 'm', |
77 values: ['all']..addAll(Mode.names), | 77 values: ['all', 'debug', 'release', 'product'], |
78 defaultsTo: Mode.debug.name), | 78 defaultsTo: 'debug'), |
79 new _Option( | 79 new _Option( |
80 'compiler', | 80 'compiler', |
81 '''Specify any compilation step (if needed). | 81 '''Specify any compilation step (if needed). |
82 | 82 |
83 none: Do not compile the Dart code (run native Dart code | 83 none: Do not compile the Dart code (run native Dart code |
84 on the VM). | 84 on the VM). |
85 (Only valid with runtimes vm, flutter, or drt.) | 85 (Only valid with runtimes vm, flutter, or drt.) |
86 | 86 |
87 precompiler: Compile into AOT snapshot before running the test. | 87 precompiler: Compile into AOT snapshot before running the test. |
88 (Only valid with runtime dart_precompiled.) | 88 (Only valid with runtime dart_precompiled.) |
(...skipping 11 matching lines...) Expand all Loading... |
100 running test. | 100 running test. |
101 (Only valid with dart_app runtime.) | 101 (Only valid with dart_app runtime.) |
102 | 102 |
103 dartk: Compile the Dart source into Kernel before running | 103 dartk: Compile the Dart source into Kernel before running |
104 test. | 104 test. |
105 | 105 |
106 dartkp: Compile the Dart source into Kernel and then Kernel | 106 dartkp: Compile the Dart source into Kernel and then Kernel |
107 into AOT snapshot before running the test. | 107 into AOT snapshot before running the test. |
108 (Only valid with runtime dart_precompiled.)''', | 108 (Only valid with runtime dart_precompiled.)''', |
109 abbr: 'c', | 109 abbr: 'c', |
110 values: Compiler.names, | 110 values: [ |
111 defaultsTo: Compiler.none.name), | 111 'none', |
| 112 'precompiler', |
| 113 'dart2js', |
| 114 'dart2analyzer', |
| 115 'app_jit', |
| 116 'dartk', |
| 117 'dartkp' |
| 118 ], |
| 119 defaultsTo: 'none'), |
112 new _Option( | 120 new _Option( |
113 'runtime', | 121 'runtime', |
114 '''Where the tests should be run. | 122 '''Where the tests should be run. |
115 vm: Run Dart code on the standalone dart vm. | 123 vm: Run Dart code on the standalone dart vm. |
116 | 124 |
117 flutter: Run Dart code on the flutter engine. | 125 flutter: Run Dart code on the flutter engine. |
118 | 126 |
119 dart_precompiled: Run a precompiled snapshot on a variant of the | 127 dart_precompiled: Run a precompiled snapshot on a variant of the |
120 standalone dart VM lacking a JIT. | 128 standalone dart VM lacking a JIT. |
121 | 129 |
(...skipping 23 matching lines...) Expand all Loading... |
145 | 153 |
146 self_check: Pass each test or its compiled output to every | 154 self_check: Pass each test or its compiled output to every |
147 file under `pkg` whose name ends with | 155 file under `pkg` whose name ends with |
148 `_self_check.dart`. Each test is given to the | 156 `_self_check.dart`. Each test is given to the |
149 self_check tester as a filename on stdin using | 157 self_check tester as a filename on stdin using |
150 the batch-mode protocol. | 158 the batch-mode protocol. |
151 | 159 |
152 none: No runtime, compile only. (For example, used | 160 none: No runtime, compile only. (For example, used |
153 for dart2analyzer static analysis tests).''', | 161 for dart2analyzer static analysis tests).''', |
154 abbr: 'r', | 162 abbr: 'r', |
155 values: Runtime.names, | 163 values: [ |
156 defaultsTo: Runtime.vm.name), | 164 'vm', |
| 165 'flutter', |
| 166 'dart_precompiled', |
| 167 'd8', |
| 168 'jsshell', |
| 169 'drt', |
| 170 'dartium', |
| 171 'ff', |
| 172 'firefox', |
| 173 'chrome', |
| 174 'safari', |
| 175 'ie9', |
| 176 'ie10', |
| 177 'ie11', |
| 178 'opera', |
| 179 'chromeOnAndroid', |
| 180 'safarimobilesim', |
| 181 'ContentShellOnAndroid', |
| 182 'DartiumOnAndroid', |
| 183 'self_check', |
| 184 'none' |
| 185 ], |
| 186 defaultsTo: 'vm'), |
157 new _Option( | 187 new _Option( |
158 'arch', | 188 'arch', |
159 '''The architecture to run tests for. | 189 '''The architecture to run tests for. |
160 | 190 |
161 Allowed values are: | 191 Allowed values are: |
162 all | 192 all |
163 ia32, x64 | 193 ia32, x64 |
164 arm, armv6, armv5te, arm64, | 194 arm, armv6, armv5te, arm64, |
165 simarm, simarmv6, simarmv5te, simarm64, | 195 simarm, simarmv6, simarmv5te, simarm64, |
166 mips, simmips | 196 mips, simmips |
167 simdbc, simdbc64''', | 197 simdbc, simdbc64''', |
168 abbr: 'a', | 198 abbr: 'a', |
169 values: ['all']..addAll(Architecture.names), | 199 values: [ |
170 defaultsTo: Architecture.x64.name), | 200 'all', |
| 201 'ia32', |
| 202 'x64', |
| 203 'arm', |
| 204 'armv6', |
| 205 'armv5te', |
| 206 'arm64', |
| 207 'mips', |
| 208 'simarm', |
| 209 'simarmv6', |
| 210 'simarmv5te', |
| 211 'simarm64', |
| 212 'simmips', |
| 213 'simdbc', |
| 214 'simdbc64', |
| 215 ], |
| 216 defaultsTo: 'x64'), |
171 new _Option('system', 'The operating system to run tests on.', | 217 new _Option('system', 'The operating system to run tests on.', |
172 abbr: 's', values: System.names, defaultsTo: Platform.operatingSystem), | 218 abbr: 's', |
| 219 values: ['linux', 'macos', 'windows', 'android'], |
| 220 defaultsTo: Platform.operatingSystem), |
173 new _Option.bool('checked', 'Run tests in checked mode.'), | 221 new _Option.bool('checked', 'Run tests in checked mode.'), |
174 new _Option.bool('strong', 'Run tests in strong mode.'), | 222 new _Option.bool('strong', 'Run tests in strong mode.'), |
175 new _Option.bool('host_checked', 'Run compiler in checked mode.'), | 223 new _Option.bool('host_checked', 'Run compiler in checked mode.'), |
176 new _Option.bool('minified', 'Enable minification in the compiler.'), | 224 new _Option.bool('minified', 'Enable minification in the compiler.'), |
177 new _Option.bool( | 225 new _Option.bool( |
178 'csp', 'Run tests under Content Security Policy restrictions.'), | 226 'csp', 'Run tests under Content Security Policy restrictions.'), |
179 new _Option.bool( | 227 new _Option.bool( |
180 'fast_startup', 'Pass the --fast-startup flag to dart2js.'), | 228 'fast_startup', 'Pass the --fast-startup flag to dart2js.'), |
181 new _Option.bool('dart2js_with_kernel', | 229 new _Option.bool('dart2js_with_kernel', |
182 'Enable the internal pipeline in dart2js to use kernel.'), | 230 'Enable the internal pipeline in dart2js to use kernel.'), |
183 new _Option.bool('hot_reload', 'Run hot reload stress tests.'), | 231 new _Option.bool('hot_reload', 'Run hot reload stress tests.'), |
184 new _Option.bool( | 232 new _Option.bool( |
185 'hot_reload_rollback', 'Run hot reload rollback stress tests.'), | 233 'hot_reload_rollback', 'Run hot reload rollback stress tests.'), |
186 new _Option.bool('use_blobs', | 234 new _Option.bool('use_blobs', |
187 'Use mmap instead of shared libraries for precompilation.'), | 235 'Use mmap instead of shared libraries for precompilation.'), |
188 new _Option.int('timeout', 'Timeout in seconds.', abbr: 't'), | 236 new _Option.int('timeout', 'Timeout in seconds.', |
| 237 abbr: 't', defaultsTo: -1), |
189 new _Option( | 238 new _Option( |
190 'progress', | 239 'progress', |
191 '''Progress indication mode. | 240 '''Progress indication mode. |
192 | 241 |
193 Allowed values are: | 242 Allowed values are: |
194 compact, color, line, verbose, silent, status, buildbot, diff | 243 compact, color, line, verbose, silent, status, buildbot, diff |
195 ''', | 244 ''', |
196 abbr: 'p', | 245 abbr: 'p', |
197 values: Progress.names, | 246 values: [ |
198 defaultsTo: Progress.compact.name), | 247 'compact', |
199 new _Option('step_name', 'Step name for use by -pbuildbot.'), | 248 'color', |
| 249 'line', |
| 250 'verbose', |
| 251 'silent', |
| 252 'status', |
| 253 'buildbot', |
| 254 'diff' |
| 255 ], |
| 256 defaultsTo: 'compact'), |
| 257 new _Option('step_name', 'Step name for use by -pbuildbot.', |
| 258 defaultsTo: null), |
200 new _Option.bool('report', | 259 new _Option.bool('report', |
201 'Print a summary report of the number of tests, by expectation.'), | 260 'Print a summary report of the number of tests, by expectation.'), |
202 new _Option.int('tasks', 'The number of parallel tasks to run.', | 261 new _Option.int('tasks', 'The number of parallel tasks to run.', |
203 abbr: 'j', defaultsTo: Platform.numberOfProcessors), | 262 abbr: 'j', defaultsTo: Platform.numberOfProcessors), |
204 new _Option.int('shards', | 263 new _Option.int('shards', |
205 'The number of instances that the tests will be sharded over.', | 264 'The number of instances that the tests will be sharded over.', |
206 defaultsTo: 1), | 265 defaultsTo: 1), |
207 new _Option.int( | 266 new _Option.int( |
208 'shard', 'The index of this instance when running in sharded mode.', | 267 'shard', 'The index of this instance when running in sharded mode.', |
209 defaultsTo: 1), | 268 defaultsTo: 1), |
210 new _Option.bool('help', 'Print list of options.', 'h'), | 269 new _Option.bool('help', 'Print list of options.', 'h'), |
211 new _Option.bool('verbose', 'Verbose output.', 'v'), | 270 new _Option.bool('verbose', 'Verbose output.', 'v'), |
212 new _Option.bool('verify-ir', 'Verify kernel IR.'), | 271 new _Option.bool('verify-ir', 'Verify kernel IR.'), |
213 new _Option.bool('no-tree-shake', 'Disable kernel IR tree shaking.'), | 272 new _Option.bool('no-tree-shake', 'Disable kernel IR tree shaking.'), |
214 new _Option.bool('list', 'List tests only, do not run them.'), | 273 new _Option.bool('list', 'List tests only, do not run them.'), |
215 new _Option.bool('report_in_json', | 274 new _Option.bool('report_in_json', |
216 'When listing with --list, output result summary in JSON.'), | 275 'When listing with --list, output result summary in JSON.'), |
217 new _Option.bool('time', 'Print timing information after running tests.'), | 276 new _Option.bool('time', 'Print timing information after running tests.'), |
218 new _Option('dart', 'Path to dart executable.'), | 277 new _Option('dart', 'Path to dart executable.'), |
219 new _Option('flutter', 'Path to flutter executable.'), | 278 new _Option('flutter', 'Path to flutter executable.'), |
220 new _Option('drt', 'Path to content shell executable.'), | 279 new _Option( |
| 280 'drt', // TODO(antonm): fix the option name. |
| 281 'Path to content shell executable.'), |
221 new _Option('dartium', 'Path to Dartium Chrome executable.'), | 282 new _Option('dartium', 'Path to Dartium Chrome executable.'), |
222 new _Option('firefox', 'Path to firefox browser executable.'), | 283 new _Option('firefox', 'Path to firefox browser executable.'), |
223 new _Option('chrome', 'Path to chrome browser executable.'), | 284 new _Option('chrome', 'Path to chrome browser executable.'), |
224 new _Option('safari', 'Path to safari browser executable.'), | 285 new _Option('safari', 'Path to safari browser executable.'), |
225 new _Option.bool( | 286 new _Option.bool( |
226 'use_sdk', | 287 'use_sdk', |
227 '''Use compiler or runtime from the SDK. | 288 '''Use compiler or runtime from the SDK. |
228 | 289 |
229 Normally, the compiler or runtimes in PRODUCT_DIR is tested, with | 290 Normally, the compiler or runtimes in PRODUCT_DIR is tested, with |
230 this option, the compiler or runtime in PRODUCT_DIR/dart-sdk/bin | 291 this option, the compiler or runtime in PRODUCT_DIR/dart-sdk/bin |
231 is tested. | 292 is tested. |
232 | 293 |
233 (Note: currently only implemented for dart2js.)'''), | 294 (Note: currently only implemented for dart2js.)'''), |
234 // TODO(rnystrom): This does not appear to be used. Remove? | |
235 new _Option('build_directory', | 295 new _Option('build_directory', |
236 'The name of the build directory, where products are placed.'), | 296 'The name of the build directory, where products are placed.'), |
237 new _Option.bool('noBatch', 'Do not run tests in batch mode.', 'n'), | 297 new _Option.bool('noBatch', 'Do not run tests in batch mode.', 'n'), |
238 new _Option.bool('dart2js_batch', 'Run dart2js tests in batch mode.'), | 298 new _Option.bool('dart2js_batch', 'Run dart2js tests in batch mode.'), |
239 new _Option.bool( | 299 new _Option.bool( |
240 'append_logs', 'Do not delete old logs but rather append to them.'), | 300 'append_logs', 'Do not delete old logs but rather append to them.'), |
241 new _Option.bool('write_debug_log', | 301 new _Option.bool('write_debug_log', |
242 'Don\'t write debug messages to stdout but rather to a logfile.'), | 302 'Don\'t write debug messages to stdout but rather to a logfile.'), |
243 new _Option.bool('write_test_outcome_log', | 303 new _Option.bool('write_test_outcome_log', |
244 'Write test outcomes to a "${TestUtils.testOutcomeFileName}" file.'), | 304 'Write test outcomes to a "${TestUtils.testOutcomeFileName}" file.'), |
(...skipping 16 matching lines...) Expand all Loading... |
261 defaultsTo: 0), | 321 defaultsTo: 0), |
262 new _Option.int('test_server_cross_origin_port', | 322 new _Option.int('test_server_cross_origin_port', |
263 'Port for test http server cross origin.', | 323 'Port for test http server cross origin.', |
264 defaultsTo: 0), | 324 defaultsTo: 0), |
265 new _Option.int('test_driver_port', 'Port for http test driver server.', | 325 new _Option.int('test_driver_port', 'Port for http test driver server.', |
266 defaultsTo: 0), | 326 defaultsTo: 0), |
267 new _Option.int( | 327 new _Option.int( |
268 'test_driver_error_port', 'Port for http test driver server errors.', | 328 'test_driver_error_port', 'Port for http test driver server errors.', |
269 defaultsTo: 0), | 329 defaultsTo: 0), |
270 new _Option('record_to_file', | 330 new _Option('record_to_file', |
271 'Records all commands to be executed and writes to a file.'), | 331 'Records all commands to be executed and writes to a file.', |
| 332 defaultsTo: null), |
272 new _Option( | 333 new _Option( |
273 'replay_from_file', 'Replays a previously recorded list of commands.'), | 334 'replay_from_file', 'Replays a previously recorded list of commands.', |
| 335 defaultsTo: null), |
274 new _Option( | 336 new _Option( |
275 'builder_tag', | 337 'builder_tag', |
276 '''Machine specific options that is not captured by the regular test | 338 '''Machine specific options that is not captured by the regular test |
277 options. Used to be able to make sane updates to the status files.'''), | 339 options. Used to be able to make sane updates to the status files.'''), |
278 new _Option('vm_options', 'Extra options to send to the vm when running.'), | 340 new _Option('vm_options', 'Extra options to send to the vm when running.', |
| 341 defaultsTo: null), |
279 new _Option( | 342 new _Option( |
280 'dart2js_options', 'Extra options for dart2js compilation step.'), | 343 'dart2js_options', 'Extra options for dart2js compilation step.', |
| 344 defaultsTo: null), |
281 new _Option( | 345 new _Option( |
282 'suite_dir', 'Additional directory to add to the testing matrix.'), | 346 'suite_dir', 'Additional directory to add to the testing matrix.', |
283 new _Option('package_root', 'The package root to use for testing.'), | 347 defaultsTo: null), |
284 new _Option('packages', 'The package spec file to use for testing.'), | 348 new _Option('package_root', 'The package root to use for testing.', |
| 349 defaultsTo: null), |
| 350 new _Option('packages', 'The package spec file to use for testing.', |
| 351 defaultsTo: null), |
285 new _Option( | 352 new _Option( |
286 'exclude_suite', | 353 'exclude_suite', |
287 '''Exclude suites from default selector, only works when no selector | 354 '''Exclude suites from default selector, only works when no selector |
288 has been specified on the command line.'''), | 355 has been specified on the command line.''', |
| 356 defaultsTo: null), |
289 new _Option.bool( | 357 new _Option.bool( |
290 'skip_compilation', | 358 'skip-compilation', |
291 ''' | 359 ''' |
292 Skip the compilation step, using the compilation artifacts left in | 360 Skip the compilation step, using the compilation artifacts left in |
293 the output folder from a previous run. This flag will often cause | 361 the output folder from a previous run. This flag will often cause |
294 false positves and negatives, but can be useful for quick and | 362 false positves and negatives, but can be useful for quick and |
295 dirty offline testing when not making changes that affect the | 363 dirty offline testing when not making changes that affect the |
296 compiler.''') | 364 compiler.''') |
297 ]; | 365 ]; |
298 | 366 |
299 /// For printing out reproducing command lines, we don't want to add these | 367 /// For printing out reproducing command lines, we don't want to add these |
300 /// options. | 368 /// options. |
(...skipping 21 matching lines...) Expand all Loading... |
322 'write_debug_log', | 390 'write_debug_log', |
323 'write_test_outcome_log', | 391 'write_test_outcome_log', |
324 ].toSet(); | 392 ].toSet(); |
325 | 393 |
326 /// Parses a list of strings as test options. | 394 /// Parses a list of strings as test options. |
327 /// | 395 /// |
328 /// Returns a list of configurations in which to run the tests. | 396 /// Returns a list of configurations in which to run the tests. |
329 /// Configurations are maps mapping from option keys to values. When | 397 /// Configurations are maps mapping from option keys to values. When |
330 /// encountering the first non-option string, the rest of the arguments are | 398 /// encountering the first non-option string, the rest of the arguments are |
331 /// stored in the returned Map under the 'rest' key. | 399 /// stored in the returned Map under the 'rest' key. |
332 List<Configuration> parse(List<String> arguments) { | 400 List<Map<String, dynamic>> parse(List<String> arguments) { |
333 // TODO(rnystrom): The builders on the buildbots still pass this even | 401 // TODO(rnystrom): The builders on the buildbots still pass this even |
334 // though it does nothing. Until those can be fixed, silently ignore the | 402 // though it does nothing. Until those can be fixed, silently ignore the |
335 // option. Remove this once the buildbot scripts are fixed. | 403 // option. Remove this once the buildbot scripts are fixed. |
336 if (arguments.contains("--failure-summary")) { | 404 if (arguments.contains("--failure-summary")) { |
337 arguments = arguments.where((arg) => arg != "--failure-summary").toList(); | 405 arguments = arguments.where((arg) => arg != "--failure-summary").toList(); |
338 print('Note: Ignoring unsupported "--failure-summary" option.'); | 406 print('Note: Ignoring unsupported "--failure-summary" option.'); |
339 } | 407 } |
340 | 408 |
341 var configuration = <String, dynamic>{}; | 409 var configuration = <String, dynamic>{}; |
342 | 410 |
(...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
442 } | 510 } |
443 } | 511 } |
444 | 512 |
445 // Apply default values for unspecified options. | 513 // Apply default values for unspecified options. |
446 for (var option in _options) { | 514 for (var option in _options) { |
447 if (!configuration.containsKey(option.name)) { | 515 if (!configuration.containsKey(option.name)) { |
448 configuration[option.name] = option.defaultValue; | 516 configuration[option.name] = option.defaultValue; |
449 } | 517 } |
450 } | 518 } |
451 | 519 |
452 return _createConfigurations(configuration); | 520 var expandedConfigs = _expandConfigurations(configuration); |
| 521 var result = expandedConfigs.where(_isValidConfig).toList(); |
| 522 for (var config in result) { |
| 523 config['_reproducing_arguments_'] = _reproducingCommand(config); |
| 524 } |
| 525 |
| 526 return result.isEmpty ? null : result; |
453 } | 527 } |
454 | 528 |
455 /// Prints [message] and exits with a non-zero exit code. | 529 /// Prints [message] and exits with a non-zero exit code. |
456 void _fail(String message) { | 530 void _fail(String message) { |
457 print(message); | 531 print(message); |
458 exit(1); | 532 exit(1); |
459 } | 533 } |
460 | 534 |
461 /// Given a set of parsed option values, returns the list of command line | 535 /// Given a configuration, returns the list of command line arguments that |
462 /// arguments that would reproduce that configuration. | 536 /// would reproduce that configuration. |
463 List<String> _reproducingCommand(Map<String, dynamic> data) { | 537 List<String> _reproducingCommand(Map config) { |
464 var arguments = <String>[]; | 538 var arguments = <String>[]; |
465 | 539 |
466 for (var option in _options) { | 540 for (var option in _options) { |
467 var name = option.name; | 541 var name = option.name; |
468 if (!data.containsKey(name) || _blacklistedOptions.contains(name)) { | 542 if (!config.containsKey(name) || _blacklistedOptions.contains(name)) { |
469 continue; | 543 continue; |
470 } | 544 } |
471 | 545 |
472 var value = data[name]; | 546 var value = config[name]; |
473 if (data[name] == option.defaultValue || | 547 if (config[name] == option.defaultValue || |
474 (name == 'packages' && | 548 (name == 'packages' && |
475 value == | 549 value == |
476 TestUtils.dartDirUri.resolve('.packages').toFilePath())) { | 550 TestUtils.dartDirUri.resolve('.packages').toFilePath())) { |
477 continue; | 551 continue; |
478 } | 552 } |
479 | 553 |
480 arguments.add(option.shortCommand); | 554 arguments.add(option.shortCommand); |
481 if (option.type != _OptionValueType.bool) { | 555 if (option.type != _OptionValueType.bool) { |
482 arguments.add(value.toString()); | 556 arguments.add(value.toString()); |
483 } | 557 } |
484 } | 558 } |
485 | 559 |
486 return arguments; | 560 return arguments; |
487 } | 561 } |
488 | 562 |
489 List<Configuration> _createConfigurations( | 563 /// Determines if a particular configuration has a valid combination of |
| 564 /// compiler and runtime elements. |
| 565 bool _isValidConfig(Map config) { |
| 566 var isValid = true; |
| 567 List<String> validRuntimes; |
| 568 switch (config['compiler'] as String) { |
| 569 case 'dart2js': |
| 570 // Note: by adding 'none' as a configuration, if the user |
| 571 // runs test.py -c dart2js -r drt,none the dart2js_none and |
| 572 // dart2js_drt will be duplicating work. If later we don't need 'none' |
| 573 // with dart2js, we should remove it from here. |
| 574 validRuntimes = const [ |
| 575 'd8', |
| 576 'jsshell', |
| 577 'drt', |
| 578 'none', |
| 579 'dartium', |
| 580 'ff', |
| 581 'chrome', |
| 582 'safari', |
| 583 'ie9', |
| 584 'ie10', |
| 585 'ie11', |
| 586 'opera', |
| 587 'chromeOnAndroid', |
| 588 'safarimobilesim' |
| 589 ]; |
| 590 break; |
| 591 case 'dart2analyzer': |
| 592 validRuntimes = const ['none']; |
| 593 break; |
| 594 case 'app_jit': |
| 595 case 'dartk': |
| 596 validRuntimes = const ['vm', 'self_check', 'none']; |
| 597 break; |
| 598 case 'precompiler': |
| 599 case 'dartkp': |
| 600 validRuntimes = const ['dart_precompiled']; |
| 601 break; |
| 602 case 'none': |
| 603 validRuntimes = const [ |
| 604 'vm', |
| 605 'flutter', |
| 606 'drt', |
| 607 'dartium', |
| 608 'ContentShellOnAndroid', |
| 609 'DartiumOnAndroid' |
| 610 ]; |
| 611 break; |
| 612 } |
| 613 |
| 614 if (!validRuntimes.contains(config['runtime'])) { |
| 615 isValid = false; |
| 616 print("Warning: combination of compiler '${config['compiler']}' and " |
| 617 "runtime '${config['runtime']}' is invalid. " |
| 618 "Skipping this combination."); |
| 619 } |
| 620 |
| 621 if ((config['ie'] as bool) && Platform.operatingSystem != 'windows') { |
| 622 isValid = false; |
| 623 print("Warning: cannot run Internet Explorer on non-Windows operating" |
| 624 " system."); |
| 625 } |
| 626 |
| 627 if ((config['shard'] as int) < 1 || |
| 628 (config['shard'] as int) > (config['shards'] as int)) { |
| 629 isValid = false; |
| 630 print("Error: shard index is ${config['shard']} out of " |
| 631 "${config['shards']} shards"); |
| 632 } |
| 633 |
| 634 if (config['runtime'] == 'flutter' && config['flutter'] == '') { |
| 635 isValid = false; |
| 636 print("-rflutter requires the flutter engine executable to " |
| 637 "be specified using --flutter="); |
| 638 } |
| 639 |
| 640 if (config['runtime'] == 'flutter' && config['arch'] != 'x64') { |
| 641 isValid = false; |
| 642 print("-rflutter is applicable only for --arch=x64"); |
| 643 } |
| 644 |
| 645 return isValid; |
| 646 } |
| 647 |
| 648 /// Recursively expands a configuration with multiple values per key into a |
| 649 /// list of configurations with exactly one value per key. |
| 650 List<Map<String, dynamic>> _expandConfigurations( |
490 Map<String, dynamic> configuration) { | 651 Map<String, dynamic> configuration) { |
491 var selectors = _expandSelectors(configuration); | 652 // Expand the pseudo-values such as 'all'. |
| 653 if (configuration['arch'] == 'all') { |
| 654 configuration['arch'] = 'ia32,x64,simarm,simarm64,simmips,simdbc64'; |
| 655 } |
| 656 |
| 657 if (configuration['mode'] == 'all') { |
| 658 configuration['mode'] = 'debug,release,product'; |
| 659 } |
| 660 |
| 661 if (configuration['report_in_json'] as bool) { |
| 662 configuration['list'] = true; |
| 663 configuration['report'] = true; |
| 664 } |
| 665 |
| 666 // Use verbose progress indication for verbose output unless buildbot |
| 667 // progress indication is requested. |
| 668 if ((configuration['verbose'] as bool) && |
| 669 (configuration['progress'] as String) != 'buildbot') { |
| 670 configuration['progress'] = 'verbose'; |
| 671 } |
| 672 |
| 673 // Create the artificial negative options that test status files |
| 674 // expect. |
| 675 configuration['unchecked'] = !(configuration['checked'] as bool); |
| 676 configuration['host_unchecked'] = !(configuration['host_checked'] as bool); |
| 677 configuration['unminified'] = !(configuration['minified'] as bool); |
| 678 configuration['nocsp'] = !(configuration['csp'] as bool); |
| 679 |
| 680 var runtime = configuration['runtime'] as String; |
| 681 if (runtime == 'firefox') { |
| 682 configuration['runtime'] == 'ff'; |
| 683 } |
| 684 |
| 685 var compiler = configuration['compiler'] as String; |
| 686 configuration['browser'] = TestUtils.isBrowserRuntime(runtime); |
| 687 configuration['analyzer'] = TestUtils.isCommandLineAnalyzer(compiler); |
| 688 |
| 689 // Set the javascript command line flag for less verbose status files. |
| 690 configuration['jscl'] = TestUtils.isJsCommandLineRuntime(runtime); |
| 691 |
| 692 // Allow suppression that is valid for all ie versions |
| 693 configuration['ie'] = runtime.startsWith('ie'); |
| 694 |
| 695 // Expand the test selectors into a suite name and a simple |
| 696 // regular expressions to be used on the full path of a test file |
| 697 // in that test suite. If no selectors are explicitly given use |
| 698 // the default suite patterns. |
| 699 var selectors = configuration['selectors']; |
| 700 if (selectors is! Map) { |
| 701 if (selectors == null) { |
| 702 if (configuration['suite_dir'] != null) { |
| 703 var suite_path = new Path(configuration['suite_dir'] as String); |
| 704 selectors = [suite_path.filename]; |
| 705 } else { |
| 706 selectors = _defaultTestSelectors.toList(); |
| 707 } |
| 708 |
| 709 var excludeSuites = configuration['exclude_suite'] != null |
| 710 ? configuration['exclude_suite'].split(',') |
| 711 : []; |
| 712 for (var exclude in excludeSuites) { |
| 713 if ((selectors as List).contains(exclude)) { |
| 714 selectors.remove(exclude); |
| 715 } else { |
| 716 print("Warning: default selectors does not contain $exclude"); |
| 717 } |
| 718 } |
| 719 } |
| 720 var selectorMap = <String, RegExp>{}; |
| 721 for (var i = 0; i < (selectors as List).length; i++) { |
| 722 var pattern = selectors[i] as String; |
| 723 var suite = pattern; |
| 724 var slashLocation = pattern.indexOf('/'); |
| 725 if (slashLocation != -1) { |
| 726 suite = pattern.substring(0, slashLocation); |
| 727 pattern = pattern.substring(slashLocation + 1); |
| 728 pattern = pattern.replaceAll('*', '.*'); |
| 729 } else { |
| 730 pattern = ".?"; |
| 731 } |
| 732 if (selectorMap.containsKey(suite)) { |
| 733 print("Error: '$suite/$pattern'. Only one test selection" |
| 734 " pattern is allowed to start with '$suite/'"); |
| 735 exit(1); |
| 736 } |
| 737 selectorMap[suite] = new RegExp(pattern); |
| 738 } |
| 739 configuration['selectors'] = selectorMap; |
| 740 } |
492 | 741 |
493 // Put observatory_ui in a configuration with its own packages override. | 742 // Put observatory_ui in a configuration with its own packages override. |
494 // Only one value in the configuration map is mutable: | 743 // Only one value in the configuration map is mutable: |
495 if (selectors.containsKey('observatory_ui')) { | 744 selectors = configuration['selectors']; |
| 745 if ((selectors as Map<String, dynamic>).containsKey('observatory_ui')) { |
496 if (selectors.length == 1) { | 746 if (selectors.length == 1) { |
497 configuration['packages'] = TestUtils.dartDirUri | 747 configuration['packages'] = TestUtils.dartDirUri |
498 .resolve('runtime/observatory/.packages') | 748 .resolve('runtime/observatory/.packages') |
499 .toFilePath(); | 749 .toFilePath(); |
500 } else { | 750 } else { |
501 // Make a new configuration whose selectors map only contains | 751 // Make a new configuration whose selectors map only contains |
502 // observatory_ui, and remove observatory_ui from the original | 752 // observatory_ui, and remove the key from the original selectors. |
503 // selectors. The only mutable value in the map is the selectors, so a | 753 // The only mutable value in the map is the selectors, so a |
504 // shallow copy is safe. | 754 // shallow copy is safe. |
505 var observatoryConfiguration = | 755 var observatoryConfiguration = |
506 new Map<String, dynamic>.from(configuration); | 756 new Map<String, dynamic>.from(configuration); |
507 var observatorySelectors = { | 757 observatoryConfiguration['selectors'] = { |
508 'observatory_ui': selectors['observatory_ui'] | 758 'observatory_ui': selectors['observatory_ui'] |
509 }; | 759 }; |
510 selectors.remove('observatory_ui'); | 760 selectors.remove('observatory_ui'); |
511 | 761 |
512 // Set the packages flag. | 762 // Set the packages flag. |
513 observatoryConfiguration['packages'] = TestUtils.dartDirUri | 763 observatoryConfiguration['packages'] = TestUtils.dartDirUri |
514 .resolve('runtime/observatory/.packages') | 764 .resolve('runtime/observatory/.packages') |
515 .toFilePath(); | 765 .toFilePath(); |
516 | 766 |
517 return _expandConfigurations(configuration, selectors) | 767 // Return the expansions of both configurations. Neither will reach |
518 ..addAll(_expandConfigurations( | 768 // this line in the recursive call to _expandConfigurations. |
519 observatoryConfiguration, observatorySelectors)); | 769 return _expandConfigurations(configuration) |
| 770 ..addAll(_expandConfigurations(observatoryConfiguration)); |
520 } | 771 } |
521 } | 772 } |
522 | 773 |
523 return _expandConfigurations(configuration, selectors); | 774 // Set the default package spec explicitly. |
524 } | 775 if (configuration['package_root'] == null && |
525 | 776 configuration['packages'] == null) { |
526 /// Recursively expands a configuration with multiple values per key into a | 777 configuration['packages'] = |
527 /// list of configurations with exactly one value per key. | 778 TestUtils.dartDirUri.resolve('.packages').toFilePath(); |
528 List<Configuration> _expandConfigurations( | |
529 Map<String, dynamic> data, Map<String, RegExp> selectors) { | |
530 var result = <Configuration>[]; | |
531 | |
532 // Handles a string option containing a space-separated list of words. | |
533 listOption(String name) { | |
534 var value = data[name] as String; | |
535 if (value == null) return const <String>[]; | |
536 return value | |
537 .split(" ") | |
538 .map((s) => s.trim()) | |
539 .where((s) => s.isNotEmpty) | |
540 .toList(); | |
541 } | 779 } |
542 | 780 |
543 var dart2jsOptions = listOption("dart2js_options"); | 781 // Expand the architectures. |
544 var vmOptions = listOption("vm_options"); | 782 if ((configuration['arch'] as String).contains(',')) { |
545 | 783 return _expandHelper('arch', configuration); |
546 // JSON reporting implies listing and reporting. | |
547 if (data['report_in_json'] as bool) { | |
548 data['list'] = true; | |
549 data['report'] = true; | |
550 } | 784 } |
551 | 785 |
552 // Use verbose progress indication for verbose output unless buildbot | 786 // Expand modes. |
553 // progress indication is requested. | 787 if ((configuration['mode'] as String).contains(',')) { |
554 if ((data['verbose'] as bool) && | 788 return _expandHelper('mode', configuration); |
555 (data['progress'] as String) != 'buildbot') { | 789 } |
556 data['progress'] = 'verbose'; | 790 |
| 791 // Expand compilers. |
| 792 if ((configuration['compiler'] as String).contains(',')) { |
| 793 return _expandHelper('compiler', configuration); |
557 } | 794 } |
558 | 795 |
559 // Expand runtimes. | 796 // Expand runtimes. |
560 for (var runtimeName in (data["runtime"] as String).split(",")) { | 797 var runtimes = configuration['runtime'] as String; |
561 var runtime = Runtime.find(runtimeName); | 798 if (runtimes.contains(',')) { |
562 | 799 return _expandHelper('runtime', configuration); |
563 // Install the runtime if needed. | 800 } else { |
564 var updater = runtimeUpdater( | 801 // All runtimes eventually go through this path, after expansion. |
565 runtime, data["drt"] as String, data["dartium"] as String); | 802 var updater = runtimeUpdater(configuration); |
566 if (updater != null) { | 803 if (updater != null) { |
567 updater.update(); | 804 updater.update(); |
568 } | 805 } |
569 | |
570 // Expand architectures. | |
571 var architectures = data["arch"] as String; | |
572 if (architectures == "all") { | |
573 architectures = "ia32,x64,simarm,simarm64,simmips,simdbc64"; | |
574 } | |
575 | |
576 for (var architectureName in architectures.split(",")) { | |
577 var architecture = Architecture.find(architectureName); | |
578 | |
579 // Expand compilers. | |
580 var compilers = data["compiler"] as String; | |
581 for (var compilerName in compilers.split(",")) { | |
582 var compiler = Compiler.find(compilerName); | |
583 | |
584 // Expand modes. | |
585 var modes = data["mode"] as String; | |
586 if (modes == "all") modes = "debug,release,product"; | |
587 for (var modeName in modes.split(",")) { | |
588 var mode = Mode.find(modeName); | |
589 | |
590 var configuration = new Configuration( | |
591 architecture: architecture, | |
592 compiler: compiler, | |
593 mode: mode, | |
594 progress: Progress.find(data["progress"] as String), | |
595 runtime: runtime, | |
596 system: System.find(data["system"] as String), | |
597 selectors: selectors, | |
598 appendLogs: data["append_logs"] as bool, | |
599 batch: !(data["noBatch"] as bool), | |
600 batchDart2JS: data["dart2js_batch"] as bool, | |
601 copyCoreDumps: data["copy_coredumps"] as bool, | |
602 hotReload: data["hot_reload"] as bool, | |
603 hotReloadRollback: data["hot_reload_rollback"] as bool, | |
604 isChecked: data["checked"] as bool, | |
605 isStrong: data["strong"] as bool, | |
606 isHostChecked: data["host_checked"] as bool, | |
607 isCsp: data["csp"] as bool, | |
608 isMinified: data["minified"] as bool, | |
609 isVerbose: data["verbose"] as bool, | |
610 listTests: data["list"] as bool, | |
611 printTiming: data["time"] as bool, | |
612 printReport: data["report"] as bool, | |
613 reportInJson: data["report_in_json"] as bool, | |
614 resetBrowser: data["reset_browser_configuration"] as bool, | |
615 skipCompilation: data["skip_compilation"] as bool, | |
616 useBlobs: data["use_blobs"] as bool, | |
617 useSdk: data["use_sdk"] as bool, | |
618 useFastStartup: data["fast_startup"] as bool, | |
619 useDart2JSWithKernel: data["dart2js_with_kernel"] as bool, | |
620 writeDebugLog: data["write_debug_log"] as bool, | |
621 writeTestOutcomeLog: data["write_test_outcome_log"] as bool, | |
622 drtPath: data["drt"] as String, | |
623 dartiumPath: data["dartium"] as String, | |
624 chromePath: data["chrome"] as String, | |
625 safariPath: data["safari"] as String, | |
626 firefoxPath: data["firefox"] as String, | |
627 dartPath: data["dart"] as String, | |
628 dartPrecompiledPath: data["dart_precompiled"] as String, | |
629 flutterPath: data["flutter"] as String, | |
630 recordingPath: data["record_to_file"] as String, | |
631 replayPath: data["replay_from_file"] as String, | |
632 taskCount: data["tasks"] as int, | |
633 timeout: data["timeout"] as int, | |
634 shardCount: data["shards"] as int, | |
635 shard: data["shard"] as int, | |
636 stepName: data["step_name"] as String, | |
637 testServerPort: data["test_server_port"] as int, | |
638 testServerCrossOriginPort: | |
639 data['test_server_cross_origin_port'] as int, | |
640 testDriverErrorPort: data["test_driver_error_port"] as int, | |
641 localIP: data["local_ip"] as String, | |
642 dart2jsOptions: dart2jsOptions, | |
643 vmOptions: vmOptions, | |
644 packages: data["packages"] as String, | |
645 packageRoot: data["package_root"] as String, | |
646 suiteDirectory: data["suite_dir"] as String, | |
647 builderTag: data["builder_tag"] as String, | |
648 reproducingArguments: _reproducingCommand(data)); | |
649 | |
650 if (configuration.validate()) { | |
651 result.add(configuration); | |
652 } | |
653 } | |
654 } | |
655 } | |
656 } | 806 } |
657 | 807 |
| 808 // Adjust default timeout based on mode, compiler, and sometimes runtime. |
| 809 if (configuration['timeout'] == -1) { |
| 810 var isReload = (configuration['hot_reload'] as bool) || |
| 811 (configuration['hot_reload_rollback'] as bool); |
| 812 int compilerMulitiplier = |
| 813 new CompilerConfiguration(configuration).computeTimeoutMultiplier(); |
| 814 int runtimeMultiplier = new RuntimeConfiguration(configuration) |
| 815 .computeTimeoutMultiplier( |
| 816 mode: configuration['mode'] as String, |
| 817 isChecked: configuration['checked'] as bool, |
| 818 isReload: isReload, |
| 819 arch: configuration['arch'] as String); |
| 820 configuration['timeout'] = 60 * compilerMulitiplier * runtimeMultiplier; |
| 821 } |
| 822 |
| 823 return [configuration]; |
| 824 } |
| 825 |
| 826 /// Helper for _expandConfigurations. Creates a new configuration and adds it |
| 827 /// to a list, for use in a case when a particular configuration has multiple |
| 828 /// results (separated by a ','). |
| 829 /// Arguments: |
| 830 /// option: The particular test option we are expanding. |
| 831 /// configuration: The map containing all test configuration information |
| 832 /// specified. |
| 833 List<Map<String, dynamic>> _expandHelper( |
| 834 String option, Map<String, dynamic> configuration) { |
| 835 var result = <Map<String, dynamic>>[]; |
| 836 var configs = configuration[option]; |
| 837 for (var config in configs.split(',')) { |
| 838 var newConfiguration = new Map<String, dynamic>.from(configuration); |
| 839 newConfiguration[option] = config; |
| 840 result.addAll(_expandConfigurations(newConfiguration)); |
| 841 } |
658 return result; | 842 return result; |
659 } | 843 } |
660 | 844 |
661 /// Expands the test selectors into a suite name and a simple regular | |
662 /// expression to be used on the full path of a test file in that test suite. | |
663 /// | |
664 /// If no selectors are explicitly given, uses the default suite patterns. | |
665 Map<String, RegExp> _expandSelectors(Map<String, dynamic> configuration) { | |
666 var selectors = configuration['selectors']; | |
667 | |
668 if (selectors == null) { | |
669 if (configuration['suite_dir'] != null) { | |
670 var suitePath = new Path(configuration['suite_dir'] as String); | |
671 selectors = [suitePath.filename]; | |
672 } else { | |
673 selectors = _defaultTestSelectors.toList(); | |
674 } | |
675 | |
676 var excludeSuites = configuration['exclude_suite'] != null | |
677 ? configuration['exclude_suite'].split(',') | |
678 : []; | |
679 for (var exclude in excludeSuites) { | |
680 if ((selectors as List).contains(exclude)) { | |
681 selectors.remove(exclude); | |
682 } else { | |
683 print("Warning: default selectors does not contain $exclude"); | |
684 } | |
685 } | |
686 } | |
687 | |
688 var selectorMap = <String, RegExp>{}; | |
689 for (var i = 0; i < (selectors as List).length; i++) { | |
690 var pattern = selectors[i] as String; | |
691 var suite = pattern; | |
692 var slashLocation = pattern.indexOf('/'); | |
693 if (slashLocation != -1) { | |
694 suite = pattern.substring(0, slashLocation); | |
695 pattern = pattern.substring(slashLocation + 1); | |
696 pattern = pattern.replaceAll('*', '.*'); | |
697 } else { | |
698 pattern = ".?"; | |
699 } | |
700 if (selectorMap.containsKey(suite)) { | |
701 print("Error: '$suite/$pattern'. Only one test selection" | |
702 " pattern is allowed to start with '$suite/'"); | |
703 exit(1); | |
704 } | |
705 selectorMap[suite] = new RegExp(pattern); | |
706 } | |
707 | |
708 return selectorMap; | |
709 } | |
710 | |
711 /// Print out usage information. | 845 /// Print out usage information. |
712 void _printHelp() { | 846 void _printHelp() { |
713 var buffer = new StringBuffer(); | 847 var buffer = new StringBuffer(); |
714 | 848 |
715 buffer.writeln('''usage: dart test.dart [options] [selector] | 849 buffer.writeln('''usage: dart test.dart [options] [selector] |
716 | 850 |
717 The optional selector limits the tests that will be run. | 851 The optional selector limits the tests that will be run. |
718 For example, the selector "language/issue", or equivalently | 852 For example, the selector "language/issue", or equivalently |
719 "language/*issue*", limits to test files matching the regexp | 853 "language/*issue*", limits to test files matching the regexp |
720 ".*issue.*\\.dart" in the "tests/language" directory. | 854 ".*issue.*\\.dart" in the "tests/language" directory. |
(...skipping 57 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
778 if (name == option.name) return option; | 912 if (name == option.name) return option; |
779 | 913 |
780 // Allow hyphens instead of underscores as the separator since they are | 914 // Allow hyphens instead of underscores as the separator since they are |
781 // more common for command line flags. | 915 // more common for command line flags. |
782 if (name == option.name.replaceAll("_", "-")) return option; | 916 if (name == option.name.replaceAll("_", "-")) return option; |
783 } | 917 } |
784 | 918 |
785 return null; | 919 return null; |
786 } | 920 } |
787 } | 921 } |
OLD | NEW |