OLD | NEW |
| (Empty) |
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | |
2 // for details. All rights reserved. Use of this source code is governed by a | |
3 // BSD-style license that can be found in the LICENSE file. | |
4 | |
5 library test_options_parser; | |
6 | |
7 import "dart:io"; | |
8 import "drt_updater.dart"; | |
9 import "test_suite.dart"; | |
10 import "path.dart"; | |
11 import "compiler_configuration.dart" show CompilerConfiguration; | |
12 import "runtime_configuration.dart" show RuntimeConfiguration; | |
13 | |
14 const List<String> defaultTestSelectors = const [ | |
15 'samples', | |
16 'standalone', | |
17 'corelib', | |
18 'co19', | |
19 'language', | |
20 'isolate', | |
21 'vm', | |
22 'html', | |
23 'benchmark_smoke', | |
24 'utils', | |
25 'lib', | |
26 'analyze_library', | |
27 'service', | |
28 'kernel', | |
29 'observatory_ui' | |
30 ]; | |
31 | |
32 /** | |
33 * Specification of a single test option. | |
34 * | |
35 * The name of the specification is used as the key for the option in | |
36 * the Map returned from the [TestOptionParser] parse method. | |
37 */ | |
38 class _TestOptionSpecification { | |
39 _TestOptionSpecification( | |
40 this.name, this.description, this.keys, this.values, this.defaultValue, | |
41 {this.type: 'string'}); | |
42 String name; | |
43 String description; | |
44 List<String> keys; | |
45 List<String> values; | |
46 Object defaultValue; | |
47 String type; | |
48 } | |
49 | |
50 /** | |
51 * Parser of test options. | |
52 */ | |
53 class TestOptionsParser { | |
54 /** | |
55 * Creates a test options parser initialized with the known options. | |
56 */ | |
57 TestOptionsParser() { | |
58 _options = [ | |
59 new _TestOptionSpecification('mode', 'Mode in which to run the tests', | |
60 ['-m', '--mode'], ['all', 'debug', 'release', 'product'], 'debug'), | |
61 new _TestOptionSpecification( | |
62 'compiler', | |
63 '''Specify any compilation step (if needed). | |
64 | |
65 none: Do not compile the Dart code (run native Dart code on the VM). | |
66 (only valid with the following runtimes: vm, flutter, drt) | |
67 | |
68 precompiler: Compile into AOT snapshot before running the test. | |
69 (only valid with the dart_precompiled runtime) | |
70 | |
71 dart2js: Compile dart code to JavaScript by running dart2js. | |
72 (only valid with the following runtimes: d8, drt, chrome, | |
73 safari, ie9, ie10, ie11, firefox, opera, chromeOnAndroid, | |
74 none (compile only)), | |
75 | |
76 dart2analyzer: Perform static analysis on Dart code by running the analyzer | |
77 (only valid with the following runtimes: none) | |
78 | |
79 app_jit: Compile the Dart code into an app snapshot before running test | |
80 (only valid with dart_app runtime) | |
81 | |
82 dartk: Compile the Dart source into Kernel before running test. | |
83 | |
84 dartkp: Compiler the Dart source into Kernel and then Kernel into AOT | |
85 snapshot before running the test. | |
86 (only valid with the dart_precompiled runtime) | |
87 ''', | |
88 ['-c', '--compiler'], | |
89 [ | |
90 'none', | |
91 'precompiler', | |
92 'dart2js', | |
93 'dart2analyzer', | |
94 'app_jit', | |
95 'dartk', | |
96 'dartkp' | |
97 ], | |
98 'none'), | |
99 // TODO(antonm): fix the option drt. | |
100 new _TestOptionSpecification( | |
101 'runtime', | |
102 '''Where the tests should be run. | |
103 vm: Run Dart code on the standalone dart vm. | |
104 | |
105 flutter: Run Dart code on the flutter engine. | |
106 | |
107 dart_precompiled: Run a precompiled snapshot on a variant of the standalone | |
108 dart vm lacking a JIT. | |
109 | |
110 d8: Run JavaScript from the command line using v8. | |
111 | |
112 jsshell: Run JavaScript from the command line using firefox js-shell. | |
113 | |
114 drt: Run Dart or JavaScript in the headless version of Chrome, | |
115 Content shell. | |
116 | |
117 dartium: Run Dart or JavaScript in Dartium. | |
118 | |
119 ContentShellOnAndroid: Run Dart or JavaScript in Dartium content shell | |
120 on Android. | |
121 | |
122 DartiumOnAndroid: Run Dart or Javascript in Dartium on Android. | |
123 | |
124 [ff | chrome | safari | ie9 | ie10 | ie11 | opera | chromeOnAndroid]: | |
125 Run JavaScript in the specified browser. | |
126 | |
127 self_check: Pass each test or its compiled output to every file under | |
128 `pkg` whose name ends with `_self_check.dart`. | |
129 Each test is given to the self_check tester as a filename on stdin using | |
130 the batch-mode protocol. | |
131 | |
132 none: No runtime, compile only (for example, used for dart2analyzer static | |
133 analysis tests).''', | |
134 ['-r', '--runtime'], | |
135 [ | |
136 'vm', | |
137 'flutter', | |
138 'dart_precompiled', | |
139 'd8', | |
140 'jsshell', | |
141 'drt', | |
142 'dartium', | |
143 'ff', | |
144 'firefox', | |
145 'chrome', | |
146 'safari', | |
147 'ie9', | |
148 'ie10', | |
149 'ie11', | |
150 'opera', | |
151 'chromeOnAndroid', | |
152 'safarimobilesim', | |
153 'ContentShellOnAndroid', | |
154 'DartiumOnAndroid', | |
155 'self_check', | |
156 'none' | |
157 ], | |
158 'vm'), | |
159 new _TestOptionSpecification( | |
160 'arch', | |
161 'The architecture to run tests for', | |
162 ['-a', '--arch'], | |
163 [ | |
164 'all', | |
165 'ia32', | |
166 'x64', | |
167 'arm', | |
168 'armv6', | |
169 'armv5te', | |
170 'arm64', | |
171 'mips', | |
172 'simarm', | |
173 'simarmv6', | |
174 'simarmv5te', | |
175 'simarm64', | |
176 'simmips', | |
177 'simdbc', | |
178 'simdbc64', | |
179 ], | |
180 'x64'), | |
181 new _TestOptionSpecification( | |
182 'system', | |
183 'The operating system to run tests on', | |
184 ['-s', '--system'], | |
185 ['linux', 'macos', 'windows', 'android'], | |
186 Platform.operatingSystem), | |
187 new _TestOptionSpecification( | |
188 'checked', 'Run tests in checked mode', ['--checked'], [], false, | |
189 type: 'bool'), | |
190 new _TestOptionSpecification( | |
191 'strong', 'Run tests in strong mode', ['--strong'], [], false, | |
192 type: 'bool'), | |
193 new _TestOptionSpecification('host_checked', | |
194 'Run compiler in checked mode', ['--host-checked'], [], false, | |
195 type: 'bool'), | |
196 new _TestOptionSpecification('minified', | |
197 'Enable minification in the compiler', ['--minified'], [], false, | |
198 type: 'bool'), | |
199 new _TestOptionSpecification( | |
200 'csp', | |
201 'Run tests under Content Security Policy restrictions', | |
202 ['--csp'], | |
203 [], | |
204 false, | |
205 type: 'bool'), | |
206 new _TestOptionSpecification( | |
207 'cps_ir', | |
208 'Run the compiler with the cps based backend', | |
209 ['--cps-ir'], | |
210 [], | |
211 false, | |
212 type: 'bool'), | |
213 new _TestOptionSpecification( | |
214 'fast_startup', | |
215 'Pass the --fast-startup flag to dart2js', | |
216 ['--fast-startup'], | |
217 [], | |
218 false, | |
219 type: 'bool'), | |
220 new _TestOptionSpecification( | |
221 'dart2js_with_kernel', | |
222 'Enable the internal pipeline in dart2js to use kernel', | |
223 ['--dart2js-with-kernel'], | |
224 [], | |
225 false, | |
226 type: 'bool'), | |
227 new _TestOptionSpecification('hot_reload', 'Run hot reload stress tests', | |
228 ['--hot-reload'], [], false, | |
229 type: 'bool'), | |
230 new _TestOptionSpecification( | |
231 'hot_reload_rollback', | |
232 'Run hot reload rollback stress tests', | |
233 ['--hot-reload-rollback'], | |
234 [], | |
235 false, | |
236 type: 'bool'), | |
237 new _TestOptionSpecification( | |
238 'use_blobs', | |
239 'Use mmap instead of shared libraries for precompilation', | |
240 ['--use-blobs'], | |
241 [], | |
242 false, | |
243 type: 'bool'), | |
244 new _TestOptionSpecification( | |
245 'timeout', 'Timeout in seconds', ['-t', '--timeout'], [], -1, | |
246 type: 'int'), | |
247 new _TestOptionSpecification( | |
248 'progress', | |
249 'Progress indication mode', | |
250 ['-p', '--progress'], | |
251 [ | |
252 'compact', | |
253 'color', | |
254 'line', | |
255 'verbose', | |
256 'silent', | |
257 'status', | |
258 'buildbot', | |
259 'diff' | |
260 ], | |
261 'compact'), | |
262 new _TestOptionSpecification('failure-summary', | |
263 'Print failure summary at the end', ['--failure-summary'], [], false, | |
264 type: 'bool'), | |
265 new _TestOptionSpecification('step_name', | |
266 'Step name for use by -pbuildbot', ['--step_name'], [], null), | |
267 new _TestOptionSpecification( | |
268 'report', | |
269 'Print a summary report of the number of tests, by expectation', | |
270 ['--report'], | |
271 [], | |
272 false, | |
273 type: 'bool'), | |
274 new _TestOptionSpecification( | |
275 'tasks', | |
276 'The number of parallel tasks to run', | |
277 ['-j', '--tasks'], | |
278 [], | |
279 Platform.numberOfProcessors, | |
280 type: 'int'), | |
281 new _TestOptionSpecification( | |
282 'shards', | |
283 'The number of instances that the tests will be sharded over', | |
284 ['--shards'], | |
285 [], | |
286 1, | |
287 type: 'int'), | |
288 new _TestOptionSpecification( | |
289 'shard', | |
290 'The index of this instance when running in sharded mode', | |
291 ['--shard'], | |
292 [], | |
293 1, | |
294 type: 'int'), | |
295 new _TestOptionSpecification( | |
296 'help', 'Print list of options', ['-h', '--help'], [], false, | |
297 type: 'bool'), | |
298 new _TestOptionSpecification( | |
299 'verbose', 'Verbose output', ['-v', '--verbose'], [], false, | |
300 type: 'bool'), | |
301 new _TestOptionSpecification( | |
302 'verify-ir', 'Verify kernel IR', ['--verify-ir'], [], false, | |
303 type: 'bool'), | |
304 new _TestOptionSpecification('no-tree-shake', | |
305 'Disable kernel IR tree shaking', ['--no-tree-shake'], [], false, | |
306 type: 'bool'), | |
307 new _TestOptionSpecification( | |
308 'list', 'List tests only, do not run them', ['--list'], [], false, | |
309 type: 'bool'), | |
310 new _TestOptionSpecification( | |
311 'report_in_json', | |
312 'When doing list, output result summary in json only.', | |
313 ['--report-in-json'], | |
314 [], | |
315 false, | |
316 type: 'bool'), | |
317 new _TestOptionSpecification('time', | |
318 'Print timing information after running tests', ['--time'], [], false, | |
319 type: 'bool'), | |
320 new _TestOptionSpecification( | |
321 'dart', 'Path to dart executable', ['--dart'], [], ''), | |
322 new _TestOptionSpecification( | |
323 'flutter', 'Path to flutter executable', ['--flutter'], [], ''), | |
324 new _TestOptionSpecification( | |
325 'drt', // TODO(antonm): fix the option name. | |
326 'Path to content shell executable', | |
327 ['--drt'], | |
328 [], | |
329 ''), | |
330 new _TestOptionSpecification('dartium', | |
331 'Path to Dartium Chrome executable', ['--dartium'], [], ''), | |
332 new _TestOptionSpecification('firefox', | |
333 'Path to firefox browser executable', ['--firefox'], [], ''), | |
334 new _TestOptionSpecification( | |
335 'chrome', 'Path to chrome browser executable', ['--chrome'], [], ''), | |
336 new _TestOptionSpecification( | |
337 'safari', 'Path to safari browser executable', ['--safari'], [], ''), | |
338 new _TestOptionSpecification( | |
339 'use_sdk', | |
340 '''Use compiler or runtime from the SDK. | |
341 | |
342 Normally, the compiler or runtimes in PRODUCT_DIR is tested, with this | |
343 option, the compiler or runtime in PRODUCT_DIR/dart-sdk/bin is tested. | |
344 | |
345 Note: currently only implemented for dart2js.''', | |
346 ['--use-sdk'], | |
347 [], | |
348 false, | |
349 type: 'bool'), | |
350 new _TestOptionSpecification( | |
351 'build_directory', | |
352 'The name of the build directory, where products are placed.', | |
353 ['--build-directory'], | |
354 [], | |
355 ''), | |
356 new _TestOptionSpecification('noBatch', 'Do not run tests in batch mode', | |
357 ['-n', '--nobatch'], [], false, | |
358 type: 'bool'), | |
359 new _TestOptionSpecification('dart2js_batch', | |
360 'Run dart2js tests in batch mode', ['--dart2js-batch'], [], false, | |
361 type: 'bool'), | |
362 new _TestOptionSpecification( | |
363 'append_logs', | |
364 'Do not delete old logs but rather append to them.', | |
365 ['--append_logs'], | |
366 [], | |
367 false, | |
368 type: 'bool'), | |
369 new _TestOptionSpecification( | |
370 'write_debug_log', | |
371 'Don\'t write debug messages to stdout but rather to a logfile.', | |
372 ['--write-debug-log'], | |
373 [], | |
374 false, | |
375 type: 'bool'), | |
376 new _TestOptionSpecification( | |
377 'write_test_outcome_log', | |
378 'Write the outcome of all tests executed to a ' | |
379 '"${TestUtils.testOutcomeFileName}" file.', | |
380 ['--write-test-outcome-log'], | |
381 [], | |
382 false, | |
383 type: 'bool'), | |
384 new _TestOptionSpecification( | |
385 'reset_browser_configuration', | |
386 'Browser specific reset of configuration. ' | |
387 'WARNING: Using this option may remove your bookmarks and ' | |
388 'other settings.', | |
389 ['--reset-browser-configuration'], | |
390 [], | |
391 false, | |
392 type: 'bool'), | |
393 new _TestOptionSpecification( | |
394 'copy_coredumps', | |
395 'If we see a crash that we did not expect, copy the core dumps. ' | |
396 'to /tmp', | |
397 ['--copy-coredumps'], | |
398 [], | |
399 false, | |
400 type: 'bool'), | |
401 new _TestOptionSpecification( | |
402 'local_ip', | |
403 'IP address the http servers should listen on.' | |
404 'This address is also used for browsers to connect.', | |
405 ['--local_ip'], | |
406 [], | |
407 '127.0.0.1'), | |
408 new _TestOptionSpecification('test_server_port', | |
409 'Port for test http server.', ['--test_server_port'], [], 0, | |
410 type: 'int'), | |
411 new _TestOptionSpecification( | |
412 'test_server_cross_origin_port', | |
413 'Port for test http server cross origin.', | |
414 ['--test_server_cross_origin_port'], | |
415 [], | |
416 0, | |
417 type: 'int'), | |
418 new _TestOptionSpecification('test_driver_port', | |
419 'Port for http test driver server.', ['--test_driver_port'], [], 0, | |
420 type: 'int'), | |
421 new _TestOptionSpecification( | |
422 'test_driver_error_port', | |
423 'Port for http test driver server errors.', | |
424 ['--test_driver_error_port'], | |
425 [], | |
426 0, | |
427 type: 'int'), | |
428 new _TestOptionSpecification( | |
429 'record_to_file', | |
430 'Records all the commands that need to be executed and writes it ' | |
431 'out to a file.', | |
432 ['--record_to_file'], | |
433 [], | |
434 null), | |
435 new _TestOptionSpecification( | |
436 'replay_from_file', | |
437 'Records all the commands that need to be executed and writes it ' | |
438 'out to a file.', | |
439 ['--replay_from_file'], | |
440 [], | |
441 null), | |
442 new _TestOptionSpecification( | |
443 'builder_tag', | |
444 'Machine specific options that is not captured by the regular ' | |
445 'test options. Used to be able to make sane updates to the ' | |
446 'status files.', | |
447 ['--builder-tag'], | |
448 [], | |
449 ''), | |
450 new _TestOptionSpecification( | |
451 'vm_options', | |
452 'Extra options to send to the vm when running', | |
453 ['--vm-options'], | |
454 [], | |
455 null), | |
456 new _TestOptionSpecification( | |
457 'dart2js_options', | |
458 'Extra options for dart2js compilation step', | |
459 ['--dart2js-options'], | |
460 [], | |
461 null), | |
462 new _TestOptionSpecification( | |
463 'suite_dir', | |
464 'Additional directory to add to the testing matrix', | |
465 ['--suite-dir'], | |
466 [], | |
467 null), | |
468 new _TestOptionSpecification('package_root', | |
469 'The package root to use for testing.', ['--package-root'], [], null), | |
470 new _TestOptionSpecification( | |
471 'packages', | |
472 'The package spec file to use for testing.', | |
473 ['--packages'], | |
474 [], | |
475 null), | |
476 new _TestOptionSpecification( | |
477 'exclude_suite', | |
478 'Exclude suites from default selector, only works when no' | |
479 ' selector has been specified on the command line', | |
480 ['--exclude-suite'], | |
481 [], | |
482 null), | |
483 new _TestOptionSpecification( | |
484 'skip-compilation', | |
485 'Skip the compilation step, using the compilation artifacts left in ' | |
486 ' the output folder from a previous run.' | |
487 'This flag will often cause false positves and negatives, but can be' | |
488 ' useful for quick-and-dirty offline testing when not making changes' | |
489 ' that affect the compiler.', | |
490 ['--skip-compilation'], | |
491 [], | |
492 false, | |
493 type: 'bool') | |
494 ]; | |
495 } | |
496 | |
497 /** | |
498 * Parse a list of strings as test options. | |
499 * | |
500 * Returns a list of configurations in which to run the | |
501 * tests. Configurations are maps mapping from option keys to | |
502 * values. When encountering the first non-option string, the rest | |
503 * of the arguments are stored in the returned Map under the 'rest' | |
504 * key. | |
505 */ | |
506 List<Map> parse(List<String> arguments) { | |
507 var configuration = new Map(); | |
508 // Fill in configuration with arguments passed to the test script. | |
509 var numArguments = arguments.length; | |
510 for (var i = 0; i < numArguments; i++) { | |
511 // Extract name and value for options. | |
512 String arg = arguments[i]; | |
513 String name = ''; | |
514 String value = ''; | |
515 _TestOptionSpecification spec; | |
516 if (arg.startsWith('--')) { | |
517 if (arg == '--help') { | |
518 _printHelp(); | |
519 return null; | |
520 } | |
521 var split = arg.indexOf('='); | |
522 if (split == -1) { | |
523 name = arg; | |
524 spec = _getSpecification(name); | |
525 // Boolean options do not have a value. | |
526 if (spec.type != 'bool') { | |
527 if ((i + 1) >= arguments.length) { | |
528 print('No value supplied for option $name'); | |
529 return null; | |
530 } | |
531 value = arguments[++i]; | |
532 } | |
533 } else { | |
534 name = arg.substring(0, split); | |
535 spec = _getSpecification(name); | |
536 value = arg.substring(split + 1, arg.length); | |
537 } | |
538 } else if (arg.startsWith('-')) { | |
539 if (arg == '-h') { | |
540 _printHelp(); | |
541 return null; | |
542 } | |
543 if (arg.length > 2) { | |
544 name = arg.substring(0, 2); | |
545 spec = _getSpecification(name); | |
546 value = arg.substring(2, arg.length); | |
547 } else { | |
548 name = arg; | |
549 spec = _getSpecification(name); | |
550 // Boolean options do not have a value. | |
551 if (spec.type != 'bool') { | |
552 if ((i + 1) >= arguments.length) { | |
553 print('No value supplied for option $name'); | |
554 return null; | |
555 } | |
556 value = arguments[++i]; | |
557 } | |
558 } | |
559 } else { | |
560 // The argument does not start with '-' or '--' and is | |
561 // therefore not an option. We use it as a test selection | |
562 // pattern. | |
563 var patterns = configuration.putIfAbsent('selectors', () => <String>[]); | |
564 patterns.add(arg); | |
565 continue; | |
566 } | |
567 | |
568 // Multiple uses of a flag are an error, because there is no | |
569 // naturally correct way to handle conflicting options. | |
570 if (configuration.containsKey(spec.name)) { | |
571 print('Error: test.dart disallows multiple "--${spec.name}" flags'); | |
572 exit(1); | |
573 } | |
574 // Parse the value for the option. | |
575 if (spec.type == 'bool') { | |
576 if (!value.isEmpty) { | |
577 print('No value expected for bool option $name'); | |
578 exit(1); | |
579 } | |
580 configuration[spec.name] = true; | |
581 } else if (spec.type == 'int') { | |
582 try { | |
583 configuration[spec.name] = int.parse(value); | |
584 } catch (e) { | |
585 print('Integer value expected for int option $name'); | |
586 exit(1); | |
587 } | |
588 } else { | |
589 assert(spec.type == 'string'); | |
590 if (!spec.values.isEmpty) { | |
591 for (var v in value.split(',')) { | |
592 if (spec.values.lastIndexOf(v) == -1) { | |
593 print('Unknown value ($v) for option $name'); | |
594 exit(1); | |
595 } | |
596 } | |
597 } | |
598 configuration[spec.name] = value; | |
599 } | |
600 } | |
601 | |
602 // Apply default values for unspecified options. | |
603 for (var option in _options) { | |
604 if (!configuration.containsKey(option.name)) { | |
605 configuration[option.name] = option.defaultValue; | |
606 } | |
607 } | |
608 | |
609 List<Map> expandedConfigs = _expandConfigurations(configuration); | |
610 List<Map> result = expandedConfigs.where(_isValidConfig).toList(); | |
611 for (var config in result) { | |
612 config['_reproducing_arguments_'] = | |
613 _constructReproducingCommandArguments(config); | |
614 } | |
615 return result.isEmpty ? null : result; | |
616 } | |
617 | |
618 // For printing out reproducing command lines, we don't want to add these | |
619 // options. | |
620 final _blacklistedOptions = [ | |
621 'append_logs', | |
622 'build_directory', | |
623 'chrome', | |
624 'copy_coredumps', | |
625 'dart', | |
626 'flutter', | |
627 'dartium', | |
628 'drt', | |
629 'exclude_suite', | |
630 'failure-summary', | |
631 'firefox', | |
632 'local_ip', | |
633 'progress', | |
634 'report', | |
635 'safari', | |
636 'shard', | |
637 'shards', | |
638 'step_name', | |
639 'tasks', | |
640 'time', | |
641 'verbose', | |
642 'write_debug_log', | |
643 'write_test_outcome_log', | |
644 ].toSet(); | |
645 | |
646 List<String> _constructReproducingCommandArguments(Map config) { | |
647 var arguments = new List<String>(); | |
648 for (var option in _options) { | |
649 var name = option.name; | |
650 if (!config.containsKey(name) || _blacklistedOptions.contains(name)) { | |
651 continue; | |
652 } | |
653 var value = config[name]; | |
654 if (config[name] == option.defaultValue || | |
655 (name == 'packages' && | |
656 value == | |
657 TestUtils.dartDirUri.resolve('.packages').toFilePath())) { | |
658 continue; | |
659 } | |
660 shortest(String a, String b) => a.length <= b.length ? a : b; | |
661 var key = option.keys.reduce(shortest); | |
662 if (option.type == 'bool') { | |
663 arguments.add(key); | |
664 } else if (key.startsWith('--')) { | |
665 // long version | |
666 arguments.add(key); | |
667 arguments.add("$value"); | |
668 } else { | |
669 // short version | |
670 assert(key.startsWith('-')); | |
671 arguments.add("$key$value"); | |
672 } | |
673 } | |
674 return arguments; | |
675 } | |
676 | |
677 /** | |
678 * Determine if a particular configuration has a valid combination of compiler | |
679 * and runtime elements. | |
680 */ | |
681 bool _isValidConfig(Map config) { | |
682 bool isValid = true; | |
683 List<String> validRuntimes; | |
684 switch (config['compiler']) { | |
685 case 'dart2js': | |
686 // Note: by adding 'none' as a configuration, if the user | |
687 // runs test.py -c dart2js -r drt,none the dart2js_none and | |
688 // dart2js_drt will be duplicating work. If later we don't need 'none' | |
689 // with dart2js, we should remove it from here. | |
690 validRuntimes = const [ | |
691 'd8', | |
692 'jsshell', | |
693 'drt', | |
694 'none', | |
695 'dartium', | |
696 'ff', | |
697 'chrome', | |
698 'safari', | |
699 'ie9', | |
700 'ie10', | |
701 'ie11', | |
702 'opera', | |
703 'chromeOnAndroid', | |
704 'safarimobilesim' | |
705 ]; | |
706 break; | |
707 case 'dart2analyzer': | |
708 validRuntimes = const ['none']; | |
709 break; | |
710 case 'app_jit': | |
711 case 'dartk': | |
712 validRuntimes = const ['vm', 'self_check', 'none']; | |
713 break; | |
714 case 'precompiler': | |
715 case 'dartkp': | |
716 validRuntimes = const ['dart_precompiled']; | |
717 break; | |
718 case 'none': | |
719 validRuntimes = const [ | |
720 'vm', | |
721 'flutter', | |
722 'drt', | |
723 'dartium', | |
724 'ContentShellOnAndroid', | |
725 'DartiumOnAndroid' | |
726 ]; | |
727 break; | |
728 } | |
729 if (!validRuntimes.contains(config['runtime'])) { | |
730 isValid = false; | |
731 print("Warning: combination of compiler '${config['compiler']}' and " | |
732 "runtime '${config['runtime']}' is invalid. " | |
733 "Skipping this combination."); | |
734 } | |
735 if (config['ie'] && Platform.operatingSystem != 'windows') { | |
736 isValid = false; | |
737 print("Warning: cannot run Internet Explorer on non-Windows operating" | |
738 " system."); | |
739 } | |
740 if (config['shard'] < 1 || config['shard'] > config['shards']) { | |
741 isValid = false; | |
742 print("Error: shard index is ${config['shard']} out of " | |
743 "${config['shards']} shards"); | |
744 } | |
745 if ((config['runtime'] == 'flutter') && (config['flutter'] == '')) { | |
746 isValid = false; | |
747 print("-rflutter requires the flutter engine executable to " | |
748 "be specified using --flutter="); | |
749 } | |
750 if ((config['runtime'] == 'flutter') && (config['arch'] != 'x64')) { | |
751 isValid = false; | |
752 print("-rflutter is applicable only for --arch=x64"); | |
753 } | |
754 | |
755 return isValid; | |
756 } | |
757 | |
758 /** | |
759 * Recursively expand a configuration with multiple values per key | |
760 * into a list of configurations with exactly one value per key. | |
761 */ | |
762 List<Map> _expandConfigurations(Map configuration) { | |
763 // Expand the pseudo-values such as 'all'. | |
764 if (configuration['arch'] == 'all') { | |
765 configuration['arch'] = 'ia32,x64,simarm,simarm64,simmips,simdbc64'; | |
766 } | |
767 if (configuration['mode'] == 'all') { | |
768 configuration['mode'] = 'debug,release,product'; | |
769 } | |
770 | |
771 if (configuration['report_in_json']) { | |
772 configuration['list'] = true; | |
773 configuration['report'] = true; | |
774 } | |
775 | |
776 // Use verbose progress indication for verbose output unless buildbot | |
777 // progress indication is requested. | |
778 if (configuration['verbose'] && configuration['progress'] != 'buildbot') { | |
779 configuration['progress'] = 'verbose'; | |
780 } | |
781 | |
782 // Create the artificial negative options that test status files | |
783 // expect. | |
784 configuration['unchecked'] = !configuration['checked']; | |
785 configuration['host_unchecked'] = !configuration['host_checked']; | |
786 configuration['unminified'] = !configuration['minified']; | |
787 configuration['nocsp'] = !configuration['csp']; | |
788 | |
789 String runtime = configuration['runtime']; | |
790 if (runtime == 'firefox') { | |
791 configuration['runtime'] == 'ff'; | |
792 } | |
793 | |
794 String compiler = configuration['compiler']; | |
795 configuration['browser'] = TestUtils.isBrowserRuntime(runtime); | |
796 configuration['analyzer'] = TestUtils.isCommandLineAnalyzer(compiler); | |
797 | |
798 // Set the javascript command line flag for less verbose status files. | |
799 configuration['jscl'] = TestUtils.isJsCommandLineRuntime(runtime); | |
800 | |
801 // Allow suppression that is valid for all ie versions | |
802 configuration['ie'] = runtime.startsWith('ie'); | |
803 | |
804 // Expand the test selectors into a suite name and a simple | |
805 // regular expressions to be used on the full path of a test file | |
806 // in that test suite. If no selectors are explicitly given use | |
807 // the default suite patterns. | |
808 var selectors = configuration['selectors']; | |
809 if (selectors is! Map) { | |
810 if (selectors == null) { | |
811 if (configuration['suite_dir'] != null) { | |
812 var suite_path = new Path(configuration['suite_dir']); | |
813 selectors = [suite_path.filename]; | |
814 } else { | |
815 selectors = defaultTestSelectors.toList(); | |
816 } | |
817 | |
818 var excludeSuites = configuration['exclude_suite'] != null | |
819 ? configuration['exclude_suite'].split(',') | |
820 : []; | |
821 for (var exclude in excludeSuites) { | |
822 if (selectors.contains(exclude)) { | |
823 selectors.remove(exclude); | |
824 } else { | |
825 print("Warning: default selectors does not contain $exclude"); | |
826 } | |
827 } | |
828 } | |
829 var selectorMap = <String, RegExp>{}; | |
830 for (var i = 0; i < selectors.length; i++) { | |
831 var pattern = selectors[i]; | |
832 var suite = pattern; | |
833 var slashLocation = pattern.indexOf('/'); | |
834 if (slashLocation != -1) { | |
835 suite = pattern.substring(0, slashLocation); | |
836 pattern = pattern.substring(slashLocation + 1); | |
837 pattern = pattern.replaceAll('*', '.*'); | |
838 } else { | |
839 pattern = ".?"; | |
840 } | |
841 if (selectorMap.containsKey(suite)) { | |
842 print("Error: '$suite/$pattern'. Only one test selection" | |
843 " pattern is allowed to start with '$suite/'"); | |
844 exit(1); | |
845 } | |
846 selectorMap[suite] = new RegExp(pattern); | |
847 } | |
848 configuration['selectors'] = selectorMap; | |
849 } | |
850 | |
851 // Put observatory_ui in a configuration with its own packages override. | |
852 // Only one value in the configuration map is mutable: | |
853 selectors = configuration['selectors']; | |
854 if (selectors.containsKey('observatory_ui')) { | |
855 if (selectors.length == 1) { | |
856 configuration['packages'] = TestUtils.dartDirUri | |
857 .resolve('runtime/observatory/.packages') | |
858 .toFilePath(); | |
859 } else { | |
860 // Make a new configuration whose selectors map only contains | |
861 // observatory_ui, and remove the key from the original selectors. | |
862 // The only mutable value in the map is the selectors, so a | |
863 // shallow copy is safe. | |
864 var observatoryConfiguration = new Map.from(configuration); | |
865 observatoryConfiguration['selectors'] = { | |
866 'observatory_ui': selectors['observatory_ui'] | |
867 }; | |
868 selectors.remove('observatory_ui'); | |
869 | |
870 // Set the packages flag. | |
871 observatoryConfiguration['packages'] = TestUtils.dartDirUri | |
872 .resolve('runtime/observatory/.packages') | |
873 .toFilePath(); | |
874 | |
875 // Return the expansions of both configurations. Neither will reach | |
876 // this line in the recursive call to _expandConfigurations. | |
877 return _expandConfigurations(configuration) | |
878 ..addAll(_expandConfigurations(observatoryConfiguration)); | |
879 } | |
880 } | |
881 // Set the default package spec explicitly. | |
882 if (configuration['package_root'] == null && | |
883 configuration['packages'] == null) { | |
884 configuration['packages'] = | |
885 TestUtils.dartDirUri.resolve('.packages').toFilePath(); | |
886 } | |
887 | |
888 // Expand the architectures. | |
889 if (configuration['arch'].contains(',')) { | |
890 return _expandHelper('arch', configuration); | |
891 } | |
892 | |
893 // Expand modes. | |
894 if (configuration['mode'].contains(',')) { | |
895 return _expandHelper('mode', configuration); | |
896 } | |
897 | |
898 // Expand compilers. | |
899 if (configuration['compiler'].contains(',')) { | |
900 return _expandHelper('compiler', configuration); | |
901 } | |
902 | |
903 // Expand runtimes. | |
904 var runtimes = configuration['runtime']; | |
905 if (runtimes.contains(',')) { | |
906 return _expandHelper('runtime', configuration); | |
907 } else { | |
908 // All runtimes eventually go through this path, after expansion. | |
909 var updater = runtimeUpdater(configuration); | |
910 if (updater != null) { | |
911 updater.update(); | |
912 } | |
913 } | |
914 | |
915 // Adjust default timeout based on mode, compiler, and sometimes runtime. | |
916 if (configuration['timeout'] == -1) { | |
917 var isReload = | |
918 configuration['hot_reload'] || configuration['hot_reload_rollback']; | |
919 int compilerMulitiplier = | |
920 new CompilerConfiguration(configuration).computeTimeoutMultiplier(); | |
921 int runtimeMultiplier = new RuntimeConfiguration(configuration) | |
922 .computeTimeoutMultiplier( | |
923 mode: configuration['mode'], | |
924 isChecked: configuration['checked'], | |
925 isReload: isReload, | |
926 arch: configuration['arch']); | |
927 configuration['timeout'] = 60 * compilerMulitiplier * runtimeMultiplier; | |
928 } | |
929 | |
930 return [configuration]; | |
931 } | |
932 | |
933 /** | |
934 * Helper for _expandConfigurations. Creates a new configuration and adds it | |
935 * to a list, for use in a case when a particular configuration has multiple | |
936 * results (separated by a ','). | |
937 * Arguments: | |
938 * option: The particular test option we are expanding. | |
939 * configuration: The map containing all test configuration information | |
940 * specified. | |
941 */ | |
942 List<Map> _expandHelper(String option, Map configuration) { | |
943 var result = new List<Map>(); | |
944 var configs = configuration[option]; | |
945 for (var config in configs.split(',')) { | |
946 var newConfiguration = new Map.from(configuration); | |
947 newConfiguration[option] = config; | |
948 result.addAll(_expandConfigurations(newConfiguration)); | |
949 } | |
950 return result; | |
951 } | |
952 | |
953 /** | |
954 * Print out usage information. | |
955 */ | |
956 void _printHelp() { | |
957 print('usage: dart test.dart [options] [selector]'); | |
958 print(''); | |
959 print('The optional selector limits the tests that will be run.'); | |
960 print('For example, the selector "language/issue", or equivalently'); | |
961 print('"language/*issue*", limits to test files matching the regexp'); | |
962 print('".*issue.*\\.dart" in the "tests/language" directory.'); | |
963 print(''); | |
964 print('Options:\n'); | |
965 for (var option in _options) { | |
966 print('${option.name}: ${option.description}.'); | |
967 for (var name in option.keys) { | |
968 assert(name.startsWith('-')); | |
969 var buffer = new StringBuffer(); | |
970 ; | |
971 buffer.write(name); | |
972 if (option.type == 'bool') { | |
973 assert(option.values.isEmpty); | |
974 } else { | |
975 buffer.write(name.startsWith('--') ? '=' : ' '); | |
976 if (option.type == 'int') { | |
977 assert(option.values.isEmpty); | |
978 buffer.write('n (default: ${option.defaultValue})'); | |
979 } else { | |
980 buffer.write('['); | |
981 bool first = true; | |
982 for (var value in option.values) { | |
983 if (!first) buffer.write(", "); | |
984 if (value == option.defaultValue) buffer.write('*'); | |
985 buffer.write(value); | |
986 first = false; | |
987 } | |
988 buffer.write(']'); | |
989 } | |
990 } | |
991 print(buffer.toString()); | |
992 } | |
993 print(''); | |
994 } | |
995 } | |
996 | |
997 /** | |
998 * Find the test option specification for a given option key. | |
999 */ | |
1000 _TestOptionSpecification _getSpecification(String name) { | |
1001 for (var option in _options) { | |
1002 if (option.keys.contains(name)) { | |
1003 return option; | |
1004 } | |
1005 } | |
1006 print('Unknown test option $name'); | |
1007 exit(1); | |
1008 return null; // Unreachable. | |
1009 } | |
1010 | |
1011 List<_TestOptionSpecification> _options; | |
1012 } | |
OLD | NEW |