OLD | NEW |
| (Empty) |
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | |
2 // for details. All rights reserved. Use of this source code is governed by a | |
3 // BSD-style license that can be found in the LICENSE file. | |
4 | |
5 import 'dart:io'; | |
6 import 'drt_updater.dart'; | |
7 import 'test_suite.dart'; | |
8 import 'path.dart'; | |
9 import 'compiler_configuration.dart' show CompilerConfiguration; | |
10 import 'runtime_configuration.dart' show RuntimeConfiguration; | |
11 | |
12 const _defaultTestSelectors = const [ | |
13 'samples', | |
14 'standalone', | |
15 'corelib', | |
16 'co19', | |
17 'language', | |
18 'isolate', | |
19 'vm', | |
20 'html', | |
21 'benchmark_smoke', | |
22 'utils', | |
23 'lib', | |
24 'analyze_library', | |
25 'service', | |
26 'kernel', | |
27 'observatory_ui' | |
28 ]; | |
29 | |
30 /// Specifies a single command line option. | |
31 /// | |
32 /// The name of the specification is used as the key for the option in the Map | |
33 /// returned from the [TestOptionParser] parse method. | |
34 class _Option { | |
35 // TODO(rnystrom): Some string options use "" to mean "no value" and others | |
36 // use null. Clean that up. | |
37 _Option(this.name, this.description, | |
38 {String abbr, List<String> values, String defaultsTo = ""}) | |
39 : abbreviation = abbr, | |
40 values = values ?? [], | |
41 defaultValue = defaultsTo, | |
42 type = _OptionValueType.string; | |
43 | |
44 _Option.bool(this.name, this.description, [this.abbreviation]) | |
45 : values = [], | |
46 defaultValue = false, | |
47 type = _OptionValueType.bool; | |
48 | |
49 _Option.int(this.name, this.description, {String abbr, int defaultsTo}) | |
50 : abbreviation = abbr, | |
51 values = [], | |
52 defaultValue = defaultsTo, | |
53 type = _OptionValueType.int; | |
54 | |
55 final String name; | |
56 final String description; | |
57 final String abbreviation; | |
58 final List<String> values; | |
59 final Object defaultValue; | |
60 final _OptionValueType type; | |
61 | |
62 /// Gets the shortest command line argument used to refer to this option. | |
63 String get shortCommand => abbreviation != null ? "-$abbreviation" : command; | |
64 | |
65 /// Gets the canonical long command line argument used to refer to this | |
66 /// option. | |
67 String get command => "--${name.replaceAll('_', '-')}"; | |
68 } | |
69 | |
70 enum _OptionValueType { bool, int, string } | |
71 | |
72 /// Parses command line arguments and produces a test runner configuration. | |
73 class OptionsParser { | |
74 static final List<_Option> _options = [ | |
75 new _Option('mode', 'Mode in which to run the tests.', | |
76 abbr: 'm', | |
77 values: ['all', 'debug', 'release', 'product'], | |
78 defaultsTo: 'debug'), | |
79 new _Option( | |
80 'compiler', | |
81 '''Specify any compilation step (if needed). | |
82 | |
83 none: Do not compile the Dart code (run native Dart code | |
84 on the VM). | |
85 (Only valid with runtimes vm, flutter, or drt.) | |
86 | |
87 precompiler: Compile into AOT snapshot before running the test. | |
88 (Only valid with runtime dart_precompiled.) | |
89 | |
90 dart2js: Compile dart code to JavaScript by running dart2js. | |
91 (Only valid with runtimes: d8, drt, chrome, safari, | |
92 ie9, ie10, ie11, firefox, opera, chromeOnAndroid, | |
93 and none [compile only].) | |
94 | |
95 dart2analyzer: Perform static analysis on Dart code by running the | |
96 analyzer. | |
97 (Only valid with runtime none.) | |
98 | |
99 app_jit: Compile the Dart code into an app snapshot before | |
100 running test. | |
101 (Only valid with dart_app runtime.) | |
102 | |
103 dartk: Compile the Dart source into Kernel before running | |
104 test. | |
105 | |
106 dartkp: Compile the Dart source into Kernel and then Kernel | |
107 into AOT snapshot before running the test. | |
108 (Only valid with runtime dart_precompiled.)''', | |
109 abbr: 'c', | |
110 values: [ | |
111 'none', | |
112 'precompiler', | |
113 'dart2js', | |
114 'dart2analyzer', | |
115 'app_jit', | |
116 'dartk', | |
117 'dartkp' | |
118 ], | |
119 defaultsTo: 'none'), | |
120 new _Option( | |
121 'runtime', | |
122 '''Where the tests should be run. | |
123 vm: Run Dart code on the standalone dart vm. | |
124 | |
125 flutter: Run Dart code on the flutter engine. | |
126 | |
127 dart_precompiled: Run a precompiled snapshot on a variant of the | |
128 standalone dart VM lacking a JIT. | |
129 | |
130 d8: Run JavaScript from the command line using v8. | |
131 | |
132 jsshell: Run JavaScript from the command line using | |
133 Firefox js-shell. | |
134 | |
135 drt: Run Dart or JavaScript in the headless version | |
136 of Chrome, Content shell. | |
137 | |
138 dartium: Run Dart or JavaScript in Dartium. | |
139 | |
140 ContentShellOnAndroid: Run Dart or JavaScript in Dartium content | |
141 shell on Android. | |
142 | |
143 DartiumOnAndroid: Run Dart or Javascript in Dartium on Android. | |
144 | |
145 ff: | |
146 chrome: | |
147 safari: | |
148 ie9: | |
149 ie10: | |
150 ie11: | |
151 opera: | |
152 chromeOnAndroid: Run JavaScript in the specified browser. | |
153 | |
154 self_check: Pass each test or its compiled output to every | |
155 file under `pkg` whose name ends with | |
156 `_self_check.dart`. Each test is given to the | |
157 self_check tester as a filename on stdin using | |
158 the batch-mode protocol. | |
159 | |
160 none: No runtime, compile only. (For example, used | |
161 for dart2analyzer static analysis tests).''', | |
162 abbr: 'r', | |
163 values: [ | |
164 'vm', | |
165 'flutter', | |
166 'dart_precompiled', | |
167 'd8', | |
168 'jsshell', | |
169 'drt', | |
170 'dartium', | |
171 'ff', | |
172 'firefox', | |
173 'chrome', | |
174 'safari', | |
175 'ie9', | |
176 'ie10', | |
177 'ie11', | |
178 'opera', | |
179 'chromeOnAndroid', | |
180 'safarimobilesim', | |
181 'ContentShellOnAndroid', | |
182 'DartiumOnAndroid', | |
183 'self_check', | |
184 'none' | |
185 ], | |
186 defaultsTo: 'vm'), | |
187 new _Option( | |
188 'arch', | |
189 '''The architecture to run tests for. | |
190 | |
191 Allowed values are: | |
192 all | |
193 ia32, x64 | |
194 arm, armv6, armv5te, arm64, | |
195 simarm, simarmv6, simarmv5te, simarm64, | |
196 mips, simmips | |
197 simdbc, simdbc64''', | |
198 abbr: 'a', | |
199 values: [ | |
200 'all', | |
201 'ia32', | |
202 'x64', | |
203 'arm', | |
204 'armv6', | |
205 'armv5te', | |
206 'arm64', | |
207 'mips', | |
208 'simarm', | |
209 'simarmv6', | |
210 'simarmv5te', | |
211 'simarm64', | |
212 'simmips', | |
213 'simdbc', | |
214 'simdbc64', | |
215 ], | |
216 defaultsTo: 'x64'), | |
217 new _Option('system', 'The operating system to run tests on.', | |
218 abbr: 's', | |
219 values: ['linux', 'macos', 'windows', 'android'], | |
220 defaultsTo: Platform.operatingSystem), | |
221 new _Option.bool('checked', 'Run tests in checked mode.'), | |
222 new _Option.bool('strong', 'Run tests in strong mode.'), | |
223 new _Option.bool('host_checked', 'Run compiler in checked mode.'), | |
224 new _Option.bool('minified', 'Enable minification in the compiler.'), | |
225 new _Option.bool( | |
226 'csp', 'Run tests under Content Security Policy restrictions.'), | |
227 new _Option.bool( | |
228 'fast_startup', 'Pass the --fast-startup flag to dart2js.'), | |
229 new _Option.bool('dart2js_with_kernel', | |
230 'Enable the internal pipeline in dart2js to use kernel.'), | |
231 new _Option.bool('hot_reload', 'Run hot reload stress tests.'), | |
232 new _Option.bool( | |
233 'hot_reload_rollback', 'Run hot reload rollback stress tests.'), | |
234 new _Option.bool('use_blobs', | |
235 'Use mmap instead of shared libraries for precompilation.'), | |
236 new _Option.int('timeout', 'Timeout in seconds.', | |
237 abbr: 't', defaultsTo: -1), | |
238 new _Option( | |
239 'progress', | |
240 '''Progress indication mode. | |
241 | |
242 Allowed values are: | |
243 compact, color, line, verbose, silent, status, buildbot, diff | |
244 ''', | |
245 abbr: 'p', | |
246 values: [ | |
247 'compact', | |
248 'color', | |
249 'line', | |
250 'verbose', | |
251 'silent', | |
252 'status', | |
253 'buildbot', | |
254 'diff' | |
255 ], | |
256 defaultsTo: 'compact'), | |
257 new _Option('step_name', 'Step name for use by -pbuildbot.', | |
258 defaultsTo: null), | |
259 new _Option.bool('report', | |
260 'Print a summary report of the number of tests, by expectation.'), | |
261 new _Option.int('tasks', 'The number of parallel tasks to run.', | |
262 abbr: 'j', defaultsTo: Platform.numberOfProcessors), | |
263 new _Option.int('shards', | |
264 'The number of instances that the tests will be sharded over.', | |
265 defaultsTo: 1), | |
266 new _Option.int( | |
267 'shard', 'The index of this instance when running in sharded mode.', | |
268 defaultsTo: 1), | |
269 new _Option.bool('help', 'Print list of options.', 'h'), | |
270 new _Option.bool('verbose', 'Verbose output.', 'v'), | |
271 new _Option.bool('verify-ir', 'Verify kernel IR.'), | |
272 new _Option.bool('no-tree-shake', 'Disable kernel IR tree shaking.'), | |
273 new _Option.bool('list', 'List tests only, do not run them.'), | |
274 new _Option.bool('report_in_json', | |
275 'When listing with --list, output result summary in JSON.'), | |
276 new _Option.bool('time', 'Print timing information after running tests.'), | |
277 new _Option('dart', 'Path to dart executable.'), | |
278 new _Option('flutter', 'Path to flutter executable.'), | |
279 new _Option( | |
280 'drt', // TODO(antonm): fix the option name. | |
281 'Path to content shell executable.'), | |
282 new _Option('dartium', 'Path to Dartium Chrome executable.'), | |
283 new _Option('firefox', 'Path to firefox browser executable.'), | |
284 new _Option('chrome', 'Path to chrome browser executable.'), | |
285 new _Option('safari', 'Path to safari browser executable.'), | |
286 new _Option.bool( | |
287 'use_sdk', | |
288 '''Use compiler or runtime from the SDK. | |
289 | |
290 Normally, the compiler or runtimes in PRODUCT_DIR is tested, with | |
291 this option, the compiler or runtime in PRODUCT_DIR/dart-sdk/bin | |
292 is tested. | |
293 | |
294 (Note: currently only implemented for dart2js.)'''), | |
295 new _Option('build_directory', | |
296 'The name of the build directory, where products are placed.'), | |
297 new _Option.bool('noBatch', 'Do not run tests in batch mode.', 'n'), | |
298 new _Option.bool('dart2js_batch', 'Run dart2js tests in batch mode.'), | |
299 new _Option.bool( | |
300 'append_logs', 'Do not delete old logs but rather append to them.'), | |
301 new _Option.bool('write_debug_log', | |
302 'Don\'t write debug messages to stdout but rather to a logfile.'), | |
303 new _Option.bool('write_test_outcome_log', | |
304 'Write test outcomes to a "${TestUtils.testOutcomeFileName}" file.'), | |
305 new _Option.bool( | |
306 'reset_browser_configuration', | |
307 '''Browser specific reset of configuration. | |
308 | |
309 Warning: Using this option may remove your bookmarks and other | |
310 settings.'''), | |
311 new _Option.bool( | |
312 'copy_coredumps', | |
313 '''If we see a crash that we did not expect, copy the core dumps to | |
314 "/tmp".'''), | |
315 new _Option( | |
316 'local_ip', | |
317 '''IP address the HTTP servers should listen on. This address is also | |
318 used for browsers to connect to.''', | |
319 defaultsTo: '127.0.0.1'), | |
320 new _Option.int('test_server_port', 'Port for test http server.', | |
321 defaultsTo: 0), | |
322 new _Option.int('test_server_cross_origin_port', | |
323 'Port for test http server cross origin.', | |
324 defaultsTo: 0), | |
325 new _Option.int('test_driver_port', 'Port for http test driver server.', | |
326 defaultsTo: 0), | |
327 new _Option.int( | |
328 'test_driver_error_port', 'Port for http test driver server errors.', | |
329 defaultsTo: 0), | |
330 new _Option('record_to_file', | |
331 'Records all commands to be executed and writes to a file.', | |
332 defaultsTo: null), | |
333 new _Option( | |
334 'replay_from_file', 'Replays a previously recorded list of commands.', | |
335 defaultsTo: null), | |
336 new _Option( | |
337 'builder_tag', | |
338 '''Machine specific options that is not captured by the regular test | |
339 options. Used to be able to make sane updates to the status files.'''), | |
340 new _Option('vm_options', 'Extra options to send to the vm when running.', | |
341 defaultsTo: null), | |
342 new _Option( | |
343 'dart2js_options', 'Extra options for dart2js compilation step.', | |
344 defaultsTo: null), | |
345 new _Option( | |
346 'suite_dir', 'Additional directory to add to the testing matrix.', | |
347 defaultsTo: null), | |
348 new _Option('package_root', 'The package root to use for testing.', | |
349 defaultsTo: null), | |
350 new _Option('packages', 'The package spec file to use for testing.', | |
351 defaultsTo: null), | |
352 new _Option( | |
353 'exclude_suite', | |
354 '''Exclude suites from default selector, only works when no selector | |
355 has been specified on the command line.''', | |
356 defaultsTo: null), | |
357 new _Option.bool( | |
358 'skip-compilation', | |
359 ''' | |
360 Skip the compilation step, using the compilation artifacts left in | |
361 the output folder from a previous run. This flag will often cause | |
362 false positves and negatives, but can be useful for quick and | |
363 dirty offline testing when not making changes that affect the | |
364 compiler.''') | |
365 ]; | |
366 | |
367 /// For printing out reproducing command lines, we don't want to add these | |
368 /// options. | |
369 static final _blacklistedOptions = [ | |
370 'append_logs', | |
371 'build_directory', | |
372 'chrome', | |
373 'copy_coredumps', | |
374 'dart', | |
375 'flutter', | |
376 'dartium', | |
377 'drt', | |
378 'exclude_suite', | |
379 'firefox', | |
380 'local_ip', | |
381 'progress', | |
382 'report', | |
383 'safari', | |
384 'shard', | |
385 'shards', | |
386 'step_name', | |
387 'tasks', | |
388 'time', | |
389 'verbose', | |
390 'write_debug_log', | |
391 'write_test_outcome_log', | |
392 ].toSet(); | |
393 | |
394 /// Parses a list of strings as test options. | |
395 /// | |
396 /// Returns a list of configurations in which to run the tests. | |
397 /// Configurations are maps mapping from option keys to values. When | |
398 /// encountering the first non-option string, the rest of the arguments are | |
399 /// stored in the returned Map under the 'rest' key. | |
400 List<Map> parse(List<String> arguments) { | |
401 // TODO(rnystrom): The builders on the buildbots still pass this even | |
402 // though it does nothing. Until those can be fixed, silently ignore the | |
403 // option. Remove this once the buildbot scripts are fixed. | |
404 if (arguments.contains("--failure-summary")) { | |
405 arguments = arguments.where((arg) => arg != "--failure-summary").toList(); | |
406 print('Note: Ignoring unsupported "--failure-summary" option.'); | |
407 } | |
408 | |
409 var configuration = {}; | |
410 | |
411 // Fill in configuration with arguments passed to the test script. | |
412 for (var i = 0; i < arguments.length; i++) { | |
413 var arg = arguments[i]; | |
414 | |
415 // Help supersedes all other arguments. | |
416 if (arg == "--help" || arg == "-h") { | |
417 _printHelp(); | |
418 return null; | |
419 } | |
420 | |
421 // Extract name and value for options. | |
422 String command; | |
423 String value; | |
424 _Option option; | |
425 | |
426 if (arg.startsWith("--")) { | |
427 // A long option name. | |
428 var equals = arg.indexOf("="); | |
429 if (equals != -1) { | |
430 // A long option with a value, like "--arch=ia32". | |
431 command = arg.substring(0, equals); | |
432 value = arg.substring(equals + 1); | |
433 } else { | |
434 command = arg; | |
435 } | |
436 | |
437 option = _findByName(command.substring(2)); | |
438 } else if (arg.startsWith("-")) { | |
439 // An abbreviated option. | |
440 if (arg.length == 1) { | |
441 _fail('Missing option name after "-".'); | |
442 } | |
443 | |
444 command = arg.substring(0, 2); | |
445 | |
446 if (arg.length > 2) { | |
447 // An abbreviated option followed by a value, like "-aia32". | |
448 value = arg.substring(2); | |
449 } | |
450 | |
451 option = _findByAbbreviation(command.substring(1)); | |
452 } else { | |
453 // The argument does not start with "-" or "--" and is therefore not an | |
454 // option. Use it as a test selector pattern. | |
455 var patterns = configuration.putIfAbsent("selectors", () => <String>[]); | |
456 patterns.add(arg); | |
457 continue; | |
458 } | |
459 | |
460 if (option == null) { | |
461 _fail('Unknown command line option "$command".'); | |
462 } | |
463 | |
464 // If we need a value, look at the next argument. | |
465 if (value == null && option.type != _OptionValueType.bool) { | |
466 if (i + 1 >= arguments.length) { | |
467 _fail('Missing value for command line option "$command".'); | |
468 } | |
469 value = arguments[++i]; | |
470 } | |
471 | |
472 // Multiple uses of a flag are an error, because there is no naturally | |
473 // correct way to handle conflicting options. | |
474 if (configuration.containsKey(option.name)) { | |
475 _fail('Already have value for command line option "$command".'); | |
476 } | |
477 | |
478 // Parse the value for the option. | |
479 switch (option.type) { | |
480 case _OptionValueType.bool: | |
481 if (value != null) { | |
482 _fail('Boolean flag "$command" does not take a value.'); | |
483 } | |
484 | |
485 configuration[option.name] = true; | |
486 break; | |
487 | |
488 case _OptionValueType.int: | |
489 try { | |
490 configuration[option.name] = int.parse(value); | |
491 } on FormatException { | |
492 _fail('Integer value expected for option "$command".'); | |
493 } | |
494 break; | |
495 | |
496 case _OptionValueType.string: | |
497 // Validate against the allowed values. | |
498 if (!option.values.isEmpty) { | |
499 for (var v in value.split(",")) { | |
500 if (!option.values.contains(v)) { | |
501 _fail('Unknown value "$v" for command line option "$command".'); | |
502 } | |
503 } | |
504 } | |
505 | |
506 // TODO(rnystrom): Store as a list instead of a comma-delimited | |
507 // string. | |
508 configuration[option.name] = value; | |
509 break; | |
510 } | |
511 } | |
512 | |
513 // Apply default values for unspecified options. | |
514 for (var option in _options) { | |
515 if (!configuration.containsKey(option.name)) { | |
516 configuration[option.name] = option.defaultValue; | |
517 } | |
518 } | |
519 | |
520 var expandedConfigs = _expandConfigurations(configuration); | |
521 var result = expandedConfigs.where(_isValidConfig).toList(); | |
522 for (var config in result) { | |
523 config['_reproducing_arguments_'] = _reproducingCommand(config); | |
524 } | |
525 | |
526 return result.isEmpty ? null : result; | |
527 } | |
528 | |
529 /// Prints [message] and exits with a non-zero exit code. | |
530 void _fail(String message) { | |
531 print(message); | |
532 exit(1); | |
533 } | |
534 | |
535 /// Given a configuration, returns the list of command line arguments that | |
536 /// would reproduce that configuration. | |
537 List<String> _reproducingCommand(Map config) { | |
538 var arguments = <String>[]; | |
539 | |
540 for (var option in _options) { | |
541 var name = option.name; | |
542 if (!config.containsKey(name) || _blacklistedOptions.contains(name)) { | |
543 continue; | |
544 } | |
545 | |
546 var value = config[name]; | |
547 if (config[name] == option.defaultValue || | |
548 (name == 'packages' && | |
549 value == | |
550 TestUtils.dartDirUri.resolve('.packages').toFilePath())) { | |
551 continue; | |
552 } | |
553 | |
554 arguments.add(option.shortCommand); | |
555 if (option.type != _OptionValueType.bool) { | |
556 arguments.add(value.toString()); | |
557 } | |
558 } | |
559 | |
560 return arguments; | |
561 } | |
562 | |
563 /// Determines if a particular configuration has a valid combination of | |
564 /// compiler and runtime elements. | |
565 bool _isValidConfig(Map config) { | |
566 var isValid = true; | |
567 List<String> validRuntimes; | |
568 switch (config['compiler']) { | |
569 case 'dart2js': | |
570 // Note: by adding 'none' as a configuration, if the user | |
571 // runs test.py -c dart2js -r drt,none the dart2js_none and | |
572 // dart2js_drt will be duplicating work. If later we don't need 'none' | |
573 // with dart2js, we should remove it from here. | |
574 validRuntimes = const [ | |
575 'd8', | |
576 'jsshell', | |
577 'drt', | |
578 'none', | |
579 'dartium', | |
580 'ff', | |
581 'chrome', | |
582 'safari', | |
583 'ie9', | |
584 'ie10', | |
585 'ie11', | |
586 'opera', | |
587 'chromeOnAndroid', | |
588 'safarimobilesim' | |
589 ]; | |
590 break; | |
591 case 'dart2analyzer': | |
592 validRuntimes = const ['none']; | |
593 break; | |
594 case 'app_jit': | |
595 case 'dartk': | |
596 validRuntimes = const ['vm', 'self_check', 'none']; | |
597 break; | |
598 case 'precompiler': | |
599 case 'dartkp': | |
600 validRuntimes = const ['dart_precompiled']; | |
601 break; | |
602 case 'none': | |
603 validRuntimes = const [ | |
604 'vm', | |
605 'flutter', | |
606 'drt', | |
607 'dartium', | |
608 'ContentShellOnAndroid', | |
609 'DartiumOnAndroid' | |
610 ]; | |
611 break; | |
612 } | |
613 | |
614 if (!validRuntimes.contains(config['runtime'])) { | |
615 isValid = false; | |
616 print("Warning: combination of compiler '${config['compiler']}' and " | |
617 "runtime '${config['runtime']}' is invalid. " | |
618 "Skipping this combination."); | |
619 } | |
620 | |
621 if (config['ie'] && Platform.operatingSystem != 'windows') { | |
622 isValid = false; | |
623 print("Warning: cannot run Internet Explorer on non-Windows operating" | |
624 " system."); | |
625 } | |
626 | |
627 if (config['shard'] < 1 || config['shard'] > config['shards']) { | |
628 isValid = false; | |
629 print("Error: shard index is ${config['shard']} out of " | |
630 "${config['shards']} shards"); | |
631 } | |
632 | |
633 if (config['runtime'] == 'flutter' && config['flutter'] == '') { | |
634 isValid = false; | |
635 print("-rflutter requires the flutter engine executable to " | |
636 "be specified using --flutter="); | |
637 } | |
638 | |
639 if (config['runtime'] == 'flutter' && config['arch'] != 'x64') { | |
640 isValid = false; | |
641 print("-rflutter is applicable only for --arch=x64"); | |
642 } | |
643 | |
644 return isValid; | |
645 } | |
646 | |
647 /// Recursively expands a configuration with multiple values per key into a | |
648 /// list of configurations with exactly one value per key. | |
649 List<Map> _expandConfigurations(Map configuration) { | |
650 // Expand the pseudo-values such as 'all'. | |
651 if (configuration['arch'] == 'all') { | |
652 configuration['arch'] = 'ia32,x64,simarm,simarm64,simmips,simdbc64'; | |
653 } | |
654 | |
655 if (configuration['mode'] == 'all') { | |
656 configuration['mode'] = 'debug,release,product'; | |
657 } | |
658 | |
659 if (configuration['report_in_json']) { | |
660 configuration['list'] = true; | |
661 configuration['report'] = true; | |
662 } | |
663 | |
664 // Use verbose progress indication for verbose output unless buildbot | |
665 // progress indication is requested. | |
666 if (configuration['verbose'] && configuration['progress'] != 'buildbot') { | |
667 configuration['progress'] = 'verbose'; | |
668 } | |
669 | |
670 // Create the artificial negative options that test status files | |
671 // expect. | |
672 configuration['unchecked'] = !configuration['checked']; | |
673 configuration['host_unchecked'] = !configuration['host_checked']; | |
674 configuration['unminified'] = !configuration['minified']; | |
675 configuration['nocsp'] = !configuration['csp']; | |
676 | |
677 String runtime = configuration['runtime']; | |
678 if (runtime == 'firefox') { | |
679 configuration['runtime'] == 'ff'; | |
680 } | |
681 | |
682 String compiler = configuration['compiler']; | |
683 configuration['browser'] = TestUtils.isBrowserRuntime(runtime); | |
684 configuration['analyzer'] = TestUtils.isCommandLineAnalyzer(compiler); | |
685 | |
686 // Set the javascript command line flag for less verbose status files. | |
687 configuration['jscl'] = TestUtils.isJsCommandLineRuntime(runtime); | |
688 | |
689 // Allow suppression that is valid for all ie versions | |
690 configuration['ie'] = runtime.startsWith('ie'); | |
691 | |
692 // Expand the test selectors into a suite name and a simple | |
693 // regular expressions to be used on the full path of a test file | |
694 // in that test suite. If no selectors are explicitly given use | |
695 // the default suite patterns. | |
696 var selectors = configuration['selectors']; | |
697 if (selectors is! Map) { | |
698 if (selectors == null) { | |
699 if (configuration['suite_dir'] != null) { | |
700 var suite_path = new Path(configuration['suite_dir']); | |
701 selectors = [suite_path.filename]; | |
702 } else { | |
703 selectors = _defaultTestSelectors.toList(); | |
704 } | |
705 | |
706 var excludeSuites = configuration['exclude_suite'] != null | |
707 ? configuration['exclude_suite'].split(',') | |
708 : []; | |
709 for (var exclude in excludeSuites) { | |
710 if (selectors.contains(exclude)) { | |
711 selectors.remove(exclude); | |
712 } else { | |
713 print("Warning: default selectors does not contain $exclude"); | |
714 } | |
715 } | |
716 } | |
717 var selectorMap = <String, RegExp>{}; | |
718 for (var i = 0; i < selectors.length; i++) { | |
719 var pattern = selectors[i]; | |
720 var suite = pattern; | |
721 var slashLocation = pattern.indexOf('/'); | |
722 if (slashLocation != -1) { | |
723 suite = pattern.substring(0, slashLocation); | |
724 pattern = pattern.substring(slashLocation + 1); | |
725 pattern = pattern.replaceAll('*', '.*'); | |
726 } else { | |
727 pattern = ".?"; | |
728 } | |
729 if (selectorMap.containsKey(suite)) { | |
730 print("Error: '$suite/$pattern'. Only one test selection" | |
731 " pattern is allowed to start with '$suite/'"); | |
732 exit(1); | |
733 } | |
734 selectorMap[suite] = new RegExp(pattern); | |
735 } | |
736 configuration['selectors'] = selectorMap; | |
737 } | |
738 | |
739 // Put observatory_ui in a configuration with its own packages override. | |
740 // Only one value in the configuration map is mutable: | |
741 selectors = configuration['selectors']; | |
742 if (selectors.containsKey('observatory_ui')) { | |
743 if (selectors.length == 1) { | |
744 configuration['packages'] = TestUtils.dartDirUri | |
745 .resolve('runtime/observatory/.packages') | |
746 .toFilePath(); | |
747 } else { | |
748 // Make a new configuration whose selectors map only contains | |
749 // observatory_ui, and remove the key from the original selectors. | |
750 // The only mutable value in the map is the selectors, so a | |
751 // shallow copy is safe. | |
752 var observatoryConfiguration = new Map.from(configuration); | |
753 observatoryConfiguration['selectors'] = { | |
754 'observatory_ui': selectors['observatory_ui'] | |
755 }; | |
756 selectors.remove('observatory_ui'); | |
757 | |
758 // Set the packages flag. | |
759 observatoryConfiguration['packages'] = TestUtils.dartDirUri | |
760 .resolve('runtime/observatory/.packages') | |
761 .toFilePath(); | |
762 | |
763 // Return the expansions of both configurations. Neither will reach | |
764 // this line in the recursive call to _expandConfigurations. | |
765 return _expandConfigurations(configuration) | |
766 ..addAll(_expandConfigurations(observatoryConfiguration)); | |
767 } | |
768 } | |
769 | |
770 // Set the default package spec explicitly. | |
771 if (configuration['package_root'] == null && | |
772 configuration['packages'] == null) { | |
773 configuration['packages'] = | |
774 TestUtils.dartDirUri.resolve('.packages').toFilePath(); | |
775 } | |
776 | |
777 // Expand the architectures. | |
778 if (configuration['arch'].contains(',')) { | |
779 return _expandHelper('arch', configuration); | |
780 } | |
781 | |
782 // Expand modes. | |
783 if (configuration['mode'].contains(',')) { | |
784 return _expandHelper('mode', configuration); | |
785 } | |
786 | |
787 // Expand compilers. | |
788 if (configuration['compiler'].contains(',')) { | |
789 return _expandHelper('compiler', configuration); | |
790 } | |
791 | |
792 // Expand runtimes. | |
793 var runtimes = configuration['runtime']; | |
794 if (runtimes.contains(',')) { | |
795 return _expandHelper('runtime', configuration); | |
796 } else { | |
797 // All runtimes eventually go through this path, after expansion. | |
798 var updater = runtimeUpdater(configuration); | |
799 if (updater != null) { | |
800 updater.update(); | |
801 } | |
802 } | |
803 | |
804 // Adjust default timeout based on mode, compiler, and sometimes runtime. | |
805 if (configuration['timeout'] == -1) { | |
806 var isReload = | |
807 configuration['hot_reload'] || configuration['hot_reload_rollback']; | |
808 int compilerMulitiplier = | |
809 new CompilerConfiguration(configuration).computeTimeoutMultiplier(); | |
810 int runtimeMultiplier = new RuntimeConfiguration(configuration) | |
811 .computeTimeoutMultiplier( | |
812 mode: configuration['mode'], | |
813 isChecked: configuration['checked'], | |
814 isReload: isReload, | |
815 arch: configuration['arch']); | |
816 configuration['timeout'] = 60 * compilerMulitiplier * runtimeMultiplier; | |
817 } | |
818 | |
819 return [configuration]; | |
820 } | |
821 | |
822 /// Helper for _expandConfigurations. Creates a new configuration and adds it | |
823 /// to a list, for use in a case when a particular configuration has multiple | |
824 /// results (separated by a ','). | |
825 /// Arguments: | |
826 /// option: The particular test option we are expanding. | |
827 /// configuration: The map containing all test configuration information | |
828 /// specified. | |
829 List<Map> _expandHelper(String option, Map configuration) { | |
830 var result = <Map>[]; | |
831 var configs = configuration[option]; | |
832 for (var config in configs.split(',')) { | |
833 var newConfiguration = new Map.from(configuration); | |
834 newConfiguration[option] = config; | |
835 result.addAll(_expandConfigurations(newConfiguration)); | |
836 } | |
837 return result; | |
838 } | |
839 | |
840 /// Print out usage information. | |
841 void _printHelp() { | |
842 var buffer = new StringBuffer(); | |
843 | |
844 buffer.writeln('''usage: dart test.dart [options] [selector] | |
845 | |
846 The optional selector limits the tests that will be run. | |
847 For example, the selector "language/issue", or equivalently | |
848 "language/*issue*", limits to test files matching the regexp | |
849 ".*issue.*\\.dart" in the "tests/language" directory. | |
850 | |
851 Options:'''); | |
852 | |
853 for (var option in _options) { | |
854 if (option.abbreviation != null) { | |
855 buffer.write("-${option.abbreviation}, "); | |
856 } else { | |
857 buffer.write(" "); | |
858 } | |
859 | |
860 buffer.write(option.command); | |
861 | |
862 switch (option.type) { | |
863 case _OptionValueType.bool: | |
864 // No value. | |
865 break; | |
866 case _OptionValueType.int: | |
867 buffer.write("=<integer>"); | |
868 break; | |
869 case _OptionValueType.string: | |
870 if (option.values.length > 6) { | |
871 // If there are many options, they won't fit nicely in one line and | |
872 // should be instead listed in the description. | |
873 buffer.write("=<...>"); | |
874 } else if (option.values.isNotEmpty) { | |
875 buffer.write("=<${option.values.join('|')}>"); | |
876 } else { | |
877 buffer.write("=<string>"); | |
878 } | |
879 break; | |
880 } | |
881 | |
882 if (option.type != _OptionValueType.bool && | |
883 option.defaultValue != null && | |
884 option.defaultValue != "") { | |
885 buffer.write(" (defaults to ${option.defaultValue})"); | |
886 } | |
887 | |
888 buffer.writeln(); | |
889 buffer | |
890 .writeln(" ${option.description.replaceAll('\n', '\n ')}"); | |
891 buffer.writeln(); | |
892 } | |
893 | |
894 print(buffer); | |
895 } | |
896 | |
897 _Option _findByAbbreviation(String abbreviation) { | |
898 for (var option in _options) { | |
899 if (abbreviation == option.abbreviation) return option; | |
900 } | |
901 | |
902 return null; | |
903 } | |
904 | |
905 _Option _findByName(String name) { | |
906 for (var option in _options) { | |
907 if (name == option.name) return option; | |
908 | |
909 // Allow hyphens instead of underscores as the separator since they are | |
910 // more common for command line flags. | |
911 if (name == option.name.replaceAll("_", "-")) return option; | |
912 } | |
913 | |
914 return null; | |
915 } | |
916 } | |
OLD | NEW |