OLD | NEW |
(Empty) | |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. |
| 4 |
| 5 import 'dart:io'; |
| 6 import 'drt_updater.dart'; |
| 7 import 'test_suite.dart'; |
| 8 import 'path.dart'; |
| 9 import 'compiler_configuration.dart' show CompilerConfiguration; |
| 10 import 'runtime_configuration.dart' show RuntimeConfiguration; |
| 11 |
| 12 const _defaultTestSelectors = const [ |
| 13 'samples', |
| 14 'standalone', |
| 15 'corelib', |
| 16 'co19', |
| 17 'language', |
| 18 'isolate', |
| 19 'vm', |
| 20 'html', |
| 21 'benchmark_smoke', |
| 22 'utils', |
| 23 'lib', |
| 24 'analyze_library', |
| 25 'service', |
| 26 'kernel', |
| 27 'observatory_ui' |
| 28 ]; |
| 29 |
| 30 /// Specifies a single command line option. |
| 31 /// |
| 32 /// The name of the specification is used as the key for the option in the Map |
| 33 /// returned from the [TestOptionParser] parse method. |
| 34 class _Option { |
| 35 // TODO(rnystrom): Some string options use "" to mean "no value" and others |
| 36 // use null. Clean that up. |
| 37 _Option(this.name, this.description, |
| 38 {String abbr, List<String> values, String defaultsTo = ""}) |
| 39 : abbreviation = abbr, |
| 40 values = values ?? [], |
| 41 defaultValue = defaultsTo, |
| 42 type = _OptionValueType.string; |
| 43 |
| 44 _Option.bool(this.name, this.description, [this.abbreviation]) |
| 45 : values = [], |
| 46 defaultValue = false, |
| 47 type = _OptionValueType.bool; |
| 48 |
| 49 _Option.int(this.name, this.description, {String abbr, int defaultsTo}) |
| 50 : abbreviation = abbr, |
| 51 values = [], |
| 52 defaultValue = defaultsTo, |
| 53 type = _OptionValueType.int; |
| 54 |
| 55 final String name; |
| 56 final String description; |
| 57 final String abbreviation; |
| 58 final List<String> values; |
| 59 final Object defaultValue; |
| 60 final _OptionValueType type; |
| 61 |
| 62 /// Gets the shortest command line argument used to refer to this option. |
| 63 String get shortCommand => abbreviation != null ? "-$abbreviation" : command; |
| 64 |
| 65 /// Gets the canonical long command line argument used to refer to this |
| 66 /// option. |
| 67 String get command => "--${name.replaceAll('_', '-')}"; |
| 68 } |
| 69 |
| 70 enum _OptionValueType { bool, int, string } |
| 71 |
| 72 /// Parses command line arguments and produces a test runner configuration. |
| 73 class OptionsParser { |
| 74 static final List<_Option> _options = [ |
| 75 new _Option('mode', 'Mode in which to run the tests.', |
| 76 abbr: 'm', |
| 77 values: ['all', 'debug', 'release', 'product'], |
| 78 defaultsTo: 'debug'), |
| 79 new _Option( |
| 80 'compiler', |
| 81 '''Specify any compilation step (if needed). |
| 82 |
| 83 none: Do not compile the Dart code (run native Dart code |
| 84 on the VM). |
| 85 (Only valid with runtimes vm, flutter, or drt.) |
| 86 |
| 87 precompiler: Compile into AOT snapshot before running the test. |
| 88 (Only valid with runtime dart_precompiled.) |
| 89 |
| 90 dart2js: Compile dart code to JavaScript by running dart2js. |
| 91 (Only valid with runtimes: d8, drt, chrome, safari, |
| 92 ie9, ie10, ie11, firefox, opera, chromeOnAndroid, |
| 93 and none [compile only].) |
| 94 |
| 95 dart2analyzer: Perform static analysis on Dart code by running the |
| 96 analyzer. |
| 97 (Only valid with runtime none.) |
| 98 |
| 99 app_jit: Compile the Dart code into an app snapshot before |
| 100 running test. |
| 101 (Only valid with dart_app runtime.) |
| 102 |
| 103 dartk: Compile the Dart source into Kernel before running |
| 104 test. |
| 105 |
| 106 dartkp: Compile the Dart source into Kernel and then Kernel |
| 107 into AOT snapshot before running the test. |
| 108 (Only valid with runtime dart_precompiled.)''', |
| 109 abbr: 'c', |
| 110 values: [ |
| 111 'none', |
| 112 'precompiler', |
| 113 'dart2js', |
| 114 'dart2analyzer', |
| 115 'app_jit', |
| 116 'dartk', |
| 117 'dartkp' |
| 118 ], |
| 119 defaultsTo: 'none'), |
| 120 new _Option( |
| 121 'runtime', |
| 122 '''Where the tests should be run. |
| 123 vm: Run Dart code on the standalone dart vm. |
| 124 |
| 125 flutter: Run Dart code on the flutter engine. |
| 126 |
| 127 dart_precompiled: Run a precompiled snapshot on a variant of the |
| 128 standalone dart VM lacking a JIT. |
| 129 |
| 130 d8: Run JavaScript from the command line using v8. |
| 131 |
| 132 jsshell: Run JavaScript from the command line using |
| 133 Firefox js-shell. |
| 134 |
| 135 drt: Run Dart or JavaScript in the headless version |
| 136 of Chrome, Content shell. |
| 137 |
| 138 dartium: Run Dart or JavaScript in Dartium. |
| 139 |
| 140 ContentShellOnAndroid: Run Dart or JavaScript in Dartium content |
| 141 shell on Android. |
| 142 |
| 143 DartiumOnAndroid: Run Dart or Javascript in Dartium on Android. |
| 144 |
| 145 ff: |
| 146 chrome: |
| 147 safari: |
| 148 ie9: |
| 149 ie10: |
| 150 ie11: |
| 151 opera: |
| 152 chromeOnAndroid: Run JavaScript in the specified browser. |
| 153 |
| 154 self_check: Pass each test or its compiled output to every |
| 155 file under `pkg` whose name ends with |
| 156 `_self_check.dart`. Each test is given to the |
| 157 self_check tester as a filename on stdin using |
| 158 the batch-mode protocol. |
| 159 |
| 160 none: No runtime, compile only. (For example, used |
| 161 for dart2analyzer static analysis tests).''', |
| 162 abbr: 'r', |
| 163 values: [ |
| 164 'vm', |
| 165 'flutter', |
| 166 'dart_precompiled', |
| 167 'd8', |
| 168 'jsshell', |
| 169 'drt', |
| 170 'dartium', |
| 171 'ff', |
| 172 'firefox', |
| 173 'chrome', |
| 174 'safari', |
| 175 'ie9', |
| 176 'ie10', |
| 177 'ie11', |
| 178 'opera', |
| 179 'chromeOnAndroid', |
| 180 'safarimobilesim', |
| 181 'ContentShellOnAndroid', |
| 182 'DartiumOnAndroid', |
| 183 'self_check', |
| 184 'none' |
| 185 ], |
| 186 defaultsTo: 'vm'), |
| 187 new _Option( |
| 188 'arch', |
| 189 '''The architecture to run tests for. |
| 190 |
| 191 Allowed values are: |
| 192 all |
| 193 ia32, x64 |
| 194 arm, armv6, armv5te, arm64, |
| 195 simarm, simarmv6, simarmv5te, simarm64, |
| 196 mips, simmips |
| 197 simdbc, simdbc64''', |
| 198 abbr: 'a', |
| 199 values: [ |
| 200 'all', |
| 201 'ia32', |
| 202 'x64', |
| 203 'arm', |
| 204 'armv6', |
| 205 'armv5te', |
| 206 'arm64', |
| 207 'mips', |
| 208 'simarm', |
| 209 'simarmv6', |
| 210 'simarmv5te', |
| 211 'simarm64', |
| 212 'simmips', |
| 213 'simdbc', |
| 214 'simdbc64', |
| 215 ], |
| 216 defaultsTo: 'x64'), |
| 217 new _Option('system', 'The operating system to run tests on.', |
| 218 abbr: 's', |
| 219 values: ['linux', 'macos', 'windows', 'android'], |
| 220 defaultsTo: Platform.operatingSystem), |
| 221 new _Option.bool('checked', 'Run tests in checked mode.'), |
| 222 new _Option.bool('strong', 'Run tests in strong mode.'), |
| 223 new _Option.bool('host_checked', 'Run compiler in checked mode.'), |
| 224 new _Option.bool('minified', 'Enable minification in the compiler.'), |
| 225 new _Option.bool( |
| 226 'csp', 'Run tests under Content Security Policy restrictions.'), |
| 227 new _Option.bool( |
| 228 'fast_startup', 'Pass the --fast-startup flag to dart2js.'), |
| 229 new _Option.bool('dart2js_with_kernel', |
| 230 'Enable the internal pipeline in dart2js to use kernel.'), |
| 231 new _Option.bool('hot_reload', 'Run hot reload stress tests.'), |
| 232 new _Option.bool( |
| 233 'hot_reload_rollback', 'Run hot reload rollback stress tests.'), |
| 234 new _Option.bool('use_blobs', |
| 235 'Use mmap instead of shared libraries for precompilation.'), |
| 236 new _Option.int('timeout', 'Timeout in seconds.', |
| 237 abbr: 't', defaultsTo: -1), |
| 238 new _Option( |
| 239 'progress', |
| 240 '''Progress indication mode. |
| 241 |
| 242 Allowed values are: |
| 243 compact, color, line, verbose, silent, status, buildbot, diff |
| 244 ''', |
| 245 abbr: 'p', |
| 246 values: [ |
| 247 'compact', |
| 248 'color', |
| 249 'line', |
| 250 'verbose', |
| 251 'silent', |
| 252 'status', |
| 253 'buildbot', |
| 254 'diff' |
| 255 ], |
| 256 defaultsTo: 'compact'), |
| 257 new _Option('step_name', 'Step name for use by -pbuildbot.', |
| 258 defaultsTo: null), |
| 259 new _Option.bool('report', |
| 260 'Print a summary report of the number of tests, by expectation.'), |
| 261 new _Option.int('tasks', 'The number of parallel tasks to run.', |
| 262 abbr: 'j', defaultsTo: Platform.numberOfProcessors), |
| 263 new _Option.int('shards', |
| 264 'The number of instances that the tests will be sharded over.', |
| 265 defaultsTo: 1), |
| 266 new _Option.int( |
| 267 'shard', 'The index of this instance when running in sharded mode.', |
| 268 defaultsTo: 1), |
| 269 new _Option.bool('help', 'Print list of options.', 'h'), |
| 270 new _Option.bool('verbose', 'Verbose output.', 'v'), |
| 271 new _Option.bool('verify-ir', 'Verify kernel IR.'), |
| 272 new _Option.bool('no-tree-shake', 'Disable kernel IR tree shaking.'), |
| 273 new _Option.bool('list', 'List tests only, do not run them.'), |
| 274 new _Option.bool('report_in_json', |
| 275 'When listing with --list, output result summary in JSON.'), |
| 276 new _Option.bool('time', 'Print timing information after running tests.'), |
| 277 new _Option('dart', 'Path to dart executable.'), |
| 278 new _Option('flutter', 'Path to flutter executable.'), |
| 279 new _Option( |
| 280 'drt', // TODO(antonm): fix the option name. |
| 281 'Path to content shell executable.'), |
| 282 new _Option('dartium', 'Path to Dartium Chrome executable.'), |
| 283 new _Option('firefox', 'Path to firefox browser executable.'), |
| 284 new _Option('chrome', 'Path to chrome browser executable.'), |
| 285 new _Option('safari', 'Path to safari browser executable.'), |
| 286 new _Option.bool( |
| 287 'use_sdk', |
| 288 '''Use compiler or runtime from the SDK. |
| 289 |
| 290 Normally, the compiler or runtimes in PRODUCT_DIR is tested, with |
| 291 this option, the compiler or runtime in PRODUCT_DIR/dart-sdk/bin |
| 292 is tested. |
| 293 |
| 294 (Note: currently only implemented for dart2js.)'''), |
| 295 new _Option('build_directory', |
| 296 'The name of the build directory, where products are placed.'), |
| 297 new _Option.bool('noBatch', 'Do not run tests in batch mode.', 'n'), |
| 298 new _Option.bool('dart2js_batch', 'Run dart2js tests in batch mode.'), |
| 299 new _Option.bool( |
| 300 'append_logs', 'Do not delete old logs but rather append to them.'), |
| 301 new _Option.bool('write_debug_log', |
| 302 'Don\'t write debug messages to stdout but rather to a logfile.'), |
| 303 new _Option.bool('write_test_outcome_log', |
| 304 'Write test outcomes to a "${TestUtils.testOutcomeFileName}" file.'), |
| 305 new _Option.bool( |
| 306 'reset_browser_configuration', |
| 307 '''Browser specific reset of configuration. |
| 308 |
| 309 Warning: Using this option may remove your bookmarks and other |
| 310 settings.'''), |
| 311 new _Option.bool( |
| 312 'copy_coredumps', |
| 313 '''If we see a crash that we did not expect, copy the core dumps to |
| 314 "/tmp".'''), |
| 315 new _Option( |
| 316 'local_ip', |
| 317 '''IP address the HTTP servers should listen on. This address is also |
| 318 used for browsers to connect to.''', |
| 319 defaultsTo: '127.0.0.1'), |
| 320 new _Option.int('test_server_port', 'Port for test http server.', |
| 321 defaultsTo: 0), |
| 322 new _Option.int('test_server_cross_origin_port', |
| 323 'Port for test http server cross origin.', |
| 324 defaultsTo: 0), |
| 325 new _Option.int('test_driver_port', 'Port for http test driver server.', |
| 326 defaultsTo: 0), |
| 327 new _Option.int( |
| 328 'test_driver_error_port', 'Port for http test driver server errors.', |
| 329 defaultsTo: 0), |
| 330 new _Option('record_to_file', |
| 331 'Records all commands to be executed and writes to a file.', |
| 332 defaultsTo: null), |
| 333 new _Option( |
| 334 'replay_from_file', 'Replays a previously recorded list of commands.', |
| 335 defaultsTo: null), |
| 336 new _Option( |
| 337 'builder_tag', |
| 338 '''Machine specific options that is not captured by the regular test |
| 339 options. Used to be able to make sane updates to the status files.'''), |
| 340 new _Option('vm_options', 'Extra options to send to the vm when running.', |
| 341 defaultsTo: null), |
| 342 new _Option( |
| 343 'dart2js_options', 'Extra options for dart2js compilation step.', |
| 344 defaultsTo: null), |
| 345 new _Option( |
| 346 'suite_dir', 'Additional directory to add to the testing matrix.', |
| 347 defaultsTo: null), |
| 348 new _Option('package_root', 'The package root to use for testing.', |
| 349 defaultsTo: null), |
| 350 new _Option('packages', 'The package spec file to use for testing.', |
| 351 defaultsTo: null), |
| 352 new _Option( |
| 353 'exclude_suite', |
| 354 '''Exclude suites from default selector, only works when no selector |
| 355 has been specified on the command line.''', |
| 356 defaultsTo: null), |
| 357 new _Option.bool( |
| 358 'skip-compilation', |
| 359 ''' |
| 360 Skip the compilation step, using the compilation artifacts left in |
| 361 the output folder from a previous run. This flag will often cause |
| 362 false positves and negatives, but can be useful for quick and |
| 363 dirty offline testing when not making changes that affect the |
| 364 compiler.''') |
| 365 ]; |
| 366 |
| 367 /// For printing out reproducing command lines, we don't want to add these |
| 368 /// options. |
| 369 static final _blacklistedOptions = [ |
| 370 'append_logs', |
| 371 'build_directory', |
| 372 'chrome', |
| 373 'copy_coredumps', |
| 374 'dart', |
| 375 'flutter', |
| 376 'dartium', |
| 377 'drt', |
| 378 'exclude_suite', |
| 379 'firefox', |
| 380 'local_ip', |
| 381 'progress', |
| 382 'report', |
| 383 'safari', |
| 384 'shard', |
| 385 'shards', |
| 386 'step_name', |
| 387 'tasks', |
| 388 'time', |
| 389 'verbose', |
| 390 'write_debug_log', |
| 391 'write_test_outcome_log', |
| 392 ].toSet(); |
| 393 |
| 394 /// Parses a list of strings as test options. |
| 395 /// |
| 396 /// Returns a list of configurations in which to run the tests. |
| 397 /// Configurations are maps mapping from option keys to values. When |
| 398 /// encountering the first non-option string, the rest of the arguments are |
| 399 /// stored in the returned Map under the 'rest' key. |
| 400 List<Map> parse(List<String> arguments) { |
| 401 // TODO(rnystrom): The builders on the buildbots still pass this even |
| 402 // though it does nothing. Until those can be fixed, silently ignore the |
| 403 // option. Remove this once the buildbot scripts are fixed. |
| 404 if (arguments.contains("--failure-summary")) { |
| 405 arguments = arguments.where((arg) => arg != "--failure-summary").toList(); |
| 406 print('Note: Ignoring unsupported "--failure-summary" option.'); |
| 407 } |
| 408 |
| 409 var configuration = {}; |
| 410 |
| 411 // Fill in configuration with arguments passed to the test script. |
| 412 for (var i = 0; i < arguments.length; i++) { |
| 413 var arg = arguments[i]; |
| 414 |
| 415 // Help supersedes all other arguments. |
| 416 if (arg == "--help" || arg == "-h") { |
| 417 _printHelp(); |
| 418 return null; |
| 419 } |
| 420 |
| 421 // Extract name and value for options. |
| 422 String command; |
| 423 String value; |
| 424 _Option option; |
| 425 |
| 426 if (arg.startsWith("--")) { |
| 427 // A long option name. |
| 428 var equals = arg.indexOf("="); |
| 429 if (equals != -1) { |
| 430 // A long option with a value, like "--arch=ia32". |
| 431 command = arg.substring(0, equals); |
| 432 value = arg.substring(equals + 1); |
| 433 } else { |
| 434 command = arg; |
| 435 } |
| 436 |
| 437 option = _findByName(command.substring(2)); |
| 438 } else if (arg.startsWith("-")) { |
| 439 // An abbreviated option. |
| 440 if (arg.length == 1) { |
| 441 _fail('Missing option name after "-".'); |
| 442 } |
| 443 |
| 444 command = arg.substring(0, 2); |
| 445 |
| 446 if (arg.length > 2) { |
| 447 // An abbreviated option followed by a value, like "-aia32". |
| 448 value = arg.substring(2); |
| 449 } |
| 450 |
| 451 option = _findByAbbreviation(command.substring(1)); |
| 452 } else { |
| 453 // The argument does not start with "-" or "--" and is therefore not an |
| 454 // option. Use it as a test selector pattern. |
| 455 var patterns = configuration.putIfAbsent("selectors", () => <String>[]); |
| 456 patterns.add(arg); |
| 457 continue; |
| 458 } |
| 459 |
| 460 if (option == null) { |
| 461 _fail('Unknown command line option "$command".'); |
| 462 } |
| 463 |
| 464 // If we need a value, look at the next argument. |
| 465 if (value == null && option.type != _OptionValueType.bool) { |
| 466 if (i + 1 >= arguments.length) { |
| 467 _fail('Missing value for command line option "$command".'); |
| 468 } |
| 469 value = arguments[++i]; |
| 470 } |
| 471 |
| 472 // Multiple uses of a flag are an error, because there is no naturally |
| 473 // correct way to handle conflicting options. |
| 474 if (configuration.containsKey(option.name)) { |
| 475 _fail('Already have value for command line option "$command".'); |
| 476 } |
| 477 |
| 478 // Parse the value for the option. |
| 479 switch (option.type) { |
| 480 case _OptionValueType.bool: |
| 481 if (value != null) { |
| 482 _fail('Boolean flag "$command" does not take a value.'); |
| 483 } |
| 484 |
| 485 configuration[option.name] = true; |
| 486 break; |
| 487 |
| 488 case _OptionValueType.int: |
| 489 try { |
| 490 configuration[option.name] = int.parse(value); |
| 491 } on FormatException { |
| 492 _fail('Integer value expected for option "$command".'); |
| 493 } |
| 494 break; |
| 495 |
| 496 case _OptionValueType.string: |
| 497 // Validate against the allowed values. |
| 498 if (!option.values.isEmpty) { |
| 499 for (var v in value.split(",")) { |
| 500 if (!option.values.contains(v)) { |
| 501 _fail('Unknown value "$v" for command line option "$command".'); |
| 502 } |
| 503 } |
| 504 } |
| 505 |
| 506 // TODO(rnystrom): Store as a list instead of a comma-delimited |
| 507 // string. |
| 508 configuration[option.name] = value; |
| 509 break; |
| 510 } |
| 511 } |
| 512 |
| 513 // Apply default values for unspecified options. |
| 514 for (var option in _options) { |
| 515 if (!configuration.containsKey(option.name)) { |
| 516 configuration[option.name] = option.defaultValue; |
| 517 } |
| 518 } |
| 519 |
| 520 var expandedConfigs = _expandConfigurations(configuration); |
| 521 var result = expandedConfigs.where(_isValidConfig).toList(); |
| 522 for (var config in result) { |
| 523 config['_reproducing_arguments_'] = _reproducingCommand(config); |
| 524 } |
| 525 |
| 526 return result.isEmpty ? null : result; |
| 527 } |
| 528 |
| 529 /// Prints [message] and exits with a non-zero exit code. |
| 530 void _fail(String message) { |
| 531 print(message); |
| 532 exit(1); |
| 533 } |
| 534 |
| 535 /// Given a configuration, returns the list of command line arguments that |
| 536 /// would reproduce that configuration. |
| 537 List<String> _reproducingCommand(Map config) { |
| 538 var arguments = <String>[]; |
| 539 |
| 540 for (var option in _options) { |
| 541 var name = option.name; |
| 542 if (!config.containsKey(name) || _blacklistedOptions.contains(name)) { |
| 543 continue; |
| 544 } |
| 545 |
| 546 var value = config[name]; |
| 547 if (config[name] == option.defaultValue || |
| 548 (name == 'packages' && |
| 549 value == |
| 550 TestUtils.dartDirUri.resolve('.packages').toFilePath())) { |
| 551 continue; |
| 552 } |
| 553 |
| 554 arguments.add(option.shortCommand); |
| 555 if (option.type != _OptionValueType.bool) { |
| 556 arguments.add(value.toString()); |
| 557 } |
| 558 } |
| 559 |
| 560 return arguments; |
| 561 } |
| 562 |
| 563 /// Determines if a particular configuration has a valid combination of |
| 564 /// compiler and runtime elements. |
| 565 bool _isValidConfig(Map config) { |
| 566 var isValid = true; |
| 567 List<String> validRuntimes; |
| 568 switch (config['compiler']) { |
| 569 case 'dart2js': |
| 570 // Note: by adding 'none' as a configuration, if the user |
| 571 // runs test.py -c dart2js -r drt,none the dart2js_none and |
| 572 // dart2js_drt will be duplicating work. If later we don't need 'none' |
| 573 // with dart2js, we should remove it from here. |
| 574 validRuntimes = const [ |
| 575 'd8', |
| 576 'jsshell', |
| 577 'drt', |
| 578 'none', |
| 579 'dartium', |
| 580 'ff', |
| 581 'chrome', |
| 582 'safari', |
| 583 'ie9', |
| 584 'ie10', |
| 585 'ie11', |
| 586 'opera', |
| 587 'chromeOnAndroid', |
| 588 'safarimobilesim' |
| 589 ]; |
| 590 break; |
| 591 case 'dart2analyzer': |
| 592 validRuntimes = const ['none']; |
| 593 break; |
| 594 case 'app_jit': |
| 595 case 'dartk': |
| 596 validRuntimes = const ['vm', 'self_check', 'none']; |
| 597 break; |
| 598 case 'precompiler': |
| 599 case 'dartkp': |
| 600 validRuntimes = const ['dart_precompiled']; |
| 601 break; |
| 602 case 'none': |
| 603 validRuntimes = const [ |
| 604 'vm', |
| 605 'flutter', |
| 606 'drt', |
| 607 'dartium', |
| 608 'ContentShellOnAndroid', |
| 609 'DartiumOnAndroid' |
| 610 ]; |
| 611 break; |
| 612 } |
| 613 |
| 614 if (!validRuntimes.contains(config['runtime'])) { |
| 615 isValid = false; |
| 616 print("Warning: combination of compiler '${config['compiler']}' and " |
| 617 "runtime '${config['runtime']}' is invalid. " |
| 618 "Skipping this combination."); |
| 619 } |
| 620 |
| 621 if (config['ie'] && Platform.operatingSystem != 'windows') { |
| 622 isValid = false; |
| 623 print("Warning: cannot run Internet Explorer on non-Windows operating" |
| 624 " system."); |
| 625 } |
| 626 |
| 627 if (config['shard'] < 1 || config['shard'] > config['shards']) { |
| 628 isValid = false; |
| 629 print("Error: shard index is ${config['shard']} out of " |
| 630 "${config['shards']} shards"); |
| 631 } |
| 632 |
| 633 if (config['runtime'] == 'flutter' && config['flutter'] == '') { |
| 634 isValid = false; |
| 635 print("-rflutter requires the flutter engine executable to " |
| 636 "be specified using --flutter="); |
| 637 } |
| 638 |
| 639 if (config['runtime'] == 'flutter' && config['arch'] != 'x64') { |
| 640 isValid = false; |
| 641 print("-rflutter is applicable only for --arch=x64"); |
| 642 } |
| 643 |
| 644 return isValid; |
| 645 } |
| 646 |
| 647 /// Recursively expands a configuration with multiple values per key into a |
| 648 /// list of configurations with exactly one value per key. |
| 649 List<Map> _expandConfigurations(Map configuration) { |
| 650 // Expand the pseudo-values such as 'all'. |
| 651 if (configuration['arch'] == 'all') { |
| 652 configuration['arch'] = 'ia32,x64,simarm,simarm64,simmips,simdbc64'; |
| 653 } |
| 654 |
| 655 if (configuration['mode'] == 'all') { |
| 656 configuration['mode'] = 'debug,release,product'; |
| 657 } |
| 658 |
| 659 if (configuration['report_in_json']) { |
| 660 configuration['list'] = true; |
| 661 configuration['report'] = true; |
| 662 } |
| 663 |
| 664 // Use verbose progress indication for verbose output unless buildbot |
| 665 // progress indication is requested. |
| 666 if (configuration['verbose'] && configuration['progress'] != 'buildbot') { |
| 667 configuration['progress'] = 'verbose'; |
| 668 } |
| 669 |
| 670 // Create the artificial negative options that test status files |
| 671 // expect. |
| 672 configuration['unchecked'] = !configuration['checked']; |
| 673 configuration['host_unchecked'] = !configuration['host_checked']; |
| 674 configuration['unminified'] = !configuration['minified']; |
| 675 configuration['nocsp'] = !configuration['csp']; |
| 676 |
| 677 String runtime = configuration['runtime']; |
| 678 if (runtime == 'firefox') { |
| 679 configuration['runtime'] == 'ff'; |
| 680 } |
| 681 |
| 682 String compiler = configuration['compiler']; |
| 683 configuration['browser'] = TestUtils.isBrowserRuntime(runtime); |
| 684 configuration['analyzer'] = TestUtils.isCommandLineAnalyzer(compiler); |
| 685 |
| 686 // Set the javascript command line flag for less verbose status files. |
| 687 configuration['jscl'] = TestUtils.isJsCommandLineRuntime(runtime); |
| 688 |
| 689 // Allow suppression that is valid for all ie versions |
| 690 configuration['ie'] = runtime.startsWith('ie'); |
| 691 |
| 692 // Expand the test selectors into a suite name and a simple |
| 693 // regular expressions to be used on the full path of a test file |
| 694 // in that test suite. If no selectors are explicitly given use |
| 695 // the default suite patterns. |
| 696 var selectors = configuration['selectors']; |
| 697 if (selectors is! Map) { |
| 698 if (selectors == null) { |
| 699 if (configuration['suite_dir'] != null) { |
| 700 var suite_path = new Path(configuration['suite_dir']); |
| 701 selectors = [suite_path.filename]; |
| 702 } else { |
| 703 selectors = _defaultTestSelectors.toList(); |
| 704 } |
| 705 |
| 706 var excludeSuites = configuration['exclude_suite'] != null |
| 707 ? configuration['exclude_suite'].split(',') |
| 708 : []; |
| 709 for (var exclude in excludeSuites) { |
| 710 if (selectors.contains(exclude)) { |
| 711 selectors.remove(exclude); |
| 712 } else { |
| 713 print("Warning: default selectors does not contain $exclude"); |
| 714 } |
| 715 } |
| 716 } |
| 717 var selectorMap = <String, RegExp>{}; |
| 718 for (var i = 0; i < selectors.length; i++) { |
| 719 var pattern = selectors[i]; |
| 720 var suite = pattern; |
| 721 var slashLocation = pattern.indexOf('/'); |
| 722 if (slashLocation != -1) { |
| 723 suite = pattern.substring(0, slashLocation); |
| 724 pattern = pattern.substring(slashLocation + 1); |
| 725 pattern = pattern.replaceAll('*', '.*'); |
| 726 } else { |
| 727 pattern = ".?"; |
| 728 } |
| 729 if (selectorMap.containsKey(suite)) { |
| 730 print("Error: '$suite/$pattern'. Only one test selection" |
| 731 " pattern is allowed to start with '$suite/'"); |
| 732 exit(1); |
| 733 } |
| 734 selectorMap[suite] = new RegExp(pattern); |
| 735 } |
| 736 configuration['selectors'] = selectorMap; |
| 737 } |
| 738 |
| 739 // Put observatory_ui in a configuration with its own packages override. |
| 740 // Only one value in the configuration map is mutable: |
| 741 selectors = configuration['selectors']; |
| 742 if (selectors.containsKey('observatory_ui')) { |
| 743 if (selectors.length == 1) { |
| 744 configuration['packages'] = TestUtils.dartDirUri |
| 745 .resolve('runtime/observatory/.packages') |
| 746 .toFilePath(); |
| 747 } else { |
| 748 // Make a new configuration whose selectors map only contains |
| 749 // observatory_ui, and remove the key from the original selectors. |
| 750 // The only mutable value in the map is the selectors, so a |
| 751 // shallow copy is safe. |
| 752 var observatoryConfiguration = new Map.from(configuration); |
| 753 observatoryConfiguration['selectors'] = { |
| 754 'observatory_ui': selectors['observatory_ui'] |
| 755 }; |
| 756 selectors.remove('observatory_ui'); |
| 757 |
| 758 // Set the packages flag. |
| 759 observatoryConfiguration['packages'] = TestUtils.dartDirUri |
| 760 .resolve('runtime/observatory/.packages') |
| 761 .toFilePath(); |
| 762 |
| 763 // Return the expansions of both configurations. Neither will reach |
| 764 // this line in the recursive call to _expandConfigurations. |
| 765 return _expandConfigurations(configuration) |
| 766 ..addAll(_expandConfigurations(observatoryConfiguration)); |
| 767 } |
| 768 } |
| 769 |
| 770 // Set the default package spec explicitly. |
| 771 if (configuration['package_root'] == null && |
| 772 configuration['packages'] == null) { |
| 773 configuration['packages'] = |
| 774 TestUtils.dartDirUri.resolve('.packages').toFilePath(); |
| 775 } |
| 776 |
| 777 // Expand the architectures. |
| 778 if (configuration['arch'].contains(',')) { |
| 779 return _expandHelper('arch', configuration); |
| 780 } |
| 781 |
| 782 // Expand modes. |
| 783 if (configuration['mode'].contains(',')) { |
| 784 return _expandHelper('mode', configuration); |
| 785 } |
| 786 |
| 787 // Expand compilers. |
| 788 if (configuration['compiler'].contains(',')) { |
| 789 return _expandHelper('compiler', configuration); |
| 790 } |
| 791 |
| 792 // Expand runtimes. |
| 793 var runtimes = configuration['runtime']; |
| 794 if (runtimes.contains(',')) { |
| 795 return _expandHelper('runtime', configuration); |
| 796 } else { |
| 797 // All runtimes eventually go through this path, after expansion. |
| 798 var updater = runtimeUpdater(configuration); |
| 799 if (updater != null) { |
| 800 updater.update(); |
| 801 } |
| 802 } |
| 803 |
| 804 // Adjust default timeout based on mode, compiler, and sometimes runtime. |
| 805 if (configuration['timeout'] == -1) { |
| 806 var isReload = |
| 807 configuration['hot_reload'] || configuration['hot_reload_rollback']; |
| 808 int compilerMulitiplier = |
| 809 new CompilerConfiguration(configuration).computeTimeoutMultiplier(); |
| 810 int runtimeMultiplier = new RuntimeConfiguration(configuration) |
| 811 .computeTimeoutMultiplier( |
| 812 mode: configuration['mode'], |
| 813 isChecked: configuration['checked'], |
| 814 isReload: isReload, |
| 815 arch: configuration['arch']); |
| 816 configuration['timeout'] = 60 * compilerMulitiplier * runtimeMultiplier; |
| 817 } |
| 818 |
| 819 return [configuration]; |
| 820 } |
| 821 |
| 822 /// Helper for _expandConfigurations. Creates a new configuration and adds it |
| 823 /// to a list, for use in a case when a particular configuration has multiple |
| 824 /// results (separated by a ','). |
| 825 /// Arguments: |
| 826 /// option: The particular test option we are expanding. |
| 827 /// configuration: The map containing all test configuration information |
| 828 /// specified. |
| 829 List<Map> _expandHelper(String option, Map configuration) { |
| 830 var result = <Map>[]; |
| 831 var configs = configuration[option]; |
| 832 for (var config in configs.split(',')) { |
| 833 var newConfiguration = new Map.from(configuration); |
| 834 newConfiguration[option] = config; |
| 835 result.addAll(_expandConfigurations(newConfiguration)); |
| 836 } |
| 837 return result; |
| 838 } |
| 839 |
| 840 /// Print out usage information. |
| 841 void _printHelp() { |
| 842 var buffer = new StringBuffer(); |
| 843 |
| 844 buffer.writeln('''usage: dart test.dart [options] [selector] |
| 845 |
| 846 The optional selector limits the tests that will be run. |
| 847 For example, the selector "language/issue", or equivalently |
| 848 "language/*issue*", limits to test files matching the regexp |
| 849 ".*issue.*\\.dart" in the "tests/language" directory. |
| 850 |
| 851 Options:'''); |
| 852 |
| 853 for (var option in _options) { |
| 854 if (option.abbreviation != null) { |
| 855 buffer.write("-${option.abbreviation}, "); |
| 856 } else { |
| 857 buffer.write(" "); |
| 858 } |
| 859 |
| 860 buffer.write(option.command); |
| 861 |
| 862 switch (option.type) { |
| 863 case _OptionValueType.bool: |
| 864 // No value. |
| 865 break; |
| 866 case _OptionValueType.int: |
| 867 buffer.write("=<integer>"); |
| 868 break; |
| 869 case _OptionValueType.string: |
| 870 if (option.values.length > 6) { |
| 871 // If there are many options, they won't fit nicely in one line and |
| 872 // should be instead listed in the description. |
| 873 buffer.write("=<...>"); |
| 874 } else if (option.values.isNotEmpty) { |
| 875 buffer.write("=<${option.values.join('|')}>"); |
| 876 } else { |
| 877 buffer.write("=<string>"); |
| 878 } |
| 879 break; |
| 880 } |
| 881 |
| 882 if (option.type != _OptionValueType.bool && |
| 883 option.defaultValue != null && |
| 884 option.defaultValue != "") { |
| 885 buffer.write(" (defaults to ${option.defaultValue})"); |
| 886 } |
| 887 |
| 888 buffer.writeln(); |
| 889 buffer |
| 890 .writeln(" ${option.description.replaceAll('\n', '\n ')}"); |
| 891 buffer.writeln(); |
| 892 } |
| 893 |
| 894 print(buffer); |
| 895 } |
| 896 |
| 897 _Option _findByAbbreviation(String abbreviation) { |
| 898 for (var option in _options) { |
| 899 if (abbreviation == option.abbreviation) return option; |
| 900 } |
| 901 |
| 902 return null; |
| 903 } |
| 904 |
| 905 _Option _findByName(String name) { |
| 906 for (var option in _options) { |
| 907 if (name == option.name) return option; |
| 908 |
| 909 // Allow hyphens instead of underscores as the separator since they are |
| 910 // more common for command line flags. |
| 911 if (name == option.name.replaceAll("_", "-")) return option; |
| 912 } |
| 913 |
| 914 return null; |
| 915 } |
| 916 } |
OLD | NEW |