OLD | NEW |
(Empty) | |
| 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. |
| 4 |
| 5 library test_options_parser; |
| 6 |
| 7 import "dart:io"; |
| 8 import "drt_updater.dart"; |
| 9 import "test_suite.dart"; |
| 10 import "path.dart"; |
| 11 import "compiler_configuration.dart" show CompilerConfiguration; |
| 12 import "runtime_configuration.dart" show RuntimeConfiguration; |
| 13 |
| 14 const List<String> defaultTestSelectors = const [ |
| 15 'samples', |
| 16 'standalone', |
| 17 'corelib', |
| 18 'co19', |
| 19 'language', |
| 20 'isolate', |
| 21 'vm', |
| 22 'html', |
| 23 'benchmark_smoke', |
| 24 'utils', |
| 25 'lib', |
| 26 'analyze_library', |
| 27 'service', |
| 28 'kernel', |
| 29 'observatory_ui' |
| 30 ]; |
| 31 |
| 32 /** |
| 33 * Specification of a single test option. |
| 34 * |
| 35 * The name of the specification is used as the key for the option in |
| 36 * the Map returned from the [TestOptionParser] parse method. |
| 37 */ |
| 38 class _TestOptionSpecification { |
| 39 _TestOptionSpecification( |
| 40 this.name, this.description, this.keys, this.values, this.defaultValue, |
| 41 {this.type: 'string'}); |
| 42 String name; |
| 43 String description; |
| 44 List<String> keys; |
| 45 List<String> values; |
| 46 Object defaultValue; |
| 47 String type; |
| 48 } |
| 49 |
| 50 /** |
| 51 * Parser of test options. |
| 52 */ |
| 53 class TestOptionsParser { |
| 54 /** |
| 55 * Creates a test options parser initialized with the known options. |
| 56 */ |
| 57 TestOptionsParser() { |
| 58 _options = [ |
| 59 new _TestOptionSpecification('mode', 'Mode in which to run the tests', |
| 60 ['-m', '--mode'], ['all', 'debug', 'release', 'product'], 'debug'), |
| 61 new _TestOptionSpecification( |
| 62 'compiler', |
| 63 '''Specify any compilation step (if needed). |
| 64 |
| 65 none: Do not compile the Dart code (run native Dart code on the VM). |
| 66 (only valid with the following runtimes: vm, flutter, drt) |
| 67 |
| 68 precompiler: Compile into AOT snapshot before running the test. |
| 69 (only valid with the dart_precompiled runtime) |
| 70 |
| 71 dart2js: Compile dart code to JavaScript by running dart2js. |
| 72 (only valid with the following runtimes: d8, drt, chrome, |
| 73 safari, ie9, ie10, ie11, firefox, opera, chromeOnAndroid, |
| 74 none (compile only)), |
| 75 |
| 76 dart2analyzer: Perform static analysis on Dart code by running the analyzer |
| 77 (only valid with the following runtimes: none) |
| 78 |
| 79 app_jit: Compile the Dart code into an app snapshot before running test |
| 80 (only valid with dart_app runtime) |
| 81 |
| 82 dartk: Compile the Dart source into Kernel before running test. |
| 83 |
| 84 dartkp: Compiler the Dart source into Kernel and then Kernel into AOT |
| 85 snapshot before running the test. |
| 86 (only valid with the dart_precompiled runtime) |
| 87 ''', |
| 88 ['-c', '--compiler'], |
| 89 [ |
| 90 'none', |
| 91 'precompiler', |
| 92 'dart2js', |
| 93 'dart2analyzer', |
| 94 'app_jit', |
| 95 'dartk', |
| 96 'dartkp' |
| 97 ], |
| 98 'none'), |
| 99 // TODO(antonm): fix the option drt. |
| 100 new _TestOptionSpecification( |
| 101 'runtime', |
| 102 '''Where the tests should be run. |
| 103 vm: Run Dart code on the standalone dart vm. |
| 104 |
| 105 flutter: Run Dart code on the flutter engine. |
| 106 |
| 107 dart_precompiled: Run a precompiled snapshot on a variant of the standalone |
| 108 dart vm lacking a JIT. |
| 109 |
| 110 d8: Run JavaScript from the command line using v8. |
| 111 |
| 112 jsshell: Run JavaScript from the command line using firefox js-shell. |
| 113 |
| 114 drt: Run Dart or JavaScript in the headless version of Chrome, |
| 115 Content shell. |
| 116 |
| 117 dartium: Run Dart or JavaScript in Dartium. |
| 118 |
| 119 ContentShellOnAndroid: Run Dart or JavaScript in Dartium content shell |
| 120 on Android. |
| 121 |
| 122 DartiumOnAndroid: Run Dart or Javascript in Dartium on Android. |
| 123 |
| 124 [ff | chrome | safari | ie9 | ie10 | ie11 | opera | chromeOnAndroid]: |
| 125 Run JavaScript in the specified browser. |
| 126 |
| 127 self_check: Pass each test or its compiled output to every file under |
| 128 `pkg` whose name ends with `_self_check.dart`. |
| 129 Each test is given to the self_check tester as a filename on stdin using |
| 130 the batch-mode protocol. |
| 131 |
| 132 none: No runtime, compile only (for example, used for dart2analyzer static |
| 133 analysis tests).''', |
| 134 ['-r', '--runtime'], |
| 135 [ |
| 136 'vm', |
| 137 'flutter', |
| 138 'dart_precompiled', |
| 139 'd8', |
| 140 'jsshell', |
| 141 'drt', |
| 142 'dartium', |
| 143 'ff', |
| 144 'firefox', |
| 145 'chrome', |
| 146 'safari', |
| 147 'ie9', |
| 148 'ie10', |
| 149 'ie11', |
| 150 'opera', |
| 151 'chromeOnAndroid', |
| 152 'safarimobilesim', |
| 153 'ContentShellOnAndroid', |
| 154 'DartiumOnAndroid', |
| 155 'self_check', |
| 156 'none' |
| 157 ], |
| 158 'vm'), |
| 159 new _TestOptionSpecification( |
| 160 'arch', |
| 161 'The architecture to run tests for', |
| 162 ['-a', '--arch'], |
| 163 [ |
| 164 'all', |
| 165 'ia32', |
| 166 'x64', |
| 167 'arm', |
| 168 'armv6', |
| 169 'armv5te', |
| 170 'arm64', |
| 171 'mips', |
| 172 'simarm', |
| 173 'simarmv6', |
| 174 'simarmv5te', |
| 175 'simarm64', |
| 176 'simmips', |
| 177 'simdbc', |
| 178 'simdbc64', |
| 179 ], |
| 180 'x64'), |
| 181 new _TestOptionSpecification( |
| 182 'system', |
| 183 'The operating system to run tests on', |
| 184 ['-s', '--system'], |
| 185 ['linux', 'macos', 'windows', 'android'], |
| 186 Platform.operatingSystem), |
| 187 new _TestOptionSpecification( |
| 188 'checked', 'Run tests in checked mode', ['--checked'], [], false, |
| 189 type: 'bool'), |
| 190 new _TestOptionSpecification( |
| 191 'strong', 'Run tests in strong mode', ['--strong'], [], false, |
| 192 type: 'bool'), |
| 193 new _TestOptionSpecification('host_checked', |
| 194 'Run compiler in checked mode', ['--host-checked'], [], false, |
| 195 type: 'bool'), |
| 196 new _TestOptionSpecification('minified', |
| 197 'Enable minification in the compiler', ['--minified'], [], false, |
| 198 type: 'bool'), |
| 199 new _TestOptionSpecification( |
| 200 'csp', |
| 201 'Run tests under Content Security Policy restrictions', |
| 202 ['--csp'], |
| 203 [], |
| 204 false, |
| 205 type: 'bool'), |
| 206 new _TestOptionSpecification( |
| 207 'cps_ir', |
| 208 'Run the compiler with the cps based backend', |
| 209 ['--cps-ir'], |
| 210 [], |
| 211 false, |
| 212 type: 'bool'), |
| 213 new _TestOptionSpecification( |
| 214 'fast_startup', |
| 215 'Pass the --fast-startup flag to dart2js', |
| 216 ['--fast-startup'], |
| 217 [], |
| 218 false, |
| 219 type: 'bool'), |
| 220 new _TestOptionSpecification( |
| 221 'dart2js_with_kernel', |
| 222 'Enable the internal pipeline in dart2js to use kernel', |
| 223 ['--dart2js-with-kernel'], |
| 224 [], |
| 225 false, |
| 226 type: 'bool'), |
| 227 new _TestOptionSpecification('hot_reload', 'Run hot reload stress tests', |
| 228 ['--hot-reload'], [], false, |
| 229 type: 'bool'), |
| 230 new _TestOptionSpecification( |
| 231 'hot_reload_rollback', |
| 232 'Run hot reload rollback stress tests', |
| 233 ['--hot-reload-rollback'], |
| 234 [], |
| 235 false, |
| 236 type: 'bool'), |
| 237 new _TestOptionSpecification( |
| 238 'use_blobs', |
| 239 'Use mmap instead of shared libraries for precompilation', |
| 240 ['--use-blobs'], |
| 241 [], |
| 242 false, |
| 243 type: 'bool'), |
| 244 new _TestOptionSpecification( |
| 245 'timeout', 'Timeout in seconds', ['-t', '--timeout'], [], -1, |
| 246 type: 'int'), |
| 247 new _TestOptionSpecification( |
| 248 'progress', |
| 249 'Progress indication mode', |
| 250 ['-p', '--progress'], |
| 251 [ |
| 252 'compact', |
| 253 'color', |
| 254 'line', |
| 255 'verbose', |
| 256 'silent', |
| 257 'status', |
| 258 'buildbot', |
| 259 'diff' |
| 260 ], |
| 261 'compact'), |
| 262 new _TestOptionSpecification('failure-summary', |
| 263 'Print failure summary at the end', ['--failure-summary'], [], false, |
| 264 type: 'bool'), |
| 265 new _TestOptionSpecification('step_name', |
| 266 'Step name for use by -pbuildbot', ['--step_name'], [], null), |
| 267 new _TestOptionSpecification( |
| 268 'report', |
| 269 'Print a summary report of the number of tests, by expectation', |
| 270 ['--report'], |
| 271 [], |
| 272 false, |
| 273 type: 'bool'), |
| 274 new _TestOptionSpecification( |
| 275 'tasks', |
| 276 'The number of parallel tasks to run', |
| 277 ['-j', '--tasks'], |
| 278 [], |
| 279 Platform.numberOfProcessors, |
| 280 type: 'int'), |
| 281 new _TestOptionSpecification( |
| 282 'shards', |
| 283 'The number of instances that the tests will be sharded over', |
| 284 ['--shards'], |
| 285 [], |
| 286 1, |
| 287 type: 'int'), |
| 288 new _TestOptionSpecification( |
| 289 'shard', |
| 290 'The index of this instance when running in sharded mode', |
| 291 ['--shard'], |
| 292 [], |
| 293 1, |
| 294 type: 'int'), |
| 295 new _TestOptionSpecification( |
| 296 'help', 'Print list of options', ['-h', '--help'], [], false, |
| 297 type: 'bool'), |
| 298 new _TestOptionSpecification( |
| 299 'verbose', 'Verbose output', ['-v', '--verbose'], [], false, |
| 300 type: 'bool'), |
| 301 new _TestOptionSpecification( |
| 302 'verify-ir', 'Verify kernel IR', ['--verify-ir'], [], false, |
| 303 type: 'bool'), |
| 304 new _TestOptionSpecification('no-tree-shake', |
| 305 'Disable kernel IR tree shaking', ['--no-tree-shake'], [], false, |
| 306 type: 'bool'), |
| 307 new _TestOptionSpecification( |
| 308 'list', 'List tests only, do not run them', ['--list'], [], false, |
| 309 type: 'bool'), |
| 310 new _TestOptionSpecification( |
| 311 'report_in_json', |
| 312 'When doing list, output result summary in json only.', |
| 313 ['--report-in-json'], |
| 314 [], |
| 315 false, |
| 316 type: 'bool'), |
| 317 new _TestOptionSpecification('time', |
| 318 'Print timing information after running tests', ['--time'], [], false, |
| 319 type: 'bool'), |
| 320 new _TestOptionSpecification( |
| 321 'dart', 'Path to dart executable', ['--dart'], [], ''), |
| 322 new _TestOptionSpecification( |
| 323 'flutter', 'Path to flutter executable', ['--flutter'], [], ''), |
| 324 new _TestOptionSpecification( |
| 325 'drt', // TODO(antonm): fix the option name. |
| 326 'Path to content shell executable', |
| 327 ['--drt'], |
| 328 [], |
| 329 ''), |
| 330 new _TestOptionSpecification('dartium', |
| 331 'Path to Dartium Chrome executable', ['--dartium'], [], ''), |
| 332 new _TestOptionSpecification('firefox', |
| 333 'Path to firefox browser executable', ['--firefox'], [], ''), |
| 334 new _TestOptionSpecification( |
| 335 'chrome', 'Path to chrome browser executable', ['--chrome'], [], ''), |
| 336 new _TestOptionSpecification( |
| 337 'safari', 'Path to safari browser executable', ['--safari'], [], ''), |
| 338 new _TestOptionSpecification( |
| 339 'use_sdk', |
| 340 '''Use compiler or runtime from the SDK. |
| 341 |
| 342 Normally, the compiler or runtimes in PRODUCT_DIR is tested, with this |
| 343 option, the compiler or runtime in PRODUCT_DIR/dart-sdk/bin is tested. |
| 344 |
| 345 Note: currently only implemented for dart2js.''', |
| 346 ['--use-sdk'], |
| 347 [], |
| 348 false, |
| 349 type: 'bool'), |
| 350 new _TestOptionSpecification( |
| 351 'build_directory', |
| 352 'The name of the build directory, where products are placed.', |
| 353 ['--build-directory'], |
| 354 [], |
| 355 ''), |
| 356 new _TestOptionSpecification('noBatch', 'Do not run tests in batch mode', |
| 357 ['-n', '--nobatch'], [], false, |
| 358 type: 'bool'), |
| 359 new _TestOptionSpecification('dart2js_batch', |
| 360 'Run dart2js tests in batch mode', ['--dart2js-batch'], [], false, |
| 361 type: 'bool'), |
| 362 new _TestOptionSpecification( |
| 363 'append_logs', |
| 364 'Do not delete old logs but rather append to them.', |
| 365 ['--append_logs'], |
| 366 [], |
| 367 false, |
| 368 type: 'bool'), |
| 369 new _TestOptionSpecification( |
| 370 'write_debug_log', |
| 371 'Don\'t write debug messages to stdout but rather to a logfile.', |
| 372 ['--write-debug-log'], |
| 373 [], |
| 374 false, |
| 375 type: 'bool'), |
| 376 new _TestOptionSpecification( |
| 377 'write_test_outcome_log', |
| 378 'Write the outcome of all tests executed to a ' |
| 379 '"${TestUtils.testOutcomeFileName}" file.', |
| 380 ['--write-test-outcome-log'], |
| 381 [], |
| 382 false, |
| 383 type: 'bool'), |
| 384 new _TestOptionSpecification( |
| 385 'reset_browser_configuration', |
| 386 'Browser specific reset of configuration. ' |
| 387 'WARNING: Using this option may remove your bookmarks and ' |
| 388 'other settings.', |
| 389 ['--reset-browser-configuration'], |
| 390 [], |
| 391 false, |
| 392 type: 'bool'), |
| 393 new _TestOptionSpecification( |
| 394 'copy_coredumps', |
| 395 'If we see a crash that we did not expect, copy the core dumps. ' |
| 396 'to /tmp', |
| 397 ['--copy-coredumps'], |
| 398 [], |
| 399 false, |
| 400 type: 'bool'), |
| 401 new _TestOptionSpecification( |
| 402 'local_ip', |
| 403 'IP address the http servers should listen on.' |
| 404 'This address is also used for browsers to connect.', |
| 405 ['--local_ip'], |
| 406 [], |
| 407 '127.0.0.1'), |
| 408 new _TestOptionSpecification('test_server_port', |
| 409 'Port for test http server.', ['--test_server_port'], [], 0, |
| 410 type: 'int'), |
| 411 new _TestOptionSpecification( |
| 412 'test_server_cross_origin_port', |
| 413 'Port for test http server cross origin.', |
| 414 ['--test_server_cross_origin_port'], |
| 415 [], |
| 416 0, |
| 417 type: 'int'), |
| 418 new _TestOptionSpecification('test_driver_port', |
| 419 'Port for http test driver server.', ['--test_driver_port'], [], 0, |
| 420 type: 'int'), |
| 421 new _TestOptionSpecification( |
| 422 'test_driver_error_port', |
| 423 'Port for http test driver server errors.', |
| 424 ['--test_driver_error_port'], |
| 425 [], |
| 426 0, |
| 427 type: 'int'), |
| 428 new _TestOptionSpecification( |
| 429 'record_to_file', |
| 430 'Records all the commands that need to be executed and writes it ' |
| 431 'out to a file.', |
| 432 ['--record_to_file'], |
| 433 [], |
| 434 null), |
| 435 new _TestOptionSpecification( |
| 436 'replay_from_file', |
| 437 'Records all the commands that need to be executed and writes it ' |
| 438 'out to a file.', |
| 439 ['--replay_from_file'], |
| 440 [], |
| 441 null), |
| 442 new _TestOptionSpecification( |
| 443 'builder_tag', |
| 444 'Machine specific options that is not captured by the regular ' |
| 445 'test options. Used to be able to make sane updates to the ' |
| 446 'status files.', |
| 447 ['--builder-tag'], |
| 448 [], |
| 449 ''), |
| 450 new _TestOptionSpecification( |
| 451 'vm_options', |
| 452 'Extra options to send to the vm when running', |
| 453 ['--vm-options'], |
| 454 [], |
| 455 null), |
| 456 new _TestOptionSpecification( |
| 457 'dart2js_options', |
| 458 'Extra options for dart2js compilation step', |
| 459 ['--dart2js-options'], |
| 460 [], |
| 461 null), |
| 462 new _TestOptionSpecification( |
| 463 'suite_dir', |
| 464 'Additional directory to add to the testing matrix', |
| 465 ['--suite-dir'], |
| 466 [], |
| 467 null), |
| 468 new _TestOptionSpecification('package_root', |
| 469 'The package root to use for testing.', ['--package-root'], [], null), |
| 470 new _TestOptionSpecification( |
| 471 'packages', |
| 472 'The package spec file to use for testing.', |
| 473 ['--packages'], |
| 474 [], |
| 475 null), |
| 476 new _TestOptionSpecification( |
| 477 'exclude_suite', |
| 478 'Exclude suites from default selector, only works when no' |
| 479 ' selector has been specified on the command line', |
| 480 ['--exclude-suite'], |
| 481 [], |
| 482 null), |
| 483 new _TestOptionSpecification( |
| 484 'skip-compilation', |
| 485 'Skip the compilation step, using the compilation artifacts left in ' |
| 486 ' the output folder from a previous run.' |
| 487 'This flag will often cause false positves and negatives, but can be' |
| 488 ' useful for quick-and-dirty offline testing when not making changes' |
| 489 ' that affect the compiler.', |
| 490 ['--skip-compilation'], |
| 491 [], |
| 492 false, |
| 493 type: 'bool') |
| 494 ]; |
| 495 } |
| 496 |
| 497 /** |
| 498 * Parse a list of strings as test options. |
| 499 * |
| 500 * Returns a list of configurations in which to run the |
| 501 * tests. Configurations are maps mapping from option keys to |
| 502 * values. When encountering the first non-option string, the rest |
| 503 * of the arguments are stored in the returned Map under the 'rest' |
| 504 * key. |
| 505 */ |
| 506 List<Map> parse(List<String> arguments) { |
| 507 var configuration = new Map(); |
| 508 // Fill in configuration with arguments passed to the test script. |
| 509 var numArguments = arguments.length; |
| 510 for (var i = 0; i < numArguments; i++) { |
| 511 // Extract name and value for options. |
| 512 String arg = arguments[i]; |
| 513 String name = ''; |
| 514 String value = ''; |
| 515 _TestOptionSpecification spec; |
| 516 if (arg.startsWith('--')) { |
| 517 if (arg == '--help') { |
| 518 _printHelp(); |
| 519 return null; |
| 520 } |
| 521 var split = arg.indexOf('='); |
| 522 if (split == -1) { |
| 523 name = arg; |
| 524 spec = _getSpecification(name); |
| 525 // Boolean options do not have a value. |
| 526 if (spec.type != 'bool') { |
| 527 if ((i + 1) >= arguments.length) { |
| 528 print('No value supplied for option $name'); |
| 529 return null; |
| 530 } |
| 531 value = arguments[++i]; |
| 532 } |
| 533 } else { |
| 534 name = arg.substring(0, split); |
| 535 spec = _getSpecification(name); |
| 536 value = arg.substring(split + 1, arg.length); |
| 537 } |
| 538 } else if (arg.startsWith('-')) { |
| 539 if (arg == '-h') { |
| 540 _printHelp(); |
| 541 return null; |
| 542 } |
| 543 if (arg.length > 2) { |
| 544 name = arg.substring(0, 2); |
| 545 spec = _getSpecification(name); |
| 546 value = arg.substring(2, arg.length); |
| 547 } else { |
| 548 name = arg; |
| 549 spec = _getSpecification(name); |
| 550 // Boolean options do not have a value. |
| 551 if (spec.type != 'bool') { |
| 552 if ((i + 1) >= arguments.length) { |
| 553 print('No value supplied for option $name'); |
| 554 return null; |
| 555 } |
| 556 value = arguments[++i]; |
| 557 } |
| 558 } |
| 559 } else { |
| 560 // The argument does not start with '-' or '--' and is |
| 561 // therefore not an option. We use it as a test selection |
| 562 // pattern. |
| 563 var patterns = configuration.putIfAbsent('selectors', () => <String>[]); |
| 564 patterns.add(arg); |
| 565 continue; |
| 566 } |
| 567 |
| 568 // Multiple uses of a flag are an error, because there is no |
| 569 // naturally correct way to handle conflicting options. |
| 570 if (configuration.containsKey(spec.name)) { |
| 571 print('Error: test.dart disallows multiple "--${spec.name}" flags'); |
| 572 exit(1); |
| 573 } |
| 574 // Parse the value for the option. |
| 575 if (spec.type == 'bool') { |
| 576 if (!value.isEmpty) { |
| 577 print('No value expected for bool option $name'); |
| 578 exit(1); |
| 579 } |
| 580 configuration[spec.name] = true; |
| 581 } else if (spec.type == 'int') { |
| 582 try { |
| 583 configuration[spec.name] = int.parse(value); |
| 584 } catch (e) { |
| 585 print('Integer value expected for int option $name'); |
| 586 exit(1); |
| 587 } |
| 588 } else { |
| 589 assert(spec.type == 'string'); |
| 590 if (!spec.values.isEmpty) { |
| 591 for (var v in value.split(',')) { |
| 592 if (spec.values.lastIndexOf(v) == -1) { |
| 593 print('Unknown value ($v) for option $name'); |
| 594 exit(1); |
| 595 } |
| 596 } |
| 597 } |
| 598 configuration[spec.name] = value; |
| 599 } |
| 600 } |
| 601 |
| 602 // Apply default values for unspecified options. |
| 603 for (var option in _options) { |
| 604 if (!configuration.containsKey(option.name)) { |
| 605 configuration[option.name] = option.defaultValue; |
| 606 } |
| 607 } |
| 608 |
| 609 List<Map> expandedConfigs = _expandConfigurations(configuration); |
| 610 List<Map> result = expandedConfigs.where(_isValidConfig).toList(); |
| 611 for (var config in result) { |
| 612 config['_reproducing_arguments_'] = |
| 613 _constructReproducingCommandArguments(config); |
| 614 } |
| 615 return result.isEmpty ? null : result; |
| 616 } |
| 617 |
| 618 // For printing out reproducing command lines, we don't want to add these |
| 619 // options. |
| 620 final _blacklistedOptions = [ |
| 621 'append_logs', |
| 622 'build_directory', |
| 623 'chrome', |
| 624 'copy_coredumps', |
| 625 'dart', |
| 626 'flutter', |
| 627 'dartium', |
| 628 'drt', |
| 629 'exclude_suite', |
| 630 'failure-summary', |
| 631 'firefox', |
| 632 'local_ip', |
| 633 'progress', |
| 634 'report', |
| 635 'safari', |
| 636 'shard', |
| 637 'shards', |
| 638 'step_name', |
| 639 'tasks', |
| 640 'time', |
| 641 'verbose', |
| 642 'write_debug_log', |
| 643 'write_test_outcome_log', |
| 644 ].toSet(); |
| 645 |
| 646 List<String> _constructReproducingCommandArguments(Map config) { |
| 647 var arguments = new List<String>(); |
| 648 for (var option in _options) { |
| 649 var name = option.name; |
| 650 if (!config.containsKey(name) || _blacklistedOptions.contains(name)) { |
| 651 continue; |
| 652 } |
| 653 var value = config[name]; |
| 654 if (config[name] == option.defaultValue || |
| 655 (name == 'packages' && |
| 656 value == |
| 657 TestUtils.dartDirUri.resolve('.packages').toFilePath())) { |
| 658 continue; |
| 659 } |
| 660 shortest(String a, String b) => a.length <= b.length ? a : b; |
| 661 var key = option.keys.reduce(shortest); |
| 662 if (option.type == 'bool') { |
| 663 arguments.add(key); |
| 664 } else if (key.startsWith('--')) { |
| 665 // long version |
| 666 arguments.add(key); |
| 667 arguments.add("$value"); |
| 668 } else { |
| 669 // short version |
| 670 assert(key.startsWith('-')); |
| 671 arguments.add("$key$value"); |
| 672 } |
| 673 } |
| 674 return arguments; |
| 675 } |
| 676 |
| 677 /** |
| 678 * Determine if a particular configuration has a valid combination of compiler |
| 679 * and runtime elements. |
| 680 */ |
| 681 bool _isValidConfig(Map config) { |
| 682 bool isValid = true; |
| 683 List<String> validRuntimes; |
| 684 switch (config['compiler']) { |
| 685 case 'dart2js': |
| 686 // Note: by adding 'none' as a configuration, if the user |
| 687 // runs test.py -c dart2js -r drt,none the dart2js_none and |
| 688 // dart2js_drt will be duplicating work. If later we don't need 'none' |
| 689 // with dart2js, we should remove it from here. |
| 690 validRuntimes = const [ |
| 691 'd8', |
| 692 'jsshell', |
| 693 'drt', |
| 694 'none', |
| 695 'dartium', |
| 696 'ff', |
| 697 'chrome', |
| 698 'safari', |
| 699 'ie9', |
| 700 'ie10', |
| 701 'ie11', |
| 702 'opera', |
| 703 'chromeOnAndroid', |
| 704 'safarimobilesim' |
| 705 ]; |
| 706 break; |
| 707 case 'dart2analyzer': |
| 708 validRuntimes = const ['none']; |
| 709 break; |
| 710 case 'app_jit': |
| 711 case 'dartk': |
| 712 validRuntimes = const ['vm', 'self_check', 'none']; |
| 713 break; |
| 714 case 'precompiler': |
| 715 case 'dartkp': |
| 716 validRuntimes = const ['dart_precompiled']; |
| 717 break; |
| 718 case 'none': |
| 719 validRuntimes = const [ |
| 720 'vm', |
| 721 'flutter', |
| 722 'drt', |
| 723 'dartium', |
| 724 'ContentShellOnAndroid', |
| 725 'DartiumOnAndroid' |
| 726 ]; |
| 727 break; |
| 728 } |
| 729 if (!validRuntimes.contains(config['runtime'])) { |
| 730 isValid = false; |
| 731 print("Warning: combination of compiler '${config['compiler']}' and " |
| 732 "runtime '${config['runtime']}' is invalid. " |
| 733 "Skipping this combination."); |
| 734 } |
| 735 if (config['ie'] && Platform.operatingSystem != 'windows') { |
| 736 isValid = false; |
| 737 print("Warning: cannot run Internet Explorer on non-Windows operating" |
| 738 " system."); |
| 739 } |
| 740 if (config['shard'] < 1 || config['shard'] > config['shards']) { |
| 741 isValid = false; |
| 742 print("Error: shard index is ${config['shard']} out of " |
| 743 "${config['shards']} shards"); |
| 744 } |
| 745 if ((config['runtime'] == 'flutter') && (config['flutter'] == '')) { |
| 746 isValid = false; |
| 747 print("-rflutter requires the flutter engine executable to " |
| 748 "be specified using --flutter="); |
| 749 } |
| 750 if ((config['runtime'] == 'flutter') && (config['arch'] != 'x64')) { |
| 751 isValid = false; |
| 752 print("-rflutter is applicable only for --arch=x64"); |
| 753 } |
| 754 |
| 755 return isValid; |
| 756 } |
| 757 |
| 758 /** |
| 759 * Recursively expand a configuration with multiple values per key |
| 760 * into a list of configurations with exactly one value per key. |
| 761 */ |
| 762 List<Map> _expandConfigurations(Map configuration) { |
| 763 // Expand the pseudo-values such as 'all'. |
| 764 if (configuration['arch'] == 'all') { |
| 765 configuration['arch'] = 'ia32,x64,simarm,simarm64,simmips,simdbc64'; |
| 766 } |
| 767 if (configuration['mode'] == 'all') { |
| 768 configuration['mode'] = 'debug,release,product'; |
| 769 } |
| 770 |
| 771 if (configuration['report_in_json']) { |
| 772 configuration['list'] = true; |
| 773 configuration['report'] = true; |
| 774 } |
| 775 |
| 776 // Use verbose progress indication for verbose output unless buildbot |
| 777 // progress indication is requested. |
| 778 if (configuration['verbose'] && configuration['progress'] != 'buildbot') { |
| 779 configuration['progress'] = 'verbose'; |
| 780 } |
| 781 |
| 782 // Create the artificial negative options that test status files |
| 783 // expect. |
| 784 configuration['unchecked'] = !configuration['checked']; |
| 785 configuration['host_unchecked'] = !configuration['host_checked']; |
| 786 configuration['unminified'] = !configuration['minified']; |
| 787 configuration['nocsp'] = !configuration['csp']; |
| 788 |
| 789 String runtime = configuration['runtime']; |
| 790 if (runtime == 'firefox') { |
| 791 configuration['runtime'] == 'ff'; |
| 792 } |
| 793 |
| 794 String compiler = configuration['compiler']; |
| 795 configuration['browser'] = TestUtils.isBrowserRuntime(runtime); |
| 796 configuration['analyzer'] = TestUtils.isCommandLineAnalyzer(compiler); |
| 797 |
| 798 // Set the javascript command line flag for less verbose status files. |
| 799 configuration['jscl'] = TestUtils.isJsCommandLineRuntime(runtime); |
| 800 |
| 801 // Allow suppression that is valid for all ie versions |
| 802 configuration['ie'] = runtime.startsWith('ie'); |
| 803 |
| 804 // Expand the test selectors into a suite name and a simple |
| 805 // regular expressions to be used on the full path of a test file |
| 806 // in that test suite. If no selectors are explicitly given use |
| 807 // the default suite patterns. |
| 808 var selectors = configuration['selectors']; |
| 809 if (selectors is! Map) { |
| 810 if (selectors == null) { |
| 811 if (configuration['suite_dir'] != null) { |
| 812 var suite_path = new Path(configuration['suite_dir']); |
| 813 selectors = [suite_path.filename]; |
| 814 } else { |
| 815 selectors = defaultTestSelectors.toList(); |
| 816 } |
| 817 |
| 818 var excludeSuites = configuration['exclude_suite'] != null |
| 819 ? configuration['exclude_suite'].split(',') |
| 820 : []; |
| 821 for (var exclude in excludeSuites) { |
| 822 if (selectors.contains(exclude)) { |
| 823 selectors.remove(exclude); |
| 824 } else { |
| 825 print("Warning: default selectors does not contain $exclude"); |
| 826 } |
| 827 } |
| 828 } |
| 829 var selectorMap = <String, RegExp>{}; |
| 830 for (var i = 0; i < selectors.length; i++) { |
| 831 var pattern = selectors[i]; |
| 832 var suite = pattern; |
| 833 var slashLocation = pattern.indexOf('/'); |
| 834 if (slashLocation != -1) { |
| 835 suite = pattern.substring(0, slashLocation); |
| 836 pattern = pattern.substring(slashLocation + 1); |
| 837 pattern = pattern.replaceAll('*', '.*'); |
| 838 } else { |
| 839 pattern = ".?"; |
| 840 } |
| 841 if (selectorMap.containsKey(suite)) { |
| 842 print("Error: '$suite/$pattern'. Only one test selection" |
| 843 " pattern is allowed to start with '$suite/'"); |
| 844 exit(1); |
| 845 } |
| 846 selectorMap[suite] = new RegExp(pattern); |
| 847 } |
| 848 configuration['selectors'] = selectorMap; |
| 849 } |
| 850 |
| 851 // Put observatory_ui in a configuration with its own packages override. |
| 852 // Only one value in the configuration map is mutable: |
| 853 selectors = configuration['selectors']; |
| 854 if (selectors.containsKey('observatory_ui')) { |
| 855 if (selectors.length == 1) { |
| 856 configuration['packages'] = TestUtils.dartDirUri |
| 857 .resolve('runtime/observatory/.packages') |
| 858 .toFilePath(); |
| 859 } else { |
| 860 // Make a new configuration whose selectors map only contains |
| 861 // observatory_ui, and remove the key from the original selectors. |
| 862 // The only mutable value in the map is the selectors, so a |
| 863 // shallow copy is safe. |
| 864 var observatoryConfiguration = new Map.from(configuration); |
| 865 observatoryConfiguration['selectors'] = { |
| 866 'observatory_ui': selectors['observatory_ui'] |
| 867 }; |
| 868 selectors.remove('observatory_ui'); |
| 869 |
| 870 // Set the packages flag. |
| 871 observatoryConfiguration['packages'] = TestUtils.dartDirUri |
| 872 .resolve('runtime/observatory/.packages') |
| 873 .toFilePath(); |
| 874 |
| 875 // Return the expansions of both configurations. Neither will reach |
| 876 // this line in the recursive call to _expandConfigurations. |
| 877 return _expandConfigurations(configuration) |
| 878 ..addAll(_expandConfigurations(observatoryConfiguration)); |
| 879 } |
| 880 } |
| 881 // Set the default package spec explicitly. |
| 882 if (configuration['package_root'] == null && |
| 883 configuration['packages'] == null) { |
| 884 configuration['packages'] = |
| 885 TestUtils.dartDirUri.resolve('.packages').toFilePath(); |
| 886 } |
| 887 |
| 888 // Expand the architectures. |
| 889 if (configuration['arch'].contains(',')) { |
| 890 return _expandHelper('arch', configuration); |
| 891 } |
| 892 |
| 893 // Expand modes. |
| 894 if (configuration['mode'].contains(',')) { |
| 895 return _expandHelper('mode', configuration); |
| 896 } |
| 897 |
| 898 // Expand compilers. |
| 899 if (configuration['compiler'].contains(',')) { |
| 900 return _expandHelper('compiler', configuration); |
| 901 } |
| 902 |
| 903 // Expand runtimes. |
| 904 var runtimes = configuration['runtime']; |
| 905 if (runtimes.contains(',')) { |
| 906 return _expandHelper('runtime', configuration); |
| 907 } else { |
| 908 // All runtimes eventually go through this path, after expansion. |
| 909 var updater = runtimeUpdater(configuration); |
| 910 if (updater != null) { |
| 911 updater.update(); |
| 912 } |
| 913 } |
| 914 |
| 915 // Adjust default timeout based on mode, compiler, and sometimes runtime. |
| 916 if (configuration['timeout'] == -1) { |
| 917 var isReload = |
| 918 configuration['hot_reload'] || configuration['hot_reload_rollback']; |
| 919 int compilerMulitiplier = |
| 920 new CompilerConfiguration(configuration).computeTimeoutMultiplier(); |
| 921 int runtimeMultiplier = new RuntimeConfiguration(configuration) |
| 922 .computeTimeoutMultiplier( |
| 923 mode: configuration['mode'], |
| 924 isChecked: configuration['checked'], |
| 925 isReload: isReload, |
| 926 arch: configuration['arch']); |
| 927 configuration['timeout'] = 60 * compilerMulitiplier * runtimeMultiplier; |
| 928 } |
| 929 |
| 930 return [configuration]; |
| 931 } |
| 932 |
| 933 /** |
| 934 * Helper for _expandConfigurations. Creates a new configuration and adds it |
| 935 * to a list, for use in a case when a particular configuration has multiple |
| 936 * results (separated by a ','). |
| 937 * Arguments: |
| 938 * option: The particular test option we are expanding. |
| 939 * configuration: The map containing all test configuration information |
| 940 * specified. |
| 941 */ |
| 942 List<Map> _expandHelper(String option, Map configuration) { |
| 943 var result = new List<Map>(); |
| 944 var configs = configuration[option]; |
| 945 for (var config in configs.split(',')) { |
| 946 var newConfiguration = new Map.from(configuration); |
| 947 newConfiguration[option] = config; |
| 948 result.addAll(_expandConfigurations(newConfiguration)); |
| 949 } |
| 950 return result; |
| 951 } |
| 952 |
| 953 /** |
| 954 * Print out usage information. |
| 955 */ |
| 956 void _printHelp() { |
| 957 print('usage: dart test.dart [options] [selector]'); |
| 958 print(''); |
| 959 print('The optional selector limits the tests that will be run.'); |
| 960 print('For example, the selector "language/issue", or equivalently'); |
| 961 print('"language/*issue*", limits to test files matching the regexp'); |
| 962 print('".*issue.*\\.dart" in the "tests/language" directory.'); |
| 963 print(''); |
| 964 print('Options:\n'); |
| 965 for (var option in _options) { |
| 966 print('${option.name}: ${option.description}.'); |
| 967 for (var name in option.keys) { |
| 968 assert(name.startsWith('-')); |
| 969 var buffer = new StringBuffer(); |
| 970 ; |
| 971 buffer.write(name); |
| 972 if (option.type == 'bool') { |
| 973 assert(option.values.isEmpty); |
| 974 } else { |
| 975 buffer.write(name.startsWith('--') ? '=' : ' '); |
| 976 if (option.type == 'int') { |
| 977 assert(option.values.isEmpty); |
| 978 buffer.write('n (default: ${option.defaultValue})'); |
| 979 } else { |
| 980 buffer.write('['); |
| 981 bool first = true; |
| 982 for (var value in option.values) { |
| 983 if (!first) buffer.write(", "); |
| 984 if (value == option.defaultValue) buffer.write('*'); |
| 985 buffer.write(value); |
| 986 first = false; |
| 987 } |
| 988 buffer.write(']'); |
| 989 } |
| 990 } |
| 991 print(buffer.toString()); |
| 992 } |
| 993 print(''); |
| 994 } |
| 995 } |
| 996 |
| 997 /** |
| 998 * Find the test option specification for a given option key. |
| 999 */ |
| 1000 _TestOptionSpecification _getSpecification(String name) { |
| 1001 for (var option in _options) { |
| 1002 if (option.keys.contains(name)) { |
| 1003 return option; |
| 1004 } |
| 1005 } |
| 1006 print('Unknown test option $name'); |
| 1007 exit(1); |
| 1008 return null; // Unreachable. |
| 1009 } |
| 1010 |
| 1011 List<_TestOptionSpecification> _options; |
| 1012 } |
OLD | NEW |