OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /** | 5 /** |
6 * Classes and methods for enumerating and preparing tests. | 6 * Classes and methods for enumerating and preparing tests. |
7 * | 7 * |
8 * This library includes: | 8 * This library includes: |
9 * | 9 * |
10 * - Creating tests by listing all the Dart files in certain directories, | 10 * - Creating tests by listing all the Dart files in certain directories, |
11 * and creating [TestCase]s for those files that meet the relevant criteria. | 11 * and creating [TestCase]s for those files that meet the relevant criteria. |
12 * - Preparing tests, including copying files and frameworks to temporary | 12 * - Preparing tests, including copying files and frameworks to temporary |
13 * directories, and computing the command line and arguments to be run. | 13 * directories, and computing the command line and arguments to be run. |
14 */ | 14 */ |
15 library test_suite; | 15 library test_suite; |
16 | 16 |
17 import "dart:async"; | 17 import "dart:async"; |
18 import "dart:io"; | 18 import "dart:io"; |
19 import "dart:math"; | 19 import "dart:math"; |
20 import "drt_updater.dart"; | 20 import "drt_updater.dart"; |
21 import "html_test.dart" as htmlTest; | 21 import "html_test.dart" as htmlTest; |
22 import "path.dart"; | 22 import "path.dart"; |
23 import "multitest.dart"; | 23 import "multitest.dart"; |
24 import "status_file_parser.dart"; | 24 import "expectation.dart"; |
| 25 import "expectation_set.dart"; |
25 import "summary_report.dart"; | 26 import "summary_report.dart"; |
26 import "test_runner.dart"; | 27 import "test_runner.dart"; |
27 import "utils.dart"; | 28 import "utils.dart"; |
28 import "http_server.dart" show PREFIX_BUILDDIR, PREFIX_DARTDIR; | 29 import "http_server.dart" show PREFIX_BUILDDIR, PREFIX_DARTDIR; |
29 | 30 |
30 import "compiler_configuration.dart" | 31 import "compiler_configuration.dart" |
31 show CommandArtifact, CompilerConfiguration; | 32 show CommandArtifact, CompilerConfiguration; |
32 | 33 |
33 import "runtime_configuration.dart" show RuntimeConfiguration; | 34 import "runtime_configuration.dart" show RuntimeConfiguration; |
34 | 35 |
(...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
111 } | 112 } |
112 | 113 |
113 /** | 114 /** |
114 * A TestSuite represents a collection of tests. It creates a [TestCase] | 115 * A TestSuite represents a collection of tests. It creates a [TestCase] |
115 * object for each test to be run, and passes the test cases to a callback. | 116 * object for each test to be run, and passes the test cases to a callback. |
116 * | 117 * |
117 * Most TestSuites represent a directory or directory tree containing tests, | 118 * Most TestSuites represent a directory or directory tree containing tests, |
118 * and a status file containing the expected results when these tests are run. | 119 * and a status file containing the expected results when these tests are run. |
119 */ | 120 */ |
120 abstract class TestSuite { | 121 abstract class TestSuite { |
121 final Map configuration; | 122 final Map<String, dynamic> configuration; |
122 final String suiteName; | 123 final String suiteName; |
123 // This function is set by subclasses before enqueueing starts. | 124 // This function is set by subclasses before enqueueing starts. |
124 Function doTest; | 125 Function doTest; |
125 Map<String, String> _environmentOverrides; | 126 Map<String, String> _environmentOverrides; |
126 RuntimeConfiguration runtimeConfiguration; | 127 RuntimeConfiguration runtimeConfiguration; |
127 | 128 |
128 TestSuite(this.configuration, this.suiteName) { | 129 TestSuite(this.configuration, this.suiteName) { |
129 TestUtils.buildDir(configuration); // Sets configuration_directory. | 130 TestUtils.buildDir(configuration); // Sets configuration_directory. |
130 if (configuration['configuration_directory'] != null) { | 131 if (configuration['configuration_directory'] != null) { |
131 _environmentOverrides = { | 132 _environmentOverrides = { |
(...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
300 return; | 301 return; |
301 } | 302 } |
302 // Test if the selector includes this test. | 303 // Test if the selector includes this test. |
303 RegExp pattern = configuration['selectors'][suiteName]; | 304 RegExp pattern = configuration['selectors'][suiteName]; |
304 if (!pattern.hasMatch(testCase.displayName)) { | 305 if (!pattern.hasMatch(testCase.displayName)) { |
305 return; | 306 return; |
306 } | 307 } |
307 | 308 |
308 if (configuration['hot_reload'] || configuration['hot_reload_rollback']) { | 309 if (configuration['hot_reload'] || configuration['hot_reload_rollback']) { |
309 // Handle reload special cases. | 310 // Handle reload special cases. |
310 if (expectations.contains(Expectation.COMPILETIME_ERROR) || | 311 if (expectations.contains(Expectation.compileTimeError) || |
311 testCase.hasCompileError || | 312 testCase.hasCompileError || |
312 testCase.expectCompileError) { | 313 testCase.expectCompileError) { |
313 // Running a test that expects a compilation error with hot reloading | 314 // Running a test that expects a compilation error with hot reloading |
314 // is redundant with a regular run of the test. | 315 // is redundant with a regular run of the test. |
315 return; | 316 return; |
316 } | 317 } |
317 } | 318 } |
318 | 319 |
319 // Update Summary report | 320 // Update Summary report |
320 if (configuration['report']) { | 321 if (configuration['report']) { |
321 if (testCase.expectCompileError && | 322 if (testCase.expectCompileError && |
322 TestUtils.isBrowserRuntime(configuration['runtime']) && | 323 TestUtils.isBrowserRuntime(configuration['runtime']) && |
323 new CompilerConfiguration(configuration).hasCompiler) { | 324 new CompilerConfiguration(configuration).hasCompiler) { |
324 summaryReport.addCompileErrorSkipTest(); | 325 summaryReport.addCompileErrorSkipTest(); |
325 return; | 326 return; |
326 } else { | 327 } else { |
327 summaryReport.add(testCase); | 328 summaryReport.add(testCase); |
328 } | 329 } |
329 } | 330 } |
330 | 331 |
331 // Handle skipped tests | 332 // Handle skipped tests |
332 if (expectations.contains(Expectation.SKIP) || | 333 if (expectations.contains(Expectation.skip) || |
333 expectations.contains(Expectation.SKIP_BY_DESIGN) || | 334 expectations.contains(Expectation.skipByDesign) || |
334 expectations.contains(Expectation.SKIP_SLOW)) { | 335 expectations.contains(Expectation.skipSlow)) { |
335 return; | 336 return; |
336 } | 337 } |
337 | 338 |
338 doTest(testCase); | 339 doTest(testCase); |
339 } | 340 } |
340 | 341 |
341 String createGeneratedTestDirectoryHelper( | 342 String createGeneratedTestDirectoryHelper( |
342 String name, String dirname, Path testPath, String optionsName) { | 343 String name, String dirname, Path testPath, String optionsName) { |
343 Path relative = testPath.relativeTo(TestUtils.dartDir); | 344 Path relative = testPath.relativeTo(TestUtils.dartDir); |
344 relative = relative.directoryPath.append(relative.filenameWithoutExtension); | 345 relative = relative.directoryPath.append(relative.filenameWithoutExtension); |
(...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
438 * The executable lists its tests when run with the --list command line flag. | 439 * The executable lists its tests when run with the --list command line flag. |
439 * Individual tests are run by specifying them on the command line. | 440 * Individual tests are run by specifying them on the command line. |
440 */ | 441 */ |
441 class CCTestSuite extends TestSuite { | 442 class CCTestSuite extends TestSuite { |
442 final String testPrefix; | 443 final String testPrefix; |
443 String targetRunnerPath; | 444 String targetRunnerPath; |
444 String hostRunnerPath; | 445 String hostRunnerPath; |
445 final String dartDir; | 446 final String dartDir; |
446 List<String> statusFilePaths; | 447 List<String> statusFilePaths; |
447 | 448 |
448 CCTestSuite(Map configuration, String suiteName, String runnerName, | 449 CCTestSuite(Map<String, dynamic> configuration, String suiteName, |
449 this.statusFilePaths, | 450 String runnerName, this.statusFilePaths, |
450 {this.testPrefix: ''}) | 451 {this.testPrefix: ''}) |
451 : dartDir = TestUtils.dartDir.toNativePath(), | 452 : dartDir = TestUtils.dartDir.toNativePath(), |
452 super(configuration, suiteName) { | 453 super(configuration, suiteName) { |
453 // For running the tests we use the given '$runnerName' binary | 454 // For running the tests we use the given '$runnerName' binary |
454 targetRunnerPath = '$buildDir/$runnerName'; | 455 targetRunnerPath = '$buildDir/$runnerName'; |
455 | 456 |
456 // For listing the tests we use the '$runnerName.host' binary if it exists | 457 // For listing the tests we use the '$runnerName.host' binary if it exists |
457 // and use '$runnerName' if it doesn't. | 458 // and use '$runnerName' if it doesn't. |
458 var binarySuffix = Platform.operatingSystem == 'windows' ? '.exe' : ''; | 459 var binarySuffix = Platform.operatingSystem == 'windows' ? '.exe' : ''; |
459 var hostBinary = '$targetRunnerPath.host$binarySuffix'; | 460 var hostBinary = '$targetRunnerPath.host$binarySuffix'; |
460 if (new File(hostBinary).existsSync()) { | 461 if (new File(hostBinary).existsSync()) { |
461 hostRunnerPath = hostBinary; | 462 hostRunnerPath = hostBinary; |
462 } else { | 463 } else { |
463 hostRunnerPath = targetRunnerPath; | 464 hostRunnerPath = targetRunnerPath; |
464 } | 465 } |
465 } | 466 } |
466 | 467 |
467 void testNameHandler(TestExpectations testExpectations, String testName) { | 468 void testNameHandler(ExpectationSet testExpectations, String testName) { |
468 // Only run the tests that match the pattern. Use the name | 469 // Only run the tests that match the pattern. Use the name |
469 // "suiteName/testName" for cc tests. | 470 // "suiteName/testName" for cc tests. |
470 String constructedName = '$suiteName/$testPrefix$testName'; | 471 String constructedName = '$suiteName/$testPrefix$testName'; |
471 | 472 |
472 var expectations = testExpectations.expectations('$testPrefix$testName'); | 473 var expectations = testExpectations.expectations('$testPrefix$testName'); |
473 | 474 |
474 var args = TestUtils.standardOptions(configuration); | 475 var args = TestUtils.standardOptions(configuration); |
475 args.add(testName); | 476 args.add(testName); |
476 | 477 |
477 var command = CommandBuilder.instance.getProcessCommand( | 478 var command = CommandBuilder.instance.getProcessCommand( |
478 'run_vm_unittest', targetRunnerPath, args, environmentOverrides); | 479 'run_vm_unittest', targetRunnerPath, args, environmentOverrides); |
479 enqueueNewTestCase( | 480 enqueueNewTestCase( |
480 new TestCase(constructedName, [command], configuration, expectations)); | 481 new TestCase(constructedName, [command], configuration, expectations)); |
481 } | 482 } |
482 | 483 |
483 void forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) { | 484 Future<Null> forEachTest(Function onTest, Map testCache, |
| 485 [VoidFunction onDone]) async { |
484 doTest = onTest; | 486 doTest = onTest; |
485 var statusFiles = | 487 var statusFiles = |
486 statusFilePaths.map((statusFile) => "$dartDir/$statusFile").toList(); | 488 statusFilePaths.map((statusFile) => "$dartDir/$statusFile").toList(); |
487 | 489 |
488 ReadTestExpectations(statusFiles, configuration) | 490 var expectations = ExpectationSet.read(statusFiles, configuration); |
489 .then((TestExpectations expectations) { | 491 |
490 ccTestLister(hostRunnerPath).then((Iterable<String> names) { | 492 try { |
491 names.forEach((testName) => testNameHandler(expectations, testName)); | 493 var names = await ccTestLister(hostRunnerPath); |
492 doTest = null; | 494 for (var name in names) { |
493 if (onDone != null) onDone(); | 495 testNameHandler(expectations, name); |
494 }).catchError((error) { | 496 } |
495 print("Fatal error occured: $error"); | 497 |
496 exit(1); | 498 doTest = null; |
497 }); | 499 if (onDone != null) onDone(); |
498 }); | 500 } catch (error) { |
| 501 print("Fatal error occured: $error"); |
| 502 exit(1); |
| 503 } |
499 } | 504 } |
500 } | 505 } |
501 | 506 |
502 class TestInformation { | 507 class TestInformation { |
503 Path filePath; | 508 Path filePath; |
504 Path originTestPath; | 509 Path originTestPath; |
505 Map optionsFromFile; | 510 Map optionsFromFile; |
506 bool hasCompileError; | 511 bool hasCompileError; |
507 bool hasRuntimeError; | 512 bool hasRuntimeError; |
508 bool isNegativeIfChecked; | 513 bool isNegativeIfChecked; |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
540 false) {} | 545 false) {} |
541 } | 546 } |
542 | 547 |
543 /** | 548 /** |
544 * A standard [TestSuite] implementation that searches for tests in a | 549 * A standard [TestSuite] implementation that searches for tests in a |
545 * directory, and creates [TestCase]s that compile and/or run them. | 550 * directory, and creates [TestCase]s that compile and/or run them. |
546 */ | 551 */ |
547 class StandardTestSuite extends TestSuite { | 552 class StandardTestSuite extends TestSuite { |
548 final Path suiteDir; | 553 final Path suiteDir; |
549 final List<String> statusFilePaths; | 554 final List<String> statusFilePaths; |
550 TestExpectations testExpectations; | 555 ExpectationSet testExpectations; |
551 List<TestInformation> cachedTests; | 556 List<TestInformation> cachedTests; |
552 final Path dartDir; | 557 final Path dartDir; |
553 Predicate<String> isTestFilePredicate; | 558 Predicate<String> isTestFilePredicate; |
554 final bool listRecursively; | 559 final bool listRecursively; |
555 final List<String> extraVmOptions; | 560 final List<String> extraVmOptions; |
556 List<Uri> _dart2JsBootstrapDependencies; | 561 List<Uri> _dart2JsBootstrapDependencies; |
557 | 562 |
558 StandardTestSuite(Map configuration, String suiteName, Path suiteDirectory, | 563 StandardTestSuite(Map<String, dynamic> configuration, String suiteName, |
559 this.statusFilePaths, | 564 Path suiteDirectory, this.statusFilePaths, |
560 {this.isTestFilePredicate, bool recursive: false}) | 565 {this.isTestFilePredicate, bool recursive: false}) |
561 : dartDir = TestUtils.dartDir, | 566 : dartDir = TestUtils.dartDir, |
562 listRecursively = recursive, | 567 listRecursively = recursive, |
563 suiteDir = TestUtils.dartDir.join(suiteDirectory), | 568 suiteDir = TestUtils.dartDir.join(suiteDirectory), |
564 extraVmOptions = TestUtils.getExtraVmOptions(configuration), | 569 extraVmOptions = TestUtils.getExtraVmOptions(configuration), |
565 super(configuration, suiteName) { | 570 super(configuration, suiteName) { |
566 if (!useSdk) { | 571 if (!useSdk) { |
567 _dart2JsBootstrapDependencies = []; | 572 _dart2JsBootstrapDependencies = []; |
568 } else { | 573 } else { |
569 var snapshotPath = TestUtils | 574 var snapshotPath = TestUtils |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
631 return filename.endsWith("Test.dart"); | 636 return filename.endsWith("Test.dart"); |
632 } | 637 } |
633 | 638 |
634 bool isHtmlTestFile(String filename) => filename.endsWith('_htmltest.html'); | 639 bool isHtmlTestFile(String filename) => filename.endsWith('_htmltest.html'); |
635 | 640 |
636 List<String> additionalOptions(Path filePath) => []; | 641 List<String> additionalOptions(Path filePath) => []; |
637 | 642 |
638 forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) async { | 643 forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) async { |
639 await updateDartium(); | 644 await updateDartium(); |
640 doTest = onTest; | 645 doTest = onTest; |
641 testExpectations = await readExpectations(); | 646 testExpectations = readExpectations(); |
642 | 647 |
643 // Check if we have already found and generated the tests for this suite. | 648 // Check if we have already found and generated the tests for this suite. |
644 if (!testCache.containsKey(suiteName)) { | 649 if (!testCache.containsKey(suiteName)) { |
645 cachedTests = testCache[suiteName] = <TestInformation>[]; | 650 cachedTests = testCache[suiteName] = <TestInformation>[]; |
646 await enqueueTests(); | 651 await enqueueTests(); |
647 } else { | 652 } else { |
648 for (var info in testCache[suiteName]) { | 653 for (var info in testCache[suiteName]) { |
649 enqueueTestCaseFromTestInformation(info); | 654 enqueueTestCaseFromTestInformation(info); |
650 } | 655 } |
651 } | 656 } |
(...skipping 16 matching lines...) Expand all Loading... |
668 | 673 |
669 assert(updater.isActive); | 674 assert(updater.isActive); |
670 updater.onUpdated.add(() => completer.complete(null)); | 675 updater.onUpdated.add(() => completer.complete(null)); |
671 | 676 |
672 return completer.future; | 677 return completer.future; |
673 } | 678 } |
674 | 679 |
675 /** | 680 /** |
676 * Reads the status files and completes with the parsed expectations. | 681 * Reads the status files and completes with the parsed expectations. |
677 */ | 682 */ |
678 Future<TestExpectations> readExpectations() { | 683 ExpectationSet readExpectations() { |
679 var statusFiles = statusFilePaths.where((String statusFilePath) { | 684 var statusFiles = statusFilePaths.where((String statusFilePath) { |
680 var file = new File(dartDir.append(statusFilePath).toNativePath()); | 685 var file = new File(dartDir.append(statusFilePath).toNativePath()); |
681 return file.existsSync(); | 686 return file.existsSync(); |
682 }).map((statusFilePath) { | 687 }).map((statusFilePath) { |
683 return dartDir.append(statusFilePath).toNativePath(); | 688 return dartDir.append(statusFilePath).toNativePath(); |
684 }).toList(); | 689 }).toList(); |
685 | 690 |
686 return ReadTestExpectations(statusFiles, configuration); | 691 return ExpectationSet.read(statusFiles, configuration); |
687 } | 692 } |
688 | 693 |
689 Future enqueueTests() { | 694 Future enqueueTests() { |
690 Directory dir = new Directory(suiteDir.toNativePath()); | 695 Directory dir = new Directory(suiteDir.toNativePath()); |
691 return dir.exists().then((exists) { | 696 return dir.exists().then((exists) { |
692 if (!exists) { | 697 if (!exists) { |
693 print('Directory containing tests missing: ${suiteDir.toNativePath()}'); | 698 print('Directory containing tests missing: ${suiteDir.toNativePath()}'); |
694 return new Future.value(null); | 699 return new Future.value(null); |
695 } else { | 700 } else { |
696 var group = new FutureGroup(); | 701 var group = new FutureGroup(); |
(...skipping 1518 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2215 } | 2220 } |
2216 if (path.length > WINDOWS_SHORTEN_PATH_LIMIT) { | 2221 if (path.length > WINDOWS_SHORTEN_PATH_LIMIT) { |
2217 ++shortNameCounter; | 2222 ++shortNameCounter; |
2218 var pathEnd = path.substring(path.length - WINDOWS_PATH_END_LENGTH); | 2223 var pathEnd = path.substring(path.length - WINDOWS_PATH_END_LENGTH); |
2219 path = "short${shortNameCounter}_$pathEnd"; | 2224 path = "short${shortNameCounter}_$pathEnd"; |
2220 } | 2225 } |
2221 } | 2226 } |
2222 return path; | 2227 return path; |
2223 } | 2228 } |
2224 } | 2229 } |
OLD | NEW |