| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for enumerating and preparing tests. | 6 * Classes and methods for enumerating and preparing tests. |
| 7 * | 7 * |
| 8 * This library includes: | 8 * This library includes: |
| 9 * | 9 * |
| 10 * - Creating tests by listing all the Dart files in certain directories, | 10 * - Creating tests by listing all the Dart files in certain directories, |
| 11 * and creating [TestCase]s for those files that meet the relevant criteria. | 11 * and creating [TestCase]s for those files that meet the relevant criteria. |
| 12 * - Preparing tests, including copying files and frameworks to temporary | 12 * - Preparing tests, including copying files and frameworks to temporary |
| 13 * directories, and computing the command line and arguments to be run. | 13 * directories, and computing the command line and arguments to be run. |
| 14 */ | 14 */ |
| 15 library test_suite; | 15 library test_suite; |
| 16 | 16 |
| 17 import "dart:async"; | 17 import "dart:async"; |
| 18 import "dart:io"; | 18 import "dart:io"; |
| 19 import "dart:math"; | 19 import "dart:math"; |
| 20 import "drt_updater.dart"; | 20 import "drt_updater.dart"; |
| 21 import "html_test.dart" as htmlTest; | 21 import "html_test.dart" as htmlTest; |
| 22 import "path.dart"; | 22 import "path.dart"; |
| 23 import "multitest.dart"; | 23 import "multitest.dart"; |
| 24 import "expectation.dart"; | 24 import "status_file_parser.dart"; |
| 25 import "expectation_set.dart"; | |
| 26 import "summary_report.dart"; | 25 import "summary_report.dart"; |
| 27 import "test_runner.dart"; | 26 import "test_runner.dart"; |
| 28 import "utils.dart"; | 27 import "utils.dart"; |
| 29 import "http_server.dart" show PREFIX_BUILDDIR, PREFIX_DARTDIR; | 28 import "http_server.dart" show PREFIX_BUILDDIR, PREFIX_DARTDIR; |
| 30 | 29 |
| 31 import "compiler_configuration.dart" | 30 import "compiler_configuration.dart" |
| 32 show CommandArtifact, CompilerConfiguration; | 31 show CommandArtifact, CompilerConfiguration; |
| 33 | 32 |
| 34 import "runtime_configuration.dart" show RuntimeConfiguration; | 33 import "runtime_configuration.dart" show RuntimeConfiguration; |
| 35 | 34 |
| (...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 112 } | 111 } |
| 113 | 112 |
| 114 /** | 113 /** |
| 115 * A TestSuite represents a collection of tests. It creates a [TestCase] | 114 * A TestSuite represents a collection of tests. It creates a [TestCase] |
| 116 * object for each test to be run, and passes the test cases to a callback. | 115 * object for each test to be run, and passes the test cases to a callback. |
| 117 * | 116 * |
| 118 * Most TestSuites represent a directory or directory tree containing tests, | 117 * Most TestSuites represent a directory or directory tree containing tests, |
| 119 * and a status file containing the expected results when these tests are run. | 118 * and a status file containing the expected results when these tests are run. |
| 120 */ | 119 */ |
| 121 abstract class TestSuite { | 120 abstract class TestSuite { |
| 122 final Map<String, dynamic> configuration; | 121 final Map configuration; |
| 123 final String suiteName; | 122 final String suiteName; |
| 124 // This function is set by subclasses before enqueueing starts. | 123 // This function is set by subclasses before enqueueing starts. |
| 125 Function doTest; | 124 Function doTest; |
| 126 Map<String, String> _environmentOverrides; | 125 Map<String, String> _environmentOverrides; |
| 127 RuntimeConfiguration runtimeConfiguration; | 126 RuntimeConfiguration runtimeConfiguration; |
| 128 | 127 |
| 129 TestSuite(this.configuration, this.suiteName) { | 128 TestSuite(this.configuration, this.suiteName) { |
| 130 TestUtils.buildDir(configuration); // Sets configuration_directory. | 129 TestUtils.buildDir(configuration); // Sets configuration_directory. |
| 131 if (configuration['configuration_directory'] != null) { | 130 if (configuration['configuration_directory'] != null) { |
| 132 _environmentOverrides = { | 131 _environmentOverrides = { |
| (...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 301 return; | 300 return; |
| 302 } | 301 } |
| 303 // Test if the selector includes this test. | 302 // Test if the selector includes this test. |
| 304 RegExp pattern = configuration['selectors'][suiteName]; | 303 RegExp pattern = configuration['selectors'][suiteName]; |
| 305 if (!pattern.hasMatch(testCase.displayName)) { | 304 if (!pattern.hasMatch(testCase.displayName)) { |
| 306 return; | 305 return; |
| 307 } | 306 } |
| 308 | 307 |
| 309 if (configuration['hot_reload'] || configuration['hot_reload_rollback']) { | 308 if (configuration['hot_reload'] || configuration['hot_reload_rollback']) { |
| 310 // Handle reload special cases. | 309 // Handle reload special cases. |
| 311 if (expectations.contains(Expectation.compileTimeError) || | 310 if (expectations.contains(Expectation.COMPILETIME_ERROR) || |
| 312 testCase.hasCompileError || | 311 testCase.hasCompileError || |
| 313 testCase.expectCompileError) { | 312 testCase.expectCompileError) { |
| 314 // Running a test that expects a compilation error with hot reloading | 313 // Running a test that expects a compilation error with hot reloading |
| 315 // is redundant with a regular run of the test. | 314 // is redundant with a regular run of the test. |
| 316 return; | 315 return; |
| 317 } | 316 } |
| 318 } | 317 } |
| 319 | 318 |
| 320 // Update Summary report | 319 // Update Summary report |
| 321 if (configuration['report']) { | 320 if (configuration['report']) { |
| 322 if (testCase.expectCompileError && | 321 if (testCase.expectCompileError && |
| 323 TestUtils.isBrowserRuntime(configuration['runtime']) && | 322 TestUtils.isBrowserRuntime(configuration['runtime']) && |
| 324 new CompilerConfiguration(configuration).hasCompiler) { | 323 new CompilerConfiguration(configuration).hasCompiler) { |
| 325 summaryReport.addCompileErrorSkipTest(); | 324 summaryReport.addCompileErrorSkipTest(); |
| 326 return; | 325 return; |
| 327 } else { | 326 } else { |
| 328 summaryReport.add(testCase); | 327 summaryReport.add(testCase); |
| 329 } | 328 } |
| 330 } | 329 } |
| 331 | 330 |
| 332 // Handle skipped tests | 331 // Handle skipped tests |
| 333 if (expectations.contains(Expectation.skip) || | 332 if (expectations.contains(Expectation.SKIP) || |
| 334 expectations.contains(Expectation.skipByDesign) || | 333 expectations.contains(Expectation.SKIP_BY_DESIGN) || |
| 335 expectations.contains(Expectation.skipSlow)) { | 334 expectations.contains(Expectation.SKIP_SLOW)) { |
| 336 return; | 335 return; |
| 337 } | 336 } |
| 338 | 337 |
| 339 doTest(testCase); | 338 doTest(testCase); |
| 340 } | 339 } |
| 341 | 340 |
| 342 String createGeneratedTestDirectoryHelper( | 341 String createGeneratedTestDirectoryHelper( |
| 343 String name, String dirname, Path testPath, String optionsName) { | 342 String name, String dirname, Path testPath, String optionsName) { |
| 344 Path relative = testPath.relativeTo(TestUtils.dartDir); | 343 Path relative = testPath.relativeTo(TestUtils.dartDir); |
| 345 relative = relative.directoryPath.append(relative.filenameWithoutExtension); | 344 relative = relative.directoryPath.append(relative.filenameWithoutExtension); |
| (...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 439 * The executable lists its tests when run with the --list command line flag. | 438 * The executable lists its tests when run with the --list command line flag. |
| 440 * Individual tests are run by specifying them on the command line. | 439 * Individual tests are run by specifying them on the command line. |
| 441 */ | 440 */ |
| 442 class CCTestSuite extends TestSuite { | 441 class CCTestSuite extends TestSuite { |
| 443 final String testPrefix; | 442 final String testPrefix; |
| 444 String targetRunnerPath; | 443 String targetRunnerPath; |
| 445 String hostRunnerPath; | 444 String hostRunnerPath; |
| 446 final String dartDir; | 445 final String dartDir; |
| 447 List<String> statusFilePaths; | 446 List<String> statusFilePaths; |
| 448 | 447 |
| 449 CCTestSuite(Map<String, dynamic> configuration, String suiteName, | 448 CCTestSuite(Map configuration, String suiteName, String runnerName, |
| 450 String runnerName, this.statusFilePaths, | 449 this.statusFilePaths, |
| 451 {this.testPrefix: ''}) | 450 {this.testPrefix: ''}) |
| 452 : dartDir = TestUtils.dartDir.toNativePath(), | 451 : dartDir = TestUtils.dartDir.toNativePath(), |
| 453 super(configuration, suiteName) { | 452 super(configuration, suiteName) { |
| 454 // For running the tests we use the given '$runnerName' binary | 453 // For running the tests we use the given '$runnerName' binary |
| 455 targetRunnerPath = '$buildDir/$runnerName'; | 454 targetRunnerPath = '$buildDir/$runnerName'; |
| 456 | 455 |
| 457 // For listing the tests we use the '$runnerName.host' binary if it exists | 456 // For listing the tests we use the '$runnerName.host' binary if it exists |
| 458 // and use '$runnerName' if it doesn't. | 457 // and use '$runnerName' if it doesn't. |
| 459 var binarySuffix = Platform.operatingSystem == 'windows' ? '.exe' : ''; | 458 var binarySuffix = Platform.operatingSystem == 'windows' ? '.exe' : ''; |
| 460 var hostBinary = '$targetRunnerPath.host$binarySuffix'; | 459 var hostBinary = '$targetRunnerPath.host$binarySuffix'; |
| 461 if (new File(hostBinary).existsSync()) { | 460 if (new File(hostBinary).existsSync()) { |
| 462 hostRunnerPath = hostBinary; | 461 hostRunnerPath = hostBinary; |
| 463 } else { | 462 } else { |
| 464 hostRunnerPath = targetRunnerPath; | 463 hostRunnerPath = targetRunnerPath; |
| 465 } | 464 } |
| 466 } | 465 } |
| 467 | 466 |
| 468 void testNameHandler(ExpectationSet testExpectations, String testName) { | 467 void testNameHandler(TestExpectations testExpectations, String testName) { |
| 469 // Only run the tests that match the pattern. Use the name | 468 // Only run the tests that match the pattern. Use the name |
| 470 // "suiteName/testName" for cc tests. | 469 // "suiteName/testName" for cc tests. |
| 471 String constructedName = '$suiteName/$testPrefix$testName'; | 470 String constructedName = '$suiteName/$testPrefix$testName'; |
| 472 | 471 |
| 473 var expectations = testExpectations.expectations('$testPrefix$testName'); | 472 var expectations = testExpectations.expectations('$testPrefix$testName'); |
| 474 | 473 |
| 475 var args = TestUtils.standardOptions(configuration); | 474 var args = TestUtils.standardOptions(configuration); |
| 476 args.add(testName); | 475 args.add(testName); |
| 477 | 476 |
| 478 var command = CommandBuilder.instance.getProcessCommand( | 477 var command = CommandBuilder.instance.getProcessCommand( |
| 479 'run_vm_unittest', targetRunnerPath, args, environmentOverrides); | 478 'run_vm_unittest', targetRunnerPath, args, environmentOverrides); |
| 480 enqueueNewTestCase( | 479 enqueueNewTestCase( |
| 481 new TestCase(constructedName, [command], configuration, expectations)); | 480 new TestCase(constructedName, [command], configuration, expectations)); |
| 482 } | 481 } |
| 483 | 482 |
| 484 Future<Null> forEachTest(Function onTest, Map testCache, | 483 void forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) { |
| 485 [VoidFunction onDone]) async { | |
| 486 doTest = onTest; | 484 doTest = onTest; |
| 487 var statusFiles = | 485 var statusFiles = |
| 488 statusFilePaths.map((statusFile) => "$dartDir/$statusFile").toList(); | 486 statusFilePaths.map((statusFile) => "$dartDir/$statusFile").toList(); |
| 489 | 487 |
| 490 var expectations = ExpectationSet.read(statusFiles, configuration); | 488 ReadTestExpectations(statusFiles, configuration) |
| 491 | 489 .then((TestExpectations expectations) { |
| 492 try { | 490 ccTestLister(hostRunnerPath).then((Iterable<String> names) { |
| 493 var names = await ccTestLister(hostRunnerPath); | 491 names.forEach((testName) => testNameHandler(expectations, testName)); |
| 494 for (var name in names) { | 492 doTest = null; |
| 495 testNameHandler(expectations, name); | 493 if (onDone != null) onDone(); |
| 496 } | 494 }).catchError((error) { |
| 497 | 495 print("Fatal error occured: $error"); |
| 498 doTest = null; | 496 exit(1); |
| 499 if (onDone != null) onDone(); | 497 }); |
| 500 } catch (error) { | 498 }); |
| 501 print("Fatal error occured: $error"); | |
| 502 exit(1); | |
| 503 } | |
| 504 } | 499 } |
| 505 } | 500 } |
| 506 | 501 |
| 507 class TestInformation { | 502 class TestInformation { |
| 508 Path filePath; | 503 Path filePath; |
| 509 Path originTestPath; | 504 Path originTestPath; |
| 510 Map optionsFromFile; | 505 Map optionsFromFile; |
| 511 bool hasCompileError; | 506 bool hasCompileError; |
| 512 bool hasRuntimeError; | 507 bool hasRuntimeError; |
| 513 bool isNegativeIfChecked; | 508 bool isNegativeIfChecked; |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 545 false) {} | 540 false) {} |
| 546 } | 541 } |
| 547 | 542 |
| 548 /** | 543 /** |
| 549 * A standard [TestSuite] implementation that searches for tests in a | 544 * A standard [TestSuite] implementation that searches for tests in a |
| 550 * directory, and creates [TestCase]s that compile and/or run them. | 545 * directory, and creates [TestCase]s that compile and/or run them. |
| 551 */ | 546 */ |
| 552 class StandardTestSuite extends TestSuite { | 547 class StandardTestSuite extends TestSuite { |
| 553 final Path suiteDir; | 548 final Path suiteDir; |
| 554 final List<String> statusFilePaths; | 549 final List<String> statusFilePaths; |
| 555 ExpectationSet testExpectations; | 550 TestExpectations testExpectations; |
| 556 List<TestInformation> cachedTests; | 551 List<TestInformation> cachedTests; |
| 557 final Path dartDir; | 552 final Path dartDir; |
| 558 Predicate<String> isTestFilePredicate; | 553 Predicate<String> isTestFilePredicate; |
| 559 final bool listRecursively; | 554 final bool listRecursively; |
| 560 final List<String> extraVmOptions; | 555 final List<String> extraVmOptions; |
| 561 List<Uri> _dart2JsBootstrapDependencies; | 556 List<Uri> _dart2JsBootstrapDependencies; |
| 562 | 557 |
| 563 StandardTestSuite(Map<String, dynamic> configuration, String suiteName, | 558 StandardTestSuite(Map configuration, String suiteName, Path suiteDirectory, |
| 564 Path suiteDirectory, this.statusFilePaths, | 559 this.statusFilePaths, |
| 565 {this.isTestFilePredicate, bool recursive: false}) | 560 {this.isTestFilePredicate, bool recursive: false}) |
| 566 : dartDir = TestUtils.dartDir, | 561 : dartDir = TestUtils.dartDir, |
| 567 listRecursively = recursive, | 562 listRecursively = recursive, |
| 568 suiteDir = TestUtils.dartDir.join(suiteDirectory), | 563 suiteDir = TestUtils.dartDir.join(suiteDirectory), |
| 569 extraVmOptions = TestUtils.getExtraVmOptions(configuration), | 564 extraVmOptions = TestUtils.getExtraVmOptions(configuration), |
| 570 super(configuration, suiteName) { | 565 super(configuration, suiteName) { |
| 571 if (!useSdk) { | 566 if (!useSdk) { |
| 572 _dart2JsBootstrapDependencies = []; | 567 _dart2JsBootstrapDependencies = []; |
| 573 } else { | 568 } else { |
| 574 var snapshotPath = TestUtils | 569 var snapshotPath = TestUtils |
| (...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 636 return filename.endsWith("Test.dart"); | 631 return filename.endsWith("Test.dart"); |
| 637 } | 632 } |
| 638 | 633 |
| 639 bool isHtmlTestFile(String filename) => filename.endsWith('_htmltest.html'); | 634 bool isHtmlTestFile(String filename) => filename.endsWith('_htmltest.html'); |
| 640 | 635 |
| 641 List<String> additionalOptions(Path filePath) => []; | 636 List<String> additionalOptions(Path filePath) => []; |
| 642 | 637 |
| 643 forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) async { | 638 forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) async { |
| 644 await updateDartium(); | 639 await updateDartium(); |
| 645 doTest = onTest; | 640 doTest = onTest; |
| 646 testExpectations = readExpectations(); | 641 testExpectations = await readExpectations(); |
| 647 | 642 |
| 648 // Check if we have already found and generated the tests for this suite. | 643 // Check if we have already found and generated the tests for this suite. |
| 649 if (!testCache.containsKey(suiteName)) { | 644 if (!testCache.containsKey(suiteName)) { |
| 650 cachedTests = testCache[suiteName] = <TestInformation>[]; | 645 cachedTests = testCache[suiteName] = <TestInformation>[]; |
| 651 await enqueueTests(); | 646 await enqueueTests(); |
| 652 } else { | 647 } else { |
| 653 for (var info in testCache[suiteName]) { | 648 for (var info in testCache[suiteName]) { |
| 654 enqueueTestCaseFromTestInformation(info); | 649 enqueueTestCaseFromTestInformation(info); |
| 655 } | 650 } |
| 656 } | 651 } |
| (...skipping 16 matching lines...) Expand all Loading... |
| 673 | 668 |
| 674 assert(updater.isActive); | 669 assert(updater.isActive); |
| 675 updater.onUpdated.add(() => completer.complete(null)); | 670 updater.onUpdated.add(() => completer.complete(null)); |
| 676 | 671 |
| 677 return completer.future; | 672 return completer.future; |
| 678 } | 673 } |
| 679 | 674 |
| 680 /** | 675 /** |
| 681 * Reads the status files and completes with the parsed expectations. | 676 * Reads the status files and completes with the parsed expectations. |
| 682 */ | 677 */ |
| 683 ExpectationSet readExpectations() { | 678 Future<TestExpectations> readExpectations() { |
| 684 var statusFiles = statusFilePaths.where((String statusFilePath) { | 679 var statusFiles = statusFilePaths.where((String statusFilePath) { |
| 685 var file = new File(dartDir.append(statusFilePath).toNativePath()); | 680 var file = new File(dartDir.append(statusFilePath).toNativePath()); |
| 686 return file.existsSync(); | 681 return file.existsSync(); |
| 687 }).map((statusFilePath) { | 682 }).map((statusFilePath) { |
| 688 return dartDir.append(statusFilePath).toNativePath(); | 683 return dartDir.append(statusFilePath).toNativePath(); |
| 689 }).toList(); | 684 }).toList(); |
| 690 | 685 |
| 691 return ExpectationSet.read(statusFiles, configuration); | 686 return ReadTestExpectations(statusFiles, configuration); |
| 692 } | 687 } |
| 693 | 688 |
| 694 Future enqueueTests() { | 689 Future enqueueTests() { |
| 695 Directory dir = new Directory(suiteDir.toNativePath()); | 690 Directory dir = new Directory(suiteDir.toNativePath()); |
| 696 return dir.exists().then((exists) { | 691 return dir.exists().then((exists) { |
| 697 if (!exists) { | 692 if (!exists) { |
| 698 print('Directory containing tests missing: ${suiteDir.toNativePath()}'); | 693 print('Directory containing tests missing: ${suiteDir.toNativePath()}'); |
| 699 return new Future.value(null); | 694 return new Future.value(null); |
| 700 } else { | 695 } else { |
| 701 var group = new FutureGroup(); | 696 var group = new FutureGroup(); |
| (...skipping 1518 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2220 } | 2215 } |
| 2221 if (path.length > WINDOWS_SHORTEN_PATH_LIMIT) { | 2216 if (path.length > WINDOWS_SHORTEN_PATH_LIMIT) { |
| 2222 ++shortNameCounter; | 2217 ++shortNameCounter; |
| 2223 var pathEnd = path.substring(path.length - WINDOWS_PATH_END_LENGTH); | 2218 var pathEnd = path.substring(path.length - WINDOWS_PATH_END_LENGTH); |
| 2224 path = "short${shortNameCounter}_$pathEnd"; | 2219 path = "short${shortNameCounter}_$pathEnd"; |
| 2225 } | 2220 } |
| 2226 } | 2221 } |
| 2227 return path; | 2222 return path; |
| 2228 } | 2223 } |
| 2229 } | 2224 } |
| OLD | NEW |