| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for enumerating and preparing tests. | 6 * Classes and methods for enumerating and preparing tests. |
| 7 * | 7 * |
| 8 * This library includes: | 8 * This library includes: |
| 9 * | 9 * |
| 10 * - Creating tests by listing all the Dart files in certain directories, | 10 * - Creating tests by listing all the Dart files in certain directories, |
| 11 * and creating [TestCase]s for those files that meet the relevant criteria. | 11 * and creating [TestCase]s for those files that meet the relevant criteria. |
| 12 * - Preparing tests, including copying files and frameworks to temporary | 12 * - Preparing tests, including copying files and frameworks to temporary |
| 13 * directories, and computing the command line and arguments to be run. | 13 * directories, and computing the command line and arguments to be run. |
| 14 */ | 14 */ |
| 15 library test_suite; | 15 library test_suite; |
| 16 | 16 |
| 17 import "dart:async"; | 17 import "dart:async"; |
| 18 import "dart:io"; | 18 import "dart:io"; |
| 19 import "dart:math"; | 19 import "dart:math"; |
| 20 import "drt_updater.dart"; | 20 import "drt_updater.dart"; |
| 21 import "html_test.dart" as htmlTest; | 21 import "html_test.dart" as htmlTest; |
| 22 import "path.dart"; | 22 import "path.dart"; |
| 23 import "multitest.dart"; | 23 import "multitest.dart"; |
| 24 import "status_file_parser.dart"; | 24 import "expectation.dart"; |
| 25 import "expectation_set.dart"; |
| 25 import "summary_report.dart"; | 26 import "summary_report.dart"; |
| 26 import "test_runner.dart"; | 27 import "test_runner.dart"; |
| 27 import "utils.dart"; | 28 import "utils.dart"; |
| 28 import "http_server.dart" show PREFIX_BUILDDIR, PREFIX_DARTDIR; | 29 import "http_server.dart" show PREFIX_BUILDDIR, PREFIX_DARTDIR; |
| 29 | 30 |
| 30 import "compiler_configuration.dart" | 31 import "compiler_configuration.dart" |
| 31 show CommandArtifact, CompilerConfiguration; | 32 show CommandArtifact, CompilerConfiguration; |
| 32 | 33 |
| 33 import "runtime_configuration.dart" show RuntimeConfiguration; | 34 import "runtime_configuration.dart" show RuntimeConfiguration; |
| 34 | 35 |
| (...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 111 } | 112 } |
| 112 | 113 |
| 113 /** | 114 /** |
| 114 * A TestSuite represents a collection of tests. It creates a [TestCase] | 115 * A TestSuite represents a collection of tests. It creates a [TestCase] |
| 115 * object for each test to be run, and passes the test cases to a callback. | 116 * object for each test to be run, and passes the test cases to a callback. |
| 116 * | 117 * |
| 117 * Most TestSuites represent a directory or directory tree containing tests, | 118 * Most TestSuites represent a directory or directory tree containing tests, |
| 118 * and a status file containing the expected results when these tests are run. | 119 * and a status file containing the expected results when these tests are run. |
| 119 */ | 120 */ |
| 120 abstract class TestSuite { | 121 abstract class TestSuite { |
| 121 final Map configuration; | 122 final Map<String, dynamic> configuration; |
| 122 final String suiteName; | 123 final String suiteName; |
| 123 // This function is set by subclasses before enqueueing starts. | 124 // This function is set by subclasses before enqueueing starts. |
| 124 Function doTest; | 125 Function doTest; |
| 125 Map<String, String> _environmentOverrides; | 126 Map<String, String> _environmentOverrides; |
| 126 RuntimeConfiguration runtimeConfiguration; | 127 RuntimeConfiguration runtimeConfiguration; |
| 127 | 128 |
| 128 TestSuite(this.configuration, this.suiteName) { | 129 TestSuite(this.configuration, this.suiteName) { |
| 129 TestUtils.buildDir(configuration); // Sets configuration_directory. | 130 TestUtils.buildDir(configuration); // Sets configuration_directory. |
| 130 if (configuration['configuration_directory'] != null) { | 131 if (configuration['configuration_directory'] != null) { |
| 131 _environmentOverrides = { | 132 _environmentOverrides = { |
| (...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 300 return; | 301 return; |
| 301 } | 302 } |
| 302 // Test if the selector includes this test. | 303 // Test if the selector includes this test. |
| 303 RegExp pattern = configuration['selectors'][suiteName]; | 304 RegExp pattern = configuration['selectors'][suiteName]; |
| 304 if (!pattern.hasMatch(testCase.displayName)) { | 305 if (!pattern.hasMatch(testCase.displayName)) { |
| 305 return; | 306 return; |
| 306 } | 307 } |
| 307 | 308 |
| 308 if (configuration['hot_reload'] || configuration['hot_reload_rollback']) { | 309 if (configuration['hot_reload'] || configuration['hot_reload_rollback']) { |
| 309 // Handle reload special cases. | 310 // Handle reload special cases. |
| 310 if (expectations.contains(Expectation.COMPILETIME_ERROR) || | 311 if (expectations.contains(Expectation.compileTimeError) || |
| 311 testCase.hasCompileError || | 312 testCase.hasCompileError || |
| 312 testCase.expectCompileError) { | 313 testCase.expectCompileError) { |
| 313 // Running a test that expects a compilation error with hot reloading | 314 // Running a test that expects a compilation error with hot reloading |
| 314 // is redundant with a regular run of the test. | 315 // is redundant with a regular run of the test. |
| 315 return; | 316 return; |
| 316 } | 317 } |
| 317 } | 318 } |
| 318 | 319 |
| 319 // Update Summary report | 320 // Update Summary report |
| 320 if (configuration['report']) { | 321 if (configuration['report']) { |
| 321 if (testCase.expectCompileError && | 322 if (testCase.expectCompileError && |
| 322 TestUtils.isBrowserRuntime(configuration['runtime']) && | 323 TestUtils.isBrowserRuntime(configuration['runtime']) && |
| 323 new CompilerConfiguration(configuration).hasCompiler) { | 324 new CompilerConfiguration(configuration).hasCompiler) { |
| 324 summaryReport.addCompileErrorSkipTest(); | 325 summaryReport.addCompileErrorSkipTest(); |
| 325 return; | 326 return; |
| 326 } else { | 327 } else { |
| 327 summaryReport.add(testCase); | 328 summaryReport.add(testCase); |
| 328 } | 329 } |
| 329 } | 330 } |
| 330 | 331 |
| 331 // Handle skipped tests | 332 // Handle skipped tests |
| 332 if (expectations.contains(Expectation.SKIP) || | 333 if (expectations.contains(Expectation.skip) || |
| 333 expectations.contains(Expectation.SKIP_BY_DESIGN) || | 334 expectations.contains(Expectation.skipByDesign) || |
| 334 expectations.contains(Expectation.SKIP_SLOW)) { | 335 expectations.contains(Expectation.skipSlow)) { |
| 335 return; | 336 return; |
| 336 } | 337 } |
| 337 | 338 |
| 338 doTest(testCase); | 339 doTest(testCase); |
| 339 } | 340 } |
| 340 | 341 |
| 341 String createGeneratedTestDirectoryHelper( | 342 String createGeneratedTestDirectoryHelper( |
| 342 String name, String dirname, Path testPath, String optionsName) { | 343 String name, String dirname, Path testPath, String optionsName) { |
| 343 Path relative = testPath.relativeTo(TestUtils.dartDir); | 344 Path relative = testPath.relativeTo(TestUtils.dartDir); |
| 344 relative = relative.directoryPath.append(relative.filenameWithoutExtension); | 345 relative = relative.directoryPath.append(relative.filenameWithoutExtension); |
| (...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 438 * The executable lists its tests when run with the --list command line flag. | 439 * The executable lists its tests when run with the --list command line flag. |
| 439 * Individual tests are run by specifying them on the command line. | 440 * Individual tests are run by specifying them on the command line. |
| 440 */ | 441 */ |
| 441 class CCTestSuite extends TestSuite { | 442 class CCTestSuite extends TestSuite { |
| 442 final String testPrefix; | 443 final String testPrefix; |
| 443 String targetRunnerPath; | 444 String targetRunnerPath; |
| 444 String hostRunnerPath; | 445 String hostRunnerPath; |
| 445 final String dartDir; | 446 final String dartDir; |
| 446 List<String> statusFilePaths; | 447 List<String> statusFilePaths; |
| 447 | 448 |
| 448 CCTestSuite(Map configuration, String suiteName, String runnerName, | 449 CCTestSuite(Map<String, dynamic> configuration, String suiteName, |
| 449 this.statusFilePaths, | 450 String runnerName, this.statusFilePaths, |
| 450 {this.testPrefix: ''}) | 451 {this.testPrefix: ''}) |
| 451 : dartDir = TestUtils.dartDir.toNativePath(), | 452 : dartDir = TestUtils.dartDir.toNativePath(), |
| 452 super(configuration, suiteName) { | 453 super(configuration, suiteName) { |
| 453 // For running the tests we use the given '$runnerName' binary | 454 // For running the tests we use the given '$runnerName' binary |
| 454 targetRunnerPath = '$buildDir/$runnerName'; | 455 targetRunnerPath = '$buildDir/$runnerName'; |
| 455 | 456 |
| 456 // For listing the tests we use the '$runnerName.host' binary if it exists | 457 // For listing the tests we use the '$runnerName.host' binary if it exists |
| 457 // and use '$runnerName' if it doesn't. | 458 // and use '$runnerName' if it doesn't. |
| 458 var binarySuffix = Platform.operatingSystem == 'windows' ? '.exe' : ''; | 459 var binarySuffix = Platform.operatingSystem == 'windows' ? '.exe' : ''; |
| 459 var hostBinary = '$targetRunnerPath.host$binarySuffix'; | 460 var hostBinary = '$targetRunnerPath.host$binarySuffix'; |
| 460 if (new File(hostBinary).existsSync()) { | 461 if (new File(hostBinary).existsSync()) { |
| 461 hostRunnerPath = hostBinary; | 462 hostRunnerPath = hostBinary; |
| 462 } else { | 463 } else { |
| 463 hostRunnerPath = targetRunnerPath; | 464 hostRunnerPath = targetRunnerPath; |
| 464 } | 465 } |
| 465 } | 466 } |
| 466 | 467 |
| 467 void testNameHandler(TestExpectations testExpectations, String testName) { | 468 void testNameHandler(ExpectationSet testExpectations, String testName) { |
| 468 // Only run the tests that match the pattern. Use the name | 469 // Only run the tests that match the pattern. Use the name |
| 469 // "suiteName/testName" for cc tests. | 470 // "suiteName/testName" for cc tests. |
| 470 String constructedName = '$suiteName/$testPrefix$testName'; | 471 String constructedName = '$suiteName/$testPrefix$testName'; |
| 471 | 472 |
| 472 var expectations = testExpectations.expectations('$testPrefix$testName'); | 473 var expectations = testExpectations.expectations('$testPrefix$testName'); |
| 473 | 474 |
| 474 var args = TestUtils.standardOptions(configuration); | 475 var args = TestUtils.standardOptions(configuration); |
| 475 args.add(testName); | 476 args.add(testName); |
| 476 | 477 |
| 477 var command = CommandBuilder.instance.getProcessCommand( | 478 var command = CommandBuilder.instance.getProcessCommand( |
| 478 'run_vm_unittest', targetRunnerPath, args, environmentOverrides); | 479 'run_vm_unittest', targetRunnerPath, args, environmentOverrides); |
| 479 enqueueNewTestCase( | 480 enqueueNewTestCase( |
| 480 new TestCase(constructedName, [command], configuration, expectations)); | 481 new TestCase(constructedName, [command], configuration, expectations)); |
| 481 } | 482 } |
| 482 | 483 |
| 483 void forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) { | 484 Future<Null> forEachTest(Function onTest, Map testCache, |
| 485 [VoidFunction onDone]) async { |
| 484 doTest = onTest; | 486 doTest = onTest; |
| 485 var statusFiles = | 487 var statusFiles = |
| 486 statusFilePaths.map((statusFile) => "$dartDir/$statusFile").toList(); | 488 statusFilePaths.map((statusFile) => "$dartDir/$statusFile").toList(); |
| 487 | 489 |
| 488 ReadTestExpectations(statusFiles, configuration) | 490 var expectations = ExpectationSet.read(statusFiles, configuration); |
| 489 .then((TestExpectations expectations) { | 491 |
| 490 ccTestLister(hostRunnerPath).then((Iterable<String> names) { | 492 try { |
| 491 names.forEach((testName) => testNameHandler(expectations, testName)); | 493 var names = await ccTestLister(hostRunnerPath); |
| 492 doTest = null; | 494 for (var name in names) { |
| 493 if (onDone != null) onDone(); | 495 testNameHandler(expectations, name); |
| 494 }).catchError((error) { | 496 } |
| 495 print("Fatal error occured: $error"); | 497 |
| 496 exit(1); | 498 doTest = null; |
| 497 }); | 499 if (onDone != null) onDone(); |
| 498 }); | 500 } catch (error) { |
| 501 print("Fatal error occured: $error"); |
| 502 exit(1); |
| 503 } |
| 499 } | 504 } |
| 500 } | 505 } |
| 501 | 506 |
| 502 class TestInformation { | 507 class TestInformation { |
| 503 Path filePath; | 508 Path filePath; |
| 504 Path originTestPath; | 509 Path originTestPath; |
| 505 Map optionsFromFile; | 510 Map optionsFromFile; |
| 506 bool hasCompileError; | 511 bool hasCompileError; |
| 507 bool hasRuntimeError; | 512 bool hasRuntimeError; |
| 508 bool isNegativeIfChecked; | 513 bool isNegativeIfChecked; |
| (...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 540 false) {} | 545 false) {} |
| 541 } | 546 } |
| 542 | 547 |
| 543 /** | 548 /** |
| 544 * A standard [TestSuite] implementation that searches for tests in a | 549 * A standard [TestSuite] implementation that searches for tests in a |
| 545 * directory, and creates [TestCase]s that compile and/or run them. | 550 * directory, and creates [TestCase]s that compile and/or run them. |
| 546 */ | 551 */ |
| 547 class StandardTestSuite extends TestSuite { | 552 class StandardTestSuite extends TestSuite { |
| 548 final Path suiteDir; | 553 final Path suiteDir; |
| 549 final List<String> statusFilePaths; | 554 final List<String> statusFilePaths; |
| 550 TestExpectations testExpectations; | 555 ExpectationSet testExpectations; |
| 551 List<TestInformation> cachedTests; | 556 List<TestInformation> cachedTests; |
| 552 final Path dartDir; | 557 final Path dartDir; |
| 553 Predicate<String> isTestFilePredicate; | 558 Predicate<String> isTestFilePredicate; |
| 554 final bool listRecursively; | 559 final bool listRecursively; |
| 555 final List<String> extraVmOptions; | 560 final List<String> extraVmOptions; |
| 556 List<Uri> _dart2JsBootstrapDependencies; | 561 List<Uri> _dart2JsBootstrapDependencies; |
| 557 | 562 |
| 558 StandardTestSuite(Map configuration, String suiteName, Path suiteDirectory, | 563 StandardTestSuite(Map<String, dynamic> configuration, String suiteName, |
| 559 this.statusFilePaths, | 564 Path suiteDirectory, this.statusFilePaths, |
| 560 {this.isTestFilePredicate, bool recursive: false}) | 565 {this.isTestFilePredicate, bool recursive: false}) |
| 561 : dartDir = TestUtils.dartDir, | 566 : dartDir = TestUtils.dartDir, |
| 562 listRecursively = recursive, | 567 listRecursively = recursive, |
| 563 suiteDir = TestUtils.dartDir.join(suiteDirectory), | 568 suiteDir = TestUtils.dartDir.join(suiteDirectory), |
| 564 extraVmOptions = TestUtils.getExtraVmOptions(configuration), | 569 extraVmOptions = TestUtils.getExtraVmOptions(configuration), |
| 565 super(configuration, suiteName) { | 570 super(configuration, suiteName) { |
| 566 if (!useSdk) { | 571 if (!useSdk) { |
| 567 _dart2JsBootstrapDependencies = []; | 572 _dart2JsBootstrapDependencies = []; |
| 568 } else { | 573 } else { |
| 569 var snapshotPath = TestUtils | 574 var snapshotPath = TestUtils |
| (...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 631 return filename.endsWith("Test.dart"); | 636 return filename.endsWith("Test.dart"); |
| 632 } | 637 } |
| 633 | 638 |
| 634 bool isHtmlTestFile(String filename) => filename.endsWith('_htmltest.html'); | 639 bool isHtmlTestFile(String filename) => filename.endsWith('_htmltest.html'); |
| 635 | 640 |
| 636 List<String> additionalOptions(Path filePath) => []; | 641 List<String> additionalOptions(Path filePath) => []; |
| 637 | 642 |
| 638 forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) async { | 643 forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) async { |
| 639 await updateDartium(); | 644 await updateDartium(); |
| 640 doTest = onTest; | 645 doTest = onTest; |
| 641 testExpectations = await readExpectations(); | 646 testExpectations = readExpectations(); |
| 642 | 647 |
| 643 // Check if we have already found and generated the tests for this suite. | 648 // Check if we have already found and generated the tests for this suite. |
| 644 if (!testCache.containsKey(suiteName)) { | 649 if (!testCache.containsKey(suiteName)) { |
| 645 cachedTests = testCache[suiteName] = <TestInformation>[]; | 650 cachedTests = testCache[suiteName] = <TestInformation>[]; |
| 646 await enqueueTests(); | 651 await enqueueTests(); |
| 647 } else { | 652 } else { |
| 648 for (var info in testCache[suiteName]) { | 653 for (var info in testCache[suiteName]) { |
| 649 enqueueTestCaseFromTestInformation(info); | 654 enqueueTestCaseFromTestInformation(info); |
| 650 } | 655 } |
| 651 } | 656 } |
| (...skipping 16 matching lines...) Expand all Loading... |
| 668 | 673 |
| 669 assert(updater.isActive); | 674 assert(updater.isActive); |
| 670 updater.onUpdated.add(() => completer.complete(null)); | 675 updater.onUpdated.add(() => completer.complete(null)); |
| 671 | 676 |
| 672 return completer.future; | 677 return completer.future; |
| 673 } | 678 } |
| 674 | 679 |
| 675 /** | 680 /** |
| 676 * Reads the status files and completes with the parsed expectations. | 681 * Reads the status files and completes with the parsed expectations. |
| 677 */ | 682 */ |
| 678 Future<TestExpectations> readExpectations() { | 683 ExpectationSet readExpectations() { |
| 679 var statusFiles = statusFilePaths.where((String statusFilePath) { | 684 var statusFiles = statusFilePaths.where((String statusFilePath) { |
| 680 var file = new File(dartDir.append(statusFilePath).toNativePath()); | 685 var file = new File(dartDir.append(statusFilePath).toNativePath()); |
| 681 return file.existsSync(); | 686 return file.existsSync(); |
| 682 }).map((statusFilePath) { | 687 }).map((statusFilePath) { |
| 683 return dartDir.append(statusFilePath).toNativePath(); | 688 return dartDir.append(statusFilePath).toNativePath(); |
| 684 }).toList(); | 689 }).toList(); |
| 685 | 690 |
| 686 return ReadTestExpectations(statusFiles, configuration); | 691 return ExpectationSet.read(statusFiles, configuration); |
| 687 } | 692 } |
| 688 | 693 |
| 689 Future enqueueTests() { | 694 Future enqueueTests() { |
| 690 Directory dir = new Directory(suiteDir.toNativePath()); | 695 Directory dir = new Directory(suiteDir.toNativePath()); |
| 691 return dir.exists().then((exists) { | 696 return dir.exists().then((exists) { |
| 692 if (!exists) { | 697 if (!exists) { |
| 693 print('Directory containing tests missing: ${suiteDir.toNativePath()}'); | 698 print('Directory containing tests missing: ${suiteDir.toNativePath()}'); |
| 694 return new Future.value(null); | 699 return new Future.value(null); |
| 695 } else { | 700 } else { |
| 696 var group = new FutureGroup(); | 701 var group = new FutureGroup(); |
| (...skipping 1518 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 2215 } | 2220 } |
| 2216 if (path.length > WINDOWS_SHORTEN_PATH_LIMIT) { | 2221 if (path.length > WINDOWS_SHORTEN_PATH_LIMIT) { |
| 2217 ++shortNameCounter; | 2222 ++shortNameCounter; |
| 2218 var pathEnd = path.substring(path.length - WINDOWS_PATH_END_LENGTH); | 2223 var pathEnd = path.substring(path.length - WINDOWS_PATH_END_LENGTH); |
| 2219 path = "short${shortNameCounter}_$pathEnd"; | 2224 path = "short${shortNameCounter}_$pathEnd"; |
| 2220 } | 2225 } |
| 2221 } | 2226 } |
| 2222 return path; | 2227 return path; |
| 2223 } | 2228 } |
| 2224 } | 2229 } |
| OLD | NEW |