OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /** | 5 /** |
6 * Classes and methods for enumerating and preparing tests. | 6 * Classes and methods for enumerating and preparing tests. |
7 * | 7 * |
8 * This library includes: | 8 * This library includes: |
9 * | 9 * |
10 * - Creating tests by listing all the Dart files in certain directories, | 10 * - Creating tests by listing all the Dart files in certain directories, |
11 * and creating [TestCase]s for those files that meet the relevant criteria. | 11 * and creating [TestCase]s for those files that meet the relevant criteria. |
12 * - Preparing tests, including copying files and frameworks to temporary | 12 * - Preparing tests, including copying files and frameworks to temporary |
13 * directories, and computing the command line and arguments to be run. | 13 * directories, and computing the command line and arguments to be run. |
14 */ | 14 */ |
15 library test_suite; | 15 library test_suite; |
16 | 16 |
17 import "dart:async"; | 17 import "dart:async"; |
18 import "dart:io"; | 18 import "dart:io"; |
19 import "dart:math"; | 19 import "dart:math"; |
20 import "drt_updater.dart"; | 20 import "drt_updater.dart"; |
21 import "html_test.dart" as htmlTest; | 21 import "html_test.dart" as htmlTest; |
22 import "path.dart"; | 22 import "path.dart"; |
23 import "multitest.dart"; | 23 import "multitest.dart"; |
24 import "expectation.dart"; | 24 import "status_file_parser.dart"; |
25 import "expectation_set.dart"; | |
26 import "summary_report.dart"; | 25 import "summary_report.dart"; |
27 import "test_runner.dart"; | 26 import "test_runner.dart"; |
28 import "utils.dart"; | 27 import "utils.dart"; |
29 import "http_server.dart" show PREFIX_BUILDDIR, PREFIX_DARTDIR; | 28 import "http_server.dart" show PREFIX_BUILDDIR, PREFIX_DARTDIR; |
30 | 29 |
31 import "compiler_configuration.dart" | 30 import "compiler_configuration.dart" |
32 show CommandArtifact, CompilerConfiguration; | 31 show CommandArtifact, CompilerConfiguration; |
33 | 32 |
34 import "runtime_configuration.dart" show RuntimeConfiguration; | 33 import "runtime_configuration.dart" show RuntimeConfiguration; |
35 | 34 |
(...skipping 76 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
112 } | 111 } |
113 | 112 |
114 /** | 113 /** |
115 * A TestSuite represents a collection of tests. It creates a [TestCase] | 114 * A TestSuite represents a collection of tests. It creates a [TestCase] |
116 * object for each test to be run, and passes the test cases to a callback. | 115 * object for each test to be run, and passes the test cases to a callback. |
117 * | 116 * |
118 * Most TestSuites represent a directory or directory tree containing tests, | 117 * Most TestSuites represent a directory or directory tree containing tests, |
119 * and a status file containing the expected results when these tests are run. | 118 * and a status file containing the expected results when these tests are run. |
120 */ | 119 */ |
121 abstract class TestSuite { | 120 abstract class TestSuite { |
122 final Map<String, dynamic> configuration; | 121 final Map configuration; |
123 final String suiteName; | 122 final String suiteName; |
124 // This function is set by subclasses before enqueueing starts. | 123 // This function is set by subclasses before enqueueing starts. |
125 Function doTest; | 124 Function doTest; |
126 Map<String, String> _environmentOverrides; | 125 Map<String, String> _environmentOverrides; |
127 RuntimeConfiguration runtimeConfiguration; | 126 RuntimeConfiguration runtimeConfiguration; |
128 | 127 |
129 TestSuite(this.configuration, this.suiteName) { | 128 TestSuite(this.configuration, this.suiteName) { |
130 TestUtils.buildDir(configuration); // Sets configuration_directory. | 129 TestUtils.buildDir(configuration); // Sets configuration_directory. |
131 if (configuration['configuration_directory'] != null) { | 130 if (configuration['configuration_directory'] != null) { |
132 _environmentOverrides = { | 131 _environmentOverrides = { |
(...skipping 168 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
301 return; | 300 return; |
302 } | 301 } |
303 // Test if the selector includes this test. | 302 // Test if the selector includes this test. |
304 RegExp pattern = configuration['selectors'][suiteName]; | 303 RegExp pattern = configuration['selectors'][suiteName]; |
305 if (!pattern.hasMatch(testCase.displayName)) { | 304 if (!pattern.hasMatch(testCase.displayName)) { |
306 return; | 305 return; |
307 } | 306 } |
308 | 307 |
309 if (configuration['hot_reload'] || configuration['hot_reload_rollback']) { | 308 if (configuration['hot_reload'] || configuration['hot_reload_rollback']) { |
310 // Handle reload special cases. | 309 // Handle reload special cases. |
311 if (expectations.contains(Expectation.compileTimeError) || | 310 if (expectations.contains(Expectation.COMPILETIME_ERROR) || |
312 testCase.hasCompileError || | 311 testCase.hasCompileError || |
313 testCase.expectCompileError) { | 312 testCase.expectCompileError) { |
314 // Running a test that expects a compilation error with hot reloading | 313 // Running a test that expects a compilation error with hot reloading |
315 // is redundant with a regular run of the test. | 314 // is redundant with a regular run of the test. |
316 return; | 315 return; |
317 } | 316 } |
318 } | 317 } |
319 | 318 |
320 // Update Summary report | 319 // Update Summary report |
321 if (configuration['report']) { | 320 if (configuration['report']) { |
322 if (testCase.expectCompileError && | 321 if (testCase.expectCompileError && |
323 TestUtils.isBrowserRuntime(configuration['runtime']) && | 322 TestUtils.isBrowserRuntime(configuration['runtime']) && |
324 new CompilerConfiguration(configuration).hasCompiler) { | 323 new CompilerConfiguration(configuration).hasCompiler) { |
325 summaryReport.addCompileErrorSkipTest(); | 324 summaryReport.addCompileErrorSkipTest(); |
326 return; | 325 return; |
327 } else { | 326 } else { |
328 summaryReport.add(testCase); | 327 summaryReport.add(testCase); |
329 } | 328 } |
330 } | 329 } |
331 | 330 |
332 // Handle skipped tests | 331 // Handle skipped tests |
333 if (expectations.contains(Expectation.skip) || | 332 if (expectations.contains(Expectation.SKIP) || |
334 expectations.contains(Expectation.skipByDesign) || | 333 expectations.contains(Expectation.SKIP_BY_DESIGN) || |
335 expectations.contains(Expectation.skipSlow)) { | 334 expectations.contains(Expectation.SKIP_SLOW)) { |
336 return; | 335 return; |
337 } | 336 } |
338 | 337 |
339 doTest(testCase); | 338 doTest(testCase); |
340 } | 339 } |
341 | 340 |
342 String createGeneratedTestDirectoryHelper( | 341 String createGeneratedTestDirectoryHelper( |
343 String name, String dirname, Path testPath, String optionsName) { | 342 String name, String dirname, Path testPath, String optionsName) { |
344 Path relative = testPath.relativeTo(TestUtils.dartDir); | 343 Path relative = testPath.relativeTo(TestUtils.dartDir); |
345 relative = relative.directoryPath.append(relative.filenameWithoutExtension); | 344 relative = relative.directoryPath.append(relative.filenameWithoutExtension); |
(...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
439 * The executable lists its tests when run with the --list command line flag. | 438 * The executable lists its tests when run with the --list command line flag. |
440 * Individual tests are run by specifying them on the command line. | 439 * Individual tests are run by specifying them on the command line. |
441 */ | 440 */ |
442 class CCTestSuite extends TestSuite { | 441 class CCTestSuite extends TestSuite { |
443 final String testPrefix; | 442 final String testPrefix; |
444 String targetRunnerPath; | 443 String targetRunnerPath; |
445 String hostRunnerPath; | 444 String hostRunnerPath; |
446 final String dartDir; | 445 final String dartDir; |
447 List<String> statusFilePaths; | 446 List<String> statusFilePaths; |
448 | 447 |
449 CCTestSuite(Map<String, dynamic> configuration, String suiteName, | 448 CCTestSuite(Map configuration, String suiteName, String runnerName, |
450 String runnerName, this.statusFilePaths, | 449 this.statusFilePaths, |
451 {this.testPrefix: ''}) | 450 {this.testPrefix: ''}) |
452 : dartDir = TestUtils.dartDir.toNativePath(), | 451 : dartDir = TestUtils.dartDir.toNativePath(), |
453 super(configuration, suiteName) { | 452 super(configuration, suiteName) { |
454 // For running the tests we use the given '$runnerName' binary | 453 // For running the tests we use the given '$runnerName' binary |
455 targetRunnerPath = '$buildDir/$runnerName'; | 454 targetRunnerPath = '$buildDir/$runnerName'; |
456 | 455 |
457 // For listing the tests we use the '$runnerName.host' binary if it exists | 456 // For listing the tests we use the '$runnerName.host' binary if it exists |
458 // and use '$runnerName' if it doesn't. | 457 // and use '$runnerName' if it doesn't. |
459 var binarySuffix = Platform.operatingSystem == 'windows' ? '.exe' : ''; | 458 var binarySuffix = Platform.operatingSystem == 'windows' ? '.exe' : ''; |
460 var hostBinary = '$targetRunnerPath.host$binarySuffix'; | 459 var hostBinary = '$targetRunnerPath.host$binarySuffix'; |
461 if (new File(hostBinary).existsSync()) { | 460 if (new File(hostBinary).existsSync()) { |
462 hostRunnerPath = hostBinary; | 461 hostRunnerPath = hostBinary; |
463 } else { | 462 } else { |
464 hostRunnerPath = targetRunnerPath; | 463 hostRunnerPath = targetRunnerPath; |
465 } | 464 } |
466 } | 465 } |
467 | 466 |
468 void testNameHandler(ExpectationSet testExpectations, String testName) { | 467 void testNameHandler(TestExpectations testExpectations, String testName) { |
469 // Only run the tests that match the pattern. Use the name | 468 // Only run the tests that match the pattern. Use the name |
470 // "suiteName/testName" for cc tests. | 469 // "suiteName/testName" for cc tests. |
471 String constructedName = '$suiteName/$testPrefix$testName'; | 470 String constructedName = '$suiteName/$testPrefix$testName'; |
472 | 471 |
473 var expectations = testExpectations.expectations('$testPrefix$testName'); | 472 var expectations = testExpectations.expectations('$testPrefix$testName'); |
474 | 473 |
475 var args = TestUtils.standardOptions(configuration); | 474 var args = TestUtils.standardOptions(configuration); |
476 args.add(testName); | 475 args.add(testName); |
477 | 476 |
478 var command = CommandBuilder.instance.getProcessCommand( | 477 var command = CommandBuilder.instance.getProcessCommand( |
479 'run_vm_unittest', targetRunnerPath, args, environmentOverrides); | 478 'run_vm_unittest', targetRunnerPath, args, environmentOverrides); |
480 enqueueNewTestCase( | 479 enqueueNewTestCase( |
481 new TestCase(constructedName, [command], configuration, expectations)); | 480 new TestCase(constructedName, [command], configuration, expectations)); |
482 } | 481 } |
483 | 482 |
484 Future<Null> forEachTest(Function onTest, Map testCache, | 483 void forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) { |
485 [VoidFunction onDone]) async { | |
486 doTest = onTest; | 484 doTest = onTest; |
487 var statusFiles = | 485 var statusFiles = |
488 statusFilePaths.map((statusFile) => "$dartDir/$statusFile").toList(); | 486 statusFilePaths.map((statusFile) => "$dartDir/$statusFile").toList(); |
489 | 487 |
490 var expectations = ExpectationSet.read(statusFiles, configuration); | 488 ReadTestExpectations(statusFiles, configuration) |
491 | 489 .then((TestExpectations expectations) { |
492 try { | 490 ccTestLister(hostRunnerPath).then((Iterable<String> names) { |
493 var names = await ccTestLister(hostRunnerPath); | 491 names.forEach((testName) => testNameHandler(expectations, testName)); |
494 for (var name in names) { | 492 doTest = null; |
495 testNameHandler(expectations, name); | 493 if (onDone != null) onDone(); |
496 } | 494 }).catchError((error) { |
497 | 495 print("Fatal error occured: $error"); |
498 doTest = null; | 496 exit(1); |
499 if (onDone != null) onDone(); | 497 }); |
500 } catch (error) { | 498 }); |
501 print("Fatal error occured: $error"); | |
502 exit(1); | |
503 } | |
504 } | 499 } |
505 } | 500 } |
506 | 501 |
507 class TestInformation { | 502 class TestInformation { |
508 Path filePath; | 503 Path filePath; |
509 Path originTestPath; | 504 Path originTestPath; |
510 Map optionsFromFile; | 505 Map optionsFromFile; |
511 bool hasCompileError; | 506 bool hasCompileError; |
512 bool hasRuntimeError; | 507 bool hasRuntimeError; |
513 bool isNegativeIfChecked; | 508 bool isNegativeIfChecked; |
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
545 false) {} | 540 false) {} |
546 } | 541 } |
547 | 542 |
548 /** | 543 /** |
549 * A standard [TestSuite] implementation that searches for tests in a | 544 * A standard [TestSuite] implementation that searches for tests in a |
550 * directory, and creates [TestCase]s that compile and/or run them. | 545 * directory, and creates [TestCase]s that compile and/or run them. |
551 */ | 546 */ |
552 class StandardTestSuite extends TestSuite { | 547 class StandardTestSuite extends TestSuite { |
553 final Path suiteDir; | 548 final Path suiteDir; |
554 final List<String> statusFilePaths; | 549 final List<String> statusFilePaths; |
555 ExpectationSet testExpectations; | 550 TestExpectations testExpectations; |
556 List<TestInformation> cachedTests; | 551 List<TestInformation> cachedTests; |
557 final Path dartDir; | 552 final Path dartDir; |
558 Predicate<String> isTestFilePredicate; | 553 Predicate<String> isTestFilePredicate; |
559 final bool listRecursively; | 554 final bool listRecursively; |
560 final List<String> extraVmOptions; | 555 final List<String> extraVmOptions; |
561 List<Uri> _dart2JsBootstrapDependencies; | 556 List<Uri> _dart2JsBootstrapDependencies; |
562 | 557 |
563 StandardTestSuite(Map<String, dynamic> configuration, String suiteName, | 558 StandardTestSuite(Map configuration, String suiteName, Path suiteDirectory, |
564 Path suiteDirectory, this.statusFilePaths, | 559 this.statusFilePaths, |
565 {this.isTestFilePredicate, bool recursive: false}) | 560 {this.isTestFilePredicate, bool recursive: false}) |
566 : dartDir = TestUtils.dartDir, | 561 : dartDir = TestUtils.dartDir, |
567 listRecursively = recursive, | 562 listRecursively = recursive, |
568 suiteDir = TestUtils.dartDir.join(suiteDirectory), | 563 suiteDir = TestUtils.dartDir.join(suiteDirectory), |
569 extraVmOptions = TestUtils.getExtraVmOptions(configuration), | 564 extraVmOptions = TestUtils.getExtraVmOptions(configuration), |
570 super(configuration, suiteName) { | 565 super(configuration, suiteName) { |
571 if (!useSdk) { | 566 if (!useSdk) { |
572 _dart2JsBootstrapDependencies = []; | 567 _dart2JsBootstrapDependencies = []; |
573 } else { | 568 } else { |
574 var snapshotPath = TestUtils | 569 var snapshotPath = TestUtils |
(...skipping 61 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
636 return filename.endsWith("Test.dart"); | 631 return filename.endsWith("Test.dart"); |
637 } | 632 } |
638 | 633 |
639 bool isHtmlTestFile(String filename) => filename.endsWith('_htmltest.html'); | 634 bool isHtmlTestFile(String filename) => filename.endsWith('_htmltest.html'); |
640 | 635 |
641 List<String> additionalOptions(Path filePath) => []; | 636 List<String> additionalOptions(Path filePath) => []; |
642 | 637 |
643 forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) async { | 638 forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) async { |
644 await updateDartium(); | 639 await updateDartium(); |
645 doTest = onTest; | 640 doTest = onTest; |
646 testExpectations = readExpectations(); | 641 testExpectations = await readExpectations(); |
647 | 642 |
648 // Check if we have already found and generated the tests for this suite. | 643 // Check if we have already found and generated the tests for this suite. |
649 if (!testCache.containsKey(suiteName)) { | 644 if (!testCache.containsKey(suiteName)) { |
650 cachedTests = testCache[suiteName] = <TestInformation>[]; | 645 cachedTests = testCache[suiteName] = <TestInformation>[]; |
651 await enqueueTests(); | 646 await enqueueTests(); |
652 } else { | 647 } else { |
653 for (var info in testCache[suiteName]) { | 648 for (var info in testCache[suiteName]) { |
654 enqueueTestCaseFromTestInformation(info); | 649 enqueueTestCaseFromTestInformation(info); |
655 } | 650 } |
656 } | 651 } |
(...skipping 16 matching lines...) Expand all Loading... |
673 | 668 |
674 assert(updater.isActive); | 669 assert(updater.isActive); |
675 updater.onUpdated.add(() => completer.complete(null)); | 670 updater.onUpdated.add(() => completer.complete(null)); |
676 | 671 |
677 return completer.future; | 672 return completer.future; |
678 } | 673 } |
679 | 674 |
680 /** | 675 /** |
681 * Reads the status files and completes with the parsed expectations. | 676 * Reads the status files and completes with the parsed expectations. |
682 */ | 677 */ |
683 ExpectationSet readExpectations() { | 678 Future<TestExpectations> readExpectations() { |
684 var statusFiles = statusFilePaths.where((String statusFilePath) { | 679 var statusFiles = statusFilePaths.where((String statusFilePath) { |
685 var file = new File(dartDir.append(statusFilePath).toNativePath()); | 680 var file = new File(dartDir.append(statusFilePath).toNativePath()); |
686 return file.existsSync(); | 681 return file.existsSync(); |
687 }).map((statusFilePath) { | 682 }).map((statusFilePath) { |
688 return dartDir.append(statusFilePath).toNativePath(); | 683 return dartDir.append(statusFilePath).toNativePath(); |
689 }).toList(); | 684 }).toList(); |
690 | 685 |
691 return ExpectationSet.read(statusFiles, configuration); | 686 return ReadTestExpectations(statusFiles, configuration); |
692 } | 687 } |
693 | 688 |
694 Future enqueueTests() { | 689 Future enqueueTests() { |
695 Directory dir = new Directory(suiteDir.toNativePath()); | 690 Directory dir = new Directory(suiteDir.toNativePath()); |
696 return dir.exists().then((exists) { | 691 return dir.exists().then((exists) { |
697 if (!exists) { | 692 if (!exists) { |
698 print('Directory containing tests missing: ${suiteDir.toNativePath()}'); | 693 print('Directory containing tests missing: ${suiteDir.toNativePath()}'); |
699 return new Future.value(null); | 694 return new Future.value(null); |
700 } else { | 695 } else { |
701 var group = new FutureGroup(); | 696 var group = new FutureGroup(); |
(...skipping 1518 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
2220 } | 2215 } |
2221 if (path.length > WINDOWS_SHORTEN_PATH_LIMIT) { | 2216 if (path.length > WINDOWS_SHORTEN_PATH_LIMIT) { |
2222 ++shortNameCounter; | 2217 ++shortNameCounter; |
2223 var pathEnd = path.substring(path.length - WINDOWS_PATH_END_LENGTH); | 2218 var pathEnd = path.substring(path.length - WINDOWS_PATH_END_LENGTH); |
2224 path = "short${shortNameCounter}_$pathEnd"; | 2219 path = "short${shortNameCounter}_$pathEnd"; |
2225 } | 2220 } |
2226 } | 2221 } |
2227 return path; | 2222 return path; |
2228 } | 2223 } |
2229 } | 2224 } |
OLD | NEW |