| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for enumerating and preparing tests. | 6 * Classes and methods for enumerating and preparing tests. |
| 7 * | 7 * |
| 8 * This library includes: | 8 * This library includes: |
| 9 * | 9 * |
| 10 * - Creating tests by listing all the Dart files in certain directories, | 10 * - Creating tests by listing all the Dart files in certain directories, |
| 11 * and creating [TestCase]s for those files that meet the relevant criteria. | 11 * and creating [TestCase]s for those files that meet the relevant criteria. |
| 12 * - Preparing tests, including copying files and frameworks to temporary | 12 * - Preparing tests, including copying files and frameworks to temporary |
| 13 * directories, and computing the command line and arguments to be run. | 13 * directories, and computing the command line and arguments to be run. |
| 14 */ | 14 */ |
| 15 library test_suite; | 15 library test_suite; |
| 16 | 16 |
| 17 import "dart:io"; | 17 import "dart:io"; |
| 18 import "dart:isolate"; | 18 import "dart:isolate"; |
| 19 import "status_file_parser.dart"; | 19 import "status_file_parser.dart"; |
| 20 import "test_runner.dart"; | 20 import "test_runner.dart"; |
| 21 import "multitest.dart"; | 21 import "multitest.dart"; |
| 22 import "drt_updater.dart"; | 22 import "drt_updater.dart"; |
| 23 import "dart:uri"; | 23 import "dart:uri"; |
| 24 import '../../../pkg/path/lib/path.dart' as pathLib; | |
| 25 | 24 |
| 26 part "browser_test.dart"; | 25 part "browser_test.dart"; |
| 27 | 26 |
| 28 | 27 |
| 29 // TODO(rnystrom): Add to dart:core? | 28 // TODO(rnystrom): Add to dart:core? |
| 30 /** | 29 /** |
| 31 * A simple function that tests [arg] and returns `true` or `false`. | 30 * A simple function that tests [arg] and returns `true` or `false`. |
| 32 */ | 31 */ |
| 33 typedef bool Predicate<T>(T arg); | 32 typedef bool Predicate<T>(T arg); |
| 34 | 33 |
| (...skipping 360 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 395 */ | 394 */ |
| 396 class StandardTestSuite extends TestSuite { | 395 class StandardTestSuite extends TestSuite { |
| 397 final Path suiteDir; | 396 final Path suiteDir; |
| 398 final List<String> statusFilePaths; | 397 final List<String> statusFilePaths; |
| 399 TestCaseEvent doTest; | 398 TestCaseEvent doTest; |
| 400 TestExpectations testExpectations; | 399 TestExpectations testExpectations; |
| 401 List<TestInformation> cachedTests; | 400 List<TestInformation> cachedTests; |
| 402 final Path dartDir; | 401 final Path dartDir; |
| 403 Predicate<String> isTestFilePredicate; | 402 Predicate<String> isTestFilePredicate; |
| 404 final bool listRecursively; | 403 final bool listRecursively; |
| 405 /** | |
| 406 * The set of servers that have been started to run these tests (Could be | |
| 407 * none). | |
| 408 */ | |
| 409 List serverList; | |
| 410 | 404 |
| 411 StandardTestSuite(Map configuration, | 405 StandardTestSuite(Map configuration, |
| 412 String suiteName, | 406 String suiteName, |
| 413 Path suiteDirectory, | 407 Path suiteDirectory, |
| 414 this.statusFilePaths, | 408 this.statusFilePaths, |
| 415 this.serverList, | |
| 416 {this.isTestFilePredicate, | 409 {this.isTestFilePredicate, |
| 417 bool recursive: false}) | 410 bool recursive: false}) |
| 418 : super(configuration, suiteName), | 411 : super(configuration, suiteName), |
| 419 dartDir = TestUtils.dartDir(), | 412 dartDir = TestUtils.dartDir(), |
| 420 listRecursively = recursive, | 413 listRecursively = recursive, |
| 421 suiteDir = TestUtils.dartDir().join(suiteDirectory); | 414 suiteDir = TestUtils.dartDir().join(suiteDirectory); |
| 422 | 415 |
| 423 /** | 416 /** |
| 424 * Creates a test suite whose file organization matches an expected structure. | 417 * Creates a test suite whose file organization matches an expected structure. |
| 425 * To use this, your suite should look like: | 418 * To use this, your suite should look like: |
| (...skipping 13 matching lines...) Expand all Loading... |
| 439 * * The status file uses the same name. | 432 * * The status file uses the same name. |
| 440 * * Test files are directly in that directory and end in "_test.dart". | 433 * * Test files are directly in that directory and end in "_test.dart". |
| 441 * | 434 * |
| 442 * If you follow that convention, then you can construct one of these like: | 435 * If you follow that convention, then you can construct one of these like: |
| 443 * | 436 * |
| 444 * new StandardTestSuite.forDirectory(configuration, 'path/to/mytestsuite'); | 437 * new StandardTestSuite.forDirectory(configuration, 'path/to/mytestsuite'); |
| 445 * | 438 * |
| 446 * instead of having to create a custom [StandardTestSuite] subclass. In | 439 * instead of having to create a custom [StandardTestSuite] subclass. In |
| 447 * particular, if you add 'path/to/mytestsuite' to [TEST_SUITE_DIRECTORIES] | 440 * particular, if you add 'path/to/mytestsuite' to [TEST_SUITE_DIRECTORIES] |
| 448 * in test.dart, this will all be set up for you. | 441 * in test.dart, this will all be set up for you. |
| 449 * | |
| 450 * The [StandardTestSuite] also optionally takes a list of servers that have | |
| 451 * been started up by the test harness, to be used by browser tests. | |
| 452 */ | 442 */ |
| 453 factory StandardTestSuite.forDirectory( | 443 factory StandardTestSuite.forDirectory( |
| 454 Map configuration, Path directory, [List serverList = const []]) { | 444 Map configuration, Path directory) { |
| 455 final name = directory.filename; | 445 final name = directory.filename; |
| 456 | 446 |
| 457 return new StandardTestSuite(configuration, | 447 return new StandardTestSuite(configuration, |
| 458 name, directory, | 448 name, directory, |
| 459 ['$directory/$name.status', '$directory/${name}_dart2js.status'], | 449 ['$directory/$name.status', '$directory/${name}_dart2js.status'], |
| 460 serverList, | |
| 461 isTestFilePredicate: (filename) => filename.endsWith('_test.dart'), | 450 isTestFilePredicate: (filename) => filename.endsWith('_test.dart'), |
| 462 recursive: true); | 451 recursive: true); |
| 463 } | 452 } |
| 464 | 453 |
| 465 Collection<Uri> get dart2JsBootstrapDependencies { | 454 Collection<Uri> get dart2JsBootstrapDependencies { |
| 466 if (!useSdk) return []; | 455 if (!useSdk) return []; |
| 467 | 456 |
| 468 var snapshotPath = TestUtils.absolutePath(new Path(buildDir).join( | 457 var snapshotPath = TestUtils.absolutePath(new Path(buildDir).join( |
| 469 new Path('dart-sdk/lib/_internal/compiler/' | 458 new Path('dart-sdk/lib/_internal/compiler/' |
| 470 'implementation/dart2js.dart.snapshot'))).toString(); | 459 'implementation/dart2js.dart.snapshot'))).toString(); |
| (...skipping 386 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 857 dartLibraryFilename = new Path('test_as_library.dart'); | 846 dartLibraryFilename = new Path('test_as_library.dart'); |
| 858 File file = new File('$tempDir/$dartLibraryFilename'); | 847 File file = new File('$tempDir/$dartLibraryFilename'); |
| 859 RandomAccessFile dartLibrary = file.openSync(FileMode.WRITE); | 848 RandomAccessFile dartLibrary = file.openSync(FileMode.WRITE); |
| 860 dartLibrary.writeStringSync(wrapDartTestInLibrary(filePath)); | 849 dartLibrary.writeStringSync(wrapDartTestInLibrary(filePath)); |
| 861 dartLibrary.closeSync(); | 850 dartLibrary.closeSync(); |
| 862 } | 851 } |
| 863 | 852 |
| 864 File file = new File(dartWrapperFilename); | 853 File file = new File(dartWrapperFilename); |
| 865 RandomAccessFile dartWrapper = file.openSync(FileMode.WRITE); | 854 RandomAccessFile dartWrapper = file.openSync(FileMode.WRITE); |
| 866 dartWrapper.writeStringSync( | 855 dartWrapper.writeStringSync( |
| 867 dartTestWrapper(dartDir, file.name, dartLibraryFilename)); | 856 dartTestWrapper(dartDir, dartLibraryFilename)); |
| 868 dartWrapper.closeSync(); | 857 dartWrapper.closeSync(); |
| 869 } else { | 858 } else { |
| 870 dartWrapperFilename = filename; | 859 dartWrapperFilename = filename; |
| 871 // TODO(whesse): Once test.py is retired, adjust the relative path in | 860 // TODO(whesse): Once test.py is retired, adjust the relative path in |
| 872 // the client/samples/dartcombat test to its css file, remove the | 861 // the client/samples/dartcombat test to its css file, remove the |
| 873 // "../../" from this path, and move this out of the isWebTest guard. | 862 // "../../" from this path, and move this out of the isWebTest guard. |
| 874 // Also remove getHtmlName, and just use test.html. | 863 // Also remove getHtmlName, and just use test.html. |
| 875 // TODO(efortuna): this shortening of htmlFilename is a band-aid until | 864 // TODO(efortuna): this shortening of htmlFilename is a band-aid until |
| 876 // the above TODO gets fixed. Windows cannot have paths that are longer | 865 // the above TODO gets fixed. Windows cannot have paths that are longer |
| 877 // than 260 characters, and without this hack, we were running past the | 866 // than 260 characters, and without this hack, we were running past the |
| 878 // the limit. | 867 // the limit. |
| 879 String htmlFilename = getHtmlName(filename); | 868 String htmlFilename = getHtmlName(filename); |
| 880 while ('$tempDir/../$htmlFilename'.length >= 260) { | 869 while ('$tempDir/../$htmlFilename'.length >= 260) { |
| 881 htmlFilename = htmlFilename.substring(htmlFilename.length~/2); | 870 htmlFilename = htmlFilename.substring(htmlFilename.length~/2); |
| 882 } | 871 } |
| 883 htmlPath = '$tempDir/../$htmlFilename'; | 872 htmlPath = '$tempDir/../$htmlFilename'; |
| 884 } | 873 } |
| 885 final String scriptPath = (compiler == 'none') ? | 874 final String scriptPath = (compiler == 'none') ? |
| 886 dartWrapperFilename : compiledDartWrapperFilename; | 875 dartWrapperFilename : compiledDartWrapperFilename; |
| 887 // Create the HTML file for the test. | 876 // Create the HTML file for the test. |
| 888 RandomAccessFile htmlTest = new File(htmlPath).openSync(FileMode.WRITE); | 877 RandomAccessFile htmlTest = new File(htmlPath).openSync(FileMode.WRITE); |
| 878 String filePrefix = ''; |
| 879 if (Platform.operatingSystem == 'windows') { |
| 880 // Firefox on Windows does not like absolute file path names that start |
| 881 // with 'C:' adding 'file:///' solves the problem. |
| 882 filePrefix = 'file:///'; |
| 883 } |
| 889 String content = null; | 884 String content = null; |
| 890 Path dir = filePath.directoryPath; | 885 Path dir = filePath.directoryPath; |
| 891 String nameNoExt = filePath.filenameWithoutExtension; | 886 String nameNoExt = filePath.filenameWithoutExtension; |
| 892 Path pngPath = dir.append('$nameNoExt.png'); | 887 Path pngPath = dir.append('$nameNoExt.png'); |
| 893 Path txtPath = dir.append('$nameNoExt.txt'); | 888 Path txtPath = dir.append('$nameNoExt.txt'); |
| 894 Path expectedOutput = null; | 889 Path expectedOutput = null; |
| 895 if (new File.fromPath(pngPath).existsSync()) { | 890 if (new File.fromPath(pngPath).existsSync()) { |
| 896 expectedOutput = pngPath; | 891 expectedOutput = pngPath; |
| 897 content = getHtmlLayoutContents(scriptType, pathLib.relative(scriptPath, | 892 content = getHtmlLayoutContents(scriptType, '$filePrefix$scriptPath'); |
| 898 from: pathLib.dirname(htmlPath))); | |
| 899 } else if (new File.fromPath(txtPath).existsSync()) { | 893 } else if (new File.fromPath(txtPath).existsSync()) { |
| 900 expectedOutput = txtPath; | 894 expectedOutput = txtPath; |
| 901 content = getHtmlLayoutContents(scriptType, pathLib.relative(scriptPath, | 895 content = getHtmlLayoutContents(scriptType, '$filePrefix$scriptPath'); |
| 902 from: pathLib.dirname(htmlPath))); | |
| 903 } else { | 896 } else { |
| 904 final htmlLocation = new Path.fromNative(htmlPath); | 897 final htmlLocation = new Path.fromNative(htmlPath); |
| 905 content = getHtmlContents( | 898 content = getHtmlContents( |
| 906 filename, | 899 filename, |
| 907 dartDir.append('pkg/unittest/test_controller.js') | 900 dartDir.append('pkg/unittest/test_controller.js') |
| 908 .relativeTo(htmlLocation), | 901 .relativeTo(htmlLocation), |
| 909 dartDir.append('client/dart.js').relativeTo(htmlLocation), | 902 dartDir.append('client/dart.js').relativeTo(htmlLocation), |
| 910 scriptType, | 903 scriptType, |
| 911 new Path.fromNative(scriptPath).relativeTo(htmlLocation)); | 904 new Path.fromNative(scriptPath).relativeTo(htmlLocation)); |
| 912 } | 905 } |
| (...skipping 29 matching lines...) Expand all Loading... |
| 942 do { | 935 do { |
| 943 List<Command> commandSet = new List<Command>.from(commands); | 936 List<Command> commandSet = new List<Command>.from(commands); |
| 944 if (subtestIndex != 0) { | 937 if (subtestIndex != 0) { |
| 945 // NOTE: The first time we enter this loop, all the compilation | 938 // NOTE: The first time we enter this loop, all the compilation |
| 946 // commands will be executed. On subsequent loop iterations, we | 939 // commands will be executed. On subsequent loop iterations, we |
| 947 // don't need to do any compilations. Thus we set "commandSet = []". | 940 // don't need to do any compilations. Thus we set "commandSet = []". |
| 948 commandSet = []; | 941 commandSet = []; |
| 949 } | 942 } |
| 950 | 943 |
| 951 List<String> args = <String>[]; | 944 List<String> args = <String>[]; |
| 952 var basePath = TestUtils.dartDir().toString(); | 945 String fullHtmlPath = htmlPath.startsWith('http:') ? htmlPath : |
| 953 htmlPath = htmlPath.startsWith(basePath) ? | 946 (htmlPath.startsWith('/') ? |
| 954 htmlPath.substring(basePath.length) : htmlPath; | 947 'file://$htmlPath' : |
| 955 String fullHtmlPath = htmlPath; | 948 'file:///$htmlPath'); |
| 956 if (!htmlPath.startsWith('http')) { | |
| 957 if (!htmlPath.startsWith('/')) htmlPath = '/$htmlPath'; | |
| 958 fullHtmlPath = 'http://127.0.0.1:${serverList[0].port}$htmlPath?' | |
| 959 'crossOriginPort=${serverList[1].port}'; | |
| 960 } | |
| 961 if (info.optionsFromFile['isMultiHtmlTest'] | 949 if (info.optionsFromFile['isMultiHtmlTest'] |
| 962 && subtestNames.length > 0) { | 950 && subtestNames.length > 0) { |
| 963 fullHtmlPath = '${fullHtmlPath}#${subtestNames[subtestIndex]}'; | 951 fullHtmlPath = '${fullHtmlPath}#${subtestNames[subtestIndex]}'; |
| 964 } | 952 } |
| 965 | 953 |
| 966 if (TestUtils.usesWebDriver(runtime)) { | 954 if (TestUtils.usesWebDriver(runtime)) { |
| 967 args = [ | 955 args = [ |
| 968 dartDir.append('tools/testing/run_selenium.py').toNativePath(), | 956 dartDir.append('tools/testing/run_selenium.py').toNativePath(), |
| 969 '--browser=$runtime', | 957 '--browser=$runtime', |
| 970 '--timeout=${configuration["timeout"] - 2}', | 958 '--timeout=${configuration["timeout"] - 2}', |
| (...skipping 21 matching lines...) Expand all Loading... |
| 992 dartFlags.add("--enable_type_checks"); | 980 dartFlags.add("--enable_type_checks"); |
| 993 } | 981 } |
| 994 dartFlags.addAll(vmOptions); | 982 dartFlags.addAll(vmOptions); |
| 995 } | 983 } |
| 996 if (compiler == 'none') { | 984 if (compiler == 'none') { |
| 997 var packageRootPath = packageRoot(optionsFromFile['packageRoot']); | 985 var packageRootPath = packageRoot(optionsFromFile['packageRoot']); |
| 998 if (packageRootPath != null) { | 986 if (packageRootPath != null) { |
| 999 var absolutePath = | 987 var absolutePath = |
| 1000 TestUtils.absolutePath(new Path(packageRootPath)); | 988 TestUtils.absolutePath(new Path(packageRootPath)); |
| 1001 packageRootUri = new Uri.fromComponents( | 989 packageRootUri = new Uri.fromComponents( |
| 1002 scheme: 'http', | 990 scheme: 'file', |
| 1003 path: '127.0.0.1:${serverList[0].port}/' | 991 path: absolutePath.toString()); |
| 1004 '${pathLib.relative(absolutePath.toString(), from: | |
| 1005 TestUtils.dartDir().toString())}'); | |
| 1006 } | 992 } |
| 1007 } | 993 } |
| 1008 | 994 |
| 1009 if (expectedOutput != null) { | 995 if (expectedOutput != null) { |
| 1010 if (expectedOutput.toNativePath().endsWith('.png')) { | 996 if (expectedOutput.toNativePath().endsWith('.png')) { |
| 1011 // pixel tests are specified by running DRT "foo.html'-p" | 997 // pixel tests are specified by running DRT "foo.html'-p" |
| 1012 dumpRenderTreeOptions.add('--notree'); | 998 dumpRenderTreeOptions.add('--notree'); |
| 1013 fullHtmlPath = "${fullHtmlPath}'-p"; | 999 fullHtmlPath = "${fullHtmlPath}'-p"; |
| 1014 } | 1000 } |
| 1015 } | 1001 } |
| (...skipping 88 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1104 } | 1090 } |
| 1105 | 1091 |
| 1106 // Create '[build dir]/generated_tests/$compiler-$runtime/$testUniqueName', | 1092 // Create '[build dir]/generated_tests/$compiler-$runtime/$testUniqueName', |
| 1107 // including any intermediate directories that don't exist. | 1093 // including any intermediate directories that don't exist. |
| 1108 // If the tests are run in checked or minified mode we add that to the | 1094 // If the tests are run in checked or minified mode we add that to the |
| 1109 // '$compile-$runtime' directory name. | 1095 // '$compile-$runtime' directory name. |
| 1110 var checked = configuration['checked'] ? '-checked' : ''; | 1096 var checked = configuration['checked'] ? '-checked' : ''; |
| 1111 var minified = configuration['minified'] ? '-minified' : ''; | 1097 var minified = configuration['minified'] ? '-minified' : ''; |
| 1112 var dirName = "${configuration['compiler']}-${configuration['runtime']}" | 1098 var dirName = "${configuration['compiler']}-${configuration['runtime']}" |
| 1113 "$checked$minified"; | 1099 "$checked$minified"; |
| 1114 Path generatedTestPath = new Path.fromNative(dartDir.toString()) | 1100 Path generatedTestPath = new Path.fromNative(buildDir) |
| 1115 .append(buildDir.toString()) | |
| 1116 .append('generated_tests') | 1101 .append('generated_tests') |
| 1117 .append(dirName) | 1102 .append(dirName) |
| 1118 .append(testUniqueName); | 1103 .append(testUniqueName); |
| 1119 | 1104 |
| 1120 TestUtils.mkdirRecursive(new Path('.'), generatedTestPath); | 1105 TestUtils.mkdirRecursive(new Path('.'), generatedTestPath); |
| 1121 return new File.fromPath(generatedTestPath).fullPathSync() | 1106 return new File.fromPath(generatedTestPath).fullPathSync() |
| 1122 .replaceAll('\\', '/'); | 1107 .replaceAll('\\', '/'); |
| 1123 } | 1108 } |
| 1124 | 1109 |
| 1125 String get scriptType { | 1110 String get scriptType { |
| (...skipping 320 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1446 List<String> _testDirs; | 1431 List<String> _testDirs; |
| 1447 | 1432 |
| 1448 DartcCompilationTestSuite(Map configuration, | 1433 DartcCompilationTestSuite(Map configuration, |
| 1449 String suiteName, | 1434 String suiteName, |
| 1450 String directoryPath, | 1435 String directoryPath, |
| 1451 List<String> this._testDirs, | 1436 List<String> this._testDirs, |
| 1452 List<String> expectations) | 1437 List<String> expectations) |
| 1453 : super(configuration, | 1438 : super(configuration, |
| 1454 suiteName, | 1439 suiteName, |
| 1455 new Path.fromNative(directoryPath), | 1440 new Path.fromNative(directoryPath), |
| 1456 expectations, | 1441 expectations); |
| 1457 []); | |
| 1458 | 1442 |
| 1459 List<String> additionalOptions(Path filePath) { | 1443 List<String> additionalOptions(Path filePath) { |
| 1460 return ['--fatal-warnings', '--fatal-type-errors']; | 1444 return ['--fatal-warnings', '--fatal-type-errors']; |
| 1461 } | 1445 } |
| 1462 | 1446 |
| 1463 Future enqueueTests() { | 1447 Future enqueueTests() { |
| 1464 var group = new FutureGroup(); | 1448 var group = new FutureGroup(); |
| 1465 | 1449 |
| 1466 for (String testDir in _testDirs) { | 1450 for (String testDir in _testDirs) { |
| 1467 Directory dir = new Directory.fromPath(suiteDir.append(testDir)); | 1451 Directory dir = new Directory.fromPath(suiteDir.append(testDir)); |
| (...skipping 354 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1822 * $pass tests are expected to pass | 1806 * $pass tests are expected to pass |
| 1823 * $failOk tests are expected to fail that we won't fix | 1807 * $failOk tests are expected to fail that we won't fix |
| 1824 * $fail tests are expected to fail that we should fix | 1808 * $fail tests are expected to fail that we should fix |
| 1825 * $crash tests are expected to crash that we should fix | 1809 * $crash tests are expected to crash that we should fix |
| 1826 * $timeout tests are allowed to timeout | 1810 * $timeout tests are allowed to timeout |
| 1827 * $compileErrorSkip tests are skipped on browsers due to compile-time error | 1811 * $compileErrorSkip tests are skipped on browsers due to compile-time error |
| 1828 """; | 1812 """; |
| 1829 print(report); | 1813 print(report); |
| 1830 } | 1814 } |
| 1831 } | 1815 } |
| OLD | NEW |