Working SimpleHTML test front end

tools/testing/dart/test_suite.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for enumerating and preparing tests.
  *
  * This library includes:
  *
  * - Creating tests by listing all the Dart files in certain directories,
  *   and creating [TestCase]s for those files that meet the relevant criteria.
  * - Preparing tests, including copying files and frameworks to temporary
  *   directories, and computing the command line and arguments to be run.
  */
 library test_suite;
 
 import "dart:async";
 import "dart:io";
 import "drt_updater.dart";
+import "html_test.dart" as htmlTest;
 import "multitest.dart";
 import "status_file_parser.dart";
 import "test_runner.dart";
 import "utils.dart";
 import "http_server.dart" show PREFIX_BUILDDIR, PREFIX_DARTDIR;
 
 import "compiler_configuration.dart" show
     CommandArtifact,
     CompilerConfiguration;
 
 import "runtime_configuration.dart" show
     RuntimeConfiguration;
 
 part "browser_test.dart";
 
 
 RegExp multiHtmlTestGroupRegExp = new RegExp(r"\s*[^/]\s*group\('[^,']*");
 RegExp multiHtmlTestRegExp = new RegExp(r"useHtmlIndividualConfiguration()");
 // Require at least one non-space character before '///'
 RegExp multiTestRegExp = new RegExp(r"\S *"
                                     r"/// \w+:(.*)");
 
 /**
  * A simple function that tests [arg] and returns `true` or `false`.
  */
 typedef bool Predicate<T>(T arg);
 
 typedef void CreateTest(Path filePath,
+                        Path originTestPath,
                         bool hasCompileError,
                         bool hasRuntimeError,
                         {bool isNegativeIfChecked,
                          bool hasCompileErrorIfChecked,
                          bool hasStaticWarning,
-                         String multitestKey,
-                         Path originTestPath});
+                         String multitestKey});
 
 typedef void VoidFunction();
 
 /**
  * Calls [function] asynchronously. Returns a future that completes with the
  * result of the function. If the function is `null`, returns a future that
  * completes immediately with `null`.
  */
 Future asynchronously(function()) {
   if (function == null) return new Future.value(null);
@@ skipping 491 matching lines @@
       }).catchError((error) {
         print("Fatal error occured: $error");
         exit(1);
       });
     });
   }
 }
 
 
 class TestInformation {
+  Path filePath;
   Path originTestPath;
-  Path filePath;
   Map optionsFromFile;
   bool hasCompileError;
   bool hasRuntimeError;
   bool isNegativeIfChecked;
   bool hasCompileErrorIfChecked;
   bool hasStaticWarning;
   String multitestKey;
 
-  TestInformation(this.filePath, this.optionsFromFile,
+  TestInformation(this.filePath, this.originTestPath, this.optionsFromFile,
                   this.hasCompileError, this.hasRuntimeError,
                   this.isNegativeIfChecked, this.hasCompileErrorIfChecked,
                   this.hasStaticWarning,
-                  {this.multitestKey, this.originTestPath}) {
+                  {this.multitestKey: ''}) {
     assert(filePath.isAbsolute);
-    if (originTestPath == null) originTestPath = filePath;
   }
 }
 
+
+class HtmlTestInformation extends TestInformation {
+  List<String> expectedMessages;
+  List<String> scripts;
+
+  HtmlTestInformation(Path filePath, this.expectedMessages, this.scripts)
+      : super(filePath, filePath,
+              {'isMultitest': false, 'isMultiHtmlTest': false},
+              false, false, false, false, false) {}
+}
+

[ricow1, 2014/11/04 09:18:03]

two newlines

[Bill Hesse, 2014/11/04 16:29:01]

Put two newlines between all classes in the file.

 /**
  * A standard [TestSuite] implementation that searches for tests in a
  * directory, and creates [TestCase]s that compile and/or run them.
  */
 class StandardTestSuite extends TestSuite {
   final Path suiteDir;
   final List<String> statusFilePaths;
   TestExpectations testExpectations;
   List<TestInformation> cachedTests;
   final Path dartDir;
@@ skipping 68 matching lines @@
 
   List<Uri> get dart2JsBootstrapDependencies => _dart2JsBootstrapDependencies;
 
   /**
    * The default implementation assumes a file is a test if
    * it ends in "Test.dart".
    */
   bool isTestFile(String filename) {
     // Use the specified predicate, if provided.
     if (isTestFilePredicate != null) return isTestFilePredicate(filename);
+    // TODO(whesse): Remove check, and make the default "_test.dart", if
+    // this line is never hit.
+    throw new Exception('This suite uses obsolete "Test.dart" filenames');

[ricow1, 2014/11/04 09:18:03]

do this in a separate cl please

[Bill Hesse, 2014/11/04 16:29:01]

Done.

+  }
 
-    return filename.endsWith("Test.dart");
-  }
+  bool isHtmlTestFile(String filename) => filename.endsWith('_htmltest.html');
 
   List<String> additionalOptions(Path filePath) => [];
 
   Map<String, String> localPackageDirectories;
 
   void forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) {
     discoverPackagesInRepository().then((Map packageDirectories) {
       localPackageDirectories = packageDirectories;
       return updateDartium();
     }).then((_) {
@@ skipping 74 matching lines @@
     group.add(listCompleter.future);
 
     var lister = dir.list(recursive: listRecursively)
         .listen((FileSystemEntity fse) {
           if (fse is File) enqueueFile(fse.path, group);
         },
         onDone: listCompleter.complete);
   }
 
   void enqueueFile(String filename, FutureGroup group) {
+    if (isHtmlTestFile(filename)) {
+      var info = htmlTest.getInformation(filename);
+      if (info == null) {
+        print("Invalid HtmlTest $filename");

[ricow1, 2014/11/04 09:18:03]

so I don't think this will catch peoples attention, shouldn't the test fail if
this is the case? (we want the actual test to fail, not the testing script to
exit)

[Bill Hesse, 2014/11/04 16:29:01]

This is rather hard to do.  At this point, we would need to create a dummy
TestInformation object, perhaps of a new subclass, that would indicate to the
BrowserTestCreator (and maybe non-browser test creators), that it should create
a TestCase object that would always fail, with a certain message.  This would
then require Command objects that would do nothing, perhaps.

Maybe a better solution is to create an HtmlTestInfo with no scripts, and a
dummy message that would not be matched, so that the test would definitely time
out when it was run, and print this dummy message.

For now, logging as an error.

+        return;
+      }
+      cachedTests.add(info);
+      enqueueTestCaseFromTestInformation(info);
+      return;
+    }
     if (!isTestFile(filename)) return;
     Path filePath = new Path(filename);
 
     // Only run the tests that match the pattern.
     if (filePath.filename.endsWith('test_config.dart')) return;
 
     var optionsFromFile = readOptionsFromFile(filePath);
     CreateTest createTestCase = makeTestCaseCreator(optionsFromFile);
 
     if (optionsFromFile['isMultitest']) {
       group.add(doMultitest(filePath, buildDir, suiteDir, createTestCase));
     } else {
       createTestCase(filePath,
+                     filePath,
                      optionsFromFile['hasCompileError'],
                      optionsFromFile['hasRuntimeError'],
                      hasStaticWarning: optionsFromFile['hasStaticWarning']);
     }
   }
 
   static Path _findPubspecYamlFile(Path filePath) {
     final existsCache = TestUtils.existsCache;
 
     Path root = TestUtils.dartDir;
     assert ("$filePath".startsWith("$root"));
 
     // We start with the parent directory of [filePath] and go up until
     // the root directory (excluding the root).
     List<String> segments =
         filePath.directoryPath.relativeTo(root).segments();
     while (segments.length > 0) {
       var pubspecYamlPath =
           new Path(segments.join('/')).append('pubspec.yaml');
       if (existsCache.doesFileExist(pubspecYamlPath.toNativePath())) {
         return root.join(pubspecYamlPath);
       }
       segments.removeLast();
     }
     return null;
   }
 
   void enqueueTestCaseFromTestInformation(TestInformation info) {
+    String testName = buildTestCaseDisplayName(suiteDir, info.originTestPath,
+        multitestName: info.multitestKey);
+    Set<Expectation> expectations = testExpectations.expectations(testName);
+    if (info is HtmlTestInformation &&
+        TestUtils.isBrowserRuntime(configuration['runtime'])) {
+      enqueueBrowserTest([], null, info, testName, expectations);
+      return;
+    }
     var filePath = info.filePath;
     var optionsFromFile = info.optionsFromFile;
 
     Map buildSpecialPackageRoot(Path pubspecYamlFile) {
       var commands = <Command>[];
       var packageDir = pubspecYamlFile.directoryPath;
       var packageName = packageDir.filename;
 
       var checkoutDirectory =
           createPubspecCheckoutDirectory(packageDir);
@@ skipping 47 matching lines @@
         'package-root' : newPackageRoot,
       };
     }
 
     // If this test is inside a package, we will check if there is a
     // pubspec.yaml file and if so, create a custom package root for it.
     List<Command> baseCommands = <Command>[];
     Path packageRoot;
     if (configuration['use_repository_packages'] ||
         configuration['use_public_packages']) {
-        Path pubspecYamlFile = _findPubspecYamlFile(filePath);
-        if (pubspecYamlFile != null) {
-          var result = buildSpecialPackageRoot(pubspecYamlFile);
-          baseCommands.addAll(result['commands']);
-          packageRoot = result['package-root'];
-          if (optionsFromFile['packageRoot'] == null ||
-              optionsFromFile['packageRoot'] == "") {
-            optionsFromFile['packageRoot'] = packageRoot.toNativePath();
-          }
+      Path pubspecYamlFile = _findPubspecYamlFile(filePath);
+      if (pubspecYamlFile != null) {
+        var result = buildSpecialPackageRoot(pubspecYamlFile);
+        baseCommands.addAll(result['commands']);
+        packageRoot = result['package-root'];
+        if (optionsFromFile['packageRoot'] == null ||
+            optionsFromFile['packageRoot'] == "") {
+          optionsFromFile['packageRoot'] = packageRoot.toNativePath();
+        }
       }
     }
     if (configuration['package_root'] != null) {
       packageRoot = new Path(configuration['package_root']);
       optionsFromFile['packageRoot'] = packageRoot.toNativePath();
     }
 
-    String testName = buildTestCaseDisplayName(suiteDir, info.originTestPath,
-        multitestName: optionsFromFile['isMultitest'] ? info.multitestKey : "");
-
-    Set<Expectation> expectations = testExpectations.expectations(testName);
     if (new CompilerConfiguration(configuration).hasCompiler &&
         expectCompileError(info)) {
       // If a compile-time error is expected, and we're testing a
       // compiler, we never need to attempt to run the program (in a
       // browser or otherwise).
       enqueueStandardTest(baseCommands, info, testName, expectations);
     } else if (TestUtils.isBrowserRuntime(configuration['runtime'])) {
       if (info.optionsFromFile['isMultiHtmlTest']) {
         // A browser multi-test has multiple expectations for one test file.
         // Find all the different sub-test expecations for one entire test file.
@@ skipping 104 matching lines @@
             runtimeConfiguration.computeRuntimeCommands(
                 this,
                 CommandBuilder.instance,
                 compilationArtifact,
                 runtimeArguments,
                 environmentOverrides));
   }
 
   CreateTest makeTestCaseCreator(Map optionsFromFile) {
     return (Path filePath,
+            Path originTestPath,
             bool hasCompileError,
             bool hasRuntimeError,
             {bool isNegativeIfChecked: false,
              bool hasCompileErrorIfChecked: false,
              bool hasStaticWarning: false,
-             String multitestKey,
-             Path originTestPath}) {
+             String multitestKey}) {
       // Cache the test information for each test case.
       var info = new TestInformation(filePath,
+                                     originTestPath,
                                      optionsFromFile,
                                      hasCompileError,
                                      hasRuntimeError,
                                      isNegativeIfChecked,
                                      hasCompileErrorIfChecked,
                                      hasStaticWarning,
-                                     multitestKey: multitestKey,
-                                     originTestPath: originTestPath);
+                                     multitestKey: multitestKey);
       cachedTests.add(info);
       enqueueTestCaseFromTestInformation(info);
     };
   }
 
   /**
    * _createUrlPathFromFile takes a [file], which is either located in the dart
    * or in the build directory, and will return a String representing
    * the relative path to either the dart or the build directory.
    * Thus, the returned [String] will be the path component of the URL
@@ skipping 77 matching lines @@
     // on to the browser (it may be test specific).
 
     // TODO(kustermann/ricow): This method should be refactored.
     Map optionsFromFile = info.optionsFromFile;
     Path filePath = info.filePath;
     String filename = filePath.toString();
 
     final String compiler = configuration['compiler'];
     final String runtime = configuration['runtime'];
 
+    if (info is HtmlTestInformation) {
+      if (compiler != 'none' || runtime != 'dartium') {
+        // TODO(whesse): Enable compilation of scripts to dart2js, and
+        // rewriting of script links in html file.  Currently unimplemented.
+        return;
+      }
+      if (info.scripts.length > 0) {
+        // TODO(whesse): Copy scripts into output directory.
+        return;
+      }
+      final String tempDir = createOutputDirectory(info.filePath, '');
+      final String htmlFile = '$tempDir/${filePath.filename}';
+      new File(htmlFile).writeAsStringSync(htmlTest.getContents(info));
+
+      String testDisplayName = '$suiteName/$testName';
+      var htmlPath = _createUrlPathFromFile(new Path(htmlFile));
+      var fullHtmlPath = _getUriForBrowserTest(info, htmlPath,
+                                               null, null);
+      var commands = [CommandBuilder.instance.getBrowserHtmlTestCommand(
+          runtime, fullHtmlPath, configuration, info.expectedMessages)];
+      var testCase = new BrowserTestCase(testDisplayName,
+                    commands, configuration, expectations,
+                    info, isNegative(info), fullHtmlPath);
+      enqueueNewTestCase(testCase);
+      return;
+    }
+
     for (var vmOptions in getVmOptions(optionsFromFile)) {
       // Create a unique temporary directory for each set of vmOptions.
       // TODO(dart:429): Replace separate replaceAlls with a RegExp when
       // replaceAll(RegExp, String) is implemented.
       String optionsName = '';
       if (getVmOptions(optionsFromFile).length > 1) {
           optionsName = vmOptions.join('-').replaceAll('-','')
                                            .replaceAll('=','')
                                            .replaceAll('/','');
       }
@@ skipping 1190 matching lines @@
  * $pass tests are expected to pass
  * $failOk tests are expected to fail that we won't fix
  * $fail tests are expected to fail that we should fix
  * $crash tests are expected to crash that we should fix
  * $timeout tests are allowed to timeout
  * $compileErrorSkip tests are skipped on browsers due to compile-time error
 """;
     print(report);
   }
 }