Working SimpleHTML test front end

tools/testing/dart/test_suite.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for enumerating and preparing tests.
  *
  * This library includes:
  *
  * - Creating tests by listing all the Dart files in certain directories,
  *   and creating [TestCase]s for those files that meet the relevant criteria.
  * - Preparing tests, including copying files and frameworks to temporary
  *   directories, and computing the command line and arguments to be run.
  */
 library test_suite;
 
 import "dart:async";
 import "dart:io";
 import "drt_updater.dart";
+import "html_test.dart" as htmlTest;
 import "multitest.dart";
 import "status_file_parser.dart";
 import "test_runner.dart";
 import "utils.dart";
 import "http_server.dart" show PREFIX_BUILDDIR, PREFIX_DARTDIR;
 
 import "compiler_configuration.dart" show
     CommandArtifact,
     CompilerConfiguration;
 
 import "runtime_configuration.dart" show
     RuntimeConfiguration;
 
 part "browser_test.dart";
 
 
 RegExp multiHtmlTestGroupRegExp = new RegExp(r"\s*[^/]\s*group\('[^,']*");
 RegExp multiHtmlTestRegExp = new RegExp(r"useHtmlIndividualConfiguration()");
 // Require at least one non-space character before '///'
 RegExp multiTestRegExp = new RegExp(r"\S *"
                                     r"/// \w+:(.*)");
 
 /**
  * A simple function that tests [arg] and returns `true` or `false`.
  */
 typedef bool Predicate<T>(T arg);
 
 typedef void CreateTest(Path filePath,
+                        Path originTestPath,
                         bool hasCompileError,
                         bool hasRuntimeError,
                         {bool isNegativeIfChecked,
                          bool hasCompileErrorIfChecked,
                          bool hasStaticWarning,
-                         String multitestKey,
-                         Path originTestPath});
+                         String multitestKey});
 
 typedef void VoidFunction();
 
 /**
  * Calls [function] asynchronously. Returns a future that completes with the
  * result of the function. If the function is `null`, returns a future that
  * completes immediately with `null`.
  */
 Future asynchronously(function()) {
   if (function == null) return new Future.value(null);
 
   var completer = new Completer();
   Timer.run(() => completer.complete(function()));
 
   return completer.future;
 }
 
+
 /** A completer that waits until all added [Future]s complete. */
 // TODO(rnystrom): Copied from web_components. Remove from here when it gets
 // added to dart:core. (See #6626.)
 class FutureGroup {
   static const _FINISHED = -1;
   int _pending = 0;
   Completer<List> _completer = new Completer<List>();
   final List<Future> futures = <Future>[];
   bool wasCompleted = false;
 
@@ skipping 21 matching lines @@
           wasCompleted = true;
         }
       }
     });
     futures.add(handledTaskFuture);
   }
 
   Future<List> get future => _completer.future;
 }
 
+
 /**
  * A TestSuite represents a collection of tests.  It creates a [TestCase]
  * object for each test to be run, and passes the test cases to a callback.
  *
  * Most TestSuites represent a directory or directory tree containing tests,
  * and a status file containing the expected results when these tests are run.
  */
 abstract class TestSuite {
   final Map configuration;
   final String suiteName;
@@ skipping 433 matching lines @@
       }).catchError((error) {
         print("Fatal error occured: $error");
         exit(1);
       });
     });
   }
 }
 
 
 class TestInformation {
+  Path filePath;
   Path originTestPath;
-  Path filePath;
   Map optionsFromFile;
   bool hasCompileError;
   bool hasRuntimeError;
   bool isNegativeIfChecked;
   bool hasCompileErrorIfChecked;
   bool hasStaticWarning;
   String multitestKey;
 
-  TestInformation(this.filePath, this.optionsFromFile,
+  TestInformation(this.filePath, this.originTestPath, this.optionsFromFile,
                   this.hasCompileError, this.hasRuntimeError,
                   this.isNegativeIfChecked, this.hasCompileErrorIfChecked,
                   this.hasStaticWarning,
-                  {this.multitestKey, this.originTestPath}) {
+                  {this.multitestKey: ''}) {
     assert(filePath.isAbsolute);
-    if (originTestPath == null) originTestPath = filePath;
   }
 }
 
+
+class HtmlTestInformation extends TestInformation {
+  List<String> expectedMessages;
+  List<String> scripts;
+
+  HtmlTestInformation(Path filePath, this.expectedMessages, this.scripts)
+      : super(filePath, filePath,
+              {'isMultitest': false, 'isMultiHtmlTest': false},
+              false, false, false, false, false) {}
+}
+
+
 /**
  * A standard [TestSuite] implementation that searches for tests in a
  * directory, and creates [TestCase]s that compile and/or run them.
  */
 class StandardTestSuite extends TestSuite {
   final Path suiteDir;
   final List<String> statusFilePaths;
   TestExpectations testExpectations;
   List<TestInformation> cachedTests;
   final Path dartDir;
@@ skipping 68 matching lines @@
 
   List<Uri> get dart2JsBootstrapDependencies => _dart2JsBootstrapDependencies;
 
   /**
    * The default implementation assumes a file is a test if
    * it ends in "Test.dart".
    */
   bool isTestFile(String filename) {
     // Use the specified predicate, if provided.
     if (isTestFilePredicate != null) return isTestFilePredicate(filename);
-
     return filename.endsWith("Test.dart");
   }
 
+  bool isHtmlTestFile(String filename) => filename.endsWith('_htmltest.html');
+
   List<String> additionalOptions(Path filePath) => [];
 
   Map<String, String> localPackageDirectories;
 
   void forEachTest(Function onTest, Map testCache, [VoidFunction onDone]) {
     discoverPackagesInRepository().then((Map packageDirectories) {
       localPackageDirectories = packageDirectories;
       return updateDartium();
     }).then((_) {
       doTest = onTest;
@@ skipping 73 matching lines @@
     group.add(listCompleter.future);
 
     var lister = dir.list(recursive: listRecursively)
         .listen((FileSystemEntity fse) {
           if (fse is File) enqueueFile(fse.path, group);
         },
         onDone: listCompleter.complete);
   }
 
   void enqueueFile(String filename, FutureGroup group) {
+    if (isHtmlTestFile(filename)) {
+      var info = htmlTest.getInformation(filename);
+      if (info == null) {
+        DebugLogger.error(
+            "HtmlTest $filename does not contain required annotations");
+        return;
+      }
+      cachedTests.add(info);
+      enqueueTestCaseFromTestInformation(info);
+      return;
+    }
     if (!isTestFile(filename)) return;
     Path filePath = new Path(filename);
 
     // Only run the tests that match the pattern.
     if (filePath.filename.endsWith('test_config.dart')) return;
 
     var optionsFromFile = readOptionsFromFile(filePath);
     CreateTest createTestCase = makeTestCaseCreator(optionsFromFile);
 
     if (optionsFromFile['isMultitest']) {
       group.add(doMultitest(filePath, buildDir, suiteDir, createTestCase));
     } else {
       createTestCase(filePath,
+                     filePath,
                      optionsFromFile['hasCompileError'],
                      optionsFromFile['hasRuntimeError'],
                      hasStaticWarning: optionsFromFile['hasStaticWarning']);
     }
   }
 
   static Path _findPubspecYamlFile(Path filePath) {
     final existsCache = TestUtils.existsCache;
 
     Path root = TestUtils.dartDir;
     assert ("$filePath".startsWith("$root"));
 
     // We start with the parent directory of [filePath] and go up until
     // the root directory (excluding the root).
     List<String> segments =
         filePath.directoryPath.relativeTo(root).segments();
     while (segments.length > 0) {
       var pubspecYamlPath =
           new Path(segments.join('/')).append('pubspec.yaml');
       if (existsCache.doesFileExist(pubspecYamlPath.toNativePath())) {
         return root.join(pubspecYamlPath);
       }
       segments.removeLast();
     }
     return null;
   }
 
   void enqueueTestCaseFromTestInformation(TestInformation info) {
+    String testName = buildTestCaseDisplayName(suiteDir, info.originTestPath,
+        multitestName: info.multitestKey);
+    Set<Expectation> expectations = testExpectations.expectations(testName);
+    if (info is HtmlTestInformation &&
+        TestUtils.isBrowserRuntime(configuration['runtime'])) {
+      enqueueBrowserTest([], null, info, testName, expectations);
+      return;
+    }
     var filePath = info.filePath;
     var optionsFromFile = info.optionsFromFile;
 
     Map buildSpecialPackageRoot(Path pubspecYamlFile) {
       var commands = <Command>[];
       var packageDir = pubspecYamlFile.directoryPath;
       var packageName = packageDir.filename;
 
       var checkoutDirectory =
           createPubspecCheckoutDirectory(packageDir);
@@ skipping 47 matching lines @@
         'package-root' : newPackageRoot,
       };
     }
 
     // If this test is inside a package, we will check if there is a
     // pubspec.yaml file and if so, create a custom package root for it.
     List<Command> baseCommands = <Command>[];
     Path packageRoot;
     if (configuration['use_repository_packages'] ||
         configuration['use_public_packages']) {
-        Path pubspecYamlFile = _findPubspecYamlFile(filePath);
-        if (pubspecYamlFile != null) {
-          var result = buildSpecialPackageRoot(pubspecYamlFile);
-          baseCommands.addAll(result['commands']);
-          packageRoot = result['package-root'];
-          if (optionsFromFile['packageRoot'] == null ||
-              optionsFromFile['packageRoot'] == "") {
-            optionsFromFile['packageRoot'] = packageRoot.toNativePath();
-          }
+      Path pubspecYamlFile = _findPubspecYamlFile(filePath);
+      if (pubspecYamlFile != null) {
+        var result = buildSpecialPackageRoot(pubspecYamlFile);
+        baseCommands.addAll(result['commands']);
+        packageRoot = result['package-root'];
+        if (optionsFromFile['packageRoot'] == null ||
+            optionsFromFile['packageRoot'] == "") {
+          optionsFromFile['packageRoot'] = packageRoot.toNativePath();
+        }
       }
     }
     if (configuration['package_root'] != null) {
       packageRoot = new Path(configuration['package_root']);
       optionsFromFile['packageRoot'] = packageRoot.toNativePath();
     }
 
-    String testName = buildTestCaseDisplayName(suiteDir, info.originTestPath,
-        multitestName: optionsFromFile['isMultitest'] ? info.multitestKey : "");
-
-    Set<Expectation> expectations = testExpectations.expectations(testName);
     if (new CompilerConfiguration(configuration).hasCompiler &&
         expectCompileError(info)) {
       // If a compile-time error is expected, and we're testing a
       // compiler, we never need to attempt to run the program (in a
       // browser or otherwise).
       enqueueStandardTest(baseCommands, info, testName, expectations);
     } else if (TestUtils.isBrowserRuntime(configuration['runtime'])) {
       if (info.optionsFromFile['isMultiHtmlTest']) {
         // A browser multi-test has multiple expectations for one test file.
         // Find all the different sub-test expecations for one entire test file.
@@ skipping 104 matching lines @@
             runtimeConfiguration.computeRuntimeCommands(
                 this,
                 CommandBuilder.instance,
                 compilationArtifact,
                 runtimeArguments,
                 environmentOverrides));
   }
 
   CreateTest makeTestCaseCreator(Map optionsFromFile) {
     return (Path filePath,
+            Path originTestPath,
             bool hasCompileError,
             bool hasRuntimeError,
             {bool isNegativeIfChecked: false,
              bool hasCompileErrorIfChecked: false,
              bool hasStaticWarning: false,
-             String multitestKey,
-             Path originTestPath}) {
+             String multitestKey}) {
       // Cache the test information for each test case.
       var info = new TestInformation(filePath,
+                                     originTestPath,
                                      optionsFromFile,
                                      hasCompileError,
                                      hasRuntimeError,
                                      isNegativeIfChecked,
                                      hasCompileErrorIfChecked,
                                      hasStaticWarning,
-                                     multitestKey: multitestKey,
-                                     originTestPath: originTestPath);
+                                     multitestKey: multitestKey);
       cachedTests.add(info);
       enqueueTestCaseFromTestInformation(info);
     };
   }
 
   /**
    * _createUrlPathFromFile takes a [file], which is either located in the dart
    * or in the build directory, and will return a String representing
    * the relative path to either the dart or the build directory.
    * Thus, the returned [String] will be the path component of the URL
@@ skipping 77 matching lines @@
     // on to the browser (it may be test specific).
 
     // TODO(kustermann/ricow): This method should be refactored.
     Map optionsFromFile = info.optionsFromFile;
     Path filePath = info.filePath;
     String filename = filePath.toString();
 
     final String compiler = configuration['compiler'];
     final String runtime = configuration['runtime'];
 
+    if (info is HtmlTestInformation) {
+      if (compiler != 'none' || runtime != 'dartium') {
+        // TODO(whesse): Enable compilation of scripts to dart2js, and
+        // rewriting of script links in html file.  Currently unimplemented.
+        return;
+      }
+      if (info.scripts.length > 0) {
+        // TODO(whesse): Copy scripts into output directory.
+        return;
+      }
+      final String tempDir = createOutputDirectory(info.filePath, '');
+      final String htmlFile = '$tempDir/${filePath.filename}';
+      new File(htmlFile).writeAsStringSync(htmlTest.getContents(info));
+
+      String testDisplayName = '$suiteName/$testName';
+      var htmlPath = _createUrlPathFromFile(new Path(htmlFile));
+      var fullHtmlPath = _getUriForBrowserTest(info, htmlPath,
+                                               null, null);
+      var commands = [CommandBuilder.instance.getBrowserHtmlTestCommand(
+          runtime, fullHtmlPath, configuration, info.expectedMessages)];
+      var testCase = new BrowserTestCase(testDisplayName,
+                    commands, configuration, expectations,
+                    info, isNegative(info), fullHtmlPath);
+      enqueueNewTestCase(testCase);
+      return;
+    }
+
     for (var vmOptions in getVmOptions(optionsFromFile)) {
       // Create a unique temporary directory for each set of vmOptions.
       // TODO(dart:429): Replace separate replaceAlls with a RegExp when
       // replaceAll(RegExp, String) is implemented.
       String optionsName = '';
       if (getVmOptions(optionsFromFile).length > 1) {
           optionsName = vmOptions.join('-').replaceAll('-','')
                                            .replaceAll('=','')
                                            .replaceAll('/','');
       }
@@ skipping 517 matching lines @@
       "otherScripts": <String>[],
       "isMultitest": isMultitest,
       "isMultiHtmlTest": false,
       "subtestNames": <String>[],
       "isolateStubs": '',
       "containsDomImport": false,
     };
   }
 }
 
+
 /// Used for testing packages in on off settings, i.e., we pass in the actual
 /// directory that we want to test.
 class PKGTestSuite extends StandardTestSuite {
 
   PKGTestSuite(Map configuration, Path directoryPath)
       : super(configuration,
               directoryPath.filename,
               directoryPath,
               ["$directoryPath/.status"],
               isTestFilePredicate: (f) => f.endsWith('_test.dart'),
@@ skipping 62 matching lines @@
       Directory dir = new Directory(suiteDir.append(testDir).toNativePath());
       if (dir.existsSync()) {
         enqueueDirectory(dir, group);
       }
     }
 
     return group.future;
   }
 }
 
+
 class AnalyzeLibraryTestSuite extends DartcCompilationTestSuite {
   AnalyzeLibraryTestSuite(Map configuration)
       : super(configuration,
               'analyze_library',
               'sdk',
               ['lib'],
               ['tests/lib/analyzer/analyze_library.status']);
 
   List<String> additionalOptions(Path filePath, {bool showSdkWarnings}) {
     var options = super.additionalOptions(filePath);
     // NOTE: This flag has been deprecated.
     options.add('--show-sdk-warnings');
     return options;
   }
 
   bool isTestFile(String filename) {
     var sep = Platform.pathSeparator;
     // NOTE: We exclude tests and patch files for now.
     return filename.endsWith(".dart") &&
         !filename.endsWith("_test.dart") &&
         !filename.contains("_internal/compiler/js_lib");
   }
 
   bool get listRecursively => true;
 }
 
+
 class PkgBuildTestSuite extends TestSuite {
   final String statusFilePath;
 
   PkgBuildTestSuite(Map configuration, String suiteName, this.statusFilePath)
       : super(configuration, suiteName) {
     assert(configuration['use_sdk']);;
   }
 
   void forEachTest(void onTest(TestCase testCase), _, [void onDone()]) {
     bool fileExists(Path path) => new File(path.toNativePath()).existsSync();
@@ skipping 114 matching lines @@
         return _cache[uri.path];
       }
       var file = new File(new Path(uri.path).toNativePath());
       _cache[uri.path] = file.existsSync() ? file.lastModifiedSync() : null;
       return _cache[uri.path];
     }
     return new DateTime.now();
   }
 }
 
+
 class ExistsCache {
   Map<String, bool> _cache = <String, bool>{};
 
   /**
    * Returns true if the file in [path] exists, false otherwise.
    *
    * The information will be cached.
    */
   bool doesFileExist(String path) {
     if (!_cache.containsKey(path)) {
       _cache[path] = new File(path).existsSync();
     }
     return _cache[path];
   }
 }
 
+
 class TestUtils {
   /**
    * Any script using TestUtils must set dartDirUri to a file:// URI
    * pointing to the root of the Dart checkout.
    */
   static setDartDirUri(uri) {
     dartDirUri = uri;
     dartDir = new Path(uri.toFilePath());
   }
   static Uri dartDirUri;
@@ skipping 326 matching lines @@
         if (path.startsWith(key)) {
           path = path.replaceFirst(key, PATH_REPLACEMENTS[key]);
           break;
         }
       }
     }
     return path;
   }
 }
 
+
 class SummaryReport {
   static int total = 0;
   static int skipped = 0;
   static int skippedByDesign = 0;
   static int noCrash = 0;
   static int pass = 0;
   static int failOk = 0;
   static int fail = 0;
   static int crash = 0;
   static int timeout = 0;
@@ skipping 49 matching lines @@
  * $pass tests are expected to pass
  * $failOk tests are expected to fail that we won't fix
  * $fail tests are expected to fail that we should fix
  * $crash tests are expected to crash that we should fix
  * $timeout tests are allowed to timeout
  * $compileErrorSkip tests are skipped on browsers due to compile-time error
 """;
     print(report);
   }
 }