Run large html tests individually.

tools/testing/dart/test_suite.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for enumerating and preparing tests.
  *
  * This library includes:
  *
  * - Creating tests by listing all the Dart files in certain directories,
@@ skipping 409 matching lines @@
     }
     if (expectations.contains(SKIP)) return;
 
     if (configuration['compiler'] != 'none' && info.hasCompileError) {
       // If a compile-time error is expected, and we're testing a
       // compiler, we never need to attempt to run the program (in a
       // browser or otherwise).
       enqueueStandardTest(info, testName, expectations);
     } else if (TestUtils.isBrowserRuntime(configuration['runtime'])) {
       bool isWrappingRequired = configuration['compiler'] != 'dart2js';
-      enqueueBrowserTest(info, testName, expectations, isWrappingRequired);
+      if (info.optionsFromFile['isMultiHtmlTest']) {
+        // A browser multi-test has multiple expectations for one test file.
+        // Find all the different sub-test expecations for one entire test file.
+        List<String> subtestNames = info.optionsFromFile['subtestNames'];
+        Map<String, Set<String>> multiHtmlTestExpectations = {};
+        for (String name in subtestNames) {
+          String fullTestName = '$testName/$name';
+          multiHtmlTestExpectations[fullTestName] =
+              testExpectations.expectations(fullTestName);
+        }
+        enqueueBrowserTest(info, testName, multiHtmlTestExpectations,
+            isWrappingRequired);
+      } else {
+        enqueueBrowserTest(info, testName, expectations, isWrappingRequired);
+      }
     } else {
       enqueueStandardTest(info, testName, expectations);
     }
   }
 
   void enqueueStandardTest(TestInformation info,
                            String testName,
                            Set<String> expectations) {
     bool isNegative = info.hasCompileError ||
         (configuration['checked'] && info.isNegativeIfChecked);
@@ skipping 131 matching lines @@
     }
   }
 
   /**
    * The [StandardTestSuite] has support for tests that
    * compile a test from Dart to JavaScript, and then run the resulting
    * JavaScript.  This function creates a working directory to hold the
    * JavaScript version of the test, and copies the appropriate framework
    * files to that directory.  It creates a [BrowserTestCase], which has
    * two sequential steps to be run by the [ProcessQueue] when the test is
-   * executed: a compilation
-   * step and an execution step, both with the appropriate executable and
-   * arguments.
+   * executed: a compilation step and an execution step, both with the 
+   * appropriate executable and arguments. The [expectations] object can be
+   * either a Set<String> if the test is a regular test, or a Map<String
+   * subTestName, Set<String>> if we are running a browser multi-test (one
+   * compilation and many browser runs).
    */
   void enqueueBrowserTest(TestInformation info,
                           String testName,
-                          Set<String> expectations,
+                          Object expectations,
                           bool isWrappingRequired) {
     Map optionsFromFile = info.optionsFromFile;
     Path filePath = info.filePath;
     String filename = filePath.toString();
     bool isWebTest = optionsFromFile['containsDomImport'];
     bool isLibraryDefinition = optionsFromFile['isLibraryDefinition'];
     if (isWrappingRequired
         && !isLibraryDefinition && optionsFromFile['containsSourceOrImport']) {
       print('Warning for $filename: Browser tests require #library '
             'in any file that uses #import, #source, or #resource');
@@ skipping 100 matching lines @@
           Path namePath = new Path(name);
           Expect.equals(namePath.extension, 'dart');
           String baseName = namePath.filenameWithoutExtension;
           Path fromPath = filePath.directoryPath.join(namePath);
           commands.add(_compileCommand(
               fromPath.toNativePath(), '$tempDir/$baseName.js',
               compiler, tempDir, vmOptions));
         }
       }
 
+      // Variables for browser multi-tests.
+      List<String> subtestNames = info.optionsFromFile['subtestNames'];
+      TestCase multitestParentTest;
+      int subtestIndex = 0;
       // Construct the command that executes the browser test
-      List<String> args;
-      if (TestUtils.usesWebDriver(runtime)) {
-        args = [dartDir.append('tools/testing/run_selenium.py').toNativePath(),
-            '--browser=$runtime',
-            '--timeout=${configuration["timeout"] - 2}',
-            '--out=$htmlPath'];
-        if (runtime == 'dartium') {
-          args.add('--executable=$dartiumFilename');
+      constructBrowserTestCommands();
+      do {
+        List<String> args = <String>[];
+        String fullHtmlPath = htmlPath;
+        if (info.optionsFromFile['isMultiHtmlTest']
+            && subtestNames.length > 0) {
+          fullHtmlPath = '${htmlPath}#${subtestNames[subtestIndex]}';
         }
-      } else {
-        args = [
-            dartDir.append('tools/testing/drt-trampoline.py').toNativePath(),
-            dumpRenderTreeFilename,
-            '--no-timeout'
-        ];
-        if (runtime == 'drt' &&
-            (compiler == 'none' || compiler == 'dart2dart')) {
-          var dartFlags = ['--ignore-unrecognized-flags'];
-          if (configuration["checked"]) {
-            dartFlags.add('--enable_asserts');
-            dartFlags.add("--enable_type_checks");
+        if (TestUtils.usesWebDriver(runtime)) {
+          args = [
+              dartDir.append('tools/testing/run_selenium.py').toNativePath(),
+              '--browser=$runtime',
+              '--timeout=${configuration["timeout"] - 2}',
+              '--out="$fullHtmlPath"'];
+          if (runtime == 'dartium') {
+            args.add('--executable=$dartiumFilename');
           }
-          dartFlags.addAll(vmOptions);
-          args.add('--dart-flags=${Strings.join(dartFlags, " ")}');
+        } else {
+          args = [
+              dartDir.append('tools/testing/drt-trampoline.py').toNativePath(),
+              dumpRenderTreeFilename,
+              '--no-timeout'
+          ];
+          if (runtime == 'drt' &&
+              (compiler == 'none' || compiler == 'dart2dart')) {
+            var dartFlags = ['--ignore-unrecognized-flags'];
+            if (configuration["checked"]) {
+              dartFlags.add('--enable_asserts');
+              dartFlags.add("--enable_type_checks");
+            }
+            dartFlags.addAll(vmOptions);
+            args.add('--dart-flags=${Strings.join(dartFlags, " ")}');
+          }
+          args.add(fullHtmlPath);
+          if (expectedOutput != null) {
+            args.add('--out-expectation=${expectedOutput.toNativePath()}');
+          }
         }
-        args.add(htmlPath);
-        if (expectedOutput != null) {
-          args.add('--out-expectation=${expectedOutput.toNativePath()}');
+        List<String> commandSet = new List<String>.from(commands);
+        if (subtestIndex != 0) {
+         commandSet = [];
+         args.add('--force-refresh');
         }
-      }
-      commands.add(new Command('python', args));
+        commandSet.add(new Command('python', args));
 
-      // Create BrowserTestCase and queue it.
-      var testCase = new BrowserTestCase('$suiteName/$testName',
-          commands, configuration, completeHandler, expectations,
-          info, info.hasCompileError || info.hasRuntimeError);
-      doTest(testCase);
+        // Create BrowserTestCase and queue it.
+        String testDisplayName = '$suiteName/$testName';
+        var testCase;
+        if (info.optionsFromFile['isMultiHtmlTest']) {
+          testDisplayName = '$testDisplayName/${subtestNames[subtestIndex]}';
+          testCase = new BrowserTestCase(testDisplayName,
+              commandSet, configuration, completeHandler,
+              expectations['$testName/${subtestNames[subtestIndex]}'],
+              info, info.hasCompileError || info.hasRuntimeError,
+              subtestIndex != 0);
+        } else {
+          testCase = new BrowserTestCase(testDisplayName,
+              commandSet, configuration, completeHandler, expectations,
+              info, info.hasCompileError || info.hasRuntimeError, false);
+        }
+        if (subtestIndex == 0) {
+          multitestParentTest = testCase;
+        } else {
+          multitestParentTest.addObserver(testCase);
+        }
+        doTest(testCase);
+        subtestIndex++;
+      } while(subtestIndex < subtestNames.length);
     }
   }
 
+  void constructBrowserTestCommands() {
+
+  }
+
   /** Helper to create a compilation command for a single input file. */
   Command _compileCommand(String inputFile, String outputFile,
       String compiler, String dir, var vmOptions) {
     String executable = TestUtils.compilerPath(configuration);
     List<String> args = TestUtils.standardOptions(configuration);
     switch (compiler) {
       case 'dart2js':
       case 'dart2dart':
         if (compiler == 'dart2dart') args.add('--out=$outputFile');
         args.add('--out=$outputFile');
@@ skipping 202 matching lines @@
    *   executing the copy command printed by the test script.
    *
    * This method is static as the map is cached and shared amongst
    * configurations, so it may not use [configuration].
    */
   static Map readOptionsFromFile(Path filePath) {
     RegExp testOptionsRegExp = const RegExp(r"// VMOptions=(.*)");
     RegExp dartOptionsRegExp = const RegExp(r"// DartOptions=(.*)");
     RegExp otherScriptsRegExp = const RegExp(r"// OtherScripts=(.*)");
     RegExp multiTestRegExp = const RegExp(r"/// [0-9][0-9]:(.*)");
+    RegExp multiHtmlTestRegExp =
+        const RegExp(r"useHtmlIndividualConfiguration()");
     RegExp staticTypeRegExp =
         const RegExp(r"/// ([0-9][0-9]:){0,1}\s*static type warning");
     RegExp compileTimeRegExp =
         const RegExp(r"/// ([0-9][0-9]:){0,1}\s*compile-time error");
     RegExp staticCleanRegExp = const RegExp(r"// @static-clean");
     RegExp leadingHashRegExp = const RegExp(r"^#", multiLine: true);
     RegExp isolateStubsRegExp = const RegExp(r"// IsolateStubs=(.*)");
     // TODO(gram) Clean these up once the old directives are not supported.
     RegExp domImportRegExp =
         const RegExp(r"^[#]?import.*dart:html", multiLine: true);
@@ skipping 46 matching lines @@
       isStaticClean = true;
     }
 
     List<String> otherScripts = new List<String>();
     matches = otherScriptsRegExp.allMatches(contents);
     for (var match in matches) {
       otherScripts.addAll(match[1].split(' ').filter((e) => e != ''));
     }
 
     bool isMultitest = multiTestRegExp.hasMatch(contents);
+    bool isMultiHtmlTest = multiHtmlTestRegExp.hasMatch(contents);
     bool containsLeadingHash = leadingHashRegExp.hasMatch(contents);
     Match isolateMatch = isolateStubsRegExp.firstMatch(contents);
     String isolateStubs = isolateMatch != null ? isolateMatch[1] : '';
     bool containsDomImport = domImportRegExp.hasMatch(contents);
     bool isLibraryDefinition = libraryDefinitionRegExp.hasMatch(contents);
     bool containsSourceOrImport = sourceOrImportRegExp.hasMatch(contents);
     int numStaticTypeAnnotations = 0;
     for (var i in staticTypeRegExp.allMatches(contents)) {
       numStaticTypeAnnotations++;
     }
     int numCompileTimeAnnotations = 0;
     for (var i in compileTimeRegExp.allMatches(contents)) {
       numCompileTimeAnnotations++;
     }
 
+    // Note: This is brittle. It's the age-old problem of having a context free
+    // language but the means to easily identify the construct is a regular
+    // expression, aka impossible. Therefore we just make an approximation of
+    // the number of top-level "group(...)" occurrences. This assumes you import
+    // unittest with no prefix and always directly call "group(". It only uses
+    // top-level "groups" so tests running nested groups will be no-ops.
+    RegExp numTests = new RegExp(r"\s*[^/]\s*group\('[^,']*");
+    List<String> subtestNames = [];
+    Iterator matchesIter = numTests.allMatches(contents).iterator();
+    while(matchesIter.hasNext && isMultiHtmlTest) {
+      String fullMatch = matchesIter.next().group(0);
+      subtestNames.add(fullMatch.substring(fullMatch.indexOf("'") + 1));
+    }
+
     return { "vmOptions": result,
              "dartOptions": dartOptions,
              "hasCompileError": hasCompileError,
              "hasRuntimeError": hasRuntimeError,
              "isStaticClean" : isStaticClean,
              "otherScripts": otherScripts,
              "isMultitest": isMultitest,
+             "isMultiHtmlTest": isMultiHtmlTest,
+             "subtestNames": subtestNames,
              "containsLeadingHash": containsLeadingHash,
              "isolateStubs": isolateStubs,
              "containsDomImport": containsDomImport,
              "isLibraryDefinition": isLibraryDefinition,
              "containsSourceOrImport": containsSourceOrImport,
              "numStaticTypeAnnotations": numStaticTypeAnnotations,
              "numCompileTimeAnnotations": numCompileTimeAnnotations };
   }
 
   List<List<String>> getVmOptions(Map optionsFromFile) {
@@ skipping 423 matching lines @@
  * $pass tests are expected to pass
  * $failOk tests are expected to fail that we won't fix
  * $fail tests are expected to fail that we should fix
  * $crash tests are expected to crash that we should fix
  * $timeout tests are allowed to timeout
  * $compileErrorSkip tests are skipped on browsers due to compile-time error
 """;
     print(report);
    }
 }