Run large html tests individually.

tools/testing/dart/test_suite.dart

Index: tools/testing/dart/test_suite.dart
===================================================================
--- tools/testing/dart/test_suite.dart	(revision 14500)
+++ tools/testing/dart/test_suite.dart	(working copy)
@@ -422,7 +422,21 @@
       enqueueStandardTest(info, testName, expectations);
     } else if (TestUtils.isBrowserRuntime(configuration['runtime'])) {
       bool isWrappingRequired = configuration['compiler'] != 'dart2js';
-      enqueueBrowserTest(info, testName, expectations, isWrappingRequired);
+      if (info.optionsFromFile['isMultiHtmlTest']) {
+        // A browser multi-test has multiple expectations for one test file.
+        // Find all the different sub-test expecations for one entire test file.
+        List<String> subtestNames = info.optionsFromFile['subtestNames'];
+        Map<String, Set<String>> multiHtmlTestExpectations = {};
+        for (String name in subtestNames) {
+          String fullTestName = '$testName/$name';
+          multiHtmlTestExpectations[fullTestName] =
+              testExpectations.expectations(fullTestName);
+        }
+        enqueueBrowserTest(info, testName, multiHtmlTestExpectations,
+            isWrappingRequired);
+      } else {
+        enqueueBrowserTest(info, testName, expectations, isWrappingRequired);
+      }
     } else {
       enqueueStandardTest(info, testName, expectations);
     }
@@ -574,13 +588,15 @@
    * JavaScript version of the test, and copies the appropriate framework
    * files to that directory.  It creates a [BrowserTestCase], which has
    * two sequential steps to be run by the [ProcessQueue] when the test is
-   * executed: a compilation
-   * step and an execution step, both with the appropriate executable and
-   * arguments.
+   * executed: a compilation step and an execution step, both with the 
+   * appropriate executable and arguments. The [expectations] object can be
+   * either a Set<String> if the test is a regular test, or a Map<String
+   * subTestName, Set<String>> if we are running a browser multi-test (one
+   * compilation and many browser runs).
    */
   void enqueueBrowserTest(TestInformation info,
                           String testName,
-                          Set<String> expectations,
+                          Object expectations,
                           bool isWrappingRequired) {
     Map optionsFromFile = info.optionsFromFile;
     Path filePath = info.filePath;
@@ -702,43 +718,76 @@
       }
 
       // Construct the command that executes the browser test
-      List<String> args;
-      if (TestUtils.usesWebDriver(runtime)) {
-        args = [dartDir.append('tools/testing/run_selenium.py').toNativePath(),
-            '--browser=$runtime',
-            '--timeout=${configuration["timeout"] - 2}',
-            '--out=$htmlPath'];
-        if (runtime == 'dartium') {
-          args.add('--executable=$dartiumFilename');
+      List<String> subtestNames = info.optionsFromFile['subtestNames'];
+      TestCase parentTest;
+      int subtestIndex = 0;
+      do {
+        List<String> args = <String>[];
+        String fullHtmlPath = htmlPath;
+        if (subtestNames.length > 0) {
+          fullHtmlPath = '${htmlPath}#${subtestNames[subtestIndex]}';
         }
-      } else {
-        args = [
-            dartDir.append('tools/testing/drt-trampoline.py').toNativePath(),
-            dumpRenderTreeFilename,
-            '--no-timeout'
-        ];
-        if (runtime == 'drt' &&
-            (compiler == 'none' || compiler == 'dart2dart')) {
-          var dartFlags = ['--ignore-unrecognized-flags'];
-          if (configuration["checked"]) {
-            dartFlags.add('--enable_asserts');
-            dartFlags.add("--enable_type_checks");
+        if (TestUtils.usesWebDriver(runtime)) {
+          args = [
+              dartDir.append('tools/testing/run_selenium.py').toNativePath(),
+              '--browser=$runtime',
+              '--timeout=${configuration["timeout"] - 2}',
+              '--out="$fullHtmlPath"'];
+          if (runtime == 'dartium') {
+            args.add('--executable=$dartiumFilename');
           }
-          dartFlags.addAll(vmOptions);
-          args.add('--dart-flags=${Strings.join(dartFlags, " ")}');
+        } else {
+          args = [
+              dartDir.append('tools/testing/drt-trampoline.py').toNativePath(),
+              dumpRenderTreeFilename,
+              '--no-timeout'
+          ];
+          if (runtime == 'drt' &&
+              (compiler == 'none' || compiler == 'dart2dart')) {
+            var dartFlags = ['--ignore-unrecognized-flags'];
+            if (configuration["checked"]) {
+              dartFlags.add('--enable_asserts');
+              dartFlags.add("--enable_type_checks");
+            }
+            dartFlags.addAll(vmOptions);
+            args.add('--dart-flags=${Strings.join(dartFlags, " ")}');
+          }
+          args.add(fullHtmlPath);
+          if (expectedOutput != null) {
+            args.add('--out-expectation=${expectedOutput.toNativePath()}');
+          }
         }
-        args.add(htmlPath);
-        if (expectedOutput != null) {
-          args.add('--out-expectation=${expectedOutput.toNativePath()}');
+        List<String> commandSet = new List<String>.from(commands);
+        if (subtestIndex != 0) {

[Bill Hesse, 2012/11/05 11:51:49]

Nit: It would be better if the sense of this test was reversed, and the
initialization was to [].

[Emily Fortuna, 2012/11/05 18:27:49]

Fixed!

+          commandSet = [];
         }
-      }
-      commands.add(new Command('python', args));
+        commandSet.add(new Command('python', args));
 
-      // Create BrowserTestCase and queue it.
-      var testCase = new BrowserTestCase('$suiteName/$testName',
-          commands, configuration, completeHandler, expectations,
-          info, info.hasCompileError || info.hasRuntimeError);
-      doTest(testCase);
+        // Create BrowserTestCase and queue it.
+        String testDisplayName = '$suiteName/$testName';
+        var testCase;
+        if (info.optionsFromFile['isMultiHtmlTest']) {
+          testDisplayName = '$testDisplayName/${subtestNames[subtestIndex]}';
+          testCase = new BrowserTestCase(testDisplayName,
+              commandSet, configuration, completeHandler,
+              expectations['$testName/${subtestNames[subtestIndex]}'],
+              info, info.hasCompileError || info.hasRuntimeError,
+              subtestIndex != 0);
+        } else {
+          testCase = new BrowserTestCase(testDisplayName,
+              commandSet, configuration, completeHandler, expectations,
+              info, info.hasCompileError || info.hasRuntimeError, false);
+        }
+        if (subtestIndex == 0) {
+          parentTest = testCase;
+        } else {
+          parentTest.addObserver(testCase);
+        }
+        doTest(testCase); // TODO in doTest, check if the hasCompiled is set for
+        // the shared tests variable. the first process can notify the other
+        // processes its ready to go.
+        subtestIndex++;
+      } while(subtestIndex < subtestNames.length);
     }
   }
 
@@ -964,6 +1013,8 @@
     RegExp dartOptionsRegExp = const RegExp(r"// DartOptions=(.*)");
     RegExp otherScriptsRegExp = const RegExp(r"// OtherScripts=(.*)");
     RegExp multiTestRegExp = const RegExp(r"/// [0-9][0-9]:(.*)");
+    RegExp multiHtmlTestRegExp =
+        const RegExp(r"useHtmlIndividualConfiguration()");
     RegExp staticTypeRegExp =
         const RegExp(r"/// ([0-9][0-9]:){0,1}\s*static type warning");
     RegExp compileTimeRegExp =
@@ -1030,6 +1081,7 @@
     }
 
     bool isMultitest = multiTestRegExp.hasMatch(contents);
+    bool isMultiHtmlTest = multiHtmlTestRegExp.hasMatch(contents);
     bool containsLeadingHash = leadingHashRegExp.hasMatch(contents);
     Match isolateMatch = isolateStubsRegExp.firstMatch(contents);
     String isolateStubs = isolateMatch != null ? isolateMatch[1] : '';
@@ -1045,6 +1097,16 @@
       numCompileTimeAnnotations++;
     }
 
+    // Note: This is brittle. It assumes you import unittest with no prefix and
+    // always directly call "group(".
+    RegExp numTests = new RegExp(r"\s*[^/]\s*group\('[^,']*");
+    List<String> subtestNames = [];
+    Iterator matchesIter = numTests.allMatches(contents).iterator();
+    while(matchesIter.hasNext && isMultiHtmlTest) {
+      String fullMatch = matchesIter.next().group(0);
+      subtestNames.add(fullMatch.substring(fullMatch.indexOf("'") + 1));
+    }
+
     return { "vmOptions": result,
              "dartOptions": dartOptions,
              "hasCompileError": hasCompileError,
@@ -1052,6 +1114,8 @@
              "isStaticClean" : isStaticClean,
              "otherScripts": otherScripts,
              "isMultitest": isMultitest,
+             "isMultiHtmlTest": isMultiHtmlTest,
+             "subtestNames": subtestNames,
              "containsLeadingHash": containsLeadingHash,
              "isolateStubs": isolateStubs,
              "containsDomImport": containsDomImport,