Run large html tests individually.

tools/testing/dart/test_suite.dart

Index: tools/testing/dart/test_suite.dart
===================================================================
--- tools/testing/dart/test_suite.dart	(revision 14350)
+++ tools/testing/dart/test_suite.dart	(working copy)
@@ -427,7 +427,21 @@
         // TODO(ahe): Investigate why this doesn't work on Windows.
         isWrappingRequired = true;
       }
-      enqueueBrowserTest(info, testName, expectations, isWrappingRequired);
+      if (info.optionsFromFile['isMultiHtmlTest']) {
+        // A browser multi-test has multiple expectations for one test file.
+        // Find all the different sub-test expecations for one entire test file.
+        List<String> subtestNames = info.optionsFromFile['subtestNames'];
+        Map<String, Set<String>> multiHtmlTestExpectations = {};
+        for (String name in subtestNames) {
+          String fullTestName = '$testName/$name';
+          multiHtmlTestExpectations[fullTestName] =
+              testExpectations.expectations(fullTestName);
+        }
+        enqueueBrowserTest(info, testName, multiHtmlTestExpectations,
+            isWrappingRequired);
+      } else {
+        enqueueBrowserTest(info, testName, expectations, isWrappingRequired);
+      }
     } else {
       enqueueStandardTest(info, testName, expectations);
     }
@@ -579,13 +593,15 @@
    * JavaScript version of the test, and copies the appropriate framework
    * files to that directory.  It creates a [BrowserTestCase], which has
    * two sequential steps to be run by the [ProcessQueue] when the test is
-   * executed: a compilation
-   * step and an execution step, both with the appropriate executable and
-   * arguments.
+   * executed: a compilation step and an execution step, both with the 
+   * appropriate executable and arguments. The [expectations] object can be
+   * either a Set<String> if the test is a regular test, or a Map<String
+   * subTestName, Set<String>> if we are running a browser multi-test (one
+   * compilation and many browser runs).
    */
   void enqueueBrowserTest(TestInformation info,
                           String testName,
-                          Set<String> expectations,
+                          Object expectations,
                           bool isWrappingRequired) {
     Map optionsFromFile = info.optionsFromFile;
     Path filePath = info.filePath;
@@ -707,43 +723,64 @@
       }
 
       // Construct the command that executes the browser test
-      List<String> args;
-      if (TestUtils.usesWebDriver(runtime)) {
-        args = [dartDir.append('tools/testing/run_selenium.py').toNativePath(),
-            '--browser=$runtime',
-            '--timeout=${configuration["timeout"] - 2}',
-            '--out=$htmlPath'];
-        if (runtime == 'dartium') {
-          args.add('--executable=$dartiumFilename');
+      List<String> subtestNames = info.optionsFromFile['subtestNames'];
+      int subtestIndex = 0;
+      do {
+        List<String> args = <String>[];
+        String fullHtmlPath = htmlPath;
+        if (subtestNames.length > 0) {
+          fullHtmlPath = '${htmlPath}#${subtestNames[subtestIndex]}';
         }
-      } else {
-        args = [
-            dartDir.append('tools/testing/drt-trampoline.py').toNativePath(),
-            dumpRenderTreeFilename,
-            '--no-timeout'
-        ];
-        if (runtime == 'drt' &&
-            (compiler == 'none' || compiler == 'dart2dart')) {
-          var dartFlags = ['--ignore-unrecognized-flags'];
-          if (configuration["checked"]) {
-            dartFlags.add('--enable_asserts');
-            dartFlags.add("--enable_type_checks");
+        if (TestUtils.usesWebDriver(runtime)) {
+          args = [
+              dartDir.append('tools/testing/run_selenium.py').toNativePath(),
+              '--browser=$runtime',
+              '--timeout=${configuration["timeout"] - 2}',
+              '--out="$fullHtmlPath"'];
+          if (runtime == 'dartium') {
+            args.add('--executable=$dartiumFilename');
           }
-          dartFlags.addAll(vmOptions);
-          args.add('--dart-flags=${Strings.join(dartFlags, " ")}');
+        } else {
+          args = [
+              dartDir.append('tools/testing/drt-trampoline.py').toNativePath(),
+              dumpRenderTreeFilename,
+              '--no-timeout'
+          ];
+          if (runtime == 'drt' &&
+              (compiler == 'none' || compiler == 'dart2dart')) {
+            var dartFlags = ['--ignore-unrecognized-flags'];
+            if (configuration["checked"]) {
+              dartFlags.add('--enable_asserts');
+              dartFlags.add("--enable_type_checks");
+            }
+            dartFlags.addAll(vmOptions);
+            args.add('--dart-flags=${Strings.join(dartFlags, " ")}');
+          }
+          args.add(fullHtmlPath);
+          if (expectedOutput != null) {
+            args.add('--out-expectation=${expectedOutput.toNativePath()}');
+          }
         }
-        args.add(htmlPath);
-        if (expectedOutput != null) {
-          args.add('--out-expectation=${expectedOutput.toNativePath()}');
+        List<String> commandSet = new List<String>.from(commands);
+        commandSet.add(new Command('python', args));

[Bill Hesse, 2012/11/01 20:26:00]

All of the test cases created from a multitest are going to include the
compilation commands, with the same directory specified.  They will be running
in parallel, writing over each other's output.  And you don't really want to
compile this browser test many times, do you?  You just want to run it on the
browser, with different URLs, many times.

I think you need some sort of preliminary phase, or some locking, to release the
many tests after the compilation is done.

+
+        // Create BrowserTestCase and queue it.
+        String testDisplayName = '$suiteName/$testName';
+        var testCase;
+        if (info.optionsFromFile['isMultiHtmlTest']) {
+          testDisplayName = '$testDisplayName/${subtestNames[subtestIndex]}';
+          testCase = new BrowserTestCase(testDisplayName,
+              commandSet, configuration, completeHandler,
+              expectations['$testName/${subtestNames[subtestIndex]}'],
+              info, info.hasCompileError || info.hasRuntimeError);
+        } else {
+          testCase = new BrowserTestCase(testDisplayName,
+              commandSet, configuration, completeHandler, expectations,
+              info, info.hasCompileError || info.hasRuntimeError);
         }
-      }
-      commands.add(new Command('python', args));
-
-      // Create BrowserTestCase and queue it.
-      var testCase = new BrowserTestCase('$suiteName/$testName',
-          commands, configuration, completeHandler, expectations,
-          info, info.hasCompileError || info.hasRuntimeError);
-      doTest(testCase);
+        doTest(testCase);
+        subtestIndex++;
+      } while(subtestIndex < subtestNames.length);
     }
   }
 
@@ -969,6 +1006,8 @@
     RegExp dartOptionsRegExp = const RegExp(r"// DartOptions=(.*)");
     RegExp otherScriptsRegExp = const RegExp(r"// OtherScripts=(.*)");
     RegExp multiTestRegExp = const RegExp(r"/// [0-9][0-9]:(.*)");
+    RegExp multiHtmlTestRegExp =
+        const RegExp(r"useHtmlIndividualConfiguration()");
     RegExp staticTypeRegExp =
         const RegExp(r"/// ([0-9][0-9]:){0,1}\s*static type warning");
     RegExp compileTimeRegExp =
@@ -1035,6 +1074,7 @@
     }
 
     bool isMultitest = multiTestRegExp.hasMatch(contents);
+    bool isMultiHtmlTest = multiHtmlTestRegExp.hasMatch(contents);
     bool containsLeadingHash = leadingHashRegExp.hasMatch(contents);
     Match isolateMatch = isolateStubsRegExp.firstMatch(contents);
     String isolateStubs = isolateMatch != null ? isolateMatch[1] : '';
@@ -1050,6 +1090,16 @@
       numCompileTimeAnnotations++;
     }
 
+    // Note: This is brittle. It assumes you import unittest with no prefix and
+    // always directly call "test(".

[Bill Hesse, 2012/11/01 20:26:00]

It would be nice if there was something more explicit: a new method
parallel_test, or a comment // Multitest, or something.

+    RegExp numTests = new RegExp(r"\s*[^/]\s*test\('[^,']*");
+    List<String> subtestNames = [];
+    Iterator matchesIter = numTests.allMatches(contents).iterator();
+    while(matchesIter.hasNext && isMultiHtmlTest) {
+      String fullMatch = matchesIter.next().group(0);
+      subtestNames.add(fullMatch.substring(fullMatch.indexOf("'") + 1));
+    }
+
     return { "vmOptions": result,
              "dartOptions": dartOptions,
              "hasCompileError": hasCompileError,
@@ -1057,6 +1107,8 @@
              "isStaticClean" : isStaticClean,
              "otherScripts": otherScripts,
              "isMultitest": isMultitest,
+             "isMultiHtmlTest": isMultiHtmlTest,
+             "subtestNames": subtestNames,
              "containsLeadingHash": containsLeadingHash,
              "isolateStubs": isolateStubs,
              "containsDomImport": containsDomImport,