Run large html tests individually.

tools/testing/dart/test_suite.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for enumerating and preparing tests.
  *
  * This library includes:
  *
  * - Creating tests by listing all the Dart files in certain directories,
@@ skipping 409 matching lines @@
       // compiler, we never need to attempt to run the program (in a
       // browser or otherwise).
       enqueueStandardTest(info, testName, expectations);
     } else if (TestUtils.isBrowserRuntime(configuration['runtime'])) {
       bool isWrappingRequired = configuration['compiler'] != 'dart2js';
       if (configuration['runtime'] == 'ff' &&
           Platform.operatingSystem == 'windows') {
         // TODO(ahe): Investigate why this doesn't work on Windows.
         isWrappingRequired = true;
       }
-      enqueueBrowserTest(info, testName, expectations, isWrappingRequired);
+      if (info.optionsFromFile['isMultiHtmlTest']) {
+        // A browser multi-test has multiple expectations for one test file.
+        // Find all the different sub-test expecations for one entire test file.
+        List<String> subtestNames = info.optionsFromFile['subtestNames'];
+        Map<String, Set<String>> multiHtmlTestExpectations = {};
+        for (String name in subtestNames) {
+          String fullTestName = '$testName/$name';
+          multiHtmlTestExpectations[fullTestName] =
+              testExpectations.expectations(fullTestName);
+        }
+        enqueueBrowserTest(info, testName, multiHtmlTestExpectations,
+            isWrappingRequired);
+      } else {
+        enqueueBrowserTest(info, testName, expectations, isWrappingRequired);
+      }
     } else {
       enqueueStandardTest(info, testName, expectations);
     }
   }
 
   void enqueueStandardTest(TestInformation info,
                            String testName,
                            Set<String> expectations) {
     bool isNegative = info.hasCompileError ||
         (configuration['checked'] && info.isNegativeIfChecked);
@@ skipping 131 matching lines @@
     }
   }
 
   /**
    * The [StandardTestSuite] has support for tests that
    * compile a test from Dart to JavaScript, and then run the resulting
    * JavaScript.  This function creates a working directory to hold the
    * JavaScript version of the test, and copies the appropriate framework
    * files to that directory.  It creates a [BrowserTestCase], which has
    * two sequential steps to be run by the [ProcessQueue] when the test is
-   * executed: a compilation
-   * step and an execution step, both with the appropriate executable and
-   * arguments.
+   * executed: a compilation step and an execution step, both with the 
+   * appropriate executable and arguments. The [expectations] object can be
+   * either a Set<String> if the test is a regular test, or a Map<String
+   * subTestName, Set<String>> if we are running a browser multi-test (one
+   * compilation and many browser runs).
    */
   void enqueueBrowserTest(TestInformation info,
                           String testName,
-                          Set<String> expectations,
+                          Object expectations,
                           bool isWrappingRequired) {
     Map optionsFromFile = info.optionsFromFile;
     Path filePath = info.filePath;
     String filename = filePath.toString();
     bool isWebTest = optionsFromFile['containsDomImport'];
     bool isLibraryDefinition = optionsFromFile['isLibraryDefinition'];
     if (isWrappingRequired
         && !isLibraryDefinition && optionsFromFile['containsSourceOrImport']) {
       print('Warning for $filename: Browser tests require #library '
             'in any file that uses #import, #source, or #resource');
@@ skipping 101 matching lines @@
           Expect.equals(namePath.extension, 'dart');
           String baseName = namePath.filenameWithoutExtension;
           Path fromPath = filePath.directoryPath.join(namePath);
           commands.add(_compileCommand(
               fromPath.toNativePath(), '$tempDir/$baseName.js',
               compiler, tempDir, vmOptions));
         }
       }
 
       // Construct the command that executes the browser test
-      List<String> args;
-      if (TestUtils.usesWebDriver(runtime)) {
-        args = [dartDir.append('tools/testing/run_selenium.py').toNativePath(),
-            '--browser=$runtime',
-            '--timeout=${configuration["timeout"] - 2}',
-            '--out=$htmlPath'];
-        if (runtime == 'dartium') {
-          args.add('--executable=$dartiumFilename');
+      List<String> subtestNames = info.optionsFromFile['subtestNames'];
+      int subtestIndex = 0;
+      do {
+        List<String> args = <String>[];
+        String fullHtmlPath = htmlPath;
+        if (subtestNames.length > 0) {
+          fullHtmlPath = '${htmlPath}#${subtestNames[subtestIndex]}';
         }
-      } else {
-        args = [
-            dartDir.append('tools/testing/drt-trampoline.py').toNativePath(),
-            dumpRenderTreeFilename,
-            '--no-timeout'
-        ];
-        if (runtime == 'drt' &&
-            (compiler == 'none' || compiler == 'dart2dart')) {
-          var dartFlags = ['--ignore-unrecognized-flags'];
-          if (configuration["checked"]) {
-            dartFlags.add('--enable_asserts');
-            dartFlags.add("--enable_type_checks");
+        if (TestUtils.usesWebDriver(runtime)) {
+          args = [
+              dartDir.append('tools/testing/run_selenium.py').toNativePath(),
+              '--browser=$runtime',
+              '--timeout=${configuration["timeout"] - 2}',
+              '--out="$fullHtmlPath"'];
+          if (runtime == 'dartium') {
+            args.add('--executable=$dartiumFilename');
           }
-          dartFlags.addAll(vmOptions);
-          args.add('--dart-flags=${Strings.join(dartFlags, " ")}');
+        } else {
+          args = [
+              dartDir.append('tools/testing/drt-trampoline.py').toNativePath(),
+              dumpRenderTreeFilename,
+              '--no-timeout'
+          ];
+          if (runtime == 'drt' &&
+              (compiler == 'none' || compiler == 'dart2dart')) {
+            var dartFlags = ['--ignore-unrecognized-flags'];
+            if (configuration["checked"]) {
+              dartFlags.add('--enable_asserts');
+              dartFlags.add("--enable_type_checks");
+            }
+            dartFlags.addAll(vmOptions);
+            args.add('--dart-flags=${Strings.join(dartFlags, " ")}');
+          }
+          args.add(fullHtmlPath);
+          if (expectedOutput != null) {
+            args.add('--out-expectation=${expectedOutput.toNativePath()}');
+          }
         }
-        args.add(htmlPath);
-        if (expectedOutput != null) {
-          args.add('--out-expectation=${expectedOutput.toNativePath()}');
+        List<String> commandSet = new List<String>.from(commands);
+        commandSet.add(new Command('python', args));

[Bill Hesse, 2012/11/01 20:26:00]

All of the test cases created from a multitest are going to include the
compilation commands, with the same directory specified.  They will be running
in parallel, writing over each other's output.  And you don't really want to
compile this browser test many times, do you?  You just want to run it on the
browser, with different URLs, many times.

I think you need some sort of preliminary phase, or some locking, to release the
many tests after the compilation is done.

+
+        // Create BrowserTestCase and queue it.
+        String testDisplayName = '$suiteName/$testName';
+        var testCase;
+        if (info.optionsFromFile['isMultiHtmlTest']) {
+          testDisplayName = '$testDisplayName/${subtestNames[subtestIndex]}';
+          testCase = new BrowserTestCase(testDisplayName,
+              commandSet, configuration, completeHandler,
+              expectations['$testName/${subtestNames[subtestIndex]}'],
+              info, info.hasCompileError || info.hasRuntimeError);
+        } else {
+          testCase = new BrowserTestCase(testDisplayName,
+              commandSet, configuration, completeHandler, expectations,
+              info, info.hasCompileError || info.hasRuntimeError);
         }
-      }
-      commands.add(new Command('python', args));
-
-      // Create BrowserTestCase and queue it.
-      var testCase = new BrowserTestCase('$suiteName/$testName',
-          commands, configuration, completeHandler, expectations,
-          info, info.hasCompileError || info.hasRuntimeError);
-      doTest(testCase);
+        doTest(testCase);
+        subtestIndex++;
+      } while(subtestIndex < subtestNames.length);
     }
   }
 
   /** Helper to create a compilation command for a single input file. */
   Command _compileCommand(String inputFile, String outputFile,
       String compiler, String dir, var vmOptions) {
     String executable = TestUtils.compilerPath(configuration);
     List<String> args = TestUtils.standardOptions(configuration);
     switch (compiler) {
       case 'dart2js':
@@ skipping 205 matching lines @@
    *   executing the copy command printed by the test script.
    *
    * This method is static as the map is cached and shared amongst
    * configurations, so it may not use [configuration].
    */
   static Map readOptionsFromFile(Path filePath) {
     RegExp testOptionsRegExp = const RegExp(r"// VMOptions=(.*)");
     RegExp dartOptionsRegExp = const RegExp(r"// DartOptions=(.*)");
     RegExp otherScriptsRegExp = const RegExp(r"// OtherScripts=(.*)");
     RegExp multiTestRegExp = const RegExp(r"/// [0-9][0-9]:(.*)");
+    RegExp multiHtmlTestRegExp =
+        const RegExp(r"useHtmlIndividualConfiguration()");
     RegExp staticTypeRegExp =
         const RegExp(r"/// ([0-9][0-9]:){0,1}\s*static type warning");
     RegExp compileTimeRegExp =
         const RegExp(r"/// ([0-9][0-9]:){0,1}\s*compile-time error");
     RegExp staticCleanRegExp = const RegExp(r"// @static-clean");
     RegExp leadingHashRegExp = const RegExp(r"^#", multiLine: true);
     RegExp isolateStubsRegExp = const RegExp(r"// IsolateStubs=(.*)");
     RegExp domImportRegExp =
         const RegExp(r"^#import.*(dart:(dom|html)|html\.dart).*\)",
                      multiLine: true);
@@ skipping 46 matching lines @@
       isStaticClean = true;
     }
 
     List<String> otherScripts = new List<String>();
     matches = otherScriptsRegExp.allMatches(contents);
     for (var match in matches) {
       otherScripts.addAll(match[1].split(' ').filter((e) => e != ''));
     }
 
     bool isMultitest = multiTestRegExp.hasMatch(contents);
+    bool isMultiHtmlTest = multiHtmlTestRegExp.hasMatch(contents);
     bool containsLeadingHash = leadingHashRegExp.hasMatch(contents);
     Match isolateMatch = isolateStubsRegExp.firstMatch(contents);
     String isolateStubs = isolateMatch != null ? isolateMatch[1] : '';
     bool containsDomImport = domImportRegExp.hasMatch(contents);
     bool isLibraryDefinition = libraryDefinitionRegExp.hasMatch(contents);
     bool containsSourceOrImport = sourceOrImportRegExp.hasMatch(contents);
     int numStaticTypeAnnotations = 0;
     for (var i in staticTypeRegExp.allMatches(contents)) {
       numStaticTypeAnnotations++;
     }
     int numCompileTimeAnnotations = 0;
     for (var i in compileTimeRegExp.allMatches(contents)) {
       numCompileTimeAnnotations++;
     }
 
+    // Note: This is brittle. It assumes you import unittest with no prefix and
+    // always directly call "test(".

[Bill Hesse, 2012/11/01 20:26:00]

It would be nice if there was something more explicit: a new method
parallel_test, or a comment // Multitest, or something.

+    RegExp numTests = new RegExp(r"\s*[^/]\s*test\('[^,']*");
+    List<String> subtestNames = [];
+    Iterator matchesIter = numTests.allMatches(contents).iterator();
+    while(matchesIter.hasNext && isMultiHtmlTest) {
+      String fullMatch = matchesIter.next().group(0);
+      subtestNames.add(fullMatch.substring(fullMatch.indexOf("'") + 1));
+    }
+
     return { "vmOptions": result,
              "dartOptions": dartOptions,
              "hasCompileError": hasCompileError,
              "hasRuntimeError": hasRuntimeError,
              "isStaticClean" : isStaticClean,
              "otherScripts": otherScripts,
              "isMultitest": isMultitest,
+             "isMultiHtmlTest": isMultiHtmlTest,
+             "subtestNames": subtestNames,
              "containsLeadingHash": containsLeadingHash,
              "isolateStubs": isolateStubs,
              "containsDomImport": containsDomImport,
              "isLibraryDefinition": isLibraryDefinition,
              "containsSourceOrImport": containsSourceOrImport,
              "numStaticTypeAnnotations": numStaticTypeAnnotations,
              "numCompileTimeAnnotations": numCompileTimeAnnotations };
   }
 
   List<List<String>> getVmOptions(Map optionsFromFile) {
@@ skipping 416 matching lines @@
  * $noCrash tests are expected to be flaky but not crash
  * $pass tests are expected to pass
  * $failOk tests are expected to fail that we won't fix
  * $fail tests are expected to fail that we should fix
  * $crash tests are expected to crash that we should fix
  * $timeout tests are allowed to timeout
 """;
     print(report);
    }
 }