Cache the tests across configurations in the test scripts.

tools/testing/dart/test_suite.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #library("test_suite");
 
 #import("status_file_parser.dart");
 #import("test_runner.dart");
 #import("multitest.dart");
 
 
 interface TestSuite {
   void forEachTest(Function onTest, [Function onDone]);

[Bill Hesse, 2011/12/09 13:20:15]

Why isn't this function signature changed?  Does this pass type checks?

[Mads Ager (google), 2011/12/09 13:21:58]

Whoops. Thanks! Done.

 }
 
 
 class CCTestListerIsolate extends Isolate {
   CCTestListerIsolate() : super.heavy();
 
   void main() {
     port.receive((String runnerPath, SendPort replyTo) {
       var p = new Process(runnerPath, ["--list"]);
       StringInputStream stdoutStream = new StringInputStream(p.stdout);
@@ skipping 68 matching lines @@
                           runnerPath,
                           args,
                           configuration,
                           completeHandler,
                           expectations));
 
       receiveTestName.receive(testNameHandler);
     }
   }
 
-  void forEachTest(Function onTest, [Function onDone]) {
+  void forEachTest(Function onTest, Map testCache, [Function onDone]) {
     doTest = onTest;
     doDone = (ignore) => (onDone != null) ? onDone() : null;
 
     testExpectations =
         new TestExpectations(complexMatching: complexStatusMatching());
     for (var statusFilePath in statusFilePaths) {
       ReadTestExpectationsInto(testExpectations,
                                statusFilePath,
                                configuration);
     }
 
     receiveTestName = new ReceivePort();
     new CCTestListerIsolate().spawn().then((port) {
         port.send(runnerPath, receiveTestName.toSendPort());
         receiveTestName.receive(testNameHandler);
     });
   }
 
   void completeHandler(TestCase testCase) {
   }
 }
 
 
+class TestInformation {
+  String filename;
+  Map optionsFromFile;
+  bool isNegative;
+  bool isNegativeIfChecked;
+  bool hasFatalTypeErrors;
+
+  TestInformation(this.filename, this.optionsFromFile, this.isNegative,
+                  this.isNegativeIfChecked, this.hasFatalTypeErrors);
+}
+
+
 class StandardTestSuite implements TestSuite {
   Map configuration;
   String suiteName;
   String directoryPath;
   List<String> statusFilePaths;
   Function doTest;
   Function doDone;
   int activeTestGenerators = 0;
   bool listingDone = false;
   TestExpectations testExpectations;
+  List<TestInformation> cachedTests;
 
   StandardTestSuite(Map this.configuration,
                     String this.suiteName,
                     String this.directoryPath,
                     List<String> this.statusFilePaths);
 
   void isTestFile(String filename) => filename.endsWith("Test.dart");
 
   void listRecursively() => false;
 
   void complexStatusMatching() => false;
 
   String shellPath() => TestUtils.dartShellFileName(configuration);
 
   List<String> additionalOptions() => [];
 
-  void forEachTest(Function onTest, [Function onDone = null]) {
+  void forEachTest(Function onTest, Map testCache, [Function onDone = null]) {
     doTest = onTest;
     doDone = (onDone != null) ? onDone : (() => null);
 
     // Read test expectations from status files.
     testExpectations =
         new TestExpectations(complexMatching: complexStatusMatching());
     for (var statusFilePath in statusFilePaths) {
       ReadTestExpectationsInto(testExpectations,
                                statusFilePath,
                                configuration);
     }
 
-    processDirectory();
+    // Checked if we have already found and generated the tests for
+    // this suite.
+    if (!testCache.containsKey(suiteName)) {
+      cachedTests = testCache[suiteName] = [];
+      processDirectory();
+    } else {
+      // We rely on enqueueing completing asynchroneously so use a

[Bill Hesse, 2011/12/09 13:20:15]

asynchronously

[Mads Ager (google), 2011/12/09 13:21:58]

Done.

+      // timer to make it so.
+      void enqueueCachedTests(Timer ignore) {
+        for (var info in testCache[suiteName]) {
+          enqueueTestCaseFromTestInformation(info);
+        }
+        doDone();
+      }
+      new Timer(enqueueCachedTests, 0, false);
+    }
   }
 
   void processDirectory() {
     directoryPath = getDirname(directoryPath);
     Directory dir = new Directory(directoryPath);
     dir.errorHandler = (s) {
       throw s;
     };
     dir.fileHandler = processFile;
     dir.doneHandler = directoryListingDone;
     dir.list(recursive: listRecursively());
   }
 
-  Function makeTestCaseCreator(Map optionsFromFile, Map configuration) {
+  void enqueueTestCaseFromTestInformation(TestInformation info) {
+    var filename = info.filename;
+    var optionsFromFile = info.optionsFromFile;
+    var isNegative = info.isNegative;
+
+    // Look up expectations in status files using a modified file path.
+    String pathSeparator = new Platform().pathSeparator();
+    String testName;
+    int start = filename.lastIndexOf('src' + pathSeparator);
+    if (start != -1) {
+      testName = filename.substring(start + 4, filename.length - 5);
+    } else if (optionsFromFile['isMultitest']) {
+      start = filename.lastIndexOf(pathSeparator);
+      int middle = filename.lastIndexOf('_');
+      testName = filename.substring(start + 1, middle) + pathSeparator +
+          filename.substring(middle + 1, filename.length - 5);
+    } else {
+      // This case is hit by the dartc client compilation
+      // tests. These tests are pretty broken compared to the
+      // rest. They use the .dart suffix in the status files. They
+      // find tests in weird ways (testing that they contain "#").
+      // They need to be redone.
+      start = filename.indexOf(directoryPath);
+      testName = filename.substring(start + directoryPath.length + 1,
+                                    filename.length);
+    }
+    Set<String> expectations = testExpectations.expectations(testName);
+    if (configuration["report"]) {
+      // Tests with multiple VMOptions are counted more than once.
+      for (var dummy in optionsFromFile["vmOptions"]) {
+        SummaryReport.add(expectations);
+      }
+    }
+    if (expectations.contains(SKIP)) return;
+
+    // Only dartc supports fatal type errors. Enable fatal type
+    // errors with a flag and treat tests that have fatal type
+    // errors as negetive.

[Bill Hesse, 2011/12/09 13:20:15]

negative

[Mads Ager (google), 2011/12/09 13:21:58]

Done.

+    var enableFatalTypeErrors =
+        (info.hasFatalTypeErrors && configuration['component'] == 'dartc');
+    var argumentLists = argumentListsFromFile(filename,
+                                              optionsFromFile,
+                                              enableFatalTypeErrors);
+    isNegative = isNegative ||
+        (configuration['checked'] && info.isNegativeIfChecked) ||
+        enableFatalTypeErrors;
+
+    for (var args in argumentLists) {
+      doTest(new TestCase('$suiteName/$testName',
+                          shellPath(),
+                          args,
+                          configuration,
+                          completeHandler,
+                          expectations,
+                          isNegative));
+    }
+  }
+
+  Function makeTestCaseCreator(Map optionsFromFile) {
     return (String filename,
             bool isNegative,
             [bool isNegativeIfChecked = false,
-             bool enableFatalTypeErrors = false]) {
-      // Look up expectations in status files using a modified file path.
-      String pathSeparator = new Platform().pathSeparator();
-      String testName;
-      int start = filename.lastIndexOf('src' + pathSeparator);
-      if (start != -1) {
-        testName = filename.substring(start + 4, filename.length - 5);
-      } else if (optionsFromFile['isMultitest']) {
-        start = filename.lastIndexOf(pathSeparator);
-        int middle = filename.lastIndexOf('_');
-        testName = filename.substring(start + 1, middle) + pathSeparator +
-            filename.substring(middle + 1, filename.length - 5);
-      } else {
-        // This case is hit by the dartc client compilation
-        // tests. These tests are pretty broken compared to the
-        // rest. They use the .dart suffix in the status files. They
-        // find tests in weird ways (testing that they contain "#"). 
-        // They need to be redone.
-        start = filename.indexOf(directoryPath);
-        testName = filename.substring(start + directoryPath.length + 1,
-                                      filename.length);
-      }
-      Set<String> expectations = testExpectations.expectations(testName);
-      if (configuration["report"]) {
-        // Tests with multiple VMOptions are counted more than once.
-        for (var dummy in optionsFromFile["vmOptions"]) {
-          SummaryReport.add(expectations);
-        }
-      }
-      if (expectations.contains(SKIP)) return;
-
-      isNegative = isNegative ||
-          (configuration['checked'] && isNegativeIfChecked);
-      var argumentLists = argumentListsFromFile(filename,
-                                                optionsFromFile,
-                                                enableFatalTypeErrors);
-      for (var args in argumentLists) {
-        doTest(new TestCase('$suiteName/$testName',
-                            shellPath(),
-                            args,
-                            configuration,
-                            completeHandler,
-                            expectations,
-                            isNegative));
-      }
+             bool hasFatalTypeErrors = false]) {
+      // Cache the test information for each test case.
+      var info = new TestInformation(filename,
+                                     optionsFromFile,
+                                     isNegative,
+                                     isNegativeIfChecked,
+                                     hasFatalTypeErrors);
+      cachedTests.add(info);
+      enqueueTestCaseFromTestInformation(info);
     };
   }
 
   void processFile(String filename) {
     if (!isTestFile(filename)) return;
 
     // Only run the tests that match the pattern.
     RegExp pattern = configuration['selectors'][suiteName];
     if (!pattern.hasMatch(filename)) return;
 
     var optionsFromFile = optionsFromFile(filename);
-    Function createTestCase =
-        makeTestCaseCreator(optionsFromFile, configuration);
+    Function createTestCase = makeTestCaseCreator(optionsFromFile);
 
     if (optionsFromFile['isMultitest']) {
-      bool supportsFatalTypeErrors = (configuration['component'] == 'dartc');
       testGeneratorStarted();
       DoMultitest(filename,
-                  TestUtils.buildDir(configuration),
+                  TestUtils.outputDir(configuration),
                   directoryPath,
-                  supportsFatalTypeErrors,
                   createTestCase,
                   testGeneratorDone);
     } else {
       createTestCase(filename, optionsFromFile['isNegative']);
     }
   }
 
   void testGeneratorStarted() {
     ++activeTestGenerators;
   }
@@ skipping 13 matching lines @@
   }
 
   void completeHandler(TestCase testCase) {
   }
 
   List<List<String>> argumentListsFromFile(String filename,
                                            Map optionsFromFile,
                                            bool enableFatalTypeErrors) {
     List args = TestUtils.standardOptions(configuration);
     args.addAll(additionalOptions());
-    if (enableFatalTypeErrors) args.add('--fatal-type-errors');
+    if (enableFatalTypeErrors && configuration['component'] == 'dartc') {
+      args.add('--fatal-type-errors');
+    }
 
     bool isMultitest = optionsFromFile["isMultitest"];
     List<String> dartOptions = optionsFromFile["dartOptions"];
     List<List<String>> vmOptionsList = optionsFromFile["vmOptions"];
     Expect.isTrue(!isMultitest || dartOptions == null);
     if (dartOptions == null) {
       args.add(filename);
     } else {
       var filename = dartOptions[0];
       // TODO(ager): Get rid of this hack when the runtime checkout goes away.
       var file = new File(filename);
       if (!file.existsSync()) {
         filename = '../$filename';
         Expect.isTrue(new File(filename).existsSync());
         dartOptions[0] = filename;
       }
       args.addAll(dartOptions);
     }
 
     var result = new List<List<String>>();
     Expect.isFalse(vmOptionsList.isEmpty(), "empty vmOptionsList");
     for (var vmOptions in vmOptionsList) {
-      if (isMultitest) {
-        // Make copy of vmOptions, since we will modify it at each iteration.
-        vmOptions = new List<String>.from(vmOptions);
-      }
-      vmOptions.addAll(args);
-      result.add(vmOptions);
+      var options = new List<String>.from(vmOptions);
+      options.addAll(args);
+      result.add(options);
     }
 
     return result;
   }
 
   Map optionsFromFile(String filename) {
     RegExp testOptionsRegExp = const RegExp(@"// VMOptions=(.*)");
     RegExp dartOptionsRegExp = const RegExp(@"// DartOptions=(.*)");
     RegExp multiTestRegExp = const RegExp(@"/// [0-9][0-9]:(.*)");
     RegExp leadingHashRegExp = const RegExp(@"^#", multiLine: true);
@@ skipping 137 matching lines @@
   }
 
   static String dartcCompilationShellPath(Map configuration) {
     var name = buildDir(configuration) + 'compiler/bin/dartc';
     if (!(new File(name)).existsSync()) {
       throw "Executable '$name' does not exist";
     }
     return name;
   }
 
-  static String buildDir(Map configuration) {
-    var buildDir = '';
+  static String outputDir(Map configuration) {
+    var outputDir = '';
     var system = configuration['system'];
     if (system == 'linux') {
-      buildDir = 'out/';
+      outputDir = 'out/';
     } else if (system == 'macos') {
-      buildDir = 'xcodebuild/';
+      outputDir = 'xcodebuild/';
     }
+    return outputDir;
+  }
+
+  static String buildDir(Map configuration) {
+    var buildDir = outputDir(configuration);
     buildDir += (configuration['mode'] == 'debug') ? 'Debug_' : 'Release_';
     buildDir += configuration['arch'] + '/';
     return buildDir;
   }
 
   static List<String> standardOptions(Map configuration) {
     List args = ["--ignore-unrecognized-flags"];
     if (configuration["checked"]) {
       args.add('--enable_asserts');
       args.add("--enable_type_checks");
@@ skipping 55 matching lines @@
  * $noCrash tests are expected to be flaky but not crash
  * $pass tests are expected to pass
  * $failOk tests are expected to fail that we won't fix
  * $fail tests are expected to fail that we should fix
  * $crash tests are expected to crash that we should fix
  * $timeout tests are allowed to timeout\
 """;
     print(report);
   }
 }