Refactor test.dart.

tools/testing/dart/test_suite.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for enumerating and preparing tests.
  *
  * This library includes:
  *
  * - Creating tests by listing all the Dart files in certain directories,
@@ skipping 22 matching lines @@
 typedef void CreateTest(Path filePath,
                         bool hasCompileError,
                         bool hasRuntimeError,
                         {bool isNegativeIfChecked,
                          bool hasFatalTypeErrors,
                          Set<String> multitestOutcome});
 
 typedef void VoidFunction();
 
 /**
+ * Calls [function] asynchronously. Returns a future that completes with the
+ * result of the function. If the function is `null`, returns a future that
+ * completes immediately with `null`.
+ */
+Future asynchronously(function()) {
+  if (function == null) return new Future.immediate(null);
+
+  var completer = new Completer();
+  new Timer(0, (_) {
+    completer.complete(function());
+  });
+
+  return completer.future;
+}
+
+/** A completer that waits until all added [Future]s complete. */
+// TODO(rnystrom): Copied from web_components. Remove from here when it gets
+// added to dart:core. (See #6626.)
+class FutureGroup {
+  const _FINISHED = -1;
+  int _pending = 0;
+  Completer<List> _completer = new Completer<List>();
+  final List<Future> futures = <Future>[];
+
+  /**
+   * Wait for [task] to complete (assuming this barrier has not already been
+   * marked as completed, otherwise you'll get an exception indicating that a
+   * future has already been completed).
+   */
+  void add(Future task) {
+    if (_pending == _FINISHED) {
+      throw new FutureAlreadyCompleteException();
+    }
+    _pending++;
+    futures.add(task);
+    task.handleException(
+        (e) => _completer.completeException(e, task.stackTrace));
+    task.then((_) {
+      _pending--;
+      if (_pending == 0) {
+        _pending = _FINISHED;
+        _completer.complete(futures);
+      }
+    });
+  }
+
+  Future<List> get future => _completer.future;
+}
+
+/**
  * A TestSuite represents a collection of tests.  It creates a [TestCase]
  * object for each test to be run, and passes the test cases to a callback.
  *
  * Most TestSuites represent a directory or directory tree containing tests,
  * and a status file containing the expected results when these tests are run.
  */
 abstract class TestSuite {
+  final Map configuration;
+  final String suiteName;
+
+  TestSuite(this.configuration, this.suiteName);
+
+  /**
+   * The output directory for this suite's configuration.
+   */
+  String get buildDir {
+    var mode = (configuration['mode'] == 'debug') ? 'Debug' : 'Release';
+    var arch = configuration['arch'].toUpperCase();
+    return "${TestUtils.outputDir(configuration)}$mode$arch";
+  }
+
+  /**
+   * The path to the compiler for this suite's configuration. Returns `null` if
+   * no compiler should be used.
+   */
+  String get compilerPath {
+    if (configuration['compiler'] == 'none') {
+      return null;  // No separate compiler for dartium tests.
+    }
+    var name = '$buildDir/${compilerName}';
+    if (!(new File(name)).existsSync() && !configuration['list']) {
+      throw "Executable '$name' does not exist";
+    }
+    return name;
+  }
+
+  /**
+   * The name of the compiler for this suite's configuration. Throws an error
+   * if the configuration does not use a compiler.
+   */
+  String get compilerName {
+    switch (configuration['compiler']) {
+      case 'dartc':
+      case 'dart2js':
+      case 'dart2dart':
+        return executableName;
+      default:
+        throw "Unknown compiler for: ${configuration['compiler']}";
+    }
+  }
+
+  /**
+   * The file name of the executable used to run this suite's tests.
+   */
+  String get executableName {
+    String suffix = getExecutableSuffix(configuration['compiler']);
+    switch (configuration['compiler']) {
+      case 'none':
+        return 'dart$suffix';
+      case 'dartc':
+        return 'analyzer/bin/dart_analyzer$suffix';
+      case 'dart2js':
+      case 'dart2dart':
+        var prefix = '';
+        if (configuration['use_sdk']) {
+          prefix = 'dart-sdk/bin/';
+        }
+        if (configuration['host_checked']) {
+          // The script dart2js_developer is not in the SDK.
+          return 'dart2js_developer$suffix';
+        } else {
+          return '${prefix}dart2js$suffix';
+        }
+        break;
+      default:
+        throw "Unknown executable for: ${configuration['compiler']}";
+    }
+  }
+
+  /**
+   * The file name of the d8 executable.
+   */
+  String get d8FileName {
+    var suffix = getExecutableSuffix('d8');
+    var d8 = '$buildDir/d8$suffix';
+    TestUtils.ensureExists(d8, configuration);
+    return d8;
+  }
+
+  String get dartShellFileName {
+    var name = configuration['dart'];
+    if (name == '') {
+      name = '$buildDir/$executableName';
+    }
+    TestUtils.ensureExists(name, configuration);
+    return name;
+  }
+
+  String get jsShellFileName {
+    var executableSuffix = getExecutableSuffix('jsshell');
+    var executable = 'jsshell$executableSuffix';
+    var jsshellDir = '${TestUtils.dartDir()}/tools/testing/bin';
+    return '$jsshellDir/$executable';
+  }
+
+  /**
+   * The file name of the Dart VM executable.
+   */
+  String get vmFileName {
+    var suffix = getExecutableSuffix('vm');
+    var vm = '$buildDir/dart$suffix';
+    TestUtils.ensureExists(vm, configuration);
+    return vm;
+  }
+
+  /**
+   * The file extension (if any) that should be added to the given executable
+   * name for the current platform.
+   */
+  String getExecutableSuffix(String executable) {
+    if (Platform.operatingSystem == 'windows') {
+      if (executable == 'd8' || executable == 'vm' || executable == 'none') {
+        return '.exe';
+      } else {
+        return '.bat';
+      }
+    }
+    return '';
+  }
+
   /**
    * Call the callback function onTest with a [TestCase] argument for each
    * test in the suite.  When all tests have been processed, call [onDone].
    *
    * The [testCache] argument provides a persistent store that can be used to
    * cache information about the test suite, so that directories do not need
    * to be listed each time.
    */
   void forEachTest(TestCaseEvent onTest, Map testCache, [VoidFunction onDone]);
 }
 
 
-// TODO(1030): remove once in the corelib.
-bool Contains(element, collection) => collection.indexOf(element) >= 0;
-
-
 void ccTestLister() {
   port.receive((String runnerPath, SendPort replyTo) {
     Future processFuture = Process.start(runnerPath, ["--list"]);
     processFuture.then((p) {
       // Drain stderr to not leak resources.
       p.stderr.onData = p.stderr.read;
       StringInputStream stdoutStream = new StringInputStream(p.stdout);
       var streamDone = false;
       var processExited = false;
       checkDone() {
@@ skipping 30 matching lines @@
 
 
 /**
  * A specialized [TestSuite] that runs tests written in C to unit test
  * the Dart virtual machine and its API.
  *
  * The tests are compiled into a monolithic executable by the build step.
  * The executable lists its tests when run with the --list command line flag.
  * Individual tests are run by specifying them on the command line.
  */
-class CCTestSuite implements TestSuite {
-  Map configuration;
-  final String suiteName;
+class CCTestSuite extends TestSuite {
   final String testPrefix;
   String runnerPath;
   final String dartDir;
   List<String> statusFilePaths;
   TestCaseEvent doTest;
   VoidFunction doDone;
   ReceivePort receiveTestName;
   TestExpectations testExpectations;
 
-  CCTestSuite(Map this.configuration,
-              String this.suiteName,
+  CCTestSuite(Map configuration,
+              String suiteName,
               String runnerName,
               List<String> this.statusFilePaths,
               {this.testPrefix: ''})
-      : dartDir = TestUtils.dartDir().toNativePath() {
-    runnerPath = '${TestUtils.buildDir(configuration)}/$runnerName';
+      : super(configuration, suiteName),
+        dartDir = TestUtils.dartDir().toNativePath() {
+    runnerPath = '$buildDir/$runnerName';
   }
 
   void testNameHandler(String testName, ignore) {
     if (testName == "") {
       receiveTestName.close();
-      doDone();
+
+      if (doDone != null) doDone();
     } else {
       // Only run the tests that match the pattern. Use the name
       // "suiteName/testName" for cc tests.
       RegExp pattern = configuration['selectors'][suiteName];
       String constructedName = '$suiteName/$testPrefix$testName';
       if (!pattern.hasMatch(constructedName)) return;
 
       var expectations = testExpectations.expectations(
           '$testPrefix$testName');
 
       if (configuration["report"]) {
         SummaryReport.add(expectations);
       }
 
       if (expectations.contains(SKIP)) return;
 
       var args = TestUtils.standardOptions(configuration);
       args.add(testName);
 
       doTest(new TestCase(constructedName,
                           [new Command(runnerPath, args)],
                           configuration,
                           completeHandler,
                           expectations));
     }
   }
 
   void forEachTest(TestCaseEvent onTest, Map testCache, [VoidFunction onDone]) {
     doTest = onTest;
-    doDone = () => (onDone != null) ? onDone() : null;
+    doDone = onDone;
 
     var filesRead = 0;
     void statusFileRead() {
       filesRead++;
       if (filesRead == statusFilePaths.length) {
         receiveTestName = new ReceivePort();
         var port = spawnFunction(ccTestLister);
         port.send(runnerPath, receiveTestName.toSendPort());
         receiveTestName.receive(testNameHandler);
       }
@@ skipping 23 matching lines @@
   Set<String> multitestOutcome;
 
   TestInformation(this.filePath, this.optionsFromFile,
                   this.hasCompileError, this.hasRuntimeError,
                   this.isNegativeIfChecked, this.hasFatalTypeErrors,
                   this.multitestOutcome) {
     Expect.isTrue(filePath.isAbsolute);
   }
 }
 
-
 /**
  * A standard [TestSuite] implementation that searches for tests in a
  * directory, and creates [TestCase]s that compile and/or run them.
  */
-class StandardTestSuite implements TestSuite {
-  Map configuration;
-  String suiteName;
-  Path suiteDir;
-  List<String> statusFilePaths;
+class StandardTestSuite extends TestSuite {
+  final Path suiteDir;
+  final List<String> statusFilePaths;
   TestCaseEvent doTest;
-  VoidFunction doDone;
-  int activeTestGenerators = 0;
-  bool listingDone = false;
   TestExpectations testExpectations;
   List<TestInformation> cachedTests;
   final Path dartDir;
   Predicate<String> isTestFilePredicate;
-  bool _listRecursive;
+  final bool listRecursively;
 
-  StandardTestSuite(this.configuration,
-                    this.suiteName,
+  StandardTestSuite(Map configuration,
+                    String suiteName,
                     Path suiteDirectory,
                     this.statusFilePaths,
                     {this.isTestFilePredicate,
                     bool recursive: false})
-  : dartDir = TestUtils.dartDir(), _listRecursive = recursive,
+  : super(configuration, suiteName),
+    dartDir = TestUtils.dartDir(),
+    listRecursively = recursive,
     suiteDir = TestUtils.dartDir().join(suiteDirectory);
 
   /**
    * Creates a test suite whose file organization matches an expected structure.
    * To use this, your suite should look like:
    *
    *     dart/
    *       path/
    *         to/
    *           mytestsuite/
@@ skipping 31 matching lines @@
    * The default implementation assumes a file is a test if
    * it ends in "Test.dart".
    */
   bool isTestFile(String filename) {
     // Use the specified predicate, if provided.
     if (isTestFilePredicate != null) return isTestFilePredicate(filename);
 
     return filename.endsWith("Test.dart");
   }
 
-  bool listRecursively() => _listRecursive;
-
-  String shellPath() => TestUtils.dartShellFileName(configuration);
-
   List<String> additionalOptions(Path filePath) => [];
 
   void forEachTest(TestCaseEvent onTest, Map testCache, [VoidFunction onDone]) {
-    // If DumpRenderTree/Dartium is required, and not yet updated,
-    // wait for update.
+    updateDartium().chain((_) {
+      doTest = onTest;
+
+      return readExpectations();
+    }).chain((expectations) {
+      testExpectations = expectations;
+
+      // Checked if we have already found and generated the tests for
+      // this suite.
+      if (!testCache.containsKey(suiteName)) {
+        cachedTests = testCache[suiteName] = [];
+        return enqueueTests();
+      } else {
+        // We rely on enqueueing completing asynchronously.
+        return asynchronously(() {
+          for (var info in testCache[suiteName]) {
+            enqueueTestCaseFromTestInformation(info);
+          }
+        });
+      }
+    }).then((_) {
+      if (onDone != null) onDone();
+    });
+  }
+
+  /**
+   * If DumpRenderTree/Dartium is required, and not yet updated, waits for
+   * the update then completes. Otherwise completes immediately.
+   */
+  Future updateDartium() {
+    var completer = new Completer();
     var updater = runtimeUpdater(configuration);
-    if (updater !== null && !updater.updated) {
-      Expect.isTrue(updater.isActive);
-      updater.onUpdated.add(() {
-        forEachTest(onTest, testCache, onDone);
-      });
-      return;
+    if (updater == null || updater.updated) {
+      return new Future.immediate(null);
     }
 
-    doTest = onTest;
-    doDone = (onDone != null) ? onDone : (() => null);
+    Expect.isTrue(updater.isActive);
+    updater.onUpdated.add(() => completer.complete(null));
+
+    return completer.future;
+  }
+
+  /**
+   * Reads the status files and completes with the parsed expectations.
+   */
+  Future<TestExpectations> readExpectations() {
+    var completer = new Completer();
+    var expectations = new TestExpectations();
 
     var filesRead = 0;
     void statusFileRead() {
       filesRead++;
       if (filesRead == statusFilePaths.length) {
-        // Checked if we have already found and generated the tests for
-        // this suite.
-        if (!testCache.containsKey(suiteName)) {
-          cachedTests = testCache[suiteName] = [];
-          processDirectory();
-        } else {
-          // We rely on enqueueing completing asynchronously so use a
-          // timer to make it so.
-          void enqueueCachedTests(Timer ignore) {
-            for (var info in testCache[suiteName]) {
-              enqueueTestCaseFromTestInformation(info);
-            }
-            doDone();
-          }
-          new Timer(0, enqueueCachedTests);
-        }
+        completer.complete(expectations);
       }
     }
 
-    // Read test expectations from status files.
-    testExpectations = new TestExpectations();
     for (var statusFilePath in statusFilePaths) {
-      // [forDirectory] adds name_dart2js.status for all tests suites, use it if
-      // it exists, but otherwise skip it and don't fail.
+      // [forDirectory] adds name_dart2js.status for all tests suites. Use it
+      // if it exists, but otherwise skip it and don't fail.
       if (statusFilePath.endsWith('_dart2js.status')) {
-        File file = new File.fromPath(dartDir.append(statusFilePath));
+        var file = new File.fromPath(dartDir.append(statusFilePath));
         if (!file.existsSync()) {
           filesRead++;
           continue;
         }
       }
-      ReadTestExpectationsInto(testExpectations,
+
+      ReadTestExpectationsInto(expectations,
                                dartDir.append(statusFilePath).toNativePath(),
-                               configuration,
-                               statusFileRead);
+                               configuration, statusFileRead);
     }
+
+    return completer.future;
   }
 
-  void processDirectory() {
+  Future enqueueTests() {
     Directory dir = new Directory.fromPath(suiteDir);
-    dir.exists().then((exists) {
+    return dir.exists().chain((exists) {
       if (!exists) {
         print('Directory containing tests not found: $suiteDir');
-        directoryListingDone(false);
+        return new Future.immediate(null);
       } else {
-        var lister = dir.list(recursive: listRecursively());
-        lister.onFile = processFile;
-        lister.onDone = directoryListingDone;
+        var group = new FutureGroup();
+        enqueueDirectory(dir, group);
+        return group.future;
       }
     });
   }
 
+  void enqueueDirectory(Directory dir, FutureGroup group) {
+    var listCompleter = new Completer();
+    group.add(listCompleter.future);
+
+    var lister = dir.list(recursive: listRecursively);
+    lister.onFile = (file) => enqueueFile(file, group);
+    lister.onDone = listCompleter.complete;
+  }
+
+  void enqueueFile(String filename, FutureGroup group) {
+    if (!isTestFile(filename)) return;
+    Path filePath = new Path.fromNative(filename);
+
+    // Only run the tests that match the pattern.
+    RegExp pattern = configuration['selectors'][suiteName];
+    if (!pattern.hasMatch('$filePath')) return;
+    if (filePath.filename.endsWith('test_config.dart')) return;
+
+    var optionsFromFile = readOptionsFromFile(filePath);
+    CreateTest createTestCase = makeTestCaseCreator(optionsFromFile);
+
+    if (optionsFromFile['isMultitest']) {
+      group.add(doMultitest(filePath, buildDir, suiteDir, createTestCase));
+    } else {
+      createTestCase(filePath,
+                     optionsFromFile['hasCompileError'],
+                     optionsFromFile['hasRuntimeError']);
+    }
+  }
+
   void enqueueTestCaseFromTestInformation(TestInformation info) {
     var filePath = info.filePath;
     var optionsFromFile = info.optionsFromFile;
     var isNegative = info.hasCompileError;
     if (info.hasRuntimeError && hasRuntime) {
       isNegative = true;
     }
 
     // Look up expectations in status files using a test name generated
     // from the test file's path.
@@ skipping 101 matching lines @@
                           info: info));
     }
   }
 
   List<Command> makeCommands(TestInformation info, var vmOptions, var args) {
     switch (configuration['compiler']) {
     case 'dart2js':
       args = new List.from(args);
       String tempDir = createOutputDirectory(info.filePath, '');
       args.add('--out=$tempDir/out.js');
-      List<Command> commands = <Command>[new Command(shellPath(), args)];
+      List<Command> commands = <Command>[new Command(dartShellFileName, args)];
       if (info.hasCompileError) {
         // Do not attempt to run the compiled result. A compilation
         // error should be reported by the compilation command.
       } else if (configuration['runtime'] == 'd8') {
-        var d8 = TestUtils.d8FileName(configuration);
-        commands.add(new Command(d8, ['$tempDir/out.js']));
+        commands.add(new Command(d8FileName, ['$tempDir/out.js']));
       } else if (configuration['runtime'] == 'jsshell') {
-        var jsshell = TestUtils.jsshellFileName(configuration);
-        commands.add(new Command(jsshell, ['$tempDir/out.js']));
+        commands.add(new Command(jsShellFileName, ['$tempDir/out.js']));
       }
       return commands;
 
     case 'dart2dart':
       var compilerArguments = new List.from(args);
       var additionalFlags =
           configuration['additional-compiler-flags'].split(' ');
       for (final flag in additionalFlags) {
         if (flag.isEmpty) continue;
         compilerArguments.add(flag);
       }
       compilerArguments.add('--output-type=dart');
       String tempDir = createOutputDirectory(info.filePath, '');
       compilerArguments.add('--out=$tempDir/out.dart');
       List<Command> commands =
-          <Command>[new Command(shellPath(), compilerArguments)];
+          <Command>[new Command(dartShellFileName, compilerArguments)];
       if (info.hasCompileError) {
         // Do not attempt to run the compiled result. A compilation
         // error should be reported by the compilation command.
       } else if (configuration['runtime'] == 'vm') {
         // TODO(antonm): support checked.
         var vmArguments = new List.from(vmOptions);
         vmArguments.addAll([
             '--ignore-unrecognized-flags', '$tempDir/out.dart']);
-        commands.add(new Command(
-            TestUtils.vmFileName(configuration),
-            vmArguments));
+        commands.add(new Command(vmFileName, vmArguments));
       } else {
         throw 'Unsupported runtime ${configuration["runtime"]} for dart2dart';
       }
       return commands;
 
     case 'none':
     case 'dartc':
       var arguments = new List.from(vmOptions);
       arguments.addAll(args);
-      return <Command>[new Command(shellPath(), arguments)];
+      return <Command>[new Command(dartShellFileName, arguments)];
 
     default:
       throw 'Unknown compiler ${configuration["compiler"]}';
     }
   }
 
   CreateTest makeTestCaseCreator(Map optionsFromFile) {
     return (Path filePath,
             bool hasCompileError,
             bool hasRuntimeError,
             {bool isNegativeIfChecked: false,
              bool hasFatalTypeErrors: false,
              Set<String> multitestOutcome: null}) {
       // Cache the test information for each test case.
       var info = new TestInformation(filePath,
                                      optionsFromFile,
                                      hasCompileError,
                                      hasRuntimeError,
                                      isNegativeIfChecked,
                                      hasFatalTypeErrors,
                                      multitestOutcome);
       cachedTests.add(info);
       enqueueTestCaseFromTestInformation(info);
     };
   }
 
-  void processFile(String filename) {
-    if (!isTestFile(filename)) return;
-    Path filePath = new Path.fromNative(filename);
-
-    // Only run the tests that match the pattern.
-    RegExp pattern = configuration['selectors'][suiteName];
-    if (!pattern.hasMatch('$filePath')) return;
-    if (filePath.filename.endsWith('test_config.dart')) return;
-
-    var optionsFromFile = readOptionsFromFile(filePath);
-    CreateTest createTestCase = makeTestCaseCreator(optionsFromFile);
-
-    if (optionsFromFile['isMultitest']) {
-      testGeneratorStarted();
-      DoMultitest(filePath,
-                  TestUtils.buildDir(configuration),
-                  suiteDir,
-                  createTestCase,
-                  testGeneratorDone);
-    } else {
-      createTestCase(filePath,
-                     optionsFromFile['hasCompileError'],
-                     optionsFromFile['hasRuntimeError']);
-    }
-  }
-
   /**
    * The [StandardTestSuite] has support for tests that
    * compile a test from Dart to JavaScript, and then run the resulting
    * JavaScript.  This function creates a working directory to hold the
    * JavaScript version of the test, and copies the appropriate framework
    * files to that directory.  It creates a [BrowserTestCase], which has
    * two sequential steps to be run by the [ProcessQueue] when the test is
-   * executed: a compilation step and an execution step, both with the 
+   * executed: a compilation step and an execution step, both with the
    * appropriate executable and arguments. The [expectations] object can be
    * either a Set<String> if the test is a regular test, or a Map<String
    * subTestName, Set<String>> if we are running a browser multi-test (one
    * compilation and many browser runs).
    */
   void enqueueBrowserTest(TestInformation info,
                           String testName,
                           Object expectations,
                           bool isWrappingRequired) {
     Map optionsFromFile = info.optionsFromFile;
@@ skipping 187 matching lines @@
         }
         doTest(testCase);
         subtestIndex++;
       } while(subtestIndex < subtestNames.length);
     }
   }
 
   /** Helper to create a compilation command for a single input file. */
   Command _compileCommand(String inputFile, String outputFile,
       String compiler, String dir, var vmOptions) {
-    String executable = TestUtils.compilerPath(configuration);
+    String executable = compilerPath;
     List<String> args = TestUtils.standardOptions(configuration);
     switch (compiler) {
       case 'dart2js':
       case 'dart2dart':
         if (compiler == 'dart2dart') args.add('--out=$outputFile');
         args.add('--out=$outputFile');
         args.add(inputFile);
         break;
       default:
         Expect.fail('unimplemented compiler $compiler');
     }
     if (executable.endsWith('.dart')) {
       // Run the compiler script via the Dart VM.
       args.insertRange(0, 1, executable);
-      executable = TestUtils.dartShellFileName(configuration);
+      executable = dartShellFileName;
     }
     return new Command(executable, args);
   }
 
   /**
    * Create a directory for the generated test.  If a Dart language test
    * needs to be run in a browser, the Dart test needs to be embedded in
    * an HTML page, with a testing framework based on scripting and DOM events.
    * These scripts and pages are written to a generated_test directory
    * inside the build directory of the checkout.
    *
    * Those tests which are already HTML web applications (web tests), with
    * resources including CSS files and HTML files, need to be compiled into
    * a work directory where the relative URLS to the resources work.
    * We use a subdirectory of the build directory that is the same number
    * of levels down in the checkout as the original path of the web test.
    */
   String createOutputDirectory(Path testPath, String optionsName) {
     Path relative = testPath.relativeTo(TestUtils.dartDir());
     relative = relative.directoryPath.append(relative.filenameWithoutExtension);
     String testUniqueName = relative.toString().replaceAll('/', '_');
     if (!optionsName.isEmpty) {
       testUniqueName = '$testUniqueName-$optionsName';
     }
 
     // Create '[build dir]/generated_tests/$compiler-$runtime/$testUniqueName',
     // including any intermediate directories that don't exist.
-    var generatedTestPath = Strings.join(
-        [TestUtils.buildDir(configuration),
-         'generated_tests',
-         "${configuration['compiler']}-${configuration['runtime']}",
-         testUniqueName], '/');
+    var generatedTestPath = Strings.join([
+        buildDir,
+        'generated_tests',
+        "${configuration['compiler']}-${configuration['runtime']}",
+        testUniqueName
+    ], '/');
 
     TestUtils.mkdirRecursive(new Path('.'), new Path(generatedTestPath));
     return new File(generatedTestPath).fullPathSync().replaceAll('\\', '/');
   }
 
   String get scriptType {
     switch (configuration['compiler']) {
       case 'none':
       case 'dart2dart':
         return 'application/dart';
@@ skipping 40 matching lines @@
     if (configuration['dartium'] != '') {
       return configuration['dartium'];
     }
     if (Platform.operatingSystem == 'macos') {
       return dartDir.append('client/tests/dartium/Chromium.app/Contents/'
                             'MacOS/Chromium').toNativePath();
     }
     return dartDir.append('client/tests/dartium/chrome').toNativePath();
   }
 
-  void testGeneratorStarted() {
-    ++activeTestGenerators;
-  }
-
-  void testGeneratorDone() {
-    --activeTestGenerators;
-    if (activeTestGenerators == 0 && listingDone) {
-      doDone();
-    }
-  }
-
-  void directoryListingDone(ignore) {
-    listingDone = true;
-    if (activeTestGenerators == 0) {
-      doDone();
-    }
-  }
-
   void completeHandler(TestCase testCase) {
   }
 
   List<String> commonArgumentsFromFile(Path filePath, Map optionsFromFile) {
     List args = TestUtils.standardOptions(configuration);
     args.addAll(additionalOptions(filePath));
     if (configuration['compiler'] == 'dartc') {
       args.add('--error_format');
       args.add('machine');
     }
@@ skipping 190 matching lines @@
              "containsLeadingHash": containsLeadingHash,
              "isolateStubs": isolateStubs,
              "containsDomImport": containsDomImport,
              "isLibraryDefinition": isLibraryDefinition,
              "containsSourceOrImport": containsSourceOrImport,
              "numStaticTypeAnnotations": numStaticTypeAnnotations,
              "numCompileTimeAnnotations": numCompileTimeAnnotations };
   }
 
   List<List<String>> getVmOptions(Map optionsFromFile) {
-    bool needsVmOptions = Contains(configuration['compiler'],
-                                   const ['none', 'dart2dart', 'dartc']) &&
-                          Contains(configuration['runtime'],
-                                   const ['none', 'vm', 'drt', 'dartium']);
+    var COMPILERS = const ['none', 'dart2dart', 'dartc'];
+    var RUNTIMES = const ['none', 'vm', 'drt', 'dartium'];
+    var needsVmOptions = COMPILERS.contains(configuration['compiler']) &&
+                         RUNTIMES.contains(configuration['runtime']);
     if (!needsVmOptions) return [[]];
     return optionsFromFile['vmOptions'];
   }
 }
 
 
 class DartcCompilationTestSuite extends StandardTestSuite {
   List<String> _testDirs;
-  int activityCount = 0;
 
   DartcCompilationTestSuite(Map configuration,
                             String suiteName,
                             String directoryPath,
                             List<String> this._testDirs,
                             List<String> expectations)
       : super(configuration,
               suiteName,
               new Path.fromNative(directoryPath),
               expectations);
 
-  void activityStarted() { ++activityCount; }
-
-  void activityCompleted() {
-    if (--activityCount == 0) {
-      directoryListingDone(true);
-    }
-  }
-
-  String shellPath() => TestUtils.compilerPath(configuration);
-
   List<String> additionalOptions(Path filePath) {
     return ['--fatal-warnings', '--fatal-type-errors'];
   }
 
-  void processDirectory() {
-    // Enqueueing the directory listers is an activity.
-    activityStarted();
+  Future enqueueTests() {
+    var group = new FutureGroup();
+
     for (String testDir in _testDirs) {
       Directory dir = new Directory.fromPath(suiteDir.append(testDir));
       if (dir.existsSync()) {
-        activityStarted();
-        var lister = dir.list(recursive: listRecursively());
-        lister.onFile = processFile;
-        lister.onDone = (ignore) => activityCompleted();
+        enqueueDirectory(dir, group);
       }
     }
-    // Completed the enqueueing of listers.
-    activityCompleted();
+
+    return group.future;
   }
 }
 
 
-class JUnitTestSuite implements TestSuite {
-  Map configuration;
-  String suiteName;
+class JUnitTestSuite extends TestSuite {
   String directoryPath;
   String statusFilePath;
   final String dartDir;
-  String buildDir;
   String classPath;
   List<String> testClasses;
   TestCaseEvent doTest;
   VoidFunction doDone;
   TestExpectations testExpectations;
 
-  JUnitTestSuite(Map this.configuration,
-                 String this.suiteName,
+  JUnitTestSuite(Map configuration,
+                 String suiteName,
                  String this.directoryPath,
                  String this.statusFilePath)
-      : dartDir = TestUtils.dartDir().toNativePath();
+      : super(configuration, suiteName),
+        dartDir = TestUtils.dartDir().toNativePath();
 
   bool isTestFile(String filename) => filename.endsWith("Tests.java") &&
       !filename.contains('com/google/dart/compiler/vm') &&
       !filename.contains('com/google/dart/corelib/SharedTests.java');
 
   void forEachTest(TestCaseEvent onTest,
                    Map testCacheIgnored,
                    [VoidFunction onDone]) {
     doTest = onTest;
-    doDone = (onDone != null) ? onDone : (() => null);
+    doDone = onDone;
 
     if (configuration['compiler'] != 'dartc') {
-      // Do nothing.  Asynchronously report that the suite is enqueued.
-      new Timer(0, (timerUnused){ doDone(); });
+      // Do nothing. Asynchronously report that the suite is enqueued.
+      asynchronously(doDone);
       return;
     }
     RegExp pattern = configuration['selectors']['dartc'];
     if (!pattern.hasMatch('junit_tests')) {
-      new Timer(0, (timerUnused){ doDone(); });
+      asynchronously(doDone);
       return;
     }
 
-    buildDir = TestUtils.buildDir(configuration);
     computeClassPath();
     testClasses = <String>[];
     // Do not read the status file.
     // All exclusions are hardcoded in this script, as they are in testcfg.py.
     processDirectory();
   }
 
   void processDirectory() {
     directoryPath = '$dartDir/$directoryPath';
     Directory dir = new Directory(directoryPath);
@@ skipping 54 matching lines @@
          '$dartDir/third_party/rhino/1_7R3/js.jar',
          '$dartDir/third_party/hamcrest/v1_3/hamcrest-core-1.3.0RC2.jar',
          '$dartDir/third_party/hamcrest/v1_3/hamcrest-generator-1.3.0RC2.jar',
          '$dartDir/third_party/hamcrest/v1_3/hamcrest-integration-1.3.0RC2.jar',
          '$dartDir/third_party/hamcrest/v1_3/hamcrest-library-1.3.0RC2.jar',
          '$dartDir/third_party/junit/v4_8_2/junit.jar'],
         Platform.operatingSystem == 'windows'? ';': ':');  // Path separator.
   }
 }
 
-
 class TestUtils {
   /**
    * The libraries in this directory relies on finding various files
    * relative to the 'test.dart' script in '.../dart/tools/test.dart'. If
    * the main script using 'test_suite.dart' is not there, the main
    * script must set this to '.../dart/tools/test.dart'.
    */
   static String testScriptPath = new Options().script;
 
   /**
@@ skipping 21 matching lines @@
    * Assumes that the directory for [dest] already exists.
    */
   static Future copyFile(Path source, Path dest) {
     var output = new File.fromPath(dest).openOutputStream();
     new File.fromPath(source).openInputStream().pipe(output);
     var completer = new Completer();
     output.onClosed = (){ completer.complete(null); };
     return completer.future;
   }
 
-  static String executableSuffix(String executable) {
-    if (Platform.operatingSystem == 'windows') {
-      if (executable == 'd8' || executable == 'vm' || executable == 'none') {
-        return '.exe';
-      } else {
-        return '.bat';
-      }
-    }
-    return '';
-  }
-
-  static String executableName(Map configuration) {
-    String suffix = executableSuffix(configuration['compiler']);
-    switch (configuration['compiler']) {
-      case 'none':
-        return 'dart$suffix';
-      case 'dartc':
-        return 'analyzer/bin/dart_analyzer$suffix';
-      case 'dart2js':
-      case 'dart2dart':
-        var prefix = '';
-        if (configuration['use_sdk']) {
-          prefix = 'dart-sdk/bin/';
-        }
-        if (configuration['host_checked']) {
-          // The script dart2js_developer is not in the SDK.
-          return 'dart2js_developer$suffix';
-        } else {
-          return '${prefix}dart2js$suffix';
-        }
-        break;
-      default:
-        throw "Unknown executable for: ${configuration['compiler']}";
-    }
-  }
-
-  static String compilerName(Map configuration) {
-    String suffix = executableSuffix(configuration['compiler']);
-    switch (configuration['compiler']) {
-      case 'dartc':
-      case 'dart2js':
-      case 'dart2dart':
-        return executableName(configuration);
-      default:
-        throw "Unknown compiler for: ${configuration['compiler']}";
-    }
-  }
-
-  static String dartShellFileName(Map configuration) {
-    var name = configuration['dart'];
-    if (name == '') {
-      name = '${buildDir(configuration)}/${executableName(configuration)}';
-    }
-    ensureExists(name, configuration);
-    return name;
-  }
-
-  static String d8FileName(Map configuration) {
-    var suffix = executableSuffix('d8');
-    var d8 = '${buildDir(configuration)}/d8$suffix';
-    ensureExists(d8, configuration);
-    return d8;
-  }
-
-  static String vmFileName(Map configuration) {
-    var suffix = executableSuffix('vm');
-    var vm = '${buildDir(configuration)}/dart$suffix';
-    ensureExists(vm, configuration);
-    return vm;
-  }
-
   static String flakyFileName() {
     // If a flaky test did fail, infos about it (i.e. test name, stdin, stdout)
     // will be written to this file. This is useful for the debugging of
     // flaky tests.
     // When running on a built bot, the file can be made visible in the waterfall UI.
     return ".flaky.log";
   }
 
   static void ensureExists(String filename, Map configuration) {
     if (!configuration['list'] && !(new File(filename).existsSync())) {
       throw "Executable '$filename' does not exist";
     }
   }
 
-  static String compilerPath(Map configuration) {
-    if (configuration['compiler'] == 'none') {
-      return null;  // No separate compiler for dartium tests.
-    }
-    var name = '${buildDir(configuration)}/${compilerName(configuration)}';
-    if (!(new File(name)).existsSync() && !configuration['list']) {
-      throw "Executable '$name' does not exist";
-    }
-    return name;
-  }
-
   static String outputDir(Map configuration) {
     var result = '';
     var system = configuration['system'];
     if (system == 'linux') {
       result = 'out/';
     } else if (system == 'macos') {
       result = 'xcodebuild/';
     } else if (system == 'windows') {
       result = 'build/';
     }
     return result;
   }
 
-  static String buildDir(Map configuration) {
-    String mode = (configuration['mode'] == 'debug') ? 'Debug' : 'Release';
-    String arch = configuration['arch'].toUpperCase();
-    return "${outputDir(configuration)}$mode$arch";
- }
-
   static Path dartDir() {
     File scriptFile = new File(testScriptPath);
     Path scriptPath = new Path.fromNative(scriptFile.fullPathSync());
     return scriptPath.directoryPath.directoryPath;
   }
 
   static List<String> standardOptions(Map configuration) {
     List args = ["--ignore-unrecognized-flags"];
     if (configuration["checked"]) {
       args.add('--enable_asserts');
@@ skipping 11 matching lines @@
       }
     }
     // TODO(riocw): Unify our minification calling convention between dart2js
     // and dart2dart.
     if (compiler == "dart2js" && configuration["minified"]) {
       args.add("--minify");
     }
     return args;
   }
 
-  static String jsshellFileName(Map configuration) {
-    var executableSuffix = executableSuffix('jsshell');
-    var executable = 'jsshell$executableSuffix';
-    var jsshellDir = '${dartDir()}/tools/testing/bin';
-    return '$jsshellDir/$executable';
+  static bool usesWebDriver(String runtime) {
+    const BROWSERS = const [
+      'dartium',
+      'ie9',
+      'ie10',
+      'safari',
+      'opera',
+      'chrome',
+      'ff'
+    ];
+    return BROWSERS.contains(runtime);
   }
 
-  static bool usesWebDriver(String runtime) => Contains(
-      runtime, const <String>['dartium',
-                              'ie9',
-                              'ie10',
-                              'safari',
-                              'opera',
-                              'chrome',
-                              'ff']);
-
   static bool isBrowserRuntime(String runtime) =>
       runtime == 'drt' || TestUtils.usesWebDriver(runtime);
 
   static bool isJsCommandLineRuntime(String runtime) =>
-      Contains(runtime, const <String>['d8', 'jsshell']);
+      const ['d8', 'jsshell'].contains(runtime);
 
 }
 
 class SummaryReport {
   static int total = 0;
   static int skipped = 0;
   static int noCrash = 0;
   static int pass = 0;
   static int failOk = 0;
   static int fail = 0;
@@ skipping 41 matching lines @@
  * $pass tests are expected to pass
  * $failOk tests are expected to fail that we won't fix
  * $fail tests are expected to fail that we should fix
  * $crash tests are expected to crash that we should fix
  * $timeout tests are allowed to timeout
  * $compileErrorSkip tests are skipped on browsers due to compile-time error
 """;
     print(report);
    }
 }