Add support for setUpAll and tearDownAll.

lib/src/runner/engine.dart

 // Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library test.runner.engine;
 
 import 'dart:async';
 import 'dart:collection';
 
 import 'package:async/async.dart' hide Result;
@@ skipping 197 matching lines @@
     return success;
   }
 
   /// Runs all the entries in [entries] in sequence.
   Future _runGroup(RunnerSuite suite, Group group) async {
     if (group.metadata.skip) {
       await _runLiveTest(_skippedTest(suite, group));
       return;
     }
 
-    for (var entry in group.entries) {
-      if (_closed) return;
+    var setUpAllSucceeded = true;
+    if (group.setUpAll != null) {
+      var liveTest = group.setUpAll.load(suite);
+      await _runLiveTest(liveTest, countSuccess: false);
+      setUpAllSucceeded = liveTest.state.result == Result.success;
+    }
 
-      if (entry is Group) {
-        await _runGroup(suite, entry);
-      } else if (entry.metadata.skip) {
-        await _runLiveTest(_skippedTest(suite, entry));
-      } else {
-        var test = entry as Test;
-        await _runLiveTest(test.load(suite));
+    if (!_closed && setUpAllSucceeded) {
+      for (var entry in group.entries) {
+        if (_closed) return;
+
+        if (entry is Group) {
+          await _runGroup(suite, entry);
+        } else if (entry.metadata.skip) {
+          await _runLiveTest(_skippedTest(suite, entry));
+        } else {
+          var test = entry as Test;
+          await _runLiveTest(test.load(suite));
+        }
       }
     }
+
+    // Even if we're closed or setUpAll failed, we want to run all the teardowns
+    // to ensure that any state is properly cleaned up.
+    if (group.tearDownAll != null) {
+      var liveTest = group.tearDownAll.load(suite);
+      await _runLiveTest(liveTest, countSuccess: false);
+      if (_closed) await liveTest.close();
+    }
   }
 
   /// Returns a dummy [LiveTest] for a test or group marked as "skip".
   LiveTest _skippedTest(RunnerSuite suite, GroupEntry entry) {
     // The netry name will be `null` for the root group.
     var test = new LocalTest(entry.name ?? "(suite)", entry.metadata, () {});
 
     var controller;
     controller = new LiveTestController(suite, test, () {
       controller.setState(const State(Status.running, Result.success));
       controller.setState(const State(Status.complete, Result.success));
       controller.completer.complete();
     }, () {});
     return controller.liveTest;
   }
 
   /// Runs [liveTest].
-  Future _runLiveTest(LiveTest liveTest) async {
+  ///
+  /// If [countSuccess] is `true` (the default), the test is put into [passed]
+  /// if it succeeds. Otherwise, it's removed from [liveTests] entirely.
+  Future _runLiveTest(LiveTest liveTest, {bool countSuccess: true}) async {
     _liveTests.add(liveTest);
     _active.add(liveTest);
 
     // If there were no active non-load tests, the current active test would
     // have been a load test. In that case, remove it, since now we have a
     // non-load test to add.
     if (_active.isNotEmpty && _active.first.suite is LoadSuite) {
       _liveTests.remove(_active.removeFirst());
     }
 
     liveTest.onStateChange.listen((state) {
       if (state.status != Status.complete) return;
       _active.remove(liveTest);
 
       // If we're out of non-load tests, surface a load test.
       if (_active.isEmpty && _activeLoadTests.isNotEmpty) {
         _active.add(_activeLoadTests.first);
         _liveTests.add(_activeLoadTests.first);
       }
 
       if (state.result != Result.success) {
         _passed.remove(liveTest);
         _failed.add(liveTest);
       } else if (liveTest.test.metadata.skip) {
         _skipped.add(liveTest);
+      } else if (countSuccess) {
+        _passed.add(liveTest);
       } else {
-        _passed.add(liveTest);
+        _liveTests.remove(liveTest);
       }
     });
 
     _onTestStartedController.add(liveTest);
 
     // First, schedule a microtask to ensure that [onTestStarted] fires before
     // the first [LiveTest.onStateChange] event. Once the test finishes, use
     // [new Future] to do a coarse-grained event loop pump to avoid starving
     // non-microtask events.
     await new Future.microtask(liveTest.run);
@@ skipping 50 matching lines @@
   ///
   /// Any actively-running tests are also closed. VM tests are allowed to finish
   /// running so that any modifications they've made to the filesystem can be
   /// cleaned up.
   ///
   /// **Note that closing the engine is not the same as closing [suiteSink].**
   /// Closing [suiteSink] indicates that no more input will be provided, closing
   /// the engine indicates that no more output should be emitted.
   Future close() async {
     _closed = true;
-    if (_closedBeforeDone == null) _closedBeforeDone = true;
+    if (_closedBeforeDone != null) _closedBeforeDone = true;
     _suiteController.close();
 
     // Close the running tests first so that we're sure to wait for them to
     // finish before we close their suites and cause them to become unloaded.
     var allLiveTests = liveTests.toSet()..addAll(_activeLoadTests);
-    await Future.wait(allLiveTests.map((liveTest) => liveTest.close()));
+    var futures = allLiveTests.map((liveTest) => liveTest.close()).toList();
 
-    var allSuites = allLiveTests.map((liveTest) => liveTest.suite).toSet();
-    await Future.wait(allSuites.map((suite) => suite.close()));
+    // Closing the load pool will close the test suites as soon as their tests
+    // are done. For browser suites this is effectively immediate since their
+    // tests shut down as soon as they're closed, but for VM suites we may need
+    // to wait for tearDowns or tearDownAlls to run.
+    futures.add(_loadPool.close());
+    await Future.wait(futures, eagerError: true);
   }
 }