Add a LoadSuite class.

lib/src/runner/engine.dart

 // Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library test.runner.engine;
 
 import 'dart:async';
 import 'dart:collection';
 
 import 'package:async/async.dart' hide Result;
 import 'package:collection/collection.dart';
 import 'package:pool/pool.dart';
 
 import '../backend/live_test.dart';
 import '../backend/live_test_controller.dart';
 import '../backend/state.dart';
 import '../backend/suite.dart';
 import '../backend/test.dart';
 import '../util/delegating_sink.dart';
+import 'load_suite.dart';
 
 /// An [Engine] manages a run that encompasses multiple test suites.
 ///
+/// Test suites are provided by passing them into [suiteSink]. Once all suites
+/// have been provided, the user should close [suiteSink] to indicate this.
+/// [run] won't terminate until [suiteSink] is closed. Suites will be run in the
+/// order they're provided to [suiteSink]. Tests within those suites will
+/// likewise be run in the order of [Suite.tests].
+///
 /// The current status of every test is visible via [liveTests]. [onTestStarted]
 /// can also be used to be notified when a test is about to be run.
 ///
-/// Suites will be run in the order they're provided to [new Engine]. Tests
-/// within those suites will likewise be run in the order of [Suite.tests].
+/// The engine has some special logic for [LoadSuite]s and the tests they
+/// contain, referred to as "load tests". Load tests exist to provide visibility
+/// into the process of loading test files, but as long as that process is
+/// proceeding normally users usually don't care about it, so the engine only
+/// surfaces running load tests (that is, includes them in [liveTests] and other
+/// collections) under specific circumstances.
+///
+/// If only load tests are running, exactly one load test will be in [active]
+/// and [liveTests]. If this test passes, it will be removed from both [active]
+/// and [liveTests] and *will not* be added to [passed]. If at any point a load
+/// test fails, it will be added to [failed] and [liveTests].
+///
+/// Load tests will always be emitted through [onTestStarted] so users can watch
+/// their event streams once they start running.
 class Engine {
   /// Whether [run] has been called yet.
   var _runCalled = false;
 
   /// Whether [close] has been called.
   var _closed = false;
 
   /// Whether [close] was called before all the tests finished running.
   ///
   /// This is `null` if close hasn't been called and the tests are still
@@ skipping 52 matching lines @@
   /// The set of tests that have completed and been marked as skipped.
   Set<LiveTest> get skipped => new UnmodifiableSetView(_skipped);
   final _skipped = new Set<LiveTest>();
 
   /// The set of tests that have completed and been marked as failing or error.
   Set<LiveTest> get failed => new UnmodifiableSetView(_failed);
   final _failed = new Set<LiveTest>();
 
   /// The tests that are still running, in the order they begain running.
   List<LiveTest> get active => new UnmodifiableListView(_active);
-  final _active = new List<LiveTest>();
+  final _active = new QueueList<LiveTest>();
+
+  /// The tests from [LoadSuite]s that are still running, in the order they
+  /// began running.
+  ///
+  /// This is separate from [active] because load tests aren't always surfaced.
+  final _activeLoadTests = new List<LiveTest>();
 
   /// Creates an [Engine] that will run all tests provided via [suiteSink].
   ///
   /// [concurrency] controls how many suites are run at once.
   Engine({int concurrency})
       : _pool = new Pool(concurrency == null ? 1 : concurrency) {
     _group.future.then((_) {
       if (_closedBeforeDone == null) _closedBeforeDone = false;
     }).catchError((_) {
       // Don't top-level errors. They'll be thrown via [success] anyway.
@@ skipping 17 matching lines @@
   /// This returns `true` if all tests succeed, and `false` otherwise. It will
   /// only return once all tests have finished running and [suiteSink] has been
   /// closed.
   Future<bool> run() {
     if (_runCalled) {
       throw new StateError("Engine.run() may not be called more than once.");
     }
     _runCalled = true;
 
     _suiteController.stream.listen((suite) {
+      if (suite is LoadSuite) {
+        _group.add(_addLoadSuite(suite));
+        return;
+      }
+
       _group.add(_pool.withResource(() {
         if (_closed) return null;
 
         // TODO(nweiz): Use a real for loop when issue 23394 is fixed.
         return Future.forEach(suite.tests, (test) async {
           if (_closed) return;
 
           var liveTest = test.metadata.skip
               ? _skippedTest(suite, test)
               : test.load(suite);
           _liveTests.add(liveTest);
           _active.add(liveTest);
 
+          // If there were no active non-load tests, the current active test
+          // would have been a load test. In that case, remove it, since now we
+          // have a non-load test to add.
+          if (_active.isNotEmpty && _active.first.suite is LoadSuite) {
+            _liveTests.remove(_active.removeFirst());
+          }
+
           liveTest.onStateChange.listen((state) {
             if (state.status != Status.complete) return;
             _active.remove(liveTest);
 
+            // If we're out of non-load tests, surface a load test.
+            if (_active.isEmpty && _activeLoadTests.isNotEmpty) {
+              _active.add(_activeLoadTests.first);
+              _liveTests.add(_activeLoadTests.first);
+            }
+
             if (state.result != Result.success) {
               _passed.remove(liveTest);
               _failed.add(liveTest);
             } else if (liveTest.test.metadata.skip) {
               _skipped.add(liveTest);
             } else {
               _passed.add(liveTest);
             }
           });
 
@@ skipping 16 matching lines @@
   LiveTest _skippedTest(Suite suite, Test test) {
     var controller;
     controller = new LiveTestController(suite, test, () {
       controller.setState(const State(Status.running, Result.success));
       controller.setState(const State(Status.complete, Result.success));
       controller.completer.complete();
     }, () {});
     return controller.liveTest;
   }
 
+  /// Adds listeners for [suite].
+  ///
+  /// Load suites have specific logic apart from normal test suites.
+  Future _addLoadSuite(LoadSuite suite) async {
+    var liveTest = await suite.tests.single.load(suite);
+
+    _activeLoadTests.add(liveTest);
+
+    // Only surface the load test if there are no other tests currently running.
+    if (_active.isEmpty) {
+      _liveTests.add(liveTest);
+      _active.add(liveTest);
+    }
+
+    liveTest.onStateChange.listen((state) {
+      if (state.status != Status.complete) return;
+      _activeLoadTests.remove(liveTest);
+
+      // Only one load test will be active at any given time, and it will always
+      // be the only active test. Remove it and, if possible, surface another
+      // load test.
+      if (_active.isNotEmpty && _active.first.suite == suite) {
+        _active.remove(liveTest);
+        _liveTests.remove(liveTest);
+
+        if (_activeLoadTests.isNotEmpty) {
+          _active.add(_activeLoadTests.last);
+          _liveTests.add(_activeLoadTests.last);
+        }
+      }
+
+      // Surface the load test if it fails so that the user can see the failure.
+      if (state.result == Result.success) return;
+      _failed.add(liveTest);
+      _liveTests.add(liveTest);
+    });
+
+    // Run the test immediately. We don't want loading to be blocked on suites
+    // that are already running.
+    _onTestStartedController.add(liveTest);
+    await liveTest.run();
+  }
+
   /// Signals that the caller is done paying attention to test results and the
   /// engine should release any resources it has allocated.
   ///
   /// Any actively-running tests are also closed. VM tests are allowed to finish
   /// running so that any modifications they've made to the filesystem can be
   /// cleaned up.
   ///
   /// **Note that closing the engine is not the same as closing [suiteSink].**
   /// Closing [suiteSink] indicates that no more input will be provided, closing
   /// the engine indicates that no more output should be emitted.
   Future close() {
     _closed = true;
     if (_closedBeforeDone == null) _closedBeforeDone = true;
     _suiteController.close();
 
-    return Future.wait(liveTests.map((liveTest) => liveTest.close()));
+    var allLiveTests = liveTests.toSet()..addAll(_activeLoadTests);
+    return Future.wait(allLiveTests.map((liveTest) => liveTest.close()));
   }
 }