Implement multitest expectations.

pkg/testing/lib/src/chain.dart

 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE.md file.
 
 library testing.chain;
 
 import 'dart:async' show
     Future,
     Stream;
 
@@ skipping 27 matching lines @@
 import 'log.dart' show
     logMessage,
     logStepComplete,
     logStepStart,
     logSuiteComplete,
     logTestComplete,
     logUnexpectedResult,
     splitLines;
 
 import 'multitest.dart' show
-    MultitestTransformer;
+    MultitestTransformer,
+    isError;
 
 typedef Future<ChainContext> CreateContext(
     Chain suite, Map<String, String> environment);
 
 /// A test suite for tool chains, for example, a compiler.
 class Chain extends Suite {
   final Uri source;
 
   final Uri uri;
 
@@ skipping 74 matching lines @@
         <TestDescription, Set<Expectation>>{};
     int completed = 0;
     List<Future> futures = <Future>[];
     for (TestDescription description in descriptions) {
       String selector = "${suite.name}/${description.shortName}";
       if (selectors.isNotEmpty &&
           !selectors.contains(selector) &&
           !selectors.contains(suite.name)) {
         continue;
       }
-      Set<Expectation> expectedOutcomes =
+      final Set<Expectation> expectedOutcomes =
           expectations.expectations(description.shortName);
+      final StringBuffer sb = new StringBuffer();
+      final Step lastStep = steps.isNotEmpty ? steps.last : null;
+      final Iterator<Step> iterator = steps.iterator;
+
       Result result;
-      StringBuffer sb = new StringBuffer();
       // Records the outcome of the last step that was run.
-      Step lastStep = null;
-      Iterator<Step> iterator = steps.iterator;
+      Step lastStepRun;
 
       /// Performs one step of [iterator].
       ///
       /// If `step.isAsync` is true, the corresponding step is said to be
       /// asynchronous.
       ///
       /// If a step is asynchrouns the future returned from this function will
       /// complete after the the first asynchronous step is scheduled.  This
       /// allows us to start processing the next test while an external process
       /// completes as steps can be interleaved. To ensure all steps are
       /// completed, wait for [futures].
       ///
       /// Otherwise, the future returned will complete when all steps are
       /// completed. This ensures that tests are run in sequence without
       /// interleaving steps.
       Future doStep(dynamic input) async {
         Future future;
         bool isAsync = false;
         if (iterator.moveNext()) {
           Step step = iterator.current;
-          lastStep = step;
+          lastStepRun = step;
           isAsync = step.isAsync;
           logStepStart(completed, unexpectedResults.length, descriptions.length,
               suite, description, step);
           future = runGuarded(() async {
             try {
               return await step.run(input, this);
             } catch (error, trace) {
               return step.unhandledError(error, trace);
             }
           }, printLineOnStdout: sb.writeln);
         } else {
           future = new Future.value(null);
         }
         future = future.then((Result currentResult) {
           if (currentResult != null) {
             logStepComplete(completed, unexpectedResults.length,
-                descriptions.length, suite, description, lastStep);
+                descriptions.length, suite, description, lastStepRun);
             result = currentResult;
             if (currentResult.outcome == Expectation.PASS) {
               // The input to the next step is the output of this step.
               return doStep(result.output);
             }
           }
-          if (steps.isNotEmpty && steps.last == lastStep &&
+          if (description.multitestExpectations != null) {
+            if (isError(description.multitestExpectations)) {
+              result = result.toNegativeTestResult();
+            }
+          } else if (lastStep == lastStepRun &&
               description.shortName.endsWith("negative_test")) {
             if (result.outcome == Expectation.PASS) {
               result.addLog("Negative test didn't report an error.\n");
             } else if (result.outcome == Expectation.FAIL) {
               result.addLog("Negative test reported an error as expeceted.\n");
             }
             result = result.toNegativeTestResult();
           }
           if (!expectedOutcomes.contains(result.outcome)) {
             result.addLog("$sb");
@@ skipping 50 matching lines @@
   }
 }
 
 abstract class Step<I, O, C extends ChainContext> {
   const Step();
 
   String get name;
 
   bool get isAsync => false;
 
+  bool get isCompiler => false;
+
+  bool get isRuntime => false;
+
   Future<Result<O>> run(I input, C context);
 
   Result<O> unhandledError(error, StackTrace trace) {
     return new Result<O>.crash(error, trace);
   }
 
   Result<O> pass(O output) => new Result<O>.pass(output);
 
   Result<O> crash(error, StackTrace trace) => new Result<O>.crash(error, trace);
 
@@ skipping 46 matching lines @@
 Future<Null> runChain(
     CreateContext f, Map<String, String> environment, Set<String> selectors,
     String json) {
   return withErrorHandling(() async {
     Chain suite = new Suite.fromJsonMap(Uri.base, JSON.decode(json));
     print("Running ${suite.name}");
     ChainContext context = await f(suite, environment);
     return context.run(suite, selectors);
   });
 }