Implement custom status file expectations.

pkg/testing/lib/src/chain.dart

 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE.md file.
 
 library testing.chain;
 
 import 'dart:async' show
     Future,
     Stream;
 
 import 'dart:convert' show
     JSON,
     JsonEncoder;
 
 import 'dart:io' show
     Directory,
     File,
     FileSystemEntity,
     exitCode;
 
 import 'suite.dart' show
     Suite;
 
 import '../testing.dart' show
     TestDescription;
 
 import 'test_dart/status_file_parser.dart' show
-    Expectation,
     ReadTestExpectations,
     TestExpectations;
 
 import 'zone_helper.dart' show
     runGuarded;
 
 import 'error_handling.dart' show
     withErrorHandling;
 
 import 'log.dart' show
     logMessage,
     logStepComplete,
     logStepStart,
     logSuiteComplete,
     logTestComplete,
     logUnexpectedResult,
     splitLines;
 
 import 'multitest.dart' show
     MultitestTransformer,
     isError;
 
+import 'expectation.dart' show
+    Expectation,
+    ExpectationSet;
+
 typedef Future<ChainContext> CreateContext(
     Chain suite, Map<String, String> environment);
 
 /// A test suite for tool chains, for example, a compiler.
 class Chain extends Suite {
   final Uri source;
 
   final Uri uri;
 
   final List<RegExp> pattern;
@@ skipping 51 matching lines @@
       "exclude": []..addAll(exclude.map((RegExp r) => r.pattern)),
     };
   }
 }
 
 abstract class ChainContext {
   const ChainContext();
 
   List<Step> get steps;
 
+  ExpectationSet get expectationSet => ExpectationSet.Default;
+
   Future<Null> run(Chain suite, Set<String> selectors) async {
     TestExpectations expectations = await ReadTestExpectations(
-        <String>[suite.statusFile.toFilePath()], {});
+        <String>[suite.statusFile.toFilePath()], {}, expectationSet);
     Stream<TestDescription> stream = list(suite);
     if (suite.processMultitests) {
       stream = stream.transform(new MultitestTransformer());
     }
     List<TestDescription> descriptions = await stream.toList();
     descriptions.sort();
     Map<TestDescription, Result> unexpectedResults =
         <TestDescription, Result>{};
     Map<TestDescription, Set<Expectation>> unexpectedOutcomes =
         <TestDescription, Set<Expectation>>{};
@@ skipping 47 matching lines @@
             }
           }, printLineOnStdout: sb.writeln);
         } else {
           future = new Future.value(null);
         }
         future = future.then((Result currentResult) {
           if (currentResult != null) {
             logStepComplete(completed, unexpectedResults.length,
                 descriptions.length, suite, description, lastStepRun);
             result = currentResult;
-            if (currentResult.outcome == Expectation.PASS) {
+            if (currentResult.outcome == Expectation.Pass) {
               // The input to the next step is the output of this step.
               return doStep(result.output);
             }
           }
           if (description.multitestExpectations != null) {
             if (isError(description.multitestExpectations)) {
               result = toNegativeTestResult(
                   result, description.multitestExpectations);
             }
           } else if (lastStep == lastStepRun &&
               description.shortName.endsWith("negative_test")) {
-            if (result.outcome == Expectation.PASS) {
+            if (result.outcome == Expectation.Pass) {
               result.addLog("Negative test didn't report an error.\n");
-            } else if (result.outcome == Expectation.FAIL) {
+            } else if (result.outcome == Expectation.Fail) {
               result.addLog("Negative test reported an error as expeceted.\n");
             }
             result = toNegativeTestResult(result);
           }
           if (!expectedOutcomes.contains(result.outcome)) {
             result.addLog("$sb");
             unexpectedResults[description] = result;
             unexpectedOutcomes[description] = expectedOutcomes;
             logUnexpectedResult(suite, description, result, expectedOutcomes);
           } else {
@@ skipping 37 matching lines @@
         String path = entity.uri.path;
         if (suite.exclude.any((RegExp r) => path.contains(r))) continue;
         if (suite.pattern.any((RegExp r) => path.contains(r))) {
           yield new TestDescription(suite.uri, entity);
         }
       }
     } else {
       throw "${suite.uri} isn't a directory";
     }
   }
+
+  Result toNegativeTestResult(Result result, [Set<String> expectations]) {
+    Expectation outcome = result.outcome;
+    if (outcome == Expectation.Pass) {
+      if (expectations == null) {
+        outcome = Expectation.Fail;
+      } else if (expectations.contains("compile-time error")) {
+        outcome = expectationSet["MissingCompileTimeError"];
+      } else if (expectations.contains("runtime error") ||
+          expectations.contains("dynamic type error")) {
+        outcome = expectationSet["MissingRuntimeError"];
+      } else {
+        outcome = Expectation.Fail;
+      }
+    } else if (outcome == Expectation.Fail) {
+      outcome = Expectation.Pass;
+    }
+    return result.copyWithOutcome(outcome);
+  }
 }
 
 abstract class Step<I, O, C extends ChainContext> {
   const Step();
 
   String get name;
 
   bool get isAsync => false;
 
   bool get isCompiler => false;
@@ skipping 22 matching lines @@
 
   final error;
 
   final StackTrace trace;
 
   final List<String> logs = <String>[];
 
   Result(this.output, this.outcome, this.error, this.trace);
 
   Result.pass(O output)
-      : this(output, Expectation.PASS, null, null);
+      : this(output, Expectation.Pass, null, null);
 
   Result.crash(error, StackTrace trace)
-      : this(null, Expectation.CRASH, error, trace);
+      : this(null, Expectation.Crash, error, trace);
 
   Result.fail(O output, [error, StackTrace trace])
-      : this(output, Expectation.FAIL, error, trace);
+      : this(output, Expectation.Fail, error, trace);
 
   String get log => logs.join();
 
   void addLog(String log) {
     logs.add(log);
   }
 
   Result<O> copyWithOutcome(Expectation outcome) {
     return new Result<O>(output, outcome, error, trace)
         ..logs.addAll(logs);
   }
 }
 
 /// This is called from generated code.
 Future<Null> runChain(
     CreateContext f, Map<String, String> environment, Set<String> selectors,
     String json) {
   return withErrorHandling(() async {
     Chain suite = new Suite.fromJsonMap(Uri.base, JSON.decode(json));
     print("Running ${suite.name}");
     ChainContext context = await f(suite, environment);
     return context.run(suite, selectors);
   });
 }
-
-Result toNegativeTestResult(Result result, [Set<String> expectations]) {
-  Expectation outcome = result.outcome;
-  if (outcome == Expectation.PASS) {
-    if (expectations == null) {
-      outcome = Expectation.FAIL;
-    } else if (expectations.contains("compile-time error")) {
-      outcome = Expectation.MISSING_COMPILETIME_ERROR;
-    } else if (expectations.contains("runtime error") ||
-        expectations.contains("dynamic type error")) {
-      outcome = Expectation.MISSING_RUNTIME_ERROR;
-    } else {
-      outcome = Expectation.FAIL;
-    }
-  } else if (outcome == Expectation.FAIL) {
-    outcome = Expectation.PASS;
-  }
-  return result.copyWithOutcome(outcome);
-}