test.py: Support for CompileTimeError,RuntimeError,MissingRuntimeError,MissingCompiletimeError mark…

tools/testing/dart/status_file_parser.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library status_file_parser;
 
 import "dart:async";
 import "dart:convert" show LineSplitter, UTF8;
 import "dart:io";
 import "status_expression.dart";
 
-/** Possible outcomes of running a test. */
-const CRASH = "crash";
-const TIMEOUT = "timeout";
-const FAIL = "fail";
-const PASS = "pass";
-/**
- * An indication to skip the test.  The caller is responsible for skipping it.
- */
-const SKIP = "skip";
-const SKIP_BY_DESIGN = "skipbydesign";
-const OK = "ok";
-/**
- * An indication that a test is slow and we should allow extra time for
- * completion.
- */
-const SLOW = "slow";
+class Expectation {
+  // Possible outcomes of running a test.
+  static Expectation PASS = byName('Pass');
+  static Expectation CRASH = byName('Crash');
+  static Expectation TIMEOUT = byName('Timeout');
+  static Expectation FAIL = byName('Fail');
+
+  // Special 'FAIL' cases
+  static Expectation RUNTIME_ERROR = byName('RuntimeError');
+  static Expectation COMPILETIME_ERROR = byName('CompileTimeError');

[ricow1, 2013/09/23 12:07:32]

nit, but there seems to be some difference in how we handle Compile{t||T}ime.
Here we call it CompileTimeError, below we call it MissingCompiletimeError

[kustermann, 2013/09/23 15:31:34]

Changed it to *CompileTime*

+  static Expectation MISSING_RUNTIME_ERROR = byName('MissingRuntimeError');
+  static Expectation MISSING_COMPILETIME_ERROR =
+      byName('MissingCompiletimeError');
+
+  // "meta expectations"
+  static Expectation OK = byName('Ok');
+  static Expectation SLOW = byName('Slow');
+  static Expectation SKIP = byName('Skip');
+  static Expectation SKIP_BY_DESIGN = byName('SkipByDesign');
+
+  static Expectation byName(String name) {
+    _initialize();
+    name = name.toLowerCase();
+    if (!_AllExpectations.containsKey(name)) {
+      throw new Exception("Expectation.byName(name='$name'): Invalid name.");
+    }
+    return _AllExpectations[name];
+  }
+
+  // Keep a map of all possible Expectation objects, initialized lazily.
+  static bool _initialized = false;
+  static final _AllExpectations = new Map<String, Expectation>();

[ricow1, 2013/09/23 12:07:32]

just let this be null and branch on that in _initialize? then you can remove the
_initialized variable above and just create the map lazily as well

[kustermann, 2013/09/23 15:31:34]

Done.

+  static void _initialize() {
+    if (!_initialized) {
+      _initialized = true;
+
+      Expectation build(prettyName, {group: null, isMetaExpectation: false}) {
+        var expectation = new Expectation._(prettyName,
+            group: group, isMetaExpectation: isMetaExpectation);
+        assert(!_AllExpectations.containsKey(expectation.name));
+        return _AllExpectations[expectation.name] = expectation;
+      }
+
+      var fail = build("Fail");
+      build("Pass");
+      build("Crash");
+      build("Timeout");
+
+      build("MissingCompiletimeError", group: fail);
+      build("MissingRuntimeError", group: fail);
+      build("CompiletimeError", group: fail);
+      build("RuntimeError", group: fail);
+
+      build("Skip", isMetaExpectation: true);
+      build("SkipByDesign", isMetaExpectation: true);
+      build("Ok", isMetaExpectation: true);
+      build("Slow", isMetaExpectation: true);
+    }
+  }
+
+  final String prettyName;
+  final String name;
+  final Expectation group;
+  // Indicates whether this expectation can actually be the outcome of a test.

[ricow1, 2013/09/23 12:07:32]

well it actually indicates the opposite 

[kustermann, 2013/09/23 15:31:34]

I clarified it a bit.

+  final bool isMetaExpectation;
+
+  Expectation._(prettyName,
+                {Expectation this.group: null,
+                 bool this.isMetaExpectation: false})
+      : prettyName = prettyName, name = prettyName.toLowerCase();
+
+  bool canBeOutcomeOf(Expectation expectation) {
+    Expectation outcome = this;
+    while (outcome != null) {
+      if (outcome == expectation) {
+        return true;
+      }
+      outcome = outcome.group;
+    }
+    return false;
+  }
+
+  String toString() => prettyName;
+}
+
 
 final RegExp SplitComment = new RegExp("^([^#]*)(#.*)?\$");
 final RegExp HeaderPattern = new RegExp(r"^\[([^\]]+)\]");
 final RegExp RulePattern = new RegExp(r"\s*([^: ]*)\s*:(.*)");
 final RegExp IssueNumberPattern =
     new RegExp("Issue ([0-9]+)|dartbug.com/([0-9]+)", caseSensitive: false);
 
 // TODO(whesse): Implement configuration_info library that contains data
 // structures for test configuration, including Section.
 class Section {
@@ skipping 117 matching lines @@
 
   /**
    * Add a rule to the expectations.
    */
   void addRule(testRule, environment) {
     // Once we have started using the expectations we cannot add more
     // rules.
     if (_preprocessed) {
       throw "TestExpectations.addRule: cannot add more rules";
     }
-    var values = testRule.expression.evaluate(environment);
-    _map.putIfAbsent(testRule.name, () => new Set()).addAll(values);
+    var names = testRule.expression.evaluate(environment);
+    var expectations = names.map((name) => Expectation.byName(name));
+    _map.putIfAbsent(testRule.name, () => new Set()).addAll(expectations);
   }
 
   /**
    * Compute the expectations for a test based on the filename.
    *
    * For every (key, expectation) pair. Match the key with the file
    * name. Return the union of the expectations for all the keys
    * that match.
    *
    * Normal matching splits the key and the filename into path
    * components and checks that the anchored regular expression
    * "^$keyComponent\$" matches the corresponding filename component.
    */
-  Set<String> expectations(String filename) {
+  Set<Expectation> expectations(String filename) {
     var result = new Set();
     var splitFilename = filename.split('/');
 
     // Create mapping from keys to list of RegExps once and for all.
     _preprocessForMatching();
 
     _map.forEach((key, expectation) {
       List regExps = _keyToRegExps[key];
       if (regExps.length > splitFilename.length) return;
       for (var i = 0; i < regExps.length; i++) {
         if (!regExps[i].hasMatch(splitFilename[i])) return;
       }
       // If all components of the status file key matches the filename
       // add the expectations to the result.
       result.addAll(expectation);
     });
 
     // If no expectations were found the expectation is that the test
     // passes.
     if (result.isEmpty) {
-      result.add(PASS);
+      result.add(Expectation.PASS);
     }
     return result;
   }
 
   // Preprocess the expectations for matching against
   // filenames. Generate lists of regular expressions once and for all
   // for each key.
   void _preprocessForMatching() {
     if (_preprocessed) return;
 
@@ skipping 14 matching lines @@
         }
         regExps[i] = regExp;
       }
       _keyToRegExps[key] = regExps;
     });
 
     _regExpCache = null;
     _preprocessed = true;
   }
 }