tools/test.dart: Add summary report of the number of tests with each expectation.

tools/testing/dart/test_suite.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 #library("test_suite");
 
 #import("status_file_parser.dart");
 #import("test_runner.dart");
 #import("multitest.dart");
 
@@ skipping 189 matching lines @@
       if (start != -1) {
         testName = filename.substring(start + 4, filename.length - 5);
       } else {
         // Only multitests in a temporary directory should reach here.
         start = filename.lastIndexOf(pathSeparator);
         int middle = filename.lastIndexOf('_');
         testName = filename.substring(start + 1, middle) + pathSeparator +
             filename.substring(middle + 1, filename.length - 5);
       }
       Set<String> expectations = testExpectations.expectations(testName);
-
+      if (configuration["report"]) {

[Mads Ager (google), 2011/12/01 19:53:53]

You want to add this above in the CCTestSuite as well. Potentially extract this
into a helper now that getting the expectations is more than a line?

+        // Tests with multiple VMOptions are counted more than once.
+        for (var dummy in optionsFromFile["vmOptions"]) {
+          SummaryReport.add(expectations);
+        }
+      }
       if (expectations.contains(SKIP)) return;
 
       isNegative = isNegative ||
           (configuration['checked'] && isNegativeIfChecked);
       var argumentLists = argumentListsFromFile(filename, optionsFromFile);
       for (var args in argumentLists) {
         doTest(new TestCase(testName,
                             shellPath,
                             args,
                             timeout,
@@ skipping 177 matching lines @@
       args.add("--enable_leg");
     }
     if (configuration["component"] == "dartc") {
       if (configuration["mode"] == "release") {
         args.add("--optimize");
       }
     }
     return args;
   }
 }
+
+class SummaryReport {
+  static int total = 0;
+  static int skipped = 0;
+  static int noCrash = 0;
+  static int pass = 0;
+  static int failOk = 0;
+  static int fail = 0;
+  static int crash = 0;
+  static int timeout = 0;
+
+  static void add(Set<String> expect) {

[Mads Ager (google), 2011/12/01 19:53:53]

I'd spell out 'expectations'.

+    ++total;
+    if (expect.contains(SKIP)) {
+      ++skipped;
+    } else {
+      if (expect.contains(PASS) && expect.contains(FAIL) &&
+          !expect.contains(CRASH) && !expect.contains(OK)) {
+        ++noCrash;
+      }
+      if (expect.contains(PASS) && expect.isSubsetOf([PASS])) {

[Mads Ager (google), 2011/12/01 19:53:53]

I would prefer:

expect.contains(PASS) && expect.length == 1

PASS is in there and there is only that element.

Similarly below:

expect.containsAll([FAIL, OK]) && expect.length == 2

I don't have strong opinions on it though and will let you decide.

+        ++pass;
+      }
+      if (expect.containsAll([FAIL, OK]) && expect.isSubsetOf([FAIL, OK])) {
+        ++failOk;
+      }
+      if (expect.contains(FAIL) && expect.isSubsetOf([FAIL])) {
+        ++fail;
+      }
+      if (expect.contains(CRASH) && expect.isSubsetOf([CRASH])) {
+        ++crash;
+      }
+      if (expect.contains(TIMEOUT)) {
+        ++timeout;
+      }
+    }

[Mads Ager (google), 2011/12/01 19:53:53]

I see what you mean about the reporting being strange. How about cases such as:

CRASH, OK
PASS, OK
CRASH, FAIL
CRASH, FAIL, PASS

In particular I think we should add reporting of the last category of flaky
tests that either crash or fail (or even pass).

+  }
+
+  static void printReport() {
+    if (total == 0) return;
+    String report = """\
+Total: $total tests
+ * $skipped tests will be skipped
+ * $noCrash tests are expected to be flaky but not crash
+ * $pass tests are expected to pass
+ * $failOk tests are expected to fail that we won't fix
+ * $fail tests are expected to fail that we should fix
+ * $crash tests are expected to crash that we should fix
+ * $timeout tests are allowed to timeout\
+""";
+    print(report);
+  }
+}