Index: tools/testing/dart/lib/summary_report.dart |
diff --git a/tools/testing/dart/lib/summary_report.dart b/tools/testing/dart/lib/summary_report.dart |
new file mode 100644 |
index 0000000000000000000000000000000000000000..47ec928119fab5cbb4f92aecebec38ef1d2f5227 |
--- /dev/null |
+++ b/tools/testing/dart/lib/summary_report.dart |
@@ -0,0 +1,106 @@ |
+library summary_report; |
+ |
+import 'status_file_parser.dart'; |
+import 'test_case.dart'; |
+ |
+class SummaryReport { |
+ static int total = 0; |
+ static int skipped = 0; |
+ static int skippedByDesign = 0; |
+ static int noCrash = 0; |
+ static int flakyCrash = 0; |
+ static int pass = 0; |
+ static int failOk = 0; |
+ static int fail = 0; |
+ static int crash = 0; |
+ static int timeout = 0; |
+ static int compileErrorSkip = 0; |
+ |
+ static List<TestCase> nonStandardTestCases = []; |
+ |
+ static void add(TestCase testCase) { |
+ var expectations = testCase.expectedOutcomes; |
+ |
+ bool containsFail = expectations.any( |
+ (expectation) => expectation.canBeOutcomeOf(Expectation.FAIL)); |
+ bool containsPass = expectations.contains(Expectation.PASS); |
+ bool containsSkip = expectations.contains(Expectation.SKIP); |
+ bool containsSkipByDesign = |
+ expectations.contains(Expectation.SKIP_BY_DESIGN); |
+ bool containsCrash = expectations.contains(Expectation.CRASH); |
+ bool containsOK = expectations.contains(Expectation.OK); |
+ bool containsSlow = expectations.contains(Expectation.SLOW); |
+ bool containsTimeout = expectations.contains(Expectation.TIMEOUT); |
+ |
+ ++total; |
+ if (containsSkip) { |
+ ++skipped; |
+ } else if (containsSkipByDesign) { |
+ ++skipped; |
+ ++skippedByDesign; |
+ } else { |
+ // We don't do if-else below because the buckets should be exclusive. |
+ // We keep a count around to guarantee that |
+ int markers = 0; |
+ |
+ // Counts the number of flaky tests. |
+ if (containsFail && containsPass && !containsCrash && !containsOK) { |
+ ++noCrash; |
+ ++markers; |
+ } |
+ if (containsCrash && !containsOK && expectations.length > 1) { |
+ ++flakyCrash; |
+ ++markers; |
+ } |
+ if ((containsPass && expectations.length == 1) || |
+ (containsPass && containsSlow && expectations.length == 2)) { |
+ ++pass; |
+ ++markers; |
+ } |
+ if (containsFail && containsOK) { |
+ ++failOk; |
+ ++markers; |
+ } |
+ if ((containsFail && expectations.length == 1) || |
+ (containsFail && containsSlow && expectations.length == 2)) { |
+ ++fail; |
+ ++markers; |
+ } |
+ if ((containsCrash && expectations.length == 1) || |
+ (containsCrash && containsSlow && expectations.length == 2)) { |
+ ++crash; |
+ ++markers; |
+ } |
+ if (containsTimeout && expectations.length == 1) { |
+ ++timeout; |
+ ++markers; |
+ } |
+ if (markers != 1) { |
+ nonStandardTestCases.add(testCase); |
+ } |
+ } |
+ } |
+ |
+ static void addCompileErrorSkipTest() { |
+ total++; |
+ compileErrorSkip++; |
+ } |
+ |
+ static void printReport() { |
+ if (total == 0) return; |
+ var bogus = nonStandardTestCases.length; |
+ String report = """Total: $total tests |
+ * $skipped tests will be skipped ($skippedByDesign skipped by design) |
+ * $noCrash tests are expected to be flaky but not crash |
+ * $flakyCrash tests are expected to flaky crash |
+ * $pass tests are expected to pass |
+ * $failOk tests are expected to fail that we won't fix |
+ * $fail tests are expected to fail that we should fix |
+ * $crash tests are expected to crash that we should fix |
+ * $timeout tests are allowed to timeout |
+ * $compileErrorSkip tests are skipped on browsers due to compile-time error |
+ * $bogus could not be categorized or are in multiple categories |
+"""; |
+ print(report); |
+ } |
+} |