| OLD | NEW |
| (Empty) | |
| 1 library summary_report; |
| 2 |
| 3 import 'status_file_parser.dart'; |
| 4 import 'test_case.dart'; |
| 5 |
| 6 class SummaryReport { |
| 7 static int total = 0; |
| 8 static int skipped = 0; |
| 9 static int skippedByDesign = 0; |
| 10 static int noCrash = 0; |
| 11 static int flakyCrash = 0; |
| 12 static int pass = 0; |
| 13 static int failOk = 0; |
| 14 static int fail = 0; |
| 15 static int crash = 0; |
| 16 static int timeout = 0; |
| 17 static int compileErrorSkip = 0; |
| 18 |
| 19 static List<TestCase> nonStandardTestCases = []; |
| 20 |
| 21 static void add(TestCase testCase) { |
| 22 var expectations = testCase.expectedOutcomes; |
| 23 |
| 24 bool containsFail = expectations.any( |
| 25 (expectation) => expectation.canBeOutcomeOf(Expectation.FAIL)); |
| 26 bool containsPass = expectations.contains(Expectation.PASS); |
| 27 bool containsSkip = expectations.contains(Expectation.SKIP); |
| 28 bool containsSkipByDesign = |
| 29 expectations.contains(Expectation.SKIP_BY_DESIGN); |
| 30 bool containsCrash = expectations.contains(Expectation.CRASH); |
| 31 bool containsOK = expectations.contains(Expectation.OK); |
| 32 bool containsSlow = expectations.contains(Expectation.SLOW); |
| 33 bool containsTimeout = expectations.contains(Expectation.TIMEOUT); |
| 34 |
| 35 ++total; |
| 36 if (containsSkip) { |
| 37 ++skipped; |
| 38 } else if (containsSkipByDesign) { |
| 39 ++skipped; |
| 40 ++skippedByDesign; |
| 41 } else { |
| 42 // We don't do if-else below because the buckets should be exclusive. |
| 43 // We keep a count around to guarantee that |
| 44 int markers = 0; |
| 45 |
| 46 // Counts the number of flaky tests. |
| 47 if (containsFail && containsPass && !containsCrash && !containsOK) { |
| 48 ++noCrash; |
| 49 ++markers; |
| 50 } |
| 51 if (containsCrash && !containsOK && expectations.length > 1) { |
| 52 ++flakyCrash; |
| 53 ++markers; |
| 54 } |
| 55 if ((containsPass && expectations.length == 1) || |
| 56 (containsPass && containsSlow && expectations.length == 2)) { |
| 57 ++pass; |
| 58 ++markers; |
| 59 } |
| 60 if (containsFail && containsOK) { |
| 61 ++failOk; |
| 62 ++markers; |
| 63 } |
| 64 if ((containsFail && expectations.length == 1) || |
| 65 (containsFail && containsSlow && expectations.length == 2)) { |
| 66 ++fail; |
| 67 ++markers; |
| 68 } |
| 69 if ((containsCrash && expectations.length == 1) || |
| 70 (containsCrash && containsSlow && expectations.length == 2)) { |
| 71 ++crash; |
| 72 ++markers; |
| 73 } |
| 74 if (containsTimeout && expectations.length == 1) { |
| 75 ++timeout; |
| 76 ++markers; |
| 77 } |
| 78 if (markers != 1) { |
| 79 nonStandardTestCases.add(testCase); |
| 80 } |
| 81 } |
| 82 } |
| 83 |
| 84 static void addCompileErrorSkipTest() { |
| 85 total++; |
| 86 compileErrorSkip++; |
| 87 } |
| 88 |
| 89 static void printReport() { |
| 90 if (total == 0) return; |
| 91 var bogus = nonStandardTestCases.length; |
| 92 String report = """Total: $total tests |
| 93 * $skipped tests will be skipped ($skippedByDesign skipped by design) |
| 94 * $noCrash tests are expected to be flaky but not crash |
| 95 * $flakyCrash tests are expected to flaky crash |
| 96 * $pass tests are expected to pass |
| 97 * $failOk tests are expected to fail that we won't fix |
| 98 * $fail tests are expected to fail that we should fix |
| 99 * $crash tests are expected to crash that we should fix |
| 100 * $timeout tests are allowed to timeout |
| 101 * $compileErrorSkip tests are skipped on browsers due to compile-time error |
| 102 * $bogus could not be categorized or are in multiple categories |
| 103 """; |
| 104 print(report); |
| 105 } |
| 106 } |
| OLD | NEW |