OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 /** | 5 /** |
6 * Classes and methods for enumerating and preparing tests. | 6 * Classes and methods for enumerating and preparing tests. |
7 * | 7 * |
8 * This library includes: | 8 * This library includes: |
9 * | 9 * |
10 * - Creating tests by listing all the Dart files in certain directories, | 10 * - Creating tests by listing all the Dart files in certain directories, |
(...skipping 337 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
348 String constructedName = '$suiteName/$testPrefix$testName'; | 348 String constructedName = '$suiteName/$testPrefix$testName'; |
349 if (!pattern.hasMatch(constructedName)) return; | 349 if (!pattern.hasMatch(constructedName)) return; |
350 | 350 |
351 var expectations = testExpectations.expectations( | 351 var expectations = testExpectations.expectations( |
352 '$testPrefix$testName'); | 352 '$testPrefix$testName'); |
353 | 353 |
354 if (configuration["report"]) { | 354 if (configuration["report"]) { |
355 SummaryReport.add(expectations); | 355 SummaryReport.add(expectations); |
356 } | 356 } |
357 | 357 |
358 if (expectations.contains(SKIP)) return; | 358 if (expectations.contains(SKIP) || |
| 359 expectations.contains(SKIP_BY_DESIGN)) { |
| 360 return; |
| 361 } |
359 | 362 |
360 var args = TestUtils.standardOptions(configuration); | 363 var args = TestUtils.standardOptions(configuration); |
361 args.add(testName); | 364 args.add(testName); |
362 | 365 |
363 var command = CommandBuilder.instance.getCommand( | 366 var command = CommandBuilder.instance.getCommand( |
364 'run_vm_unittest', targetRunnerPath, args, configurationDir); | 367 'run_vm_unittest', targetRunnerPath, args, configurationDir); |
365 doTest( | 368 doTest( |
366 new TestCase(constructedName, | 369 new TestCase(constructedName, |
367 [command], | 370 [command], |
368 configuration, | 371 configuration, |
(...skipping 307 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
676 configuration['report']) { | 679 configuration['report']) { |
677 SummaryReport.addCompileErrorSkipTest(); | 680 SummaryReport.addCompileErrorSkipTest(); |
678 return; | 681 return; |
679 } | 682 } |
680 if (configuration['report']) { | 683 if (configuration['report']) { |
681 // Tests with multiple VMOptions are counted more than once. | 684 // Tests with multiple VMOptions are counted more than once. |
682 for (var dummy in getVmOptions(optionsFromFile)) { | 685 for (var dummy in getVmOptions(optionsFromFile)) { |
683 SummaryReport.add(expectations); | 686 SummaryReport.add(expectations); |
684 } | 687 } |
685 } | 688 } |
686 if (expectations.contains(SKIP)) return; | 689 if (expectations.contains(SKIP) || |
| 690 expectations.contains(SKIP_BY_DESIGN)) { |
| 691 return; |
| 692 } |
687 | 693 |
688 if (configuration['compiler'] != 'none' && info.hasCompileError) { | 694 if (configuration['compiler'] != 'none' && info.hasCompileError) { |
689 // If a compile-time error is expected, and we're testing a | 695 // If a compile-time error is expected, and we're testing a |
690 // compiler, we never need to attempt to run the program (in a | 696 // compiler, we never need to attempt to run the program (in a |
691 // browser or otherwise). | 697 // browser or otherwise). |
692 enqueueStandardTest(info, testName, expectations); | 698 enqueueStandardTest(info, testName, expectations); |
693 } else if (TestUtils.isBrowserRuntime(configuration['runtime'])) { | 699 } else if (TestUtils.isBrowserRuntime(configuration['runtime'])) { |
694 bool isWrappingRequired = configuration['compiler'] != 'dart2js'; | 700 bool isWrappingRequired = configuration['compiler'] != 'dart2js'; |
695 if (info.optionsFromFile['isMultiHtmlTest']) { | 701 if (info.optionsFromFile['isMultiHtmlTest']) { |
696 // A browser multi-test has multiple expectations for one test file. | 702 // A browser multi-test has multiple expectations for one test file. |
(...skipping 1203 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1900 } | 1906 } |
1901 return extraVmOptions; | 1907 return extraVmOptions; |
1902 } | 1908 } |
1903 | 1909 |
1904 | 1910 |
1905 } | 1911 } |
1906 | 1912 |
1907 class SummaryReport { | 1913 class SummaryReport { |
1908 static int total = 0; | 1914 static int total = 0; |
1909 static int skipped = 0; | 1915 static int skipped = 0; |
| 1916 static int skippedByDesign = 0; |
1910 static int noCrash = 0; | 1917 static int noCrash = 0; |
1911 static int pass = 0; | 1918 static int pass = 0; |
1912 static int failOk = 0; | 1919 static int failOk = 0; |
1913 static int fail = 0; | 1920 static int fail = 0; |
1914 static int crash = 0; | 1921 static int crash = 0; |
1915 static int timeout = 0; | 1922 static int timeout = 0; |
1916 static int compileErrorSkip = 0; | 1923 static int compileErrorSkip = 0; |
1917 | 1924 |
1918 static void add(Set<String> expectations) { | 1925 static void add(Set<String> expectations) { |
1919 ++total; | 1926 ++total; |
1920 if (expectations.contains(SKIP)) { | 1927 if (expectations.contains(SKIP)) { |
1921 ++skipped; | 1928 ++skipped; |
| 1929 } else if (expectations.contains(SKIP_BY_DESIGN)) { |
| 1930 ++skipped; |
| 1931 ++skippedByDesign; |
1922 } else { | 1932 } else { |
1923 if (expectations.contains(PASS) && expectations.contains(FAIL) && | 1933 if (expectations.contains(PASS) && expectations.contains(FAIL) && |
1924 !expectations.contains(CRASH) && !expectations.contains(OK)) { | 1934 !expectations.contains(CRASH) && !expectations.contains(OK)) { |
1925 ++noCrash; | 1935 ++noCrash; |
1926 } | 1936 } |
1927 if (expectations.contains(PASS) && expectations.length == 1) { | 1937 if (expectations.contains(PASS) && expectations.length == 1) { |
1928 ++pass; | 1938 ++pass; |
1929 } | 1939 } |
1930 if (expectations.containsAll([FAIL, OK]) && expectations.length == 2) { | 1940 if (expectations.containsAll([FAIL, OK]) && expectations.length == 2) { |
1931 ++failOk; | 1941 ++failOk; |
(...skipping 11 matching lines...) Expand all Loading... |
1943 } | 1953 } |
1944 | 1954 |
1945 static void addCompileErrorSkipTest() { | 1955 static void addCompileErrorSkipTest() { |
1946 total++; | 1956 total++; |
1947 compileErrorSkip++; | 1957 compileErrorSkip++; |
1948 } | 1958 } |
1949 | 1959 |
1950 static void printReport() { | 1960 static void printReport() { |
1951 if (total == 0) return; | 1961 if (total == 0) return; |
1952 String report = """Total: $total tests | 1962 String report = """Total: $total tests |
1953 * $skipped tests will be skipped | 1963 * $skipped tests will be skipped ($skippedByDesign skipped by design) |
1954 * $noCrash tests are expected to be flaky but not crash | 1964 * $noCrash tests are expected to be flaky but not crash |
1955 * $pass tests are expected to pass | 1965 * $pass tests are expected to pass |
1956 * $failOk tests are expected to fail that we won't fix | 1966 * $failOk tests are expected to fail that we won't fix |
1957 * $fail tests are expected to fail that we should fix | 1967 * $fail tests are expected to fail that we should fix |
1958 * $crash tests are expected to crash that we should fix | 1968 * $crash tests are expected to crash that we should fix |
1959 * $timeout tests are allowed to timeout | 1969 * $timeout tests are allowed to timeout |
1960 * $compileErrorSkip tests are skipped on browsers due to compile-time error | 1970 * $compileErrorSkip tests are skipped on browsers due to compile-time error |
1961 """; | 1971 """; |
1962 print(report); | 1972 print(report); |
1963 } | 1973 } |
1964 } | 1974 } |
OLD | NEW |