| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 /** | 5 /** |
| 6 * Classes and methods for enumerating and preparing tests. | 6 * Classes and methods for enumerating and preparing tests. |
| 7 * | 7 * |
| 8 * This library includes: | 8 * This library includes: |
| 9 * | 9 * |
| 10 * - Creating tests by listing all the Dart files in certain directories, | 10 * - Creating tests by listing all the Dart files in certain directories, |
| (...skipping 337 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 348 String constructedName = '$suiteName/$testPrefix$testName'; | 348 String constructedName = '$suiteName/$testPrefix$testName'; |
| 349 if (!pattern.hasMatch(constructedName)) return; | 349 if (!pattern.hasMatch(constructedName)) return; |
| 350 | 350 |
| 351 var expectations = testExpectations.expectations( | 351 var expectations = testExpectations.expectations( |
| 352 '$testPrefix$testName'); | 352 '$testPrefix$testName'); |
| 353 | 353 |
| 354 if (configuration["report"]) { | 354 if (configuration["report"]) { |
| 355 SummaryReport.add(expectations); | 355 SummaryReport.add(expectations); |
| 356 } | 356 } |
| 357 | 357 |
| 358 if (expectations.contains(SKIP)) return; | 358 if (expectations.contains(SKIP) || |
| 359 expectations.contains(SKIP_BY_DESIGN)) { |
| 360 return; |
| 361 } |
| 359 | 362 |
| 360 var args = TestUtils.standardOptions(configuration); | 363 var args = TestUtils.standardOptions(configuration); |
| 361 args.add(testName); | 364 args.add(testName); |
| 362 | 365 |
| 363 var command = CommandBuilder.instance.getCommand( | 366 var command = CommandBuilder.instance.getCommand( |
| 364 'run_vm_unittest', targetRunnerPath, args, configurationDir); | 367 'run_vm_unittest', targetRunnerPath, args, configurationDir); |
| 365 doTest( | 368 doTest( |
| 366 new TestCase(constructedName, | 369 new TestCase(constructedName, |
| 367 [command], | 370 [command], |
| 368 configuration, | 371 configuration, |
| (...skipping 307 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 676 configuration['report']) { | 679 configuration['report']) { |
| 677 SummaryReport.addCompileErrorSkipTest(); | 680 SummaryReport.addCompileErrorSkipTest(); |
| 678 return; | 681 return; |
| 679 } | 682 } |
| 680 if (configuration['report']) { | 683 if (configuration['report']) { |
| 681 // Tests with multiple VMOptions are counted more than once. | 684 // Tests with multiple VMOptions are counted more than once. |
| 682 for (var dummy in getVmOptions(optionsFromFile)) { | 685 for (var dummy in getVmOptions(optionsFromFile)) { |
| 683 SummaryReport.add(expectations); | 686 SummaryReport.add(expectations); |
| 684 } | 687 } |
| 685 } | 688 } |
| 686 if (expectations.contains(SKIP)) return; | 689 if (expectations.contains(SKIP) || |
| 690 expectations.contains(SKIP_BY_DESIGN)) { |
| 691 return; |
| 692 } |
| 687 | 693 |
| 688 if (configuration['compiler'] != 'none' && info.hasCompileError) { | 694 if (configuration['compiler'] != 'none' && info.hasCompileError) { |
| 689 // If a compile-time error is expected, and we're testing a | 695 // If a compile-time error is expected, and we're testing a |
| 690 // compiler, we never need to attempt to run the program (in a | 696 // compiler, we never need to attempt to run the program (in a |
| 691 // browser or otherwise). | 697 // browser or otherwise). |
| 692 enqueueStandardTest(info, testName, expectations); | 698 enqueueStandardTest(info, testName, expectations); |
| 693 } else if (TestUtils.isBrowserRuntime(configuration['runtime'])) { | 699 } else if (TestUtils.isBrowserRuntime(configuration['runtime'])) { |
| 694 bool isWrappingRequired = configuration['compiler'] != 'dart2js'; | 700 bool isWrappingRequired = configuration['compiler'] != 'dart2js'; |
| 695 if (info.optionsFromFile['isMultiHtmlTest']) { | 701 if (info.optionsFromFile['isMultiHtmlTest']) { |
| 696 // A browser multi-test has multiple expectations for one test file. | 702 // A browser multi-test has multiple expectations for one test file. |
| (...skipping 1203 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1900 } | 1906 } |
| 1901 return extraVmOptions; | 1907 return extraVmOptions; |
| 1902 } | 1908 } |
| 1903 | 1909 |
| 1904 | 1910 |
| 1905 } | 1911 } |
| 1906 | 1912 |
| 1907 class SummaryReport { | 1913 class SummaryReport { |
| 1908 static int total = 0; | 1914 static int total = 0; |
| 1909 static int skipped = 0; | 1915 static int skipped = 0; |
| 1916 static int skippedByDesign = 0; |
| 1910 static int noCrash = 0; | 1917 static int noCrash = 0; |
| 1911 static int pass = 0; | 1918 static int pass = 0; |
| 1912 static int failOk = 0; | 1919 static int failOk = 0; |
| 1913 static int fail = 0; | 1920 static int fail = 0; |
| 1914 static int crash = 0; | 1921 static int crash = 0; |
| 1915 static int timeout = 0; | 1922 static int timeout = 0; |
| 1916 static int compileErrorSkip = 0; | 1923 static int compileErrorSkip = 0; |
| 1917 | 1924 |
| 1918 static void add(Set<String> expectations) { | 1925 static void add(Set<String> expectations) { |
| 1919 ++total; | 1926 ++total; |
| 1920 if (expectations.contains(SKIP)) { | 1927 if (expectations.contains(SKIP)) { |
| 1921 ++skipped; | 1928 ++skipped; |
| 1929 } else if (expectations.contains(SKIP_BY_DESIGN)) { |
| 1930 ++skipped; |
| 1931 ++skippedByDesign; |
| 1922 } else { | 1932 } else { |
| 1923 if (expectations.contains(PASS) && expectations.contains(FAIL) && | 1933 if (expectations.contains(PASS) && expectations.contains(FAIL) && |
| 1924 !expectations.contains(CRASH) && !expectations.contains(OK)) { | 1934 !expectations.contains(CRASH) && !expectations.contains(OK)) { |
| 1925 ++noCrash; | 1935 ++noCrash; |
| 1926 } | 1936 } |
| 1927 if (expectations.contains(PASS) && expectations.length == 1) { | 1937 if (expectations.contains(PASS) && expectations.length == 1) { |
| 1928 ++pass; | 1938 ++pass; |
| 1929 } | 1939 } |
| 1930 if (expectations.containsAll([FAIL, OK]) && expectations.length == 2) { | 1940 if (expectations.containsAll([FAIL, OK]) && expectations.length == 2) { |
| 1931 ++failOk; | 1941 ++failOk; |
| (...skipping 11 matching lines...) Expand all Loading... |
| 1943 } | 1953 } |
| 1944 | 1954 |
| 1945 static void addCompileErrorSkipTest() { | 1955 static void addCompileErrorSkipTest() { |
| 1946 total++; | 1956 total++; |
| 1947 compileErrorSkip++; | 1957 compileErrorSkip++; |
| 1948 } | 1958 } |
| 1949 | 1959 |
| 1950 static void printReport() { | 1960 static void printReport() { |
| 1951 if (total == 0) return; | 1961 if (total == 0) return; |
| 1952 String report = """Total: $total tests | 1962 String report = """Total: $total tests |
| 1953 * $skipped tests will be skipped | 1963 * $skipped tests will be skipped ($skippedByDesign skipped by design) |
| 1954 * $noCrash tests are expected to be flaky but not crash | 1964 * $noCrash tests are expected to be flaky but not crash |
| 1955 * $pass tests are expected to pass | 1965 * $pass tests are expected to pass |
| 1956 * $failOk tests are expected to fail that we won't fix | 1966 * $failOk tests are expected to fail that we won't fix |
| 1957 * $fail tests are expected to fail that we should fix | 1967 * $fail tests are expected to fail that we should fix |
| 1958 * $crash tests are expected to crash that we should fix | 1968 * $crash tests are expected to crash that we should fix |
| 1959 * $timeout tests are allowed to timeout | 1969 * $timeout tests are allowed to timeout |
| 1960 * $compileErrorSkip tests are skipped on browsers due to compile-time error | 1970 * $compileErrorSkip tests are skipped on browsers due to compile-time error |
| 1961 """; | 1971 """; |
| 1962 print(report); | 1972 print(report); |
| 1963 } | 1973 } |
| 1964 } | 1974 } |
| OLD | NEW |