test.py: Support for CompileTimeError,RuntimeError,MissingRuntimeError,MissingCompiletimeError mark…

tools/testing/dart/test_suite.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for enumerating and preparing tests.
  *
  * This library includes:
  *
  * - Creating tests by listing all the Dart files in certain directories,
@@ skipping 280 matching lines @@
       if (testCase.info != null &&
           testCase.info.hasCompileError &&
           TestUtils.isBrowserRuntime(configuration['runtime']) &&
           configuration['compiler'] != 'none') {
         SummaryReport.addCompileErrorSkipTest();
         return;
       }
     }
 
     // Handle skipped tests
-    if (expectations.contains(SKIP) ||
-        expectations.contains(SKIP_BY_DESIGN)) {
+    if (expectations.contains(Expectation.SKIP) ||
+        expectations.contains(Expectation.SKIP_BY_DESIGN)) {
       return;
     }
 
     doTest(testCase);
   }
 
   String buildTestCaseDisplayName(Path suiteDir,
                                   Path originTestPath,
                                   {String multitestName: ""}) {
     Path testNamePath = originTestPath.relativeTo(suiteDir);
@@ skipping 384 matching lines @@
     }
   }
 
   void enqueueTestCaseFromTestInformation(TestInformation info) {
     var filePath = info.filePath;
     var optionsFromFile = info.optionsFromFile;
 
     String testName = buildTestCaseDisplayName(suiteDir, info.originTestPath,
         multitestName: optionsFromFile['isMultitest'] ? info.multitestKey : "");
 
-    Set<String> expectations = testExpectations.expectations(testName);
+    Set<Expectation> expectations = testExpectations.expectations(testName);
     if (configuration['compiler'] != 'none' && info.hasCompileError) {
       // If a compile-time error is expected, and we're testing a
       // compiler, we never need to attempt to run the program (in a
       // browser or otherwise).
       enqueueStandardTest(info, testName, expectations);
     } else if (TestUtils.isBrowserRuntime(configuration['runtime'])) {
       bool isWrappingRequired = configuration['compiler'] != 'dart2js';
       if (info.optionsFromFile['isMultiHtmlTest']) {
         // A browser multi-test has multiple expectations for one test file.
         // Find all the different sub-test expecations for one entire test file.
         List<String> subtestNames = info.optionsFromFile['subtestNames'];
-        Map<String, Set<String>> multiHtmlTestExpectations = {};
+        Map<String, Set<Expectation>> multiHtmlTestExpectations = {};
         for (String name in subtestNames) {
           String fullTestName = '$testName/$name';
           multiHtmlTestExpectations[fullTestName] =
               testExpectations.expectations(fullTestName);
         }
         enqueueBrowserTest(info, testName, multiHtmlTestExpectations,
             isWrappingRequired);
       } else {
         enqueueBrowserTest(info, testName, expectations, isWrappingRequired);
       }
     } else {
       enqueueStandardTest(info, testName, expectations);
     }
   }
 
   void enqueueStandardTest(TestInformation info,
                            String testName,
-                           Set<String> expectations) {
+                           Set<Expectation> expectations) {
     var commonArguments = commonArgumentsFromFile(info.filePath,
                                                   info.optionsFromFile);
 
     List<List<String>> vmOptionsList = getVmOptions(info.optionsFromFile);
     assert(!vmOptionsList.isEmpty);
 
     for (var vmOptions in vmOptionsList) {
       var allVmOptions = vmOptions;
       if (!extraVmOptions.isEmpty) {
         allVmOptions = new List.from(vmOptions)..addAll(extraVmOptions);
@@ skipping 35 matching lines @@
 
       var command = CommandBuilder.instance.getCompilationCommand(
           compiler, "$tempDir/out.js", !useSdk,
           dart2JsBootstrapDependencies, compilerPath, args, configurationDir);
 
       List<Command> commands = <Command>[command];
       if (info.hasCompileError) {
         // Do not attempt to run the compiled result. A compilation
         // error should be reported by the compilation command.
       } else if (configuration['runtime'] == 'd8') {
-        commands.add(CommandBuilder.instance.getCommand(
+        commands.add(CommandBuilder.instance.getJSCommandlineCommand(
             "d8", d8FileName, ['$tempDir/out.js'], configurationDir));
       } else if (configuration['runtime'] == 'jsshell') {
-        commands.add(CommandBuilder.instance.getCommand(
+        commands.add(CommandBuilder.instance.getJSCommandlineCommand(
             "jsshell", jsShellFileName, ['$tempDir/out.js'], configurationDir));
       }
       return commands;
 
     case 'dart2dart':
       args = new List.from(args);
       args.add('--output-type=dart');
       String tempDir = createOutputDirectory(info.filePath, '');
       args.add('--out=$tempDir/out.dart');
 
       List<Command> commands =
           <Command>[CommandBuilder.instance.getCompilationCommand(
               compiler, "$tempDir/out.dart", !useSdk,
               dart2JsBootstrapDependencies, compilerPath, args,
               configurationDir)];
       if (info.hasCompileError) {
         // Do not attempt to run the compiled result. A compilation
         // error should be reported by the compilation command.
       } else if (configuration['runtime'] == 'vm') {
         // TODO(antonm): support checked.
         var vmArguments = new List.from(vmOptions);
         vmArguments.addAll([
             '--ignore-unrecognized-flags', '$tempDir/out.dart']);
-        commands.add(CommandBuilder.instance.getCommand(
-            "vm", vmFileName, vmArguments, configurationDir));
+        commands.add(CommandBuilder.instance.getVmCommand(
+            vmFileName, vmArguments, configurationDir));
       } else {
         throw 'Unsupported runtime ${configuration["runtime"]} for dart2dart';
       }
       return commands;
 
     case 'none':
       var arguments = new List.from(vmOptions);
       arguments.addAll(args);
-      return <Command>[CommandBuilder.instance.getCommand(
-          'vm', dartShellFileName, arguments, configurationDir)];
+      return <Command>[CommandBuilder.instance.getVmCommand(
+          dartShellFileName, arguments, configurationDir)];
 
     case 'dartanalyzer':
     case 'dart2analyzer':
       return <Command>[CommandBuilder.instance.getAnalysisCommand(
           compiler, dartShellFileName, args, configurationDir,
           flavor: compiler)];
 
     default:
       throw 'Unknown compiler ${configuration["compiler"]}';
     }
@@ skipping 1118 matching lines @@
    * Gets extra vm options passed to the testing script.
    */
   static List<String> getExtraVmOptions(Map configuration) {
     var extraVmOptions = [];
     if (configuration['vm-options'] != null) {
       extraVmOptions = configuration['vm-options'].split(" ");
       extraVmOptions.removeWhere((s) => s.trim() == "");
     }
     return extraVmOptions;
   }
-
-
 }
 
 class SummaryReport {
   static int total = 0;
   static int skipped = 0;
   static int skippedByDesign = 0;
   static int noCrash = 0;
   static int pass = 0;
   static int failOk = 0;
   static int fail = 0;
   static int crash = 0;
   static int timeout = 0;
   static int compileErrorSkip = 0;
 
-  static void add(Set<String> expectations) {
+  static void add(Set<Expectation> expectations) {
     ++total;
-    if (expectations.contains(SKIP)) {
+    if (expectations.contains(Expectation.SKIP)) {
       ++skipped;
-    } else if (expectations.contains(SKIP_BY_DESIGN)) {
+    } else if (expectations.contains(Expectation.SKIP_BY_DESIGN)) {
       ++skipped;
       ++skippedByDesign;
     } else {
-      if (expectations.contains(PASS) && expectations.contains(FAIL) &&
-          !expectations.contains(CRASH) && !expectations.contains(OK)) {
+      if (expectations.contains(Expectation.PASS) &&
+          expectations.contains(Expectation.FAIL) &&
+          !expectations.contains(Expectation.CRASH) &&
+          !expectations.contains(Expectation.OK)) {
         ++noCrash;
       }
-      if (expectations.contains(PASS) && expectations.length == 1) {
+      if (expectations.contains(Expectation.PASS) && expectations.length == 1) {
         ++pass;
       }
-      if (expectations.containsAll([FAIL, OK]) && expectations.length == 2) {
+      if (expectations.containsAll([Expectation.FAIL, Expectation.OK]) &&
+          expectations.length == 2) {
         ++failOk;
       }
-      if (expectations.contains(FAIL) && expectations.length == 1) {
+      if (expectations.contains(Expectation.FAIL) && expectations.length == 1) {
         ++fail;
       }
-      if (expectations.contains(CRASH) && expectations.length == 1) {
+      if (expectations.contains(Expectation.CRASH) &&
+          expectations.length == 1) {
         ++crash;
       }
-      if (expectations.contains(TIMEOUT)) {
+      if (expectations.contains(Expectation.TIMEOUT)) {
         ++timeout;
       }
     }
   }
 
   static void addCompileErrorSkipTest() {
     total++;
     compileErrorSkip++;
   }
 
   static void printReport() {
     if (total == 0) return;
     String report = """Total: $total tests
  * $skipped tests will be skipped ($skippedByDesign skipped by design)
  * $noCrash tests are expected to be flaky but not crash
  * $pass tests are expected to pass
  * $failOk tests are expected to fail that we won't fix
  * $fail tests are expected to fail that we should fix
  * $crash tests are expected to crash that we should fix
  * $timeout tests are allowed to timeout
  * $compileErrorSkip tests are skipped on browsers due to compile-time error
 """;
     print(report);
    }
 }