Add 'self_check' runtime for running self-checking unit tests

tools/testing/dart/test_runner.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Classes and methods for executing tests.
  *
  * This module includes:
  * - Managing parallel execution of tests, including timeout checks.
  * - Evaluating the output of each test as pass/fail/crash/timeout.
@@ skipping 121 matching lines @@
       deepJsonCompare(arguments, other.arguments) &&
       workingDirectory == other.workingDirectory &&
       deepJsonCompare(environmentOverrides, other.environmentOverrides);
 
   String get reproductionCommand {
     var env = new StringBuffer();
     environmentOverrides?.forEach((key, value) =>
         (io.Platform.operatingSystem == 'windows')
             ? env.write('set $key=${escapeCommandLineArgument(value)} & ')
             : env.write('$key=${escapeCommandLineArgument(value)} '));
-    var command = ([executable]..addAll(arguments))
+    var command = ([executable]..addAll(batchArguments)..addAll(arguments))
         .map(escapeCommandLineArgument)
         .join(' ');
     if (workingDirectory != null) {
       command = "$command (working directory: $workingDirectory)";
     }
     return "$env$command";
   }
 
   Future<bool> get outputIsUpToDate => new Future.value(false);
+
+  /// Arguments that are passed to the process when starting batch mode.
+  ///
+  /// In non-batch mode, they should be passed before [arguments].
+  List<String> get batchArguments => const [];
 }
 
 class CompilationCommand extends ProcessCommand {
   final String _outputFile;
   final bool _neverSkipCompilation;
   final List<Uri> _bootstrapDependencies;
 
   CompilationCommand._(
       String displayName,
       this._outputFile,
@@ skipping 193 matching lines @@
   bool _equal(AnalysisCommand other) =>
       super._equal(other) && flavor == other.flavor;
 }
 
 class VmCommand extends ProcessCommand {
   VmCommand._(String executable, List<String> arguments,
       Map<String, String> environmentOverrides)
       : super._("vm", executable, arguments, environmentOverrides);
 }
 
+class VmBatchCommand extends ProcessCommand implements VmCommand {

[kustermann, 2016/12/06 10:39:05]

If you make a [VmBatchCommandOutputImpl] as well for this command (see e.g. 
KernelCompilationCommandOutputImpl) then this class can define was is 'success'
and what is 'failure'.

By doing so I think you might be able to get rid of shouldSkipNegativeTests and
the following code in test_suite.dart:

if (testCase.isNegative && runtimeConfiguration.shouldSkipNegativeTests) {
  return;
}

[asgerf, 2016/12/06 13:42:08]

I gave this a shot but couldn't make it work. The VmBatchCommandOutputImpl only
gets to choose the outcome if it actually executes, which it doesn't if the
preceding compiler command fails.

Also the VmBatchCommand was intended to run Dart programs in batch mode without
being tied to the self_check runtime, but it could of course just be renamed to
SelfCheckCommand.

+  final String dartFile;
+  final bool checked;
+
+  VmBatchCommand._(String executable, String dartFile, List<String> arguments,
+      Map<String, String> environmentOverrides, {this.checked: true})
+      : this.dartFile = dartFile,
+        super._('vm-batch', executable, arguments, environmentOverrides);
+
+  @override
+  List<String> get batchArguments => checked
+      ? ['--checked', dartFile]
+      : [dartFile];
+
+  @override
+  bool _equal(Command other) {
+    return super._equal(other) &&
+        other is VmBatchCommand &&

[kustermann, 2016/12/06 10:39:05]

The "other is VmBatchCommand" is unnecessary I think, because "operator==" will
take care of it.

[asgerf, 2016/12/06 13:42:08]

Done.

I also added 'checked' to the comparison because I noticed it was missing
(although not strictly necessary at this time, it ought to be there).

+        dartFile == other.dartFile;
+  }
+
+  @override
+  void _buildHashCode(HashCodeBuilder builder) {
+    super._buildHashCode(builder);
+    builder.addJson(dartFile);
+  }
+}
+
 class AdbPrecompilationCommand extends Command {
   final String precompiledRunnerFilename;
   final String processTestFilename;
   final String precompiledTestDirectory;
   final List<String> arguments;
   final bool useBlobs;
 
   AdbPrecompilationCommand._(this.precompiledRunnerFilename,
                              this.processTestFilename,
                              this.precompiledTestDirectory,
@@ skipping 319 matching lines @@
         flavor, displayName, executable, arguments, environmentOverrides);
     return _getUniqueCommand(command);
   }
 
   VmCommand getVmCommand(String executable, List<String> arguments,
       Map<String, String> environmentOverrides) {
     var command = new VmCommand._(executable, arguments, environmentOverrides);
     return _getUniqueCommand(command);
   }
 
+  VmBatchCommand getVmBatchCommand(String executable, String tester,
+      List<String> arguments, Map<String, String> environmentOverrides,
+      {bool checked: true}) {
+    var command =
+        new VmBatchCommand._(executable, tester, arguments, environmentOverrides,
+            checked: checked);
+    return _getUniqueCommand(command);
+  }
+
   AdbPrecompilationCommand getAdbPrecompiledCommand(String precompiledRunner,
                                                     String processTest,
                                                     String testDirectory,
                                                     List<String> arguments,
                                                     bool useBlobs) {
     var command = new AdbPrecompilationCommand._(
         precompiledRunner, processTest, testDirectory, arguments, useBlobs);
     return _getUniqueCommand(command);
   }
 
@@ skipping 127 matching lines @@
   bool get isNegative => _expectations & IS_NEGATIVE != 0;
   bool get hasRuntimeError => _expectations & HAS_RUNTIME_ERROR != 0;
   bool get hasStaticWarning => _expectations & HAS_STATIC_WARNING != 0;
   bool get isNegativeIfChecked => _expectations & IS_NEGATIVE_IF_CHECKED != 0;
   bool get hasCompileError => _expectations & HAS_COMPILE_ERROR != 0;
   bool get hasCompileErrorIfChecked =>
       _expectations & HAS_COMPILE_ERROR_IF_CHECKED != 0;
   bool get expectCompileError => _expectations & EXPECT_COMPILE_ERROR != 0;
 
   bool get unexpectedOutput {
-    var outcome = lastCommandOutput.result(this);
+    var outcome = this.result;
     return !expectedOutcomes.any((expectation) {
       return outcome.canBeOutcomeOf(expectation);
     });
   }
 
   Expectation get result => lastCommandOutput.result(this);
 
   CommandOutput get lastCommandOutput {
     if (commandOutputs.length == 0) {
       throw new Exception("CommandOutputs is empty, maybe no command was run? ("
@@ skipping 1312 matching lines @@
   }
 
   void _timeoutHandler() {
     _processExitHandler = makeExitHandler(">>> TEST TIMEOUT");
     _process.kill();
   }
 
   _startProcess(callback) {
     assert(_command is ProcessCommand);
     var executable = _command.executable;
-    var arguments = ['--batch'];
+    var arguments = []..addAll(_command.batchArguments)..add('--batch');
     var environment = new Map.from(io.Platform.environment);
     if (_processEnvironmentOverrides != null) {
       for (var key in _processEnvironmentOverrides.keys) {
         environment[key] = _processEnvironmentOverrides[key];
       }
     }
     Future processFuture =
         io.Process.start(executable, arguments, environment: environment);
     processFuture.then((io.Process p) {
       _process = p;
@@ skipping 440 matching lines @@
     } else if (command is ScriptCommand) {
       return command.run();
     } else if (command is AdbPrecompilationCommand) {
       assert(adbDevicePool != null);
       return adbDevicePool.acquireDevice().then((AdbDevice device) {
         return _runAdbPrecompilationCommand(device, command, timeout)
             .whenComplete(() {
           adbDevicePool.releaseDevice(device);
         });
       });
+    } else if (command is VmBatchCommand) {
+      var name = command.displayName;
+      return _getBatchRunner(command.displayName + command.dartFile)
+          .runCommand(name, command, timeout, command.arguments);
     } else {
       return new RunningProcess(command, timeout).run();
     }
   }
 
   Future<CommandOutput> _runAdbPrecompilationCommand(
       AdbDevice device, AdbPrecompilationCommand command, int timeout) async {
     var runner = command.precompiledRunnerFilename;
     var processTest = command.processTestFilename;
     var testdir = command.precompiledTestDirectory;
@@ skipping 499 matching lines @@
     }
   }
 
   void eventAllTestsDone() {
     for (var listener in _eventListener) {
       listener.allDone();
     }
     _allDone();
   }
 }