VM: [Kernel] Switch to DFE for testing -c dartk configuration.

tools/testing/dart/test_runner.dart

Index: tools/testing/dart/test_runner.dart
diff --git a/tools/testing/dart/test_runner.dart b/tools/testing/dart/test_runner.dart
index ac80516b978b71fcb18f1862368ef165fcd78661..0c15413f515eb868d0dc07cff8b9c5817b4f9b50 100644
--- a/tools/testing/dart/test_runner.dart
+++ b/tools/testing/dart/test_runner.dart
@@ -365,14 +365,20 @@ class AnalysisCommand extends ProcessCommand {
 }
 
 class VmCommand extends ProcessCommand {
+  final bool needsDFERunner;
   VmCommand._(String executable, List<String> arguments,
-      Map<String, String> environmentOverrides)
+      Map<String, String> environmentOverrides,
+      bool this.needsDFERunner)
       : super._("vm", executable, arguments, environmentOverrides);
+
+  bool _equal(VmCommand other) =>
+      super._equal(other) && needsDFERunner == other.needsDFERunner;

[kustermann, 2017/01/31 15:39:00]

whenever you overwrite _equal, also overwrite _buildHashCode

[Vyacheslav Egorov (Google), 2017/01/31 17:06:56]

Done.

 }
 
 class VmBatchCommand extends ProcessCommand implements VmCommand {
   final String dartFile;
   final bool checked;
+  final needsDFERunner = false;
 
   VmBatchCommand._(String executable, String dartFile, List<String> arguments,
       Map<String, String> environmentOverrides, {this.checked: true})
@@ -733,8 +739,8 @@ class CommandBuilder {
   }
 
   VmCommand getVmCommand(String executable, List<String> arguments,
-      Map<String, String> environmentOverrides) {
-    var command = new VmCommand._(executable, arguments, environmentOverrides);
+      Map<String, String> environmentOverrides, {bool needsDFERunner: false}) {
+    var command = new VmCommand._(executable, arguments, environmentOverrides, needsDFERunner);
     return _getUniqueCommand(command);
   }
 
@@ -1592,6 +1598,7 @@ class AnalysisCommandOutputImpl extends CommandOutputImpl {
 
 class VmCommandOutputImpl extends CommandOutputImpl
     with UnittestSuiteMessagesMixin {
+  static const DART_VM_EXITCODE_DFE_ERROR = 252;
   static const DART_VM_EXITCODE_COMPILE_TIME_ERROR = 254;
   static const DART_VM_EXITCODE_UNCAUGHT_EXCEPTION = 255;
 
@@ -1601,6 +1608,7 @@ class VmCommandOutputImpl extends CommandOutputImpl
 
   Expectation result(TestCase testCase) {
     // Handle crashes and timeouts first
+    if (exitCode == DART_VM_EXITCODE_DFE_ERROR) return Expectation.DARTK_CRASH;
     if (hasCrashed) return Expectation.CRASH;
     if (hasTimedOut) return Expectation.TIMEOUT;
 
@@ -1915,8 +1923,9 @@ class RunningProcess {
   List<String> diagnostics = <String>[];
   bool compilationSkipped = false;
   Completer<CommandOutput> completer;
+  List<String> preArguments;
 
-  RunningProcess(this.command, this.timeout);
+  RunningProcess(this.command, this.timeout, {this.preArguments});
 
   Future<CommandOutput> run() {
     completer = new Completer<CommandOutput>();
@@ -1932,8 +1941,12 @@ class RunningProcess {
         _commandComplete(0);
       } else {
         var processEnvironment = _createProcessEnvironment();
+        var args = command.arguments;
+        if (preArguments != null) {
+          args = []..addAll(preArguments)..addAll(args);
+        }
         Future processFuture = io.Process.start(
-            command.executable, command.arguments,
+            command.executable, args,
             environment: processEnvironment,
             workingDirectory: command.workingDirectory);
         processFuture.then((io.Process process) {
@@ -2087,7 +2100,8 @@ class RunningProcess {
   }
 }
 
-class BatchRunnerProcess {
+

[kustermann, 2017/01/31 15:39:00]

one new line is enough

[Vyacheslav Egorov (Google), 2017/01/31 17:06:56]

Done.

+class BatchRunnerProcess  {
   Completer<CommandOutput> _completer;
   ProcessCommand _command;
   List<String> _arguments;
@@ -2108,8 +2122,6 @@ class BatchRunnerProcess {
   DateTime _startTime;
   Timer _timer;
 
-  BatchRunnerProcess();
-
   Future<CommandOutput> runCommand(String runnerType, ProcessCommand command,
       int timeout, List<String> arguments) {
     assert(_completer == null);
@@ -2306,6 +2318,84 @@ class BatchRunnerProcess {
   }
 }
 
+class BatchDFEProcess  {
+  io.Process _process;
+  int _port = -1;
+  Function _processExitHandler;
+
+  Completer terminating = null;
+
+  bool locked = false;
+
+  Future<int> acquire() async {
+    try {
+      assert(!locked);
+      locked = true;
+      if (_process == null) {
+        await _startProcess();
+      }
+      return _port;
+    } catch(e) {
+      locked = false;
+      rethrow;
+    }
+  }
+
+  void release() {
+    locked = false;
+  }
+
+  Future terminate() {
+    locked = true;
+    if (_process == null) {
+      return new Future.value(true);
+    }
+    if (terminating == null) {
+      terminating = new Completer();
+      _process.kill();
+    }
+    return terminating.future;
+  }
+
+  _onExit(exitCode) {
+    if (terminating != null) {
+      terminating.complete();
+      return;
+    }
+
+    _process = null;
+    locked = false;
+    _port = -1;
+  }
+
+  _startProcess() async {

[kustermann, 2017/01/31 15:39:00]

Future 

[Vyacheslav Egorov (Google), 2017/01/31 17:06:56]

Done.

+    final executable = 'tools/dartk_wrappers/dfe_worker';
+    final arguments = [];
+
+    try {
+      _port = -1;
+      _process = await io.Process.start(executable, arguments);

[kustermann, 2017/01/31 15:39:00]

You can just use Platform.executable and 'tools/dartk_wrappers/dfe_worker' here.
The reason we have a wrapper script for dartk is different and not necessary
here.

[Vyacheslav Egorov (Google), 2017/01/31 17:06:56]

Done.

+      _process.exitCode.then(_onExit);
+
+      final readyMsg = await _process.stdout.transform(UTF8.decoder).transform(new LineSplitter()).first;

[kustermann, 2017/01/31 15:39:00]

long line

this will potentially crash the subprocess if it prints more than one line
(because the write causes SIGPIPE / broken pipe).

The .first will cancel the subscription after the first line.

[Vyacheslav Egorov (Google), 2017/01/31 17:06:56]

Done.

[Vyacheslav Egorov (Google), 2017/01/31 17:06:56]

Done.

+      final data = readyMsg.split(' ');
+      assert(data[0] == 'READY');

[kustermann, 2017/01/31 15:39:00]

Consider draining _process.stderr as well (maybe just using DebugLogger or
io.stderr to report it)

[Vyacheslav Egorov (Google), 2017/01/31 17:06:56]

Done.

+
+      _port = int.parse(data[1]);
+    } catch (e) {
+      // TODO(floitsch): should we try to report the stacktrace?

[kustermann, 2017/01/31 15:39:00]

floitsch :)

[Vyacheslav Egorov (Google), 2017/01/31 17:06:56]

Done.

+      print("Process error:");
+      print("  Command: $executable ${arguments.join(' ')}");
+      print("  Error: $e");
+      // If there is an error starting a batch process, chances are that
+      // it will always fail. So rather than re-trying a 1000+ times, we
+      // exit.
+      io.exit(1);
+      return true;
+    }
+  }
+}
+
 /**
  * [TestCaseEnqueuer] takes a list of TestSuites, generates TestCases and
  * builds a dependency graph of all commands in every TestSuite.
@@ -2614,6 +2704,8 @@ class CommandExecutorImpl implements CommandExecutor {
   // We keep a BrowserTestRunner for every configuration.
   final _browserTestRunners = new Map<Map, BrowserTestRunner>();
 
+  List<BatchDFEProcess> _dfeProcesses = null;
+
   bool _finishing = false;
 
   CommandExecutorImpl(
@@ -2638,7 +2730,12 @@ class CommandExecutorImpl implements CommandExecutor {
       return Future.wait(futures);
     }
 
-    return Future.wait([_terminateBatchRunners(), _terminateBrowserRunners()]);
+    Future _terminateDFEWorkers() =>
+      Future.wait((_dfeProcesses ?? []).map((p) => p.terminate()));
+
+    return Future.wait([_terminateBatchRunners(),
+    _terminateBrowserRunners(),
+    _terminateDFEWorkers()]);

[kustermann, 2017/01/31 15:39:00]

indentation

[Vyacheslav Egorov (Google), 2017/01/31 17:06:56]

Done.

   }
 
   Future<CommandOutput> runCommand(node, Command command, int timeout) {
@@ -2686,6 +2783,11 @@ class CommandExecutorImpl implements CommandExecutor {
           adbDevicePool.releaseDevice(device);
         });
       });
+    } else if (command is VmCommand && command.needsDFERunner) {
+      final runner = _getDFEProcess();
+      return runner.acquire().then((port) {
+        return new RunningProcess(command, timeout, preArguments: ['-DDFE_WORKER_PORT=${port}']).run();
+      }).whenComplete(() => runner.release());
     } else if (command is VmBatchCommand) {
       var name = command.displayName;
       return _getBatchRunner(command.displayName + command.dartFile)
@@ -2788,6 +2890,12 @@ class CommandExecutorImpl implements CommandExecutor {
     throw new Exception('Unable to find inactive batch runner.');
   }
 
+  BatchDFEProcess _getDFEProcess() {
+    _dfeProcesses ??= new List<BatchDFEProcess>.generate(maxProcesses,
+        (_) => new BatchDFEProcess());
+    return _dfeProcesses.firstWhere((runner) => !runner.locked);
+  }
+
   Future<CommandOutput> _startBrowserControllerTest(
       BrowserTestCommand browserCommand, int timeout) {
     var completer = new Completer<CommandOutput>();