VM: [Kernel] Switch to DFE for testing -c dartk configuration.

runtime/tools/kernel-service.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 library runtime.tools.kernel_service;
 
 // This is an interface to the Dart Kernel parser and Kernel binary generator.
 //
 // It is used by the kernel-isolate to load Dart source code and generate
 // Kernel binary format.
 //
+// This is either invoked as the root script of the Kernel isolate when used
+// as a part of
+//
+//           dart --dfe=runtime/tools/kernel-service.dart ...
+//
+// invocation or it is invoked as a standalone script to perform batch mode
+// compilation requested via an HTTP interface
+//
+//           dart runtime/tools/kernel-service.dart

[kustermann, 2017/01/31 15:38:59]

This should be:


    dart runtime/tools/kernel-service.dart --batch

[Vyacheslav Egorov (Google), 2017/01/31 17:06:55]

Done.

[Vyacheslav Egorov (Google), 2017/01/31 17:06:55]

Done.

+//
+// The port for the batch mode worker is controlled by DFE_WORKER_PORT
+// environment variable. When not set (or set to 0) an ephemeral port returned

[kustermann, 2017/01/31 15:38:59]

"environment variable" aren't these -D... flags to the VM instead of environment
variables?

[Vyacheslav Egorov (Google), 2017/01/31 17:06:55]

Done.

+// by the OS is used instead.
+//
+// When this script is used as a Kernel isolate root script and DFE_WORKER_PORT
+// is set to non-zero value then Kernel isolate will forward all compilation
+// requests it receives to the batch worker on the given port.
+//
 
 import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
 import 'dart:isolate';
 import 'dart:typed_data';
 
 import 'package:kernel/analyzer/loader.dart';
 import 'package:kernel/binary/ast_to_binary.dart';
 import 'package:kernel/kernel.dart';
 import 'package:kernel/target/targets.dart';
 
 const bool verbose = const bool.fromEnvironment('DFE_VERBOSE') ?? false;
+const int workerPort = const int.fromEnvironment('DFE_WORKER_PORT') ?? 0;
 
 class DataSink implements Sink<List<int>> {
   final BytesBuilder builder = new BytesBuilder();
 
   void add(List<int> data) {
     builder.add(data);
   }
 
   void close() {
     // Nothing to do.
@@ skipping 14 matching lines @@
 
   CompilationOk(this.binary);
 
   List toResponse() => [STATUS_OK, binary];
 
   String toString() => "CompilationOk(${binary.length} bytes)";
 }
 
 abstract class CompilationFail extends CompilationResult {
   String get errorString;
+
+  Map<String, dynamic> toJson();
+
+  static CompilationFail fromJson(Map m) {
+    switch (m['status']) {
+      case STATUS_ERROR:
+        return new CompilationError(m['errors']);
+      case STATUS_CRASH:
+        return new CompilationCrash(m['exception'], m['stack']);
+    }
+  }
 }
 
 class CompilationError extends CompilationFail {
   final List<String> errors;
 
   CompilationError(this.errors);
 
   List toResponse() => [STATUS_ERROR, errorString];
 
+  Map<String, dynamic> toJson() => {
+        "status": STATUS_ERROR,
+        "errors": errors,
+      };
+
   String get errorString => errors.take(10).join('\n');
 
   String toString() => "CompilationError(${errorString})";
 }
 
 class CompilationCrash extends CompilationFail {
   final String exception;
   final String stack;
 
   CompilationCrash(this.exception, this.stack);
 
   List toResponse() => [STATUS_CRASH, errorString];
 
+  Map<String, dynamic> toJson() => {
+        "status": STATUS_CRASH,
+        "exception": exception,
+        "stack": stack,
+      };
+
   String get errorString => "${exception}\n${stack}";
 
   String toString() => "CompilationCrash(${errorString})";
 }
 
 Future<CompilationResult> parseScriptImpl(DartLoaderBatch batch_loader,
     Uri fileName, String packageConfig, String sdkPath) async {
   if (!FileSystemEntity.isFileSync(fileName.path)) {
     throw "Input file '${fileName.path}' does not exist.";
   }
@@ skipping 65 matching lines @@
       Uri.parse(Platform.resolvedExecutable).resolve("patched_sdk");
 
   if (verbose) {
     print("""DFE: Requesting compilation {
   scriptUri: ${scriptUri}
   packagesUri: ${packagesUri}
   patchedSdk: ${patchedSdk}
 }""");
   }
 
-  return await parseScript(
-      new DartLoaderBatch(), scriptUri, packagesUri.path, patchedSdk.path);
+  if (workerPort != 0) {
+    return await requestParse(scriptUri, packagesUri, patchedSdk);
+  } else {
+    return await parseScript(
+        new DartLoaderBatch(), scriptUri, packagesUri.path, patchedSdk.path);
+  }
 }
 
+
 // Process a request from the runtime. See KernelIsolate::CompileToKernel in
 // kernel_isolate.cc and Loader::SendKernelRequest in loader.cc.
 Future _processLoadRequest(request) async {
   if (verbose) {
     print("DFE: request: $request");
     print("DFE: Platform.packageConfig: ${Platform.packageConfig}");
     print("DFE: Platform.resolvedExecutable: ${Platform.resolvedExecutable}");
   }
 
   final int tag = request[0];
@@ skipping 19 matching lines @@
   } else {
     // See loader.cc for the code that handles these replies.
     if (result is CompilationOk) {
       port.send([tag, inputFileUrl, inputFileUrl, null, result]);
     } else {
       port.send([-tag, inputFileUrl, inputFileUrl, null, result.errorString]);
     }
   }
 }
 
-main() => new RawReceivePort()..handler = _processLoadRequest;
+Future<CompilationResult> requestParse(
+    Uri scriptUri, Uri packagesUri, Uri patchedSdk) async {
+  if (verbose) {
+    print(
+        "DFE: forwarding request to worker at http://localhost:${workerPort}/");
+  }
+
+  HttpClient client = new HttpClient();
+  final rq = await client
+      .postUrl(new Uri(host: 'localhost', port: workerPort, scheme: 'http'));
+  rq.headers.contentType =
+      new ContentType("application", "json", charset: "utf-8");

[kustermann, 2017/01/31 15:38:59]

Isn't there a constant for it in dart:io?

ContentType.JSON

[Vyacheslav Egorov (Google), 2017/01/31 17:06:55]

Done.

+  rq.write(JSON.encode({
+    "inputFileUrl": scriptUri.toString(),
+    "packagesUri": packagesUri.toString(),
+    "patchedSdk": patchedSdk.toString(),

[kustermann, 2017/01/31 15:39:00]

If these can be relative, you should Platform.currentWorkingDirectory (or
whatever it's called) as well.

[Vyacheslav Egorov (Google), 2017/01/31 17:06:55]

These are all absolute.

+  }));
+  final rs = await rq.close();
+  try {
+    if (rs.statusCode == HttpStatus.OK) {
+      final ds = new DataSink();
+      await rs.forEach(ds.add);

[kustermann, 2017/01/31 15:38:59]

Either use something like (and make DataSink be able to accept it)

  await rw.pipe(ds);

or get rid of [DataSink] and juse say

final buffer = new BytesBuffer();
await rs.ForEach(buffer.add);
buffer.takeBytes();

[Vyacheslav Egorov (Google), 2017/01/31 17:06:55]

Done.

+      return new CompilationOk(ds.builder.takeBytes());
+    } else {
+      return CompilationFail.fromJson(JSON.decode(await UTF8.decodeStream(rs)));
+    }
+  } finally {
+    client.close();

[kustermann, 2017/01/31 15:38:59]

await client.close()

[Vyacheslav Egorov (Google), 2017/01/31 17:06:55]

Done.

+  }
+}
+
+void startBatchServer() {
+  final loader = new DartLoaderBatch();
+  HttpServer.bind(InternetAddress.LOOPBACK_IP_V6, workerPort).then((server) {
+    print('READY ${server.port}');
+    server.listen((HttpRequest request) async {

[kustermann, 2017/01/31 15:38:59]

For a graceful shutdown:

ProcessSignal.SIGTERM.first.then((_) {
  server.close();
});

which will gracefully shutdown the server.

[Vyacheslav Egorov (Google), 2017/01/31 17:06:55]

Done.

+      final rq = JSON.decode(await UTF8.decodeStream(request));
+
+      final Uri scriptUri = Uri.parse(rq['inputFileUrl']);
+      final Uri packagesUri = Uri.parse(rq['packagesUri']);
+      final Uri patchedSdk = Uri.parse(rq['patchedSdk']);
+
+      final CompilationResult result = await parseScript(
+          loader, scriptUri, packagesUri.path, patchedSdk.path);
+
+      if (result is CompilationOk) {
+        request.response.headers.contentType = ContentType.BINARY;
+        request.response.add(result.binary);
+        request.response.close();
+      } else {
+        request.response.headers.contentType = ContentType.TEXT;
+        request.response.statusCode = HttpStatus.INTERNAL_SERVER_ERROR;

[kustermann, 2017/01/31 15:38:59]

Maybe set the statusCode to be OK in the other branch to be symmetric.

[Vyacheslav Egorov (Google), 2017/01/31 17:06:55]

Done.

+        request.response.write(JSON.encode(result.toJson()));
+        request.response.close();
+      }
+    });
+  });
+}
+
+main([args]) {
+  if (args?.length == 1 && args[0] == '--batch') {
+    return startBatchServer();
+  }
+
+  return new RawReceivePort()..handler = _processLoadRequest;

[kustermann, 2017/01/31 15:39:00]

maybe use 

if () {
  ....
  return null;
} else {
  return new RawReceivePort()..handler = ...;
}

It's kind of weired that one branch returns a Future and the other one a
RawReceivePort.

[Vyacheslav Egorov (Google), 2017/01/31 17:06:55]

Done.

+}
 
 // This duplicates functionality from the Loader which we can't easily
 // access from here.
 Uri _findPackagesFile(Uri base) async {
   var dir = new File.fromUri(base).parent;
   while (true) {
     final packagesFile = dir.uri.resolve(".packages");
     if (await new File.fromUri(packagesFile).exists()) {
       return packagesFile;
     }
-    if (dir.parent == dir) {
+    if (dir.parent.path == dir.path) {
       break;
     }
     dir = dir.parent;
   }
   return null;
 }