Move patch file generation to fasta.

tools/patch_sdk.dart

 #!/usr/bin/env dart
 // Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /// Command line tool to merge the SDK libraries and our patch files.
 /// This is currently designed as an offline tool, but we could automate it.
 
 import 'dart:io';
+import 'dart:isolate' show RawReceivePort;
 import 'dart:async';
 import 'dart:math' as math;
 
 import 'package:analyzer/analyzer.dart';
 import 'package:analyzer/src/generated/sdk.dart';
 import 'package:path/path.dart' as path;
-import 'package:front_end/src/fasta/compile_platform.dart' as compile_platform;
 
-import 'package:front_end/src/fasta/fasta.dart' show CompileTask;
-
-import 'package:front_end/src/fasta/compiler_command_line.dart'
-    show CompilerCommandLine;
-
-import 'package:front_end/src/fasta/compiler_context.dart' show CompilerContext;
-
-import 'package:front_end/src/fasta/ticker.dart' show Ticker;
+import 'package:front_end/src/fasta/fasta.dart'
+    show compilePlatform, writeDepsFile;
 
 /// Set of input files that were read by this script to generate patched SDK.
 /// We will dump it out into the depfile for ninja to use.
 ///
 /// For more information see GN and Ninja references:
 ///    https://chromium.googlesource.com/chromium/src/+/56807c6cb383140af0c03da8f6731d77785d7160/tools/gn/docs/reference.md#depfile_string_File-name-for-input-dependencies-for-actions
 ///    https://ninja-build.org/manual.html#_depfile
 ///
-final deps = new Set<String>();
+final deps = new Set<Uri>();
 
 /// Create [File] object from the given path and register it as a dependency.
 File getInputFile(String path, {canBeMissing: false}) {
   final file = new File(path);
   if (!file.existsSync()) {
     if (!canBeMissing) throw "patch_sdk.dart expects all inputs to exist";
     return null;
   }
-  deps.add(file.absolute.path);
+  deps.add(Uri.base.resolveUri(file.uri));
   return file;
 }
 
 /// Read the given file synchronously as a string and register this path as
 /// a dependency.
 String readInputFile(String path, {canBeMissing: false}) =>
     getInputFile(path, canBeMissing: canBeMissing)?.readAsStringSync();
 
 Future main(List<String> argv) async {
-  var base = path.fromUri(Platform.script);
-  var dartDir = path.dirname(path.dirname(path.absolute(base)));
+  var port = new RawReceivePort();
+  try {

[Vyacheslav Egorov (Google), 2017/03/15 13:43:59]

With such a gigantic try { } finally { } it might be better to split it like
this:

Future main(List<String> argv) async {
  var port = new ...;
  try {
    return mainImpl(argv, port);
  } finally {
    port.close();
  }
}

[ahe, 2017/03/31 15:05:02]

Done.

+    var base = path.fromUri(Platform.script);
+    var dartDir = path.dirname(path.dirname(path.absolute(base)));
 
-  if (argv.length != 5 || argv.first != 'vm') {
-    final self = path.relative(base);
-    print('Usage: $self vm SDK_DIR PATCH_DIR OUTPUT_DIR PACKAGES');
+    if (argv.length != 5 || argv.first != 'vm') {
+      final self = path.relative(base);
+      print('Usage: $self vm SDK_DIR PATCH_DIR OUTPUT_DIR PACKAGES');
 
-    final repositoryDir = path.relative(path.dirname(path.dirname(base)));
-    final sdkExample = path.relative(path.join(repositoryDir, 'sdk'));
-    final patchExample = path.relative(
-        path.join(repositoryDir, 'out', 'DebugX64', 'obj', 'gen', 'patch'));
-    final outExample = path.relative(path.join(
-        repositoryDir, 'out', 'DebugX64', 'obj', 'gen', 'patched_sdk'));
-    print('For example:');
-    print('\$ $self vm $sdkExample $patchExample $outExample');
+      final repositoryDir = path.relative(path.dirname(path.dirname(base)));
+      final sdkExample = path.relative(path.join(repositoryDir, 'sdk'));
+      final patchExample = path.relative(
+          path.join(repositoryDir, 'out', 'DebugX64', 'obj', 'gen', 'patch'));
+      final outExample = path.relative(path.join(
+          repositoryDir, 'out', 'DebugX64', 'obj', 'gen', 'patched_sdk'));
+      print('For example:');
+      print('\$ $self vm $sdkExample $patchExample $outExample');
 
-    exit(1);
-  }
+      exit(1);
+    }
 
-  var mode = argv[0];
-  assert(mode == "vm");
-  var input = argv[1];
-  var sdkLibIn = path.join(input, 'lib');
-  var patchIn = argv[2];
-  var outDir = argv[3];
-  var sdkOut = path.join(outDir, 'lib');
-  var packagesFile = argv[4];
+    var mode = argv[0];
+    assert(mode == "vm");
+    var input = argv[1];
+    var sdkLibIn = path.join(input, 'lib');
+    var patchIn = argv[2];
+    var outDir = argv[3];
+    var sdkOut = path.join(outDir, 'lib');
+    var packagesFile = argv[4];
 
-  // Copy and patch libraries.dart and version
-  var libContents = readInputFile(path.join(
-      sdkLibIn, '_internal', 'sdk_library_metadata', 'lib', 'libraries.dart'));
-  libContents = libContents.replaceAll(
-      ' libraries = const {',
-      ''' libraries = const {
+    // Copy and patch libraries.dart and version
+    var libContents = readInputFile(path.join(sdkLibIn, '_internal',
+        'sdk_library_metadata', 'lib', 'libraries.dart'));
+    libContents = libContents.replaceAll(
+        ' libraries = const {',
+        ''' libraries = const {
 
   "_builtin": const LibraryInfo(
       "_builtin/_builtin.dart",
       categories: "Client,Server",
       implementation: true,
       documented: false,
       platforms: VM_PLATFORM),
 
   "profiler": const LibraryInfo(
       "profiler/profiler.dart",
       maturity: Maturity.DEPRECATED,
       documented: false),
 
   "_vmservice": const LibraryInfo(
       "vmservice/vmservice.dart",
       implementation: true,
       documented: false,
       platforms: VM_PLATFORM),
 
   "vmservice_io": const LibraryInfo(
       "vmservice_io/vmservice_io.dart",
       implementation: true,
       documented: false,
       platforms: VM_PLATFORM),
 
 ''');
-  _writeSync(
-      path.join(
-          sdkOut, '_internal', 'sdk_library_metadata', 'lib', 'libraries.dart'),
-      libContents);
+    _writeSync(
+        path.join(sdkOut, '_internal', 'sdk_library_metadata', 'lib',
+            'libraries.dart'),
+        libContents);
 
-  // Parse libraries.dart
-  var sdkLibraries = _getSdkLibraries(libContents);
+    // Parse libraries.dart
+    var sdkLibraries = _getSdkLibraries(libContents);
 
-  // Enumerate core libraries and apply patches
-  for (SdkLibrary library in sdkLibraries) {
-    if (library.isDart2JsLibrary) {
-      continue;
-    }
-
-    var libraryOut = path.join(sdkLibIn, library.path);
-    var libraryIn = libraryOut;
-
-    var libraryFile = getInputFile(libraryIn, canBeMissing: true);
-    if (libraryFile != null) {
-      var outPaths = <String>[libraryOut];
-      var libraryContents = libraryFile.readAsStringSync();
-
-      int inputModifyTime =
-          libraryFile.lastModifiedSync().millisecondsSinceEpoch;
-      var partFiles = <File>[];
-      for (var part in parseDirectives(libraryContents).directives) {
-        if (part is PartDirective) {
-          var partPath = part.uri.stringValue;
-          outPaths.add(path.join(path.dirname(libraryOut), partPath));
-
-          var partFile =
-              getInputFile(path.join(path.dirname(libraryIn), partPath));
-          partFiles.add(partFile);
-          inputModifyTime = math.max(inputModifyTime,
-              partFile.lastModifiedSync().millisecondsSinceEpoch);
-        }
+    // Enumerate core libraries and apply patches
+    for (SdkLibrary library in sdkLibraries) {
+      if (library.isDart2JsLibrary) {
+        continue;
       }
 
-      // See if we can find a patch file.
-      var patchPath = path.join(
-          patchIn, path.basenameWithoutExtension(libraryIn) + '_patch.dart');
+      var libraryOut = path.join(sdkLibIn, library.path);
+      var libraryIn = libraryOut;
 
-      var patchFile = getInputFile(patchPath, canBeMissing: true);
-      if (patchFile != null) {
-        inputModifyTime = math.max(inputModifyTime,
-            patchFile.lastModifiedSync().millisecondsSinceEpoch);
-      }
+      var libraryFile = getInputFile(libraryIn, canBeMissing: true);
+      if (libraryFile != null) {
+        var outPaths = <String>[libraryOut];
+        var libraryContents = libraryFile.readAsStringSync();
 
-      // Compute output paths
-      outPaths = outPaths
-          .map((p) => path.join(sdkOut, path.relative(p, from: sdkLibIn)))
-          .toList();
+        int inputModifyTime =
+            libraryFile.lastModifiedSync().millisecondsSinceEpoch;
+        var partFiles = <File>[];
+        for (var part in parseDirectives(libraryContents).directives) {
+          if (part is PartDirective) {
+            var partPath = part.uri.stringValue;
+            outPaths.add(path.join(path.dirname(libraryOut), partPath));
 
-      // Compare output modify time with input modify time.
-      bool needsUpdate = false;
-      for (var outPath in outPaths) {
-        var outFile = new File(outPath);
-        if (!outFile.existsSync() ||
-            outFile.lastModifiedSync().millisecondsSinceEpoch <
-                inputModifyTime) {
-          needsUpdate = true;
-          break;
-        }
-      }
-
-      if (needsUpdate) {
-        var contents = <String>[libraryContents];
-        contents.addAll(partFiles.map((f) => f.readAsStringSync()));
-        if (patchFile != null) {
-          var patchContents = patchFile.readAsStringSync();
-          contents = _patchLibrary(patchFile.path, contents, patchContents);
+            var partFile =
+                getInputFile(path.join(path.dirname(libraryIn), partPath));
+            partFiles.add(partFile);
+            inputModifyTime = math.max(inputModifyTime,
+                partFile.lastModifiedSync().millisecondsSinceEpoch);
+          }
         }
 
-        for (var i = 0; i < outPaths.length; i++) {
-          _writeSync(outPaths[i], contents[i]);
+        // See if we can find a patch file.
+        var patchPath = path.join(
+            patchIn, path.basenameWithoutExtension(libraryIn) + '_patch.dart');
+
+        var patchFile = getInputFile(patchPath, canBeMissing: true);
+        if (patchFile != null) {
+          inputModifyTime = math.max(inputModifyTime,
+              patchFile.lastModifiedSync().millisecondsSinceEpoch);
+        }
+
+        // Compute output paths
+        outPaths = outPaths
+            .map((p) => path.join(sdkOut, path.relative(p, from: sdkLibIn)))
+            .toList();
+
+        // Compare output modify time with input modify time.
+        bool needsUpdate = false;
+        for (var outPath in outPaths) {
+          var outFile = new File(outPath);
+          if (!outFile.existsSync() ||
+              outFile.lastModifiedSync().millisecondsSinceEpoch <
+                  inputModifyTime) {
+            needsUpdate = true;
+            break;
+          }
+        }
+
+        if (needsUpdate) {
+          var contents = <String>[libraryContents];
+          contents.addAll(partFiles.map((f) => f.readAsStringSync()));
+          if (patchFile != null) {
+            var patchContents = patchFile.readAsStringSync();
+            contents = _patchLibrary(patchFile.path, contents, patchContents);
+          }
+
+          for (var i = 0; i < outPaths.length; i++) {
+            _writeSync(outPaths[i], contents[i]);
+          }
         }
       }
     }
-  }
 
-  for (var tuple in [
-    ['_builtin', 'builtin.dart']
-  ]) {
-    var vmLibrary = tuple[0];
-    var dartFile = tuple[1];
+    for (var tuple in [
+      ['_builtin', 'builtin.dart']
+    ]) {
+      var vmLibrary = tuple[0];
+      var dartFile = tuple[1];
 
-    // The "dart:_builtin" library is only available for the DartVM.
-    var builtinLibraryIn = path.join(dartDir, 'runtime', 'bin', dartFile);
-    var builtinLibraryOut = path.join(sdkOut, vmLibrary, '${vmLibrary}.dart');
-    _writeSync(builtinLibraryOut, readInputFile(builtinLibraryIn));
-  }
+      // The "dart:_builtin" library is only available for the DartVM.
+      var builtinLibraryIn = path.join(dartDir, 'runtime', 'bin', dartFile);
+      var builtinLibraryOut = path.join(sdkOut, vmLibrary, '${vmLibrary}.dart');
+      _writeSync(builtinLibraryOut, readInputFile(builtinLibraryIn));
+    }
 
-  for (var file in ['loader.dart', 'server.dart', 'vmservice_io.dart']) {
-    var libraryIn = path.join(dartDir, 'runtime', 'bin', 'vmservice', file);
-    var libraryOut = path.join(sdkOut, 'vmservice_io', file);
-    _writeSync(libraryOut, readInputFile(libraryIn));
-  }
+    for (var file in ['loader.dart', 'server.dart', 'vmservice_io.dart']) {
+      var libraryIn = path.join(dartDir, 'runtime', 'bin', 'vmservice', file);
+      var libraryOut = path.join(sdkOut, 'vmservice_io', file);
+      _writeSync(libraryOut, readInputFile(libraryIn));
+    }
 
-  final platform = path.join(outDir, 'platform.dill');
+    Uri platform = Uri.base
+        .resolveUri(new Uri.directory(outDir).resolve('platform.dill.tmp'));
+    Uri packages = Uri.base.resolveUri(new Uri.file(packagesFile));
+    await compilePlatform(
+        Uri.base.resolveUri(new Uri.directory(outDir)), platform,
+        packages: packages, verbose: false);
 
-  await compile_platform.mainEntryPoint(<String>[
-    '--packages',
-    new Uri.file(packagesFile).toString(),
-    new Uri.directory(outDir).toString(),
-    platform,
-  ]);
+    Uri platformFinalLocation =
+        Uri.base.resolveUri(new Uri.directory(outDir).resolve('platform.dill'));
 
-  // TODO(kustermann): We suppress compiler hints/warnings/errors temporarily
-  // because everyone building the `runtime` target will get these now.
-  // We should remove the suppression again once the underlying issues have
-  // been fixed (either in fasta or the dart files in the patched_sdk).
-  final capturedLines = <String>[];
-  try {
-    await runZoned(() async {
-      // platform.dill was generated, now generate platform.dill.d depfile
-      // that captures all dependencies that participated in the generation.
-      // There are two types of dependencies:
-      //   (1) all Dart sources that constitute this tool itself
-      //   (2) Dart SDK and patch sources.
-      // We already collected all inputs from the second category in the deps
-      // set. To collect inputs from the first category we actually use Fasta:
-      // we ask Fasta to outline patch_sdk.dart and generate a depfile which
-      // would list all the sources.
-      final depfile = "${outDir}.d";
-      await CompilerCommandLine.withGlobalOptions("outline", [
-        '--packages',
-        new Uri.file(packagesFile).toString(),
-        '--platform',
-        new Uri.file(platform).toString(), // platform.dill
-        Platform.script.toString() // patch_sdk.dart
-      ], (CompilerContext c) async {
-        CompileTask task =
-            new CompileTask(c, new Ticker(isVerbose: c.options.verbose));
-        final kernelTarget = await task.buildOutline(null);
-        await kernelTarget.writeDepsFile(
-            new Uri.file(platform), new Uri.file(depfile));
-      });
+    await writeDepsFile(Platform.script,
+        Uri.base.resolveUri(new Uri.file("$outDir.d")), platformFinalLocation,
+        packages: packages,
+        platform: platform,
+        extraDependencies: deps,
+        verbose: false);
 
-      // Read depfile generated by Fasta and append deps that we have collected
-      // during generation of patched_sdk to it.
-      // Note: we are splitting by ': ' because Windows paths can start with
-      // drive letter followed by a colon.
-      final list = new File(depfile).readAsStringSync().split(': ');
-      assert(list.length == 2);
-      deps.addAll(list[1].split(' ').where((str) => str.isNotEmpty));
-      assert(list[0] == path.join('patched_sdk', 'platform.dill'));
-      new File(depfile).writeAsStringSync("${list[0]}: ${deps.join(' ')}\n");
-    }, zoneSpecification: new ZoneSpecification(print: (_, _2, _3, line) {
-      capturedLines.add(line);
-    }));
-  } catch (_) {
-    for (final line in capturedLines) {
-      print(line);
-    }
-    rethrow;
+    await new File.fromUri(platform).rename(platformFinalLocation.toFilePath());
+  } finally {
+    port.close();
   }
 }
 
 /// Writes a file, creating the directory if needed.
 void _writeSync(String filePath, String contents) {
   var outDir = new Directory(path.dirname(filePath));
   if (!outDir.existsSync()) outDir.createSync(recursive: true);
 
   new File(filePath).writeAsStringSync(contents);
 }
@@ skipping 328 matching lines @@
     if (diff != 0) return diff;
     return end - other.end;
   }
 }
 
 List<SdkLibrary> _getSdkLibraries(String contents) {
   var libraryBuilder = new SdkLibrariesReader_LibraryBuilder(true);
   parseCompilationUnit(contents).accept(libraryBuilder);
   return libraryBuilder.librariesMap.sdkLibraries;
 }