Tweak public APIs and use them in patch_sdk, dart2js, and kernel-service

tools/patch_sdk.dart

 #!/usr/bin/env dart
 // Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /// Command line tool to merge the SDK libraries and our patch files.
 /// This is currently designed as an offline tool, but we could automate it.
 
 import 'dart:io';
 import 'dart:isolate' show RawReceivePort;
 import 'dart:async';
 import 'dart:math' as math;
 import 'dart:convert' show JSON;
 
 import 'package:analyzer/analyzer.dart';
 import 'package:analyzer/src/generated/sdk.dart';
 import 'package:path/path.dart' as path;
 
-import 'package:front_end/src/fasta/fasta.dart' as fasta
-    show compile, compilePlatform, writeDepsFile;
+import 'package:front_end/front_end.dart';
+import 'package:front_end/src/kernel_generator_impl.dart';
+import 'package:front_end/src/fasta/util/relativize.dart' show relativizeUri;
 
-import 'package:compiler/src/kernel/fasta_support.dart' as dart2js
-    show compilePlatform;
+import 'package:front_end/src/fasta/fasta.dart' as fasta show getDependencies;
+import 'package:front_end/src/fasta/kernel/utils.dart' show writeProgramToFile;
+
+import 'package:kernel/target/targets.dart';
+import 'package:kernel/target/vm_fasta.dart';
+import 'package:kernel/target/flutter_fasta.dart';
+import 'package:compiler/src/kernel/dart2js_target.dart' show Dart2jsTarget;
 
 /// Set of input files that were read by this script to generate patched SDK.
 /// We will dump it out into the depfile for ninja to use.
 ///
 /// For more information see GN and Ninja references:
 ///    https://chromium.googlesource.com/chromium/src/+/56807c6cb383140af0c03da8f6731d77785d7160/tools/gn/docs/reference.md#depfile_string_File-name-for-input-dependencies-for-actions
 ///    https://ninja-build.org/manual.html#_depfile
 ///
 final deps = new Set<Uri>();
 
@@ skipping 78 matching lines @@
   _copyExtraLibraries(sdkOut, locations);
 
   Uri platform = outDirUri.resolve('platform.dill.tmp');
   Uri outline = outDirUri.resolve('outline.dill');
   Uri librariesJson = outDirUri.resolve("lib/libraries.json");
   Uri packages = Uri.base.resolveUri(new Uri.file(packagesFile));
 
   await _writeSync(
       librariesJson.toFilePath(), JSON.encode({"libraries": locations}));
 
-  if (forVm || forFlutter) {
-    await fasta.compilePlatform(outDirUri, platform,
-        packages: packages,
-        outlineOutput: outline,
-        backendTarget: forVm ? 'vm_fasta' : 'flutter_fasta');
-  } else {
-    await dart2js.compilePlatform(outDirUri, platform,
-        packages: packages, outlineOutput: outline);
-  }
+  var flags = new TargetFlags();
+  var target = forVm

[ahe, 2017/07/11 09:34:26]

This seems brittle compared to:

switch (mode) {
  case "vm": ...
  case "flutter": ...
  case "dart2js" ...
  default:
    throw "Unhandled mode: $mode";
}

[Siggi Cherem (dart-lang), 2017/07/11 23:16:32]

will do

+      ? new VmFastaTarget(flags)
+      : (forFlutter ? new FlutterFastaTarget(flags) : new Dart2jsTarget(flags));
+  var platformDeps =
+      await compilePlatform(outDirUri, target, packages, platform, outline);
+  deps.addAll(platformDeps);
 
   if (forVm) {
+    // TODO(sigmund): add support for the flutter vmservice_sky as well.
+    var vmserviceName = 'vmservice_io';
     var base = path.fromUri(Platform.script);
-    Uri repositoryDir =
+    Uri dartDir =
         new Uri.directory(path.dirname(path.dirname(path.absolute(base))));
-    var vmserviceName = 'vmservice_io';
-    Uri vmserviceSdk = repositoryDir.resolve('runtime/bin/vmservice_sdk/');
+    var program = await kernelForProgram(
+        Uri.parse('dart:$vmserviceName'),
+        new CompilerOptions()
+          ..sdkSummary = outline
+          ..dartLibraries = <String, Uri>{
+            '_vmservice': outDirUri.resolve('lib/vmservice/vmservice.dart'),
+            'vmservice_io':
+                dartDir.resolve('runtime/bin/vmservice/vmservice_io.dart'),
+          }
+          ..packagesFileUri = packages);
     Uri vmserviceUri = outDirUri.resolve('$vmserviceName.dill');
-    // TODO(sigmundch): Specify libraries.json directly instead of "--sdk"
-    // after #29882 is fixed.
-    await fasta.compile([
-      "--sdk=$vmserviceSdk",
-      "--platform=$outline",
-      "--target=vm_fasta",
-      "--packages=$packages",
-      "dart:$vmserviceName",
-      "-o",
-      "$vmserviceUri",
-    ]);
+    await writeProgramToFile(program, vmserviceUri);
   }
 
   Uri platformFinalLocation = outDirUri.resolve('platform.dill');
 
   // We generate a dependency file for GN to properly regenerate the patched sdk
   // folder, outline.dill and platform.dill files when necessary: either when
   // the sdk sources change or when this script is updated. In particular:
   //
   //  - sdk changes: we track the actual sources we are compiling. If we are
   //    building the dart2js sdk, this includes the dart2js-specific patch
   //    files.
   //
   //    These files are tracked by [deps] and passed below to [writeDepsFile] in
   //    the extraDependencies argument.
   //
   //  - script updates: we track this script file and any code it imports (even
   //    sdk libraries). Note that this script runs on the standalone VM, so any
   //    sdk library used by this script indirectly depends on a VM-specific
   //    patch file.
   //
-  //    These set of files is discovered by `writeDepsFile` below, and the
+  //    These set of files is discovered by `getDependencies` below, and the
   //    [platformForDeps] is always the VM-specific `platform.dill` file.
-  //
-  //    TODO(sigmund): we should change this:
-  //      - we should rewrite writeDepsFile: fasta could provide an API to crawl
-  //        the dependencies, but anything that is GN specific, should be on
-  //        this file instead.
-  //
-  //      - We don't need to include sdk dependencies of the script because
-  //        those are already included indirectly (either in [deps] when
-  //        building the sdk for the VM, or via the .GN dependencies in the
-  //        build files for dart2js and flutter).
   var platformForDeps = platform;
   var sdkDir = outDirUri;
   if (forDart2js || forFlutter) {
     // Note: this would fail if `../patched_sdk/platform.dill` doesn't exist. We
     // added an explicit dependency in the .GN rules so patched_dart2js_sdk (and
     // patched_flutter_sdk) depend on patched_sdk to ensure that it exists.
     platformForDeps = outDirUri.resolve('../patched_sdk/platform.dill');
     sdkDir = outDirUri.resolve('../patched_sdk/');
   }
-  await fasta.writeDepsFile(Platform.script,
-      Uri.base.resolveUri(new Uri.file("$outDir.d")), platformFinalLocation,
-      sdk: sdkDir,
-      packages: packages,
-      platform: platformForDeps,
-      extraDependencies: deps);
+  deps.addAll(await fasta.getDependencies(Platform.script,
+      sdk: sdkDir, packages: packages, platform: platformForDeps));
+  await writeDepsFile(Uri.base.resolveUri(new Uri.file("$outDir.d")),
+      platformFinalLocation, deps);
+  await new File.fromUri(platform).rename(platformFinalLocation.toFilePath());
+}
 
-  await new File.fromUri(platform).rename(platformFinalLocation.toFilePath());
+/// Generates an outline.dill and platform.dill file containing the result of
+/// compiling a platform's SDK.
+///
+/// Returns a list of dependencies read by the compiler. This list can be used
+/// to create GN dependency files.
+Future<List<Uri>> compilePlatform(Uri patchedSdk, Target target, Uri packages,
+    Uri fullOutput, Uri outlineOutput) async {
+  var options = new CompilerOptions()
+    ..strongMode = false
+    ..compileSdk = true
+    ..sdkRoot = patchedSdk
+    ..packagesFileUri = packages
+    ..chaseDependencies = true
+    ..target = target;
+
+  var result = await generateKernel([Uri.parse('dart:core')], options,
+      buildSummary: true, buildProgram: true);
+  new File.fromUri(outlineOutput).writeAsBytesSync(result.summary);
+  await writeProgramToFile(result.program, fullOutput);
+  return result.deps;
+}
+
+Future writeDepsFile(
+    Uri output, Uri depsFile, Iterable<Uri> allDependencies) async {
+  if (allDependencies.isEmpty) return;
+  String toRelativeFilePath(Uri uri) {
+    // Ninja expects to find file names relative to the current working
+    // directory. We've tried making them relative to the deps file, but that
+    // doesn't work for downstream projects. Making them absolute also
+    // doesn't work.
+    //
+    // We can test if it works by running ninja twice, for example:
+    //
+    //     ninja -C xcodebuild/ReleaseX64 runtime_kernel -d explain
+    //     ninja -C xcodebuild/ReleaseX64 runtime_kernel -d explain
+    //
+    // The second time, ninja should say:
+    //
+    //     ninja: Entering directory `xcodebuild/ReleaseX64'
+    //     ninja: no work to do.
+    //
+    // It's broken if it says something like this:
+    //
+    //     ninja explain: expected depfile 'patched_sdk.d' to mention
+    //     'patched_sdk/platform.dill', got
+    //     '/.../xcodebuild/ReleaseX64/patched_sdk/platform.dill'
+    return Uri.parse(relativizeUri(uri, base: Uri.base)).toFilePath();
+  }
+
+  StringBuffer sb = new StringBuffer();
+  sb.write(toRelativeFilePath(output));
+  sb.write(":");
+  for (Uri uri in allDependencies) {
+    sb.write(" ");
+    sb.write(toRelativeFilePath(uri));
+  }
+  sb.writeln();
+  await new File.fromUri(depsFile).writeAsString("$sb");
 }
 
 /// Updates the contents of
 /// sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart to include
 /// declarations for vm internal libraries.
 String _updateLibraryMetadata(String sdkOut, String libContents) {
   if (!forVm && !forFlutter) return libContents;
   var extraLibraries = new StringBuffer();
   extraLibraries.write('''
   "_builtin": const LibraryInfo(
@@ skipping 465 matching lines @@
     if (diff != 0) return diff;
     return end - other.end;
   }
 }
 
 List<SdkLibrary> _getSdkLibraries(String contents) {
   var libraryBuilder = new SdkLibrariesReader_LibraryBuilder(forDart2js);
   parseCompilationUnit(contents).accept(libraryBuilder);
   return libraryBuilder.librariesMap.sdkLibraries;
 }