Global compile option

bin/commands/global_compile.dart

Index: bin/commands/global_compile.dart
diff --git a/bin/commands/global_compile.dart b/bin/commands/global_compile.dart
new file mode 100644
index 0000000000000000000000000000000000000000..0f452a01bad6640042b91eb205906d3929609472
--- /dev/null
+++ b/bin/commands/global_compile.dart
@@ -0,0 +1,198 @@
+// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+import 'package:analyzer/src/generated/engine.dart' show AnalysisContext;
+import 'package:analyzer/src/generated/source.dart' show Source;
+import 'package:analyzer/src/summary/package_bundle_reader.dart'
+    show InSummarySource;
+import 'package:args/command_runner.dart';
+import 'package:dev_compiler/src/analyzer/context.dart' show
+    createAnalysisContextWithSources;
+import 'package:dev_compiler/src/compiler/command.dart';
+import 'package:dev_compiler/src/compiler/compiler.dart'
+    show BuildUnit, CompilerOptions, JSModuleFile, ModuleCompiler, ModuleFormat;
+import 'package:dev_compiler/src/analyzer/context.dart' show AnalyzerOptions;
+import 'package:path/path.dart' as path;
+import 'package:yaml/yaml.dart' show loadYaml;
+
+
+/// The command for invoking the modular compiler on the whole program.
+class GlobalCommand extends Command {

[Jennifer Messerly, 2016/04/18 21:45:45]

WholeProgramCompileCommand? or GlobalCompileCommand

("global" is a bit ambiguous)

+  get name => 'global-compile';

[Jennifer Messerly, 2016/04/18 21:45:45]

in the original design this was called "build". The idea was "compile" is a word
we use for a single unit. "link" combines the units. "build" orchestrates the
process.

In the long run I think a finer grained (per-package at minimum) build+link step
will stress the compiler less. It's not designed to have every file in your
program in memory at once.

+  get description => 'Compile all files reachable from an entry point.';
+
+  GlobalCommand() {
+    argParser.addOption('out', abbr: 'o', help: 'Output file (required)');
+    CompilerOptions.addArguments(argParser);
+    AnalyzerOptions.addArguments(argParser);
+  }
+
+  String canonicalize(String uri, String root) {

[Jennifer Messerly, 2016/04/18 21:45:45]

is this is the exact code we have in ModuleCompiler? if so should probably move
it into our analyzer helpers file (currently called
compiler/element_helpers.dart, maybe we need a generic analyzer/helpers.dart)
... I'm trying to collect Analyzer workarounds to make it easier to see them and
upstream at some point.

+    var sourceUri = Uri.parse(uri);
+    if (sourceUri.scheme == '') {
+      sourceUri = path.toUri(path.isAbsolute(uri) ? path.absolute(uri) : path.join(root, uri));

[Jennifer Messerly, 2016/04/18 21:45:45]

looks like this needs dartfmt

+    }
+    return sourceUri.toString();
+  }
+
+  void transitiveFiles(Set<String> results, AnalysisContext context, String entryPoint, String root) {
+    entryPoint = canonicalize(entryPoint, root);
+    if (entryPoint.startsWith('dart:')) return;
+    var entryDir = path.dirname(entryPoint);
+    if (results.add(entryPoint)) {
+      // Process this
+      var source = context.sourceFactory.forUri(entryPoint);
+      if (source == null) {
+        throw new Exception('could not create a source for $entryPoint.'
+            ' The file name is in the wrong format or was not found.');
+      }
+      var library = context.computeLibraryElement(source);

[Jennifer Messerly, 2016/04/18 21:45:45]

I'm curious, why use resolution?
You can figure out URIs much faster via import parsing:
https://github.com/dart-lang/dev_compiler/blob/39d038366f9080bf00ab242202a19d...

as it is, this will trip on Analyzer's worst case AST caching/eviction behavior
(slower perf & more bugs)

+      for (var entry in library.imports) {
+        if (entry.uri == null) continue;
+        transitiveFiles(results, context, entry.uri, entryDir);
+      }
+      for (var entry in library.exports) {
+        transitiveFiles(results, context, entry.uri, entryDir);
+      }
+      for (var part in library.parts) {
+        results.add(canonicalize(part.uri, entryDir));
+      }
+    }
+  }
+
+  @override
+  void run() {
+    var analyzerOptions = new AnalyzerOptions.fromArguments(argResults);
+
+    var context = createAnalysisContextWithSources(analyzerOptions);
+    var inputSet = new Set<String>();
+    for (var entry in argResults.rest) {
+      transitiveFiles(inputSet, context, entry, Directory.current.path);
+    }
+    compile(
+      new ModuleCompiler.withContext(context),

[Jennifer Messerly, 2016/04/18 21:45:45]

IMO, it would be better to have it use the command line interface. That way it
won't rely on implementation details and is more like what a build tool would
do.

Unlike a watcher/server, this is really just a single compile, so it won't
really benefit from keeping the compiler warm anyway.

[Jennifer Messerly, 2016/04/18 21:49:30]

addendum -- a lot depends on where we are going with this.

For a quick hack to get Angular tests working, I don't think we need to check it
in to dev_compiler repo at all. It'll be much better to keep over there. So
that's one option.

If this is a longer term "build" command UI we want to support, then it makes
sense to have it deeply integrated, and try and be smarter about the compilation
units it handles (e.g. per-package).

If we'd rather avoid the long term questions, IMHO it would be better to keep it
localized in the tests, not add it as a fully public command interface. If it's
fully public we need to think about these hard UI questions now.

+      new CompilerOptions.fromArguments(argResults),
+      argResults['out'],
+      inputSet.toList());
+  }
+
+  void compile(ModuleCompiler compiler, CompilerOptions compilerOptions,
+      String outPath, List<String> extraArgs,
+      {void forEachError(String error): print}) {
+    if (outPath == null) {
+      usageException('Please include the output file location. For example:\n'
+          '    -o PATH/TO/OUTPUT_FILE.js');
+    }
+    if (compilerOptions.sourceMap) {
+      usageException('Source maps are not supported in global mode');
+    }
+    if (compilerOptions.summarizeApi) {
+      usageException('Summaries are not supported in global mode');
+    }
+    if (compilerOptions.moduleFormat != ModuleFormat.legacy) {
+      usageException('Only legacy modules are supported in global mode');
+    }
+    var unit = new BuildUnit(
+        path.basenameWithoutExtension(outPath), extraArgs, _moduleForLibrary);
+
+    JSModuleFile module = compiler.compile(unit, compilerOptions);
+    module.errors.forEach(forEachError);
+
+    if (!module.isValid) throw new CompileErrorException();
+
+    // Write JS file, as well as source map and summary (if requested).
+    var file = new File(outPath);
+    if (file.existsSync()) file.delete();
+    _copyLegacyDartRuntime(file);
+    file.writeAsStringSync(module.code, mode: FileMode.APPEND);

[Jennifer Messerly, 2016/04/18 21:45:45]

I don't think you want .APPEND, won't this break after the first build?

+  }
+
+  String _moduleForLibrary(Source source) {
+    if (source is InSummarySource) {
+      return path.basenameWithoutExtension(source.summaryPath);
+    }
+
+    throw usageException(
+        'Imported file "${source.uri}" was not found as a summary or source '
+        'file. Please pass in either the summary or the source file '
+        'for this import.');
+  }
+
+  void _copyLegacyDartRuntime(File outfile) {
+    var runtimeDir = _computeRuntimeDir();
+    if (runtimeDir == null) {
+      usageException('Cannot compute runtime directory for global mode');
+    }
+    for (var runtimeFile in _RUNTIME_FILES) {
+      var inPath = path.join(runtimeDir, runtimeFile);
+      var infile = new File(inPath);
+      var contents = infile.readAsStringSync();
+      outfile.writeAsStringSync(contents, mode: FileMode.APPEND);
+    }
+  }
+}
+
+const _ENTRY_POINTS = const [
+  'dartdevc.dart',
+];
+
+const _RUNTIME_FILES = const [
+  'dart_library.js',
+  'dart_sdk.js'
+];
+
+final _ENTRY_POINT_SNAPSHOTS = _ENTRY_POINTS.map((f) => "$f.snapshot");
+
+/// Tries to find the `lib/runtime/` directory of the dev_compiler package. This
+/// works when running devc from it's sources or from a snapshot that is
+/// activated via `pub global activate`.
+String _computeRuntimeDir() {

[Jennifer Messerly, 2016/04/18 21:45:45]

Vader "NOOOOOOOooooooooooo" meme ... ;)

We just got rid of this after so long. This code is totally broken.

Here are some options:

* Since this is just for Angular test logic, I'm not sure why we need fancy
auto-SDK finding. Make it the script's job. Surely it's easy in the test script
to copy dart_sdk.js from packages/dev_compiler/runtime...?

* Or, move the auto-finding logic to another repository with a "global build
tool" thingy. As long as no one complains when it breaks, because it will break
(it relies on all kinds of implementation details of Pub/snapshots/DDC.)

* It would fine, and probably a lot easier, to embed the JS string so the
command knows the String to output. We had discussed that a while back? It would
be very easy to implement, just tweak ./tool/build_sdk.sh (or the .dart,
whatever makes sense) to wrap dart_sdk.js in a .dart file.

+  var scriptPath = path.fromUri(Platform.script);
+  var file = path.basename(scriptPath);
+  var dir = path.dirname(scriptPath);
+  var lastdir = path.basename(dir);
+  dir = path.dirname(dir);
+
+  // Both the source dartdevc.dart and the snapshot generated by pub global activate
+  // are under a bin folder.
+  if (lastdir != 'bin') return null;
+
+  // And both under a project directory containing a pubspec.lock file.
+  var lockfile = path.join(dir, 'pubspec.lock');
+  if (!new File(lockfile).existsSync()) return null;
+
+  // If running from sources we found it!
+  if (_ENTRY_POINTS.contains(file)) {
+    return path.join(dir, 'lib', 'runtime');
+  }
+
+  // If running from a pub global snapshot, we need to read the lock file to
+  // find where the actual sources are located in the pub cache.
+  if (_ENTRY_POINT_SNAPSHOTS.contains(file)) {
+    // Note: this depends on implementation details of pub.
+    var yaml = loadYaml(new File(lockfile).readAsStringSync());
+    var info = yaml['packages']['dev_compiler'];
+    if (info == null) return null;
+
+    var cacheDir;
+    if (info['source'] == 'hosted') {
+      cacheDir = path.join(
+          'hosted', 'pub.dartlang.org', 'dev_compiler-${info["version"]}');
+    } else if (info['source'] == 'git') {
+      var ref = info['description']['resolved-ref'];
+      cacheDir = path.join('git', 'dev_compiler-${ref}');
+    }
+
+    // We should be under "/path/to/pub-cache/global_packages/dev_compiler".
+    // The pub-cache directory is two levels up, but we verify that the layout
+    // looks correct.
+    if (path.basename(dir) != 'dev_compiler') return null;
+    dir = path.dirname(dir);
+    if (path.basename(dir) != 'global_packages') return null;
+    dir = path.dirname(dir);
+    return path.join(dir, cacheDir, 'lib', 'runtime');
+  }
+  return null;
+}