Modify global compile tool to compile by package

tool/global_compile.dart

 #!/usr/bin/env dart
 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:async';
 import 'dart:io';
 
 import 'package:analyzer/src/generated/engine.dart' show AnalysisContext;
 import 'package:args/args.dart' show ArgParser;
 import 'package:dev_compiler/src/analyzer/context.dart'
     show createAnalysisContextWithSources;
 import 'package:dev_compiler/src/analyzer/context.dart'
     show createAnalysisContextWithSources, AnalyzerOptions;
 import 'package:path/path.dart' as path;
 
+const ENTRY = "main";

[Jennifer Messerly, 2016/05/10 20:43:00]

style nit: this should not be upper camel case :)
https://www.dartlang.org/effective-dart/style/#identifiers

+
 void main(List<String> args) {
   // Parse flags.
   var parser = new ArgParser()
-    ..addOption('out', abbr: 'o', defaultsTo: 'out.js')
-    ..addFlag('unsafe-force-compile', negatable: false)
-    ..addOption('package-root', abbr: 'p', defaultsTo: 'packages/');
+    ..addOption('out',
+        help: 'Output file (defaults to "out.js")',
+        abbr: 'o',
+        defaultsTo: 'out.js')
+    ..addFlag('unsafe-force-compile',

[Jennifer Messerly, 2016/05/10 20:43:00]

I may have asked this before but, shouldn't this just populate with our existing
args from src/compiler/command.dart and src/analyzer/context.dart?

they both have static methods to make it easy to populate any ArgParser for just
this reason :)

+        help: 'Generate code with undefined behavior', negatable: false)
+    ..addOption('package-root',
+        help: 'Directory containing packages',
+        abbr: 'p',
+        defaultsTo: 'packages/')
+    ..addFlag('log', help: 'Show individual build commands')
+    ..addOption('tmp',

[Jennifer Messerly, 2016/05/10 20:43:00]

any reason to not always use system temp? I'm trying to think what Unix
conventions are for these kinds of commands. I think usually temp is usually
controlled by an environment variable right?
https://gcc.gnu.org/onlinedocs/gcc/Environment-Variables.html

there is also a flag in GCC/Clang to save temp files to the output dir:
http://clang.llvm.org/docs/CommandGuide/clang.html#cmdoption-save-temps

+        help:
+            'Directory for temporary artifacts (defaults to a system tmp directory)');

[Jennifer Messerly, 2016/05/10 20:43:00]

long line

 
   var options = parser.parse(args);
   if (options.rest.length != 1) {
     throw 'Expected a single dart entrypoint.';
   }
   var entry = options.rest.first;
-  var outfile = options['out'];
-  var packageRoot = options['package-root'];
-  var unsafe = options['unsafe-force-compile'];
-
-  // Build an invocation to dartdevc.
+  var outfile = options['out'] as String;
+  var packageRoot = options['package-root'] as String;
+  var unsafe = options['unsafe-force-compile'] as bool;
+  var log = options['log'] as bool;
+  var tmp = options['tmp'] as String;
+
+  // Build an invocation to dartdevc
   var dartPath = Platform.resolvedExecutable;
   var ddcPath = path.dirname(path.dirname(Platform.script.toFilePath()));
-  var command = [
+  var template = [
     '$ddcPath/bin/dartdevc.dart',
     'compile',
     '--no-source-map', // Invalid as we're just concatenating files below
     '-p',
-    packageRoot,
-    '-o',
-    outfile
+    packageRoot
   ];
   if (unsafe) {
-    command.add('--unsafe-force-compile');
+    template.add('--unsafe-force-compile');
   }
 
   // Compute the transitive closure
-  var watch = new Stopwatch()..start();
+  var total = new Stopwatch()..start();
+  var partial = new Stopwatch()..start();
+
+  // TODO(vsm): We're using the analyzer just to compute the import/export/part
+  // dependence graph.  This is expensive.  Is there a lighterweight way to do
+  // this?

[Jennifer Messerly, 2016/05/10 20:43:00]

yes indeed there is! See

https://github.com/dart-lang/dev_compiler/blob/335c43428ee67b93cda49b9dc9279b...

=)

   var context = createAnalysisContextWithSources(new AnalyzerOptions());
-  var inputSet = new Set<String>();
-  transitiveFiles(inputSet, context, entry, Directory.current.path);
-  command.addAll(inputSet);
-  var result = Process.runSync(dartPath, command);
-
-  if (result.exitCode == 0) {
-    print(result.stdout);
-  } else {
-    print('ERROR:');
-    print(result.stdout);
-    print(result.stderr);
-    exit(1);
-  }
-  var time = watch.elapsedMilliseconds / 1000;
-  print('Successfully compiled ${inputSet.length} files in $time seconds');
-
-  // Prepend Dart runtime files to the output.
+  transitiveFiles(context, entry, Directory.current.path);
+  orderModules();
+  computeTransitiveDependences();
+
+  var graphTime = partial.elapsedMilliseconds / 1000;
+  print('Computed global build graph in $graphTime seconds');
+
+  // Prepend Dart runtime files to the output
   var out = new File(outfile);
-  var code = out.readAsStringSync();
   var dartLibrary =
       new File(path.join(ddcPath, 'lib', 'runtime', 'dart_library.js'))
           .readAsStringSync();
+  out.writeAsStringSync(dartLibrary);
   var dartSdk = new File(path.join(ddcPath, 'lib', 'runtime', 'dart_sdk.js'))
       .readAsStringSync();
-  out.writeAsStringSync(dartLibrary);
   out.writeAsStringSync(dartSdk, mode: FileMode.APPEND);
-  out.writeAsStringSync(code, mode: FileMode.APPEND);
-
-  // Append the entry point invocation.
-  var moduleName = path.basenameWithoutExtension(outfile);
-  var libraryName =
-      path.withoutExtension(entry).replaceAll(path.separator, '__');
-  out.writeAsStringSync('dart_library.start("$moduleName", "$libraryName");\n',
-      mode: FileMode.APPEND);
+
+  // Linearize module concatenation for deterministic output
+  var last = new Future.value();
+  for (var module in orderedModules) {
+    linearizerMap[module] = last;
+    var completer = new Completer();
+    completerMap[module] = completer;
+    last = completer.future;
+  }
+
+  // Build modules asynchronously

[Jennifer Messerly, 2016/05/10 20:43:00]

Would it be easier to generate a Makefile and just call "make"?

The makefile might even be a useful artifact, then when you're iterating, you
can just call "make" and get incrementalism "for free"

That's also in the Unix philosophy of keeping commands small and modular :)

+  var tmpdir = (tmp == null)
+      ? Directory.systemTemp
+          .createTempSync(outfile.replaceAll(path.separator, '__'))
+      : new Directory(tmp)..createSync();
+  for (var module in orderedModules) {
+    var file = tmpdir.path + path.separator + module + '.js';
+    var command = new List.from(template)..addAll(['-o', file]);
+    var dependences = transitiveDependenceMap[module];
+    for (var dependence in dependences) {
+      var summary = tmpdir.path + path.separator + dependence + '.sum';
+      command.addAll(['-s', summary]);
+    }
+    var infiles = fileMap[module];
+    command.addAll(infiles);
+
+    var immediateDeps =
+        dependenceMap.containsKey(module) ? dependenceMap[module] : <String>[];
+    var waitList = immediateDeps.map((dep) => readyMap[dep]);
+    var future = Future.wait(waitList);
+    readyMap[module] = future.then((_) {
+      var ready = Process.run(dartPath, command);
+      if (log) {
+        print(command.join(' '));
+      }
+      return ready.then((result) {

[Jennifer Messerly, 2016/05/10 20:43:00]

this could be an `await`
https://www.dartlang.org/effective-dart/usage/#prefer-asyncawait-over-using-r...

(I know the outer one can't because you're trying on purpose to spawn things in
parallel, but this one looks like it can be)

+        if (result.exitCode != 0) {
+          print('ERROR: compiling $module');
+          print(result.stdout);
+          print(result.stderr);
+          out.deleteSync();
+          exit(1);
+        }
+        print('Compiled $module (${infiles.length} files)');
+        print(result.stdout);
+
+        // Schedule module append once the previous module is written
+        var codefile = new File(file);
+        linearizerMap[module]
+            .then((_) => codefile.readAsString())
+            .then((code) =>
+                out.writeAsString(code, mode: FileMode.APPEND, flush: true))
+            .then((_) => completerMap[module].complete());
+      });
+    });
+  }
+
+  last.then((_) {

[Jennifer Messerly, 2016/05/10 20:43:00]

same this could be async/await

+    var time = total.elapsedMilliseconds / 1000;
+    print('Successfully compiled ${inputSet.length} files in $time seconds');
+
+    // Append the entry point invocation.
+    var libraryName =
+        path.withoutExtension(entry).replaceAll(path.separator, '__');
+    out.writeAsStringSync('dart_library.start("$ENTRY", "$libraryName");\n',
+        mode: FileMode.APPEND);
+  });
+}
+
+final inputSet = new Set<String>();
+final dependenceMap = new Map<String, Set<String>>();

[Jennifer Messerly, 2016/05/10 20:43:00]

all of these maps do make me wonder if we should have a graph abstraction?

Past a certain point, is it really that we have a graph node here and we're just
splitting out its fields into different Map values?

+final transitiveDependenceMap = new Map<String, Set<String>>();
+final fileMap = new Map<String, Set<String>>();
+
+final readyMap = new Map<String, Future>();
+final linearizerMap = new Map<String, Future>();
+final completerMap = new Map<String, Completer>();
+
+final orderedModules = new List<String>();
+final visitedModules = new Set<String>();
+
+void orderModules(

[Jennifer Messerly, 2016/05/10 20:43:00]

fyi ... I didn't look at all this build graph stuff too closely, let me know if
you want me to, I feel like eventually we'll replace with a real build too
(Make/Bazel) so it doesn't seem like something we need to worry much about.

+    [String module = ENTRY, List<String> stack, Set<String> visited]) {
+  if (stack == null) {
+    assert(visited == null);
+    stack = new List<String>();
+    visited = new Set<String>();
+  }
+  if (visited.contains(module)) return;
+  visited.add(module);
+  if (stack.contains(module)) {
+    print(stack);
+    throw 'Circular dependence on $module';
+  }
+  stack.add(module);
+  var dependences = dependenceMap[module];
+  if (dependences != null) {
+    for (var dependence in dependences) {
+      orderModules(dependence, stack, visited);
+    }
+  }
+  orderedModules.add(module);
+  assert(module == stack.last);
+  stack.removeLast();
+}
+
+void computeTransitiveDependences() {
+  for (var module in orderedModules) {
+    var transitiveSet = new Set<String>();
+    if (dependenceMap.containsKey(module)) {
+      transitiveSet.addAll(dependenceMap[module]);
+      for (var dependence in dependenceMap[module]) {
+        transitiveSet.addAll(transitiveDependenceMap[dependence]);
+      }
+    }
+    transitiveDependenceMap[module] = transitiveSet;
+  }
+}
+
+String getModule(String uri) {
+  var sourceUri = Uri.parse(uri);
+  if (sourceUri.scheme == 'dart') {
+    return 'dart';
+  } else if (sourceUri.scheme == 'package') {
+    return path.split(sourceUri.path)[0];
+  } else {
+    return ENTRY;
+  }
+}
+
+bool processFile(String file) {
+  inputSet.add(file);
+
+  var module = getModule(file);
+  fileMap.putIfAbsent(module, () => new Set<String>());
+  return fileMap[module].add(file);
+}
+
+void processDependence(String from, String to) {
+  var fromModule = getModule(from);
+  var toModule = getModule(to);
+  if (fromModule == toModule || toModule == 'dart') return;
+  dependenceMap.putIfAbsent(fromModule, () => new Set<String>());
+  dependenceMap[fromModule].add(toModule);
 }
 
 String canonicalize(String uri, String root) {
   var sourceUri = Uri.parse(uri);
   if (sourceUri.scheme == '') {
     sourceUri = path.toUri(
         path.isAbsolute(uri) ? path.absolute(uri) : path.join(root, uri));
   }
   return sourceUri.toString();
 }
 
-void transitiveFiles(Set<String> results, AnalysisContext context,
-    String entryPoint, String root) {
+void transitiveFiles(AnalysisContext context, String entryPoint, String root) {
   entryPoint = canonicalize(entryPoint, root);
   if (entryPoint.startsWith('dart:')) return;
   var entryDir = path.dirname(entryPoint);
-  if (results.add(entryPoint)) {
+  if (processFile(entryPoint)) {
     // Process this
     var source = context.sourceFactory.forUri(entryPoint);
     if (source == null) {
       throw new Exception('could not create a source for $entryPoint.'
           ' The file name is in the wrong format or was not found.');
     }
     var library = context.computeLibraryElement(source);
     for (var entry in library.imports) {
       if (entry.uri == null) continue;
-      transitiveFiles(results, context, entry.uri, entryDir);
+      processDependence(entryPoint, canonicalize(entry.uri, entryDir));
+      transitiveFiles(context, entry.uri, entryDir);
     }
     for (var entry in library.exports) {
-      transitiveFiles(results, context, entry.uri, entryDir);
+      processDependence(entryPoint, canonicalize(entry.uri, entryDir));
+      transitiveFiles(context, entry.uri, entryDir);
     }
     for (var part in library.parts) {
-      results.add(canonicalize(part.uri, entryDir));
+      processFile(canonicalize(part.uri, entryDir));
     }
   }
 }