DDC: mostly incremental compilation, fixes #223

lib/src/compiler.dart

 // Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /// Command line tool to run the checker on a Dart program.
 library dev_compiler.src.compiler;
 
 import 'dart:async';
 import 'dart:collection';
 import 'dart:math' as math;
 import 'dart:io';
 
-import 'package:analyzer/src/generated/ast.dart' show CompilationUnit;
-import 'package:analyzer/src/generated/element.dart';
+import 'package:analyzer/src/generated/ast.dart'
+    show CompilationUnit, NamespaceDirective, PartDirective, UriBasedDirective;
 import 'package:analyzer/src/generated/engine.dart'
     show AnalysisEngine, AnalysisContext, ChangeSet, ParseDartTask;
 import 'package:analyzer/src/generated/error.dart'
     show AnalysisError, ErrorSeverity, ErrorType;
 import 'package:analyzer/src/generated/error.dart';
-import 'package:analyzer/src/generated/resolver.dart' show TypeProvider;
 import 'package:analyzer/src/generated/source.dart' show Source;
 import 'package:analyzer/src/task/html.dart';
 import 'package:html/dom.dart' as html;
 import 'package:html/parser.dart' as html;
 import 'package:logging/logging.dart' show Level, Logger, LogRecord;
 import 'package:path/path.dart' as path;
 
-import 'package:dev_compiler/strong_mode.dart' show StrongModeOptions;
-
 import 'analysis_context.dart';
 import 'checker/checker.dart';
 import 'checker/rules.dart';
 import 'codegen/html_codegen.dart' as html_codegen;
 import 'codegen/js_codegen.dart';
 import 'info.dart'
     show AnalyzerMessage, CheckerResults, LibraryInfo, LibraryUnit;
 import 'options.dart';
 import 'report.dart';
 
@@ skipping 26 matching lines @@
 bool compile(CompilerOptions options) {
   assert(!options.serverMode);
   var context = createAnalysisContextWithSources(
       options.strongOptions, options.sourceOptions);
   var reporter = createErrorReporter(context, options);
   return new BatchCompiler(context, options, reporter: reporter).run();
 }
 
 class BatchCompiler extends AbstractCompiler {
   JSGenerator _jsGen;
-  LibraryElement _dartCore;
+  Source _dartCore;
   String _runtimeOutputDir;
 
   /// Already compiled sources, so we don't compile them again.
-  final _compiled = new HashSet<LibraryElement>();
+  final _compiled = new HashSet<Uri>();
   bool _sdkCopied = false;
 
   bool _failure = false;
   bool get failure => _failure;
 
   BatchCompiler(AnalysisContext context, CompilerOptions options,
       {AnalysisErrorListener reporter})
       : super(
             context,
             options,
-            new ErrorCollector(
-                reporter ?? AnalysisErrorListener.NULL_LISTENER)) {
-    _inputBaseDir = options.inputBaseDir;
+            new ErrorCollector(context, reporter, options.logLevel,
+                saveMessages: options.saveMessages)) {
     if (outputDir != null) {
-      _jsGen = new JSGenerator(this);
       _runtimeOutputDir = path.join(outputDir, 'dev_compiler', 'runtime');
     }
-    _dartCore = context.typeProvider.objectType.element.library;
+    _dartCore = context.sourceFactory.forUri('dart:core');
   }
 
-  ErrorCollector get reporter => checker.reporter;
+  ErrorCollector get reporter => super.reporter;
 
   void reset() {
     _compiled.clear();
     _sdkCopied = false;
   }
 
   /// Compiles every file in [options.inputs].
   /// Returns true on successful compile.
   bool run() {
     var clock = new Stopwatch()..start();
     options.inputs.forEach(compileFromUriString);
     clock.stop();
     var time = (clock.elapsedMilliseconds / 1000).toStringAsFixed(2);
     _log.fine('Compiled ${_compiled.length} libraries in ${time} s\n');
-
     return !_failure;
   }
 
   void compileFromUriString(String uriString) {
     _compileFromUri(stringToUri(uriString));
   }
 
   void _compileFromUri(Uri uri) {
     if (!uri.isAbsolute) {
       throw new ArgumentError.value('$uri', 'uri', 'must be absolute');
     }
     var source = context.sourceFactory.forUri(Uri.encodeFull('$uri'));
     if (source == null) {
       throw new ArgumentError.value('$uri', 'uri', 'could not find source for');
     }
     compileSource(source);
   }
 
   void compileSource(Source source) {
     if (AnalysisEngine.isHtmlFileName(source.uri.path)) {
       _compileHtml(source);
     } else {
-      _compileLibrary(context.computeLibraryElement(source));
-    }
-    reporter.flush();
-  }
-
-  void _compileLibrary(LibraryElement library) {
-    if (!_compiled.add(library)) return;
-
-    if (!options.checkSdk && library.source.uri.scheme == 'dart') {
-      if (_jsGen != null) _copyDartRuntime();
-      return;
-    }
-
-    // TODO(jmesserly): in incremental mode, we can skip the transitive
-    // compile of imports/exports.
-    _compileLibrary(_dartCore); // implicit dart:core dependency
-    library.importedLibraries.forEach(_compileLibrary);
-    library.exportedLibraries.forEach(_compileLibrary);
-
-    var unitElements = [library.definingCompilationUnit]..addAll(library.parts);
-    var units = <CompilationUnit>[];
-
-    bool failureInLib = false;
-    for (var element in unitElements) {
-      var unit = context.resolveCompilationUnit(element.source, library);
-      units.add(unit);
-      failureInLib = logErrors(element.source) || failureInLib;
-      checker.visitCompilationUnit(unit);
-      if (checker.failure) failureInLib = true;
-    }
-
-    if (failureInLib) {
-      _failure = true;
-      if (!options.codegenOptions.forceCompile) return;
-    }
-
-    if (_jsGen != null) {
-      // TODO(jmesserly): full incremental support would avoid checking as well,
-      // however, we'd lose compiler messages in that case.
-
-      // Note: analyzer's modification stamp is millisecondsSinceEpoch
-      int lastModifyTime = unitElements
-          .map((e) => context.getModificationStamp(e.source))
-          .reduce(math.max);
-      var outFile = new File(getOutputPath(library.source.uri));
-      if (outFile.existsSync() &&
-          outFile.lastModifiedSync().millisecondsSinceEpoch >= lastModifyTime) {
-        // Output already up to date.
-        return;
-      }
-
-      var unit = units.first;
-      var parts = units.skip(1).toList();
-      _jsGen.generateLibrary(new LibraryUnit(unit, parts));
+      _compileLibrary(source);
     }
   }
 
-  void _copyDartRuntime() {
-    if (_sdkCopied) return;
+  bool _compileLibrary(Source source) {
+    if (!_compiled.add(source.uri)) return false;

[Leaf, 2015/09/04 21:46:31]

I'm worried about this doing the right thing in the presence of cycles.  If I
have a cycle of libraries A and B, and I modify A, then I think this code does
the following:

1) Adds A to _compiled
2) Tries to compile B
3) Adds B to _compiled
4) Tries to compile A
5) Notices that A is in _compiled, so returns false for changed,
6) Does a time stamp check on B, sees that it hasn't itself changed, so doesn't
recompile it
7) Recompiles A based on timestamps

Am I missing something? 

+
+    if (!options.checkSdk && source.uri.scheme == 'dart') {
+      return outputDir != null && _copyDartRuntime();
+    }
+
+    var sources = <Source>[source];
+    if (!source.exists()) return false;
+
+    int lastEdit = context.getModificationStamp(source);
+
+    // implicit dart:core dependency
+    bool changed = _compileLibrary(_dartCore);
+
+    var definingUnit = context.parseCompilationUnit(source);
+    for (var d in definingUnit.directives) {
+      if (d is UriBasedDirective) {
+        var src = context.sourceFactory.resolveUri(source, d.uri.stringValue);
+        if (src == null) continue;
+
+        if (d is NamespaceDirective) {
+          if (_compileLibrary(src)) {
+            changed = true;
+          }
+        } else if (d is PartDirective) {
+          sources.add(src);
+          lastEdit = math.max(lastEdit, context.getModificationStamp(src));
+        }
+      }
+    }
+
+    // Take into account if the compiler itself was edited.
+    if (compilerLastModified != null) {
+      lastEdit =
+          math.max(lastEdit, compilerLastModified.millisecondsSinceEpoch);
+    }
+
+    String messageFilePath;
+    File outFile;
+    if (outputDir != null) {
+      messageFilePath =
+          path.withoutExtension(getOutputPath(source.uri)) + '.txt';
+      outFile = new File(getOutputPath(source.uri));
+    } else {
+      // if no output directory is specified, we're in checker mode, and should
+      // run the full compilation.
+      // TODO(jmesserly): deprecate check only mode; this should go through
+      // analyzer_cli instead. BatchCompiler would be simpler if it was always
+      // a compiler that produced output.
+      changed = true;
+    }
+    if (!changed) {
+      var msgFile = new File(messageFilePath);
+
+      if (outFile.existsSync()) {
+        changed = outFile.lastModifiedSync().millisecondsSinceEpoch < lastEdit;
+      } else if (msgFile.existsSync()) {
+        changed = msgFile.lastModifiedSync().millisecondsSinceEpoch < lastEdit;
+      } else {
+        // Output files do not exist.
+        changed = true;
+      }
+      // If output is already up to date, we can skip remaining steps.
+      if (!changed) return false;
+    }
+
+    var units = <CompilationUnit>[];
+    for (var src in sources) {
+      var unit = context.resolveCompilationUnit2(src, source);
+      units.add(unit);
+      if (logErrors(src)) _failure = true;
+      checker.visitCompilationUnit(unit);
+      if (checker.failure) _failure = true;
+    }
+
+    if (outputDir != null) {
+      if (!_failure || options.codegenOptions.forceCompile) {
+        if (_jsGen == null) _jsGen = new JSGenerator(this);
+        var unit = units.first;
+        var parts = units.skip(1).toList();
+        _jsGen.generateLibrary(new LibraryUnit(unit, parts));
+      } else {
+        // Delete stale output.
+        if (outFile.existsSync()) outFile.deleteSync();
+      }
+    }
+
+    reporter.flush(messageFilePath);
+    return true;
+  }
+
+  bool _copyDartRuntime() {
+    if (_sdkCopied) return false;
+    if (options.runtimeDir == null) return false;
+
     _sdkCopied = true;
+
+    bool changed = false;
     for (var file in defaultRuntimeFiles) {
       var input = new File(path.join(options.runtimeDir, file));
       var output = new File(path.join(_runtimeOutputDir, file));
       if (output.existsSync() &&
           output.lastModifiedSync() == input.lastModifiedSync()) {
         continue;
       }
+
+      changed = true;
       new Directory(path.dirname(output.path)).createSync(recursive: true);
       input.copySync(output.path);
     }
+    return changed;
   }
 
-  void _compileHtml(Source source) {
+  bool _compileHtml(Source source) {
     // TODO(jmesserly): reuse DartScriptsTask instead of copy/paste.
     var contents = context.getContents(source);
     var document = html.parse(contents.data, generateSpans: true);
     var scripts = document.querySelectorAll('script[type="application/dart"]');
 
-    var loadedLibs = new LinkedHashSet<Uri>();
+    var outFile = new File(getOutputPath(source.uri));
 
+    bool changed = !outFile.existsSync() ||
+        context.getModificationStamp(source) >
+            outFile.lastModifiedSync().millisecondsSinceEpoch;
+
+    var scriptSources = <Source>[];
     var htmlOutDir = path.dirname(getOutputPath(source.uri));
     for (var script in scripts) {
       Source scriptSource = null;
       var srcAttr = script.attributes['src'];
       if (srcAttr == null) {
         if (script.hasContent()) {
           var fragments = <ScriptFragment>[];
           for (var node in script.nodes) {
             if (node is html.Text) {
               var start = node.sourceSpan.start;
               fragments.add(new ScriptFragment(
                   start.offset, start.line, start.column, node.data));
             }
           }
           scriptSource = new DartScript(source, fragments);
         }
       } else if (AnalysisEngine.isDartFileName(srcAttr)) {
         scriptSource = context.sourceFactory.resolveUri(source, srcAttr);
       }
+      scriptSources.add(scriptSource);
 
-      if (scriptSource != null) {
-        var lib = context.computeLibraryElement(scriptSource);
-        _compileLibrary(lib);
-        script.replaceWith(_linkLibraries(lib, loadedLibs, from: htmlOutDir));
-      }
+      if (scriptSource != null && _compileLibrary(scriptSource)) changed = true;
     }
 
-    new File(getOutputPath(source.uri)).openSync(mode: FileMode.WRITE)
+    if (!changed) {
+      return false;
+    }
+
+    var loadedLibs = new LinkedHashSet<Uri>();
+    for (int i = 0; i < scripts.length; i++) {
+      var src = scriptSources[i];
+      if (src == null) continue;
+      scripts[i].replaceWith(_linkLibraries(src, loadedLibs, from: htmlOutDir));
+    }
+
+    outFile.openSync(mode: FileMode.WRITE)
       ..writeStringSync(document.outerHtml)
       ..writeStringSync('\n')
       ..closeSync();
+
+    reporter.flush(getOutputPath(source.uri) + '.txt');
+    return true;
   }
 
   html.DocumentFragment _linkLibraries(
-      LibraryElement mainLib, LinkedHashSet<Uri> loaded,
+      Source mainLib, LinkedHashSet<Uri> loaded,
       {String from}) {
     assert(from != null);
     var alreadyLoaded = loaded.length;
     _collectLibraries(mainLib, loaded);
 
     var newLibs = loaded.skip(alreadyLoaded);
     var df = new html.DocumentFragment();
 
     for (var uri in newLibs) {
       if (uri.scheme == 'dart') {
         if (uri.path == 'core') {
           // TODO(jmesserly): it would be nice to not special case these.
           for (var file in defaultRuntimeFiles) {
             file = path.join(_runtimeOutputDir, file);
             df.append(
                 html_codegen.libraryInclude(path.relative(file, from: from)));
           }
         }
       } else {
         var file = path.join(outputDir, getModulePath(uri));
         df.append(html_codegen.libraryInclude(path.relative(file, from: from)));
       }
     }
 
     df.append(html_codegen.invokeMain(getModuleName(mainLib.source.uri)));
     return df;
   }
 
-  void _collectLibraries(LibraryElement lib, LinkedHashSet<Uri> loaded) {
-    var uri = lib.source.uri;
+  void _collectLibraries(Source source, LinkedHashSet<Uri> loaded) {
+    var uri = source.uri;
     if (!loaded.add(uri)) return;
     _collectLibraries(_dartCore, loaded);
-    for (var l in lib.importedLibraries) _collectLibraries(l, loaded);
-    for (var l in lib.exportedLibraries) _collectLibraries(l, loaded);
+
+    var definingUnit = context.parseCompilationUnit(source);
+    for (var d in definingUnit.directives) {
+      if (d is NamespaceDirective) {
+        var src = context.sourceFactory.resolveUri(source, d.uri.stringValue);
+        if (src != null) _collectLibraries(src, loaded);
+      }
+    }
+
     // Move the item to the end of the list.
     loaded.remove(uri);
     loaded.add(uri);
   }
 }
 
 abstract class AbstractCompiler {
   final CompilerOptions options;
   final AnalysisContext context;
-  final CodeChecker checker;
+  final AnalysisErrorListener reporter;
+  CodeChecker _checker;
 
-  AbstractCompiler(AnalysisContext context, CompilerOptions options,
+  AbstractCompiler(this.context, CompilerOptions options,
       [AnalysisErrorListener reporter])
-      : context = context,
+      : reporter = reporter ?? AnalysisErrorListener.NULL_LISTENER,
         options = options,
-        checker = createChecker(context.typeProvider, options.strongOptions,
-            reporter ?? AnalysisErrorListener.NULL_LISTENER) {
+        _inputBaseDir = options.inputBaseDir {
     enableDevCompilerInference(context, options.strongOptions);
   }
 
-  static CodeChecker createChecker(TypeProvider typeProvider,
-      StrongModeOptions options, AnalysisErrorListener reporter) {
-    return new CodeChecker(
-        new RestrictedRules(typeProvider, options: options), reporter, options);
+  CodeChecker get checker {
+    if (_checker == null) {
+      var opts = options.strongOptions;
+      _checker = new CodeChecker(
+          new RestrictedRules(context.typeProvider, options: opts),
+          reporter,
+          opts);
+    }
+    return _checker;
   }
 
   String get outputDir => options.codegenOptions.outputDir;
   TypeRules get rules => checker.rules;
-  AnalysisErrorListener get reporter => checker.reporter;
 
   Uri stringToUri(String uriString) {
     var uri = uriString.startsWith('dart:') || uriString.startsWith('package:')
         ? Uri.parse(uriString)
         : new Uri.file(path.absolute(uriString));
     return uri;
   }
 
   /// Directory presumed to be the common prefix for all input file:// URIs.
   /// Used when computing output paths.
@@ skipping 87 matching lines @@
         failure = true;
         reporter.onError(error);
       }
     }
     return failure;
   }
 }
 
 AnalysisErrorListener createErrorReporter(
     AnalysisContext context, CompilerOptions options) {
-  return options.dumpInfo
-      ? new SummaryReporter(context, options.logLevel)
-      : new LogReporter(context, useColors: options.useColors);
+  if (options.dumpInfo) return new SummaryReporter(context, options.logLevel);
+  return new LogReporter(context);
 }
 
 // TODO(jmesserly): find a better home for these.
 /// Curated order to minimize lazy classes needed by dart:core and its
 /// transitive SDK imports.
 const corelibOrder = const [
   'dart.core',
   'dart.collection',
   'dart._internal',
   'dart.math',
@@ skipping 31 matching lines @@
     '_rtti.js',
     '_classes.js',
     '_operations.js',
     'dart_runtime.js',
   ];
   files.addAll(corelibOrder.map((l) => l.replaceAll('.', '/') + '.js'));
   return files;
 }();
 
 final _log = new Logger('dev_compiler.src.compiler');