Initial cut for a development server

lib/devc.dart

 // Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /// Command line tool to run the checker on a Dart program.
 library ddc.devc;

[Jennifer Messerly, 2015/03/03 02:24:21]

I wonder if this should be in "src"? just noticed we've had a decent amount of
churn in this lib

unrelated note: we forgot to rename "ddc" here

[Siggi Cherem (dart-lang), 2015/03/04 04:44:23]

good point - now that I made most things private, it seems ok to keep it here,
but I'd be OK moving things further down in /src/

Overall, I'm not sure there is much we want to have in lib, almost everything we
want to expose in this package will come from bin/ 
filed: https://github.com/dart-lang/dev_compiler/issues/84

 
 import 'dart:async';
 import 'dart:convert';
 import 'dart:io';
 
+import 'package:analyzer/src/generated/engine.dart' show ChangeSet;
 import 'package:logging/logging.dart' show Level, Logger, LogRecord;
 import 'package:path/path.dart' as path;
-import 'package:html5lib/parser.dart' show parse;
+import 'package:shelf/shelf.dart' as shelf;
+import 'package:shelf/shelf_io.dart' as shelf;
+import 'package:shelf_static/shelf_static.dart' as shelf_static;
 
+import 'src/checker/checker.dart';
+import 'src/checker/dart_sdk.dart' show mockSdkSources;
 import 'src/checker/resolver.dart';
-import 'src/checker/checker.dart';
 import 'src/checker/rules.dart';
 import 'src/codegen/code_generator.dart' show CodeGenerator;
 import 'src/codegen/dart_codegen.dart';
+import 'src/codegen/html_codegen.dart';
 import 'src/codegen/js_codegen.dart';
-import 'src/codegen/html_codegen.dart';
+import 'src/dependency_graph.dart';
+import 'src/info.dart' show LibraryInfo, CheckerResults;
 import 'src/options.dart';
 import 'src/report.dart';
-import 'src/info.dart' show LibraryInfo, CheckerResults;
-import 'src/utils.dart' show reachableSources, partsOf, colorOf;
 
 /// Sets up the type checker logger to print a span that highlights error
 /// messages.
 StreamSubscription setupLogger(Level level, printFn) {
   Logger.root.level = level;
   return Logger.root.onRecord.listen((LogRecord rec) {
     printFn('${rec.level.name.toLowerCase()}: ${rec.message}');
   });
 }
 
-/// Compiles [inputFile] writing output as specified by the arguments.
-CheckerResults compile(
-    String inputFile, TypeResolver resolver, CompilerOptions options,
-    [CheckerReporter reporter]) {
-  if (inputFile.endsWith('.html')) {
-    return _compileHtml(inputFile, resolver, options, reporter);
-  } else {
-    return _compileDart(inputFile, resolver, options, reporter);
-  }
-}
-
-CheckerResults _compileHtml(
-    String inputFile, TypeResolver resolver, CompilerOptions options,
-    [CheckerReporter reporter]) {
-  var doc = parse(new File(inputFile).readAsStringSync(), generateSpans: true);
-  var scripts = doc.querySelectorAll('script[type="application/dart"]');
-  if (scripts.isEmpty) {
-    _log.severe('No <script type="application/dart"> found in $inputFile');
-    return _earlyErrorResult;
-  }
-  var mainScriptTag = scripts[0];
-  scripts.skip(1).forEach((s) {
-    _log.warning(s.sourceSpan.message(
-        'unexpected script. Only one Dart script tag allowed '
-        '(see https://github.com/dart-lang/dart-dev-compiler/issues/53).',
-        color: options.useColors ? colorOf('warning') : false));
-  });
-
-  var url = mainScriptTag.attributes['src'];
-  if (url == null) {
-    _log.severe(mainScriptTag.sourceSpan.message(
-        'inlined script tags not supported at this time '
-        '(see https://github.com/dart-lang/dart-dev-compiler/issues/54).',
-        color: options.useColors ? colorOf('error') : false));
-    return _earlyErrorResult;
-  }
-
-  var dartInputFile = path.join(path.dirname(inputFile), url);
-
-  if (!new File(dartInputFile).existsSync()) {
-    _log.severe(mainScriptTag.sourceSpan.message(
-        'Script file $dartInputFile not found',
-        color: options.useColors ? colorOf('error') : false));
-    return _earlyErrorResult;
-  }
-
-  var results = _compileDart(dartInputFile, resolver, options, reporter);
-  if (results.failure && !options.forceCompile) return results;
-
-  if (options.outputDir != null) {
-    generateEntryHtml(inputFile, options, results, doc);
-  }
-  return results;
-}
-
-CheckerResults _compileDart(
-    String inputFile, TypeResolver resolver, CompilerOptions options,
-    [CheckerReporter reporter]) {
-  Uri uri;
-  if (inputFile.startsWith('dart:') || inputFile.startsWith('package:')) {
-    uri = Uri.parse(inputFile);
-  } else {
-    uri = new Uri.file(path.absolute(inputFile));
-  }
-
-  if (reporter == null) {
-    reporter = options.dumpInfo
-        ? new SummaryReporter()
-        : new LogReporter(options.useColors);
-  }
-
-  var libraries = <LibraryInfo>[];
-  var rules = new RestrictedRules(resolver.context.typeProvider, reporter,
-      options: options);
-  var codeChecker = new CodeChecker(rules, reporter, options);
-  var generators = <CodeGenerator>[];
-  if (options.dumpSrcDir != null) {
-    generators
-        .add(new EmptyDartGenerator(options.dumpSrcDir, uri, rules, options));
-  }
-  var outputDir = options.outputDir;
-  if (outputDir != null) {
-    var cg = options.outputDart
-        ? new DartGenerator(outputDir, uri, rules, options)
-        : new JSGenerator(outputDir, uri, rules, options);
-    generators.add(cg);
-  }
-
+/// Encapsulates the logic to do a one-off compilation or a partial compilation
+/// when the compiler is run as a development server.
+class Compiler {
+  final CompilerOptions options;
+  final TypeResolver resolver;
+  final CheckerReporter reporter;
+  final TypeRules rules;
+  final CodeChecker checker;
+  final SourceGraph graph;
+  final SourceNode entryNode;
+  List<LibraryInfo> libraries = <LibraryInfo>[];

[Jennifer Messerly, 2015/03/03 02:24:21]

since this is in a public library, do we want to protect any of this state?
I'm thinking specifically "libraries" ... not sure if it makes sense to mutate
it except from inside the class

[Siggi Cherem (dart-lang), 2015/03/04 04:44:23]

Good point - actually all of the properties here should be private. Done.

+  final List<CodeGenerator> generators;
   bool failure = false;
-  var rootSource = resolver.findSource(uri);
-  // TODO(sigmund): switch to use synchronous codegen?
-  for (var source in reachableSources(rootSource, resolver.context)) {
+
+  factory Compiler(CompilerOptions options,
+      [TypeResolver resolver, CheckerReporter reporter]) {
+    if (resolver == null) {
+      resolver = options.useMockSdk
+          ? new TypeResolver.fromMock(mockSdkSources, options)
+          : new TypeResolver.fromDir(options.dartSdkPath, options);
+    }
+
+    if (reporter == null) {
+      reporter = options.dumpInfo
+          ? new SummaryReporter()
+          : new LogReporter(options.useColors);
+    }
+    var graph = new SourceGraph(resolver.context, options);
+    var rules = new RestrictedRules(resolver.context.typeProvider, reporter,
+        options: options);
+    var checker = new CodeChecker(rules, reporter, options);
+    var inputFile = options.entryPointFile;
+    var uri = inputFile.startsWith('dart:') || inputFile.startsWith('package:')
+        ? Uri.parse(inputFile)
+        : new Uri.file(path.absolute(inputFile));
+    var entryNode = graph.nodeFromUri(uri);
+
+    var outputDir = options.outputDir;
+    var generators = <CodeGenerator>[];
+    if (options.dumpSrcDir != null) {
+      generators.add(new EmptyDartGenerator(
+          options.dumpSrcDir, entryNode.uri, rules, options));
+    }
+    if (outputDir != null) {
+      generators.add(options.outputDart
+          ? new DartGenerator(outputDir, entryNode.uri, rules, options)
+          : new JSGenerator(outputDir, entryNode.uri, rules, options));
+    }
+    return new Compiler._(options, resolver, reporter, rules, checker, graph,
+        entryNode, generators);
+  }
+
+  Compiler._(this.options, this.resolver, this.reporter, this.rules,
+      this.checker, this.graph, this.entryNode, this.generators);
+
+  bool _buildSource(SourceNode node) {
+    if (node is HtmlSourceNode) {
+      _buildHtmlFile(node);
+    } else if (node is LibrarySourceNode) {
+      _buildDartLibrary(node);
+    }

[Jennifer Messerly, 2015/03/03 02:24:21]

what other kinds of nodes are possible?

[Siggi Cherem (dart-lang), 2015/03/04 04:44:23]

True, at this point no other kind of node should reach here (PartSourceNode will
not reach here). I added an assertion to make this more clear.

+
+    // TODO(sigmund): don't always return true. Use summarization to better
+    // determine when rebuilding is needed.
+    return true;
+  }
+
+  bool _devCompilerRuntimeCopied = false;
+
+  void _buildHtmlFile(HtmlSourceNode node) {
+    if (options.outputDir == null) return;
+    var output = generateEntryHtml(node, options);
+    if (output == null) {
+      failure = true;
+      return;
+    }
+    var filename = path.basename(node.uri.path);
+    String outputFile = path.join(options.outputDir, filename);
+    new File(outputFile).writeAsStringSync(output);
+
+    if (options.outputDart || _devCompilerRuntimeCopied) return;
+    // Copy the dev_compiler runtime (implicit dependency for js codegen)
+    // TODO(sigmund): split this out as a separate node in our dependency graph?

[Jennifer Messerly, 2015/03/03 02:24:21]

+1 .. also that way it could have mtime tracking

[Siggi Cherem (dart-lang), 2015/03/04 04:44:23]

Good point - I've opened https://github.com/dart-lang/dev_compiler/issues/85 to
track. 

+    var runtimeDir = path.join(
+        path.dirname(path.dirname(Platform.script.path)), 'lib/runtime/');
+    var runtimeOutput = path.join(options.outputDir, 'dev_compiler/runtime/');
+    new Directory(runtimeOutput).createSync(recursive: true);
+    new File(path.join(runtimeDir, 'harmony_feature_check.js'))
+        .copy(path.join(runtimeOutput, 'harmony_feature_check.js'));
+    new File(path.join(runtimeDir, 'dart_runtime.js'))
+        .copy(path.join(runtimeOutput, 'dart_runtime.js'));
+    _devCompilerRuntimeCopied = true;
+  }
+
+  bool _isEntry(LibrarySourceNode node) {
+    if (entryNode is LibrarySourceNode) return entryNode == node;
+    return (entryNode as HtmlSourceNode).scripts.contains(node);
+  }
+
+  void _buildDartLibrary(LibrarySourceNode node) {
+    var source = node.source;
+    // TODO(sigmund): find out from analyzer team if there is a better way
+    resolver.context.applyChanges(new ChangeSet()..changedSource(source));
     var entryUnit = resolver.context.resolveCompilationUnit2(source, source);
     var lib = entryUnit.element.enclosingElement;
-    if (!options.checkSdk && lib.isInSdk) continue;
-    var current = new LibraryInfo(lib, source.uri == uri);
+    if (!options.checkSdk && lib.isInSdk) return;
+    var current = node.info;
+    if (current != null) {
+      assert(current.library == lib);
+    } else {
+      node.info = current = new LibraryInfo(lib, _isEntry(node));
+    }
     reporter.enterLibrary(current);
     libraries.add(current);
     rules.currentLibraryInfo = current;
 
     var units = [entryUnit]
-      ..addAll(partsOf(entryUnit, resolver.context)
-          .map((p) => resolver.context.resolveCompilationUnit2(p, source)));
+      ..addAll(node.parts.map(
+          (p) => resolver.context.resolveCompilationUnit2(p.source, source)));
     bool failureInLib = false;
     for (var unit in units) {
       var unitSource = unit.element.source;
       reporter.enterSource(unitSource);
       // TODO(sigmund): integrate analyzer errors with static-info (issue #6).
       failureInLib = resolver.logErrors(unitSource, reporter) || failureInLib;
-      unit.visitChildren(codeChecker);
-      if (codeChecker.failure) failureInLib = true;
+      unit.visitChildren(checker);
+      if (checker.failure) failureInLib = true;
       reporter.leaveSource();
     }
-    reporter.leaveLibrary();
-
     if (failureInLib) {
       failure = true;
-      if (!options.forceCompile) continue;
+      if (!options.forceCompile) return;
     }
     for (var cg in generators) {
       cg.generateLibrary(units, current, reporter);
     }
-  }
-
-  if (options.dumpInfo && reporter is SummaryReporter) {
-    print(summaryToString(reporter.result));
-    if (options.dumpInfoFile != null) {
-      new File(options.dumpInfoFile)
-          .writeAsStringSync(JSON.encode(reporter.result.toJsonMap()));
-    }
-  }
-  return new CheckerResults(libraries, rules, failure || options.forceCompile);
+    reporter.leaveLibrary();
+  }
+
+  CheckerResults runOnce() {

[Jennifer Messerly, 2015/03/03 02:24:21]

made this comment elsewhere, but I think "run" would be a good name for this

[Siggi Cherem (dart-lang), 2015/03/04 04:44:23]

Done.

+    var clock = new Stopwatch()..start();
+
+    // TODO(sigmund): we are missing a couple failures here. The
+    // dependendency_graph now detects broken imports or unsupported features
+    // like more than one script tag (see .severe messages in
+    // dependency_graph.dart). Such failures should be reported back
+    // here so we can mark failure=true in the CheckerResutls.
+    rebuild(entryNode, graph, _buildSource);
+    if (options.dumpInfo && reporter is SummaryReporter) {
+      var result = (reporter as SummaryReporter).result;
+      print(summaryToString(result));
+      if (options.dumpInfoFile != null) {
+        new File(options.dumpInfoFile)
+            .writeAsStringSync(JSON.encode(result.toJsonMap()));
+      }
+    }
+    clock.stop();
+    if (options.serverMode) {
+      var time = (clock.elapsedMilliseconds / 1000).toStringAsFixed(2);
+      print('Compiled ${libraries.length} libraries in ${time} s\n');
+    }
+    return new CheckerResults(
+        libraries, rules, failure || options.forceCompile);
+  }
+
+  void _runAgain() {
+    var clock = new Stopwatch()..start();
+    if (reporter is SummaryReporter) (reporter as SummaryReporter).clear();

[Jennifer Messerly, 2015/03/03 02:24:21]

WAT :)

can we file this somewhere?

another idea to maybe help, is have a getter for `summaryReporter` that is typed
... or have these methods on CheckerReporter but have them do nothing ...

[Siggi Cherem (dart-lang), 2015/03/04 04:44:23]

It's sad, the promotion rules only apply to local variables. Even though this
field is final. What's crazy is that I can write this:

  CheckerReporter reporter2 = reporter;
  if (reporter2 is SummaryReporter) reporter2.clear();

and get no warnings.

Maybe we should create a DEP to improve type-promotion?

Meanwhile, I'm inclined to keep it as it is, this is likely going to change
again soon :)

+    libraries = <LibraryInfo>[];
+    int changed = 0;
+
+    // TODO(sigmund): propagate failures here (see TODO in runOnce).
+    rebuild(entryNode, graph, (n) {
+      changed++;
+      return _buildSource(n);
+    });
+    if (reporter is SummaryReporter) {
+      print(summaryToString((reporter as SummaryReporter).result));
+    }
+    clock.stop();
+    var time = (clock.elapsedMilliseconds / 1000).toStringAsFixed(2);
+    print("Compiled ${changed} libraries in ${time} s\n");
+  }
 }
 
+class CompilerServer {
+  final Compiler compiler;
+  final String outDir;
+  final int port;
+  final String _entryPath;
+
+  factory CompilerServer(CompilerOptions options) {
+    var entryPath = path.basename(options.entryPointFile);
+    if (path.extension(entryPath) != '.html') {
+      print('error: devc in server mode requires an HTML entry point.');
+      exit(1);
+    }
+
+    // TODO(sigmund): allow running without a dir, but keep output in memory?
+    var outDir = options.outputDir;
+    if (outDir == null) {
+      print('error: devc in server mode also requires specifying and '
+          'output location for generated code.');
+      exit(1);
+    }
+    var port = options.port;
+    print('[dev_compiler]: Serving $entryPath at http://0.0.0.0:$port/');
+    var compiler = new Compiler(options);
+    return new CompilerServer._(compiler, outDir, port, entryPath);
+  }
+
+  CompilerServer._(this.compiler, this.outDir, this.port, this._entryPath);
+
+  Future start() async {
+    var handler = const shelf.Pipeline()
+        .addMiddleware(shelf.createMiddleware(requestHandler: rebuildIfNeeded))
+        .addHandler(shelf_static.createStaticHandler(outDir,
+            defaultDocument: _entryPath));
+    await shelf.serve(handler, '0.0.0.0', port);
+    compiler.runOnce();
+  }
+
+  rebuildIfNeeded(shelf.Request request) {
+    var filepath = request.url.path;
+    if (filepath == '/$_entryPath' || filepath == '/') compiler._runAgain();
+  }
+}
+
 final _log = new Logger('ddc');
 final _earlyErrorResult = new CheckerResults(const [], null, true);