First pass at asynchronous input loading in dart2js.

sdk/lib/_internal/compiler/implementation/compiler.dart

Index: sdk/lib/_internal/compiler/implementation/compiler.dart
diff --git a/sdk/lib/_internal/compiler/implementation/compiler.dart b/sdk/lib/_internal/compiler/implementation/compiler.dart
index 96490f5c03790dbf7f869857c9fc4a0d3e50e52b..ec8d0655d6caa4c070868a862db92e52fbc286f0 100644
--- a/sdk/lib/_internal/compiler/implementation/compiler.dart
+++ b/sdk/lib/_internal/compiler/implementation/compiler.dart
@@ -379,6 +379,7 @@ abstract class Compiler implements DiagnosticListener {
    * error occurs then report it as having occurred during compilation of
    * [element].  Can be nested.
    */
+  // TODO(rnystrom): Handle futures.

[Bob Nystrom, 2013/06/26 01:03:24]

This method assumes the f() is completely done by the time it returns, which
isn't the case for futures.

In the places that use this, I've manually called withCurrentElement() again
isn't the future's then() callback to restore the "current" element, but that's
a bit ugly. There may be a cleaner solution to this that's more future-aware.

   withCurrentElement(Element element, f()) {
     Element old = currentElement;
     _currentElement = element;
@@ -648,14 +649,17 @@ abstract class Compiler implements DiagnosticListener {
     reportDiagnostic(null, message, api.Diagnostic.VERBOSE_INFO);
   }
 
-  bool run(Uri uri) {
+  Future<bool> run(Uri uri) {
     totalCompileTime.start();
-    try {
-      runCompiler(uri);
-    } on CompilerCancelledException catch (exception) {
-      log('Error: $exception');
-      return false;
-    } catch (exception) {
+
+    return runCompiler(uri).then((_) {
+      return true;
+    }).catchError((error) {
+      if (error is CompilerCancelledException) {
+        log('Error: $error');
+        return false;
+      }
+
       try {
         if (!hasCrashed) {
           hasCrashed = true;
@@ -667,12 +671,13 @@ abstract class Compiler implements DiagnosticListener {
       } catch (doubleFault) {
         // Ignoring exceptions in exception handling.
       }
-      rethrow;
-    } finally {
+      throw error;
+    }).whenComplete(() {
       tracer.close();
       totalCompileTime.stop();
-    }
-    return true;
+    }).then((_) {
+      return true;
+    });
   }
 
   bool hasIsolateSupport() => isolateLibrary != null;
@@ -711,7 +716,7 @@ abstract class Compiler implements DiagnosticListener {
     }
   }
 
-  LibraryElement scanBuiltinLibrary(String filename);
+  Future<LibraryElement> scanBuiltinLibrary(String filename);
 
   void initializeSpecialClasses() {
     final List missingCoreClasses = [];
@@ -787,29 +792,37 @@ abstract class Compiler implements DiagnosticListener {
         listClass.lookupConstructor(callConstructor);
   }
 
-  void scanBuiltinLibraries() {
-    jsHelperLibrary = scanBuiltinLibrary('_js_helper');
-    interceptorsLibrary = scanBuiltinLibrary('_interceptors');
-    foreignLibrary = scanBuiltinLibrary('_foreign_helper');
-    isolateHelperLibrary = scanBuiltinLibrary('_isolate_helper');
-
-    assertMethod = jsHelperLibrary.find(const SourceString('assertHelper'));
-    identicalFunction = coreLibrary.find(const SourceString('identical'));
-
-    initializeSpecialClasses();
-
-    functionClass.ensureResolved(this);
-    functionApplyMethod =
-        functionClass.lookupLocalMember(const SourceString('apply'));
-    jsInvocationMirrorClass.ensureResolved(this);
-    invokeOnMethod = jsInvocationMirrorClass.lookupLocalMember(INVOKE_ON);
-
-    if (preserveComments) {
-      var uri = new Uri(scheme: 'dart', path: 'mirrors');
-      LibraryElement libraryElement =
-          libraryLoader.loadLibrary(uri, null, uri);
-      documentClass = libraryElement.find(const SourceString('Comment'));
-    }
+  Future scanBuiltinLibraries() {
+    return scanBuiltinLibrary('_js_helper').then((library) {
+      jsHelperLibrary = library;
+      return scanBuiltinLibrary('_interceptors');
+    }).then((library) {
+      interceptorsLibrary = library;
+      return scanBuiltinLibrary('_foreign_helper');
+    }).then((library) {
+      foreignLibrary = library;
+      return scanBuiltinLibrary('_isolate_helper');
+    }).then((library) {
+      isolateHelperLibrary = library;

[Bob Nystrom, 2013/06/26 01:03:24]

This is a bit ugly, but it ensures that these libraries are scanned
sequentially. If you don't care about that, I can use Future.wait() here and it
will actually load them in parallel.

That might make the order of error messages non-deterministic, though.

With this patch, I've tried to keep everything sequential and in the same order,
even though it's async.

[ahe, 2013/06/26 07:02:00]

I think we can use Future.wait here.  Normally these libraries will not produce
error messages.

[Bob Nystrom, 2013/06/27 00:38:18]

I just tried that but it apparently causes some other issues. I think it gets
confused and ends up loading "dart:core" multiple times because there are
multiple loads of it in flight at the same time.

I can try to clean this up to use Future.forEach() or leave it with explicit
chained .then()s like this.

+
+      assertMethod = jsHelperLibrary.find(const SourceString('assertHelper'));
+      identicalFunction = coreLibrary.find(const SourceString('identical'));
+
+      initializeSpecialClasses();
+
+      functionClass.ensureResolved(this);
+      functionApplyMethod =
+          functionClass.lookupLocalMember(const SourceString('apply'));
+      jsInvocationMirrorClass.ensureResolved(this);
+      invokeOnMethod = jsInvocationMirrorClass.lookupLocalMember(INVOKE_ON);
+
+      if (preserveComments) {
+        var uri = new Uri(scheme: 'dart', path: 'mirrors');
+        return libraryLoader.loadLibrary(uri, null, uri).then((libraryElement) {

[ahe, 2013/06/26 07:02:00]

Please restore the explicit type of libraryElement.

[Bob Nystrom, 2013/06/27 00:38:18]

Done.

+          documentClass = libraryElement.find(const SourceString('Comment'));
+        });
+      }
+    });
   }
 
   void importHelperLibrary(LibraryElement library) {
@@ -824,110 +837,117 @@ abstract class Compiler implements DiagnosticListener {
    */
   Uri resolvePatchUri(String dartLibraryPath);
 
-  void runCompiler(Uri uri) {
+  Future runCompiler(Uri uri) {
     assert(uri != null || analyzeOnly);
-    scanBuiltinLibraries();
-    if (librariesToAnalyzeWhenRun != null) {
-      for (Uri libraryUri in librariesToAnalyzeWhenRun) {
-        log('analyzing $libraryUri ($buildId)');
-        libraryLoader.loadLibrary(libraryUri, null, libraryUri);
-      }
-    }
-    if (uri != null) {
-      if (analyzeOnly) {
-        log('analyzing $uri ($buildId)');
-      } else {
-        log('compiling $uri ($buildId)');
+    return scanBuiltinLibraries().then((_) {
+      if (librariesToAnalyzeWhenRun != null) {
+        return Future.forEach(librariesToAnalyzeWhenRun, (libraryUri) {
+          log('analyzing $libraryUri ($buildId)');
+          return libraryLoader.loadLibrary(libraryUri, null, libraryUri);
+        });
       }
-      mainApp = libraryLoader.loadLibrary(uri, null, uri);
-    }
-    Element main = null;
-    if (mainApp != null) {
-      main = mainApp.find(MAIN);
-      if (main == null) {
-        if (!analyzeOnly) {
-          // Allow analyze only of libraries with no main.
-          reportFatalError('Could not find $MAIN', mainApp);
-        } else if (!analyzeAll) {
-          reportFatalError(
-              "Could not find $MAIN. "
-              "No source will be analyzed. "
-              "Use '--analyze-all' to analyze all code in the library.",
-              mainApp);
-        }
-      } else {
-        if (!main.isFunction()) {
-          reportFatalError('main is not a function', main);
+    }).then((_) {
+      if (uri != null) {
+        if (analyzeOnly) {
+          log('analyzing $uri ($buildId)');
+        } else {
+          log('compiling $uri ($buildId)');
         }
-        FunctionElement mainMethod = main;
-        FunctionSignature parameters = mainMethod.computeSignature(this);
-        parameters.forEachParameter((Element parameter) {
-          reportFatalError('main cannot have parameters', parameter);
+        return libraryLoader.loadLibrary(uri, null, uri).then((library) {
+          mainApp = library;
         });
       }
+    }).then((_) {
+      Element main = null;
+      if (mainApp != null) {
+        main = mainApp.find(MAIN);
+        if (main == null) {
+          if (!analyzeOnly) {
+            // Allow analyze only of libraries with no main.
+            reportFatalError('Could not find $MAIN', mainApp);
+          } else if (!analyzeAll) {
+            reportFatalError(
+                "Could not find $MAIN. "
+                "No source will be analyzed. "
+                "Use '--analyze-all' to analyze all code in the library.",
+                mainApp);
+          }
+        } else {
+          if (!main.isFunction()) {
+            reportFatalError('main is not a function', main);
+          }
+          FunctionElement mainMethod = main;
+          FunctionSignature parameters = mainMethod.computeSignature(this);
+          parameters.forEachParameter((Element parameter) {
+            reportFatalError('main cannot have parameters', parameter);
+          });
+        }
 
-      // In order to see if a library is deferred, we must compute the
-      // compile-time constants that are metadata.  This means adding
-      // something to the resolution queue.  So we cannot wait with
-      // this until after the resolution queue is processed.
-      // TODO(ahe): Clean this up, for example, by not enqueueing
-      // classes only used for metadata.
-      deferredLoadTask.findDeferredLibraries(mainApp);
-    }
-
-    log('Resolving...');
-    phase = PHASE_RESOLVING;
-    if (analyzeAll) {
-      libraries.forEach(
-          (_, lib) => fullyEnqueueLibrary(lib, enqueuer.resolution));
-    }
-    // Elements required by enqueueHelpers are global dependencies
-    // that are not pulled in by a particular element.
-    backend.enqueueHelpers(enqueuer.resolution, globalDependencies);
-    resolveReflectiveDataIfNeeded();
-    processQueue(enqueuer.resolution, main);
-    enqueuer.resolution.logSummary(log);
+        // In order to see if a library is deferred, we must compute the
+        // compile-time constants that are metadata.  This means adding
+        // something to the resolution queue.  So we cannot wait with
+        // this until after the resolution queue is processed.
+        // TODO(ahe): Clean this up, for example, by not enqueueing
+        // classes only used for metadata.
+        deferredLoadTask.findDeferredLibraries(mainApp);
+      }
 
-    if (compilationFailed) return;
-    if (analyzeOnly) return;
-    assert(main != null);
-    phase = PHASE_DONE_RESOLVING;
-
-    // TODO(ahe): Remove this line. Eventually, enqueuer.resolution
-    // should know this.
-    world.populate();
-    // Compute whole-program-knowledge that the backend needs. (This might
-    // require the information computed in [world.populate].)
-    backend.onResolutionComplete();
-
-    deferredLoadTask.onResolutionComplete(main);
-
-    log('Inferring types...');
-    typesTask.onResolutionComplete(main);
-
-    log('Compiling...');
-    phase = PHASE_COMPILING;
-    // TODO(johnniwinther): Move these to [CodegenEnqueuer].
-    if (hasIsolateSupport()) {
-      enqueuer.codegen.addToWorkList(
-          isolateHelperLibrary.find(Compiler.START_ROOT_ISOLATE));
-      enqueuer.codegen.registerGetOfStaticFunction(mainApp.find(MAIN));
-    }
-    if (enabledNoSuchMethod) {
-      enqueuer.codegen.registerInvocation(NO_SUCH_METHOD, noSuchMethodSelector);
-      enqueuer.codegen.addToWorkList(createInvocationMirrorElement);
-    }
-    if (compileAll) {
-      libraries.forEach((_, lib) => fullyEnqueueLibrary(lib, enqueuer.codegen));
-    }
-    processQueue(enqueuer.codegen, main);
-    enqueuer.codegen.logSummary(log);
+      log('Resolving...');
+      phase = PHASE_RESOLVING;
+      if (analyzeAll) {
+        libraries.forEach(
+            (_, lib) => fullyEnqueueLibrary(lib, enqueuer.resolution));
+      }
+      // Elements required by enqueueHelpers are global dependencies
+      // that are not pulled in by a particular element.
+      backend.enqueueHelpers(enqueuer.resolution, globalDependencies);
+      resolveReflectiveDataIfNeeded();
+      processQueue(enqueuer.resolution, main);
+      enqueuer.resolution.logSummary(log);
+
+      if (compilationFailed) return;
+      if (analyzeOnly) return;
+      assert(main != null);
+      phase = PHASE_DONE_RESOLVING;
+
+      // TODO(ahe): Remove this line. Eventually, enqueuer.resolution
+      // should know this.
+      world.populate();
+      // Compute whole-program-knowledge that the backend needs. (This might
+      // require the information computed in [world.populate].)
+      backend.onResolutionComplete();
+
+      deferredLoadTask.onResolutionComplete(main);
+
+      log('Inferring types...');
+      typesTask.onResolutionComplete(main);
+
+      log('Compiling...');
+      phase = PHASE_COMPILING;
+      // TODO(johnniwinther): Move these to [CodegenEnqueuer].
+      if (hasIsolateSupport()) {
+        enqueuer.codegen.addToWorkList(
+            isolateHelperLibrary.find(Compiler.START_ROOT_ISOLATE));
+        enqueuer.codegen.registerGetOfStaticFunction(mainApp.find(MAIN));
+      }
+      if (enabledNoSuchMethod) {
+        enqueuer.codegen.registerInvocation(NO_SUCH_METHOD,
+            noSuchMethodSelector);
+        enqueuer.codegen.addToWorkList(createInvocationMirrorElement);
+      }
+      if (compileAll) {
+        libraries.forEach((_, lib) => fullyEnqueueLibrary(lib,
+            enqueuer.codegen));
+      }
+      processQueue(enqueuer.codegen, main);
+      enqueuer.codegen.logSummary(log);
 
-    if (compilationFailed) return;
+      if (compilationFailed) return;
 
-    backend.assembleProgram();
+      backend.assembleProgram();
 
-    checkQueues();
+      checkQueues();
+    });
   }
 
   void resolveReflectiveDataIfNeeded() {
@@ -1264,7 +1284,7 @@ abstract class Compiler implements DiagnosticListener {
    *
    * See [LibraryLoader] for terminology on URIs.
    */
-  Script readScript(Uri readableUri, [Node node]) {
+  Future<Script> readScript(Uri readableUri, [Node node]) {
     unimplemented('Compiler.readScript');
   }
 
@@ -1342,6 +1362,22 @@ class CompilerTask {
   measureElement(Element element, action()) {
     compiler.withCurrentElement(element, () => measure(action));
   }
+
+  Future measureAsync(Future action()) {
+    if (watch == null) return action();
+    CompilerTask previous = compiler.measuredTask;
+    if (identical(this, previous)) return action();
+    compiler.measuredTask = this;
+    if (previous != null) previous.watch.stop();
+    watch.start();
+    print("start measuring async $this");
+    return action().whenComplete(() {

[ahe, 2013/06/26 07:02:00]

I'm not sure measureAsync actually works.

The problem I see is that when you return from this method, previousTask is
still "running", but whatever work it does before yielding to the main event
loop will be counted as belonging to this task.

As far as I can tell, we can only count how much time is being spent reading
files by having a global counter that covers the time from the beginning of the
compiler until it reaches a point when all libraries are scanned, and subtract
the time spent in other tasks.

I think you should drop measureAsync. I think we can measure the time spent
reading files in dart2js.dart (you can just add a TODO).

[Bob Nystrom, 2013/06/27 00:38:18]

That sounds good to me.
Done.
I put this in compiler.dart above ReadingFilesTask since that seemed related.
Does that work?

+      print("done measuring async $this");

[Bob Nystrom, 2013/06/26 01:03:24]

Oops, I'll delete these debug prints.

+      watch.stop();
+      if (previous != null) previous.watch.start();
+      compiler.measuredTask = previous;
+    });
+  }
 }
 
 class CompilerCancelledException implements Exception {