First pass at asynchronous input loading in dart2js.

sdk/lib/_internal/compiler/implementation/compiler.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 part of dart2js;
 
 /**
  * If true, print a warning for each method that was resolved, but not
  * compiled.
  */
@@ skipping 361 matching lines @@
   Element invokeOnMethod;
   Element createInvocationMirrorElement;
 
   Element get currentElement => _currentElement;
 
   /**
    * Perform an operation, [f], returning the return value from [f].  If an
    * error occurs then report it as having occurred during compilation of
    * [element].  Can be nested.
    */
+  // TODO(rnystrom): Handle futures.

[Bob Nystrom, 2013/06/26 01:03:24]

This method assumes the f() is completely done by the time it returns, which
isn't the case for futures.

In the places that use this, I've manually called withCurrentElement() again
isn't the future's then() callback to restore the "current" element, but that's
a bit ugly. There may be a cleaner solution to this that's more future-aware.

   withCurrentElement(Element element, f()) {
     Element old = currentElement;
     _currentElement = element;
     try {
       return f();
     } on SpannableAssertionFailure catch (ex) {
       if (!hasCrashed) {
         SourceSpan span = spanFromSpannable(ex.node);
         reportDiagnostic(span, ex.message, api.Diagnostic.ERROR);
         pleaseReportCrash();
@@ skipping 249 matching lines @@
     withCurrentElement(element, () {
       cancel(reason, node: node, token: token, instruction: instruction,
              element: element);
     });
   }
 
   void log(message) {
     reportDiagnostic(null, message, api.Diagnostic.VERBOSE_INFO);
   }
 
-  bool run(Uri uri) {
+  Future<bool> run(Uri uri) {
     totalCompileTime.start();
-    try {
-      runCompiler(uri);
-    } on CompilerCancelledException catch (exception) {
-      log('Error: $exception');
-      return false;
-    } catch (exception) {
+
+    return runCompiler(uri).then((_) {
+      return true;
+    }).catchError((error) {
+      if (error is CompilerCancelledException) {
+        log('Error: $error');
+        return false;
+      }
+
       try {
         if (!hasCrashed) {
           hasCrashed = true;
           reportDiagnostic(new SourceSpan(uri, 0, 0),
                            MessageKind.COMPILER_CRASHED.error().toString(),
                            api.Diagnostic.CRASH);
           pleaseReportCrash();
         }
       } catch (doubleFault) {
         // Ignoring exceptions in exception handling.
       }
-      rethrow;
-    } finally {
+      throw error;
+    }).whenComplete(() {
       tracer.close();
       totalCompileTime.stop();
-    }
-    return true;
+    }).then((_) {
+      return true;
+    });
   }
 
   bool hasIsolateSupport() => isolateLibrary != null;
 
   /**
    * This method is called before [library] import and export scopes have been
    * set up.
    */
   void onLibraryScanned(LibraryElement library, Uri uri) {
     if (dynamicClass != null) {
@@ skipping 18 matching lines @@
       mirrorSystemGetNameFunction =
         cls.lookupLocalMember(const SourceString('getName'));
     } else if (symbolClass == cls) {
       symbolConstructor = cls.constructors.head;
     } else if (symbolImplementationClass == cls) {
       symbolValidatedConstructor = symbolImplementationClass.lookupConstructor(
           symbolValidatedConstructorSelector);
     }
   }
 
-  LibraryElement scanBuiltinLibrary(String filename);
+  Future<LibraryElement> scanBuiltinLibrary(String filename);
 
   void initializeSpecialClasses() {
     final List missingCoreClasses = [];
     ClassElement lookupCoreClass(String name) {
       ClassElement result = coreLibrary.find(new SourceString(name));
       if (result == null) {
         missingCoreClasses.add(name);
       }
       return result;
     }
@@ skipping 55 matching lines @@
 
   Element _filledListConstructor;
   Element get filledListConstructor {
     if (_filledListConstructor != null) return _filledListConstructor;
     Selector callConstructor = new Selector.callConstructor(
         const SourceString("filled"), listClass.getLibrary());
     return _filledListConstructor =
         listClass.lookupConstructor(callConstructor);
   }
 
-  void scanBuiltinLibraries() {
-    jsHelperLibrary = scanBuiltinLibrary('_js_helper');
-    interceptorsLibrary = scanBuiltinLibrary('_interceptors');
-    foreignLibrary = scanBuiltinLibrary('_foreign_helper');
-    isolateHelperLibrary = scanBuiltinLibrary('_isolate_helper');
+  Future scanBuiltinLibraries() {
+    return scanBuiltinLibrary('_js_helper').then((library) {
+      jsHelperLibrary = library;
+      return scanBuiltinLibrary('_interceptors');
+    }).then((library) {
+      interceptorsLibrary = library;
+      return scanBuiltinLibrary('_foreign_helper');
+    }).then((library) {
+      foreignLibrary = library;
+      return scanBuiltinLibrary('_isolate_helper');
+    }).then((library) {
+      isolateHelperLibrary = library;

[Bob Nystrom, 2013/06/26 01:03:24]

This is a bit ugly, but it ensures that these libraries are scanned
sequentially. If you don't care about that, I can use Future.wait() here and it
will actually load them in parallel.

That might make the order of error messages non-deterministic, though.

With this patch, I've tried to keep everything sequential and in the same order,
even though it's async.

[ahe, 2013/06/26 07:02:00]

I think we can use Future.wait here.  Normally these libraries will not produce
error messages.

[Bob Nystrom, 2013/06/27 00:38:18]

I just tried that but it apparently causes some other issues. I think it gets
confused and ends up loading "dart:core" multiple times because there are
multiple loads of it in flight at the same time.

I can try to clean this up to use Future.forEach() or leave it with explicit
chained .then()s like this.

 
-    assertMethod = jsHelperLibrary.find(const SourceString('assertHelper'));
-    identicalFunction = coreLibrary.find(const SourceString('identical'));
+      assertMethod = jsHelperLibrary.find(const SourceString('assertHelper'));
+      identicalFunction = coreLibrary.find(const SourceString('identical'));
 
-    initializeSpecialClasses();
+      initializeSpecialClasses();
 
-    functionClass.ensureResolved(this);
-    functionApplyMethod =
-        functionClass.lookupLocalMember(const SourceString('apply'));
-    jsInvocationMirrorClass.ensureResolved(this);
-    invokeOnMethod = jsInvocationMirrorClass.lookupLocalMember(INVOKE_ON);
+      functionClass.ensureResolved(this);
+      functionApplyMethod =
+          functionClass.lookupLocalMember(const SourceString('apply'));
+      jsInvocationMirrorClass.ensureResolved(this);
+      invokeOnMethod = jsInvocationMirrorClass.lookupLocalMember(INVOKE_ON);
 
-    if (preserveComments) {
-      var uri = new Uri(scheme: 'dart', path: 'mirrors');
-      LibraryElement libraryElement =
-          libraryLoader.loadLibrary(uri, null, uri);
-      documentClass = libraryElement.find(const SourceString('Comment'));
-    }
+      if (preserveComments) {
+        var uri = new Uri(scheme: 'dart', path: 'mirrors');
+        return libraryLoader.loadLibrary(uri, null, uri).then((libraryElement) {

[ahe, 2013/06/26 07:02:00]

Please restore the explicit type of libraryElement.

[Bob Nystrom, 2013/06/27 00:38:18]

Done.

+          documentClass = libraryElement.find(const SourceString('Comment'));
+        });
+      }
+    });
   }
 
   void importHelperLibrary(LibraryElement library) {
     if (jsHelperLibrary != null) {
       libraryLoader.importLibrary(library, jsHelperLibrary, null);
     }
   }
 
   /**
    * Get an [Uri] pointing to a patch for the dart: library with
    * the given path. Returns null if there is no patch.
    */
   Uri resolvePatchUri(String dartLibraryPath);
 
-  void runCompiler(Uri uri) {
+  Future runCompiler(Uri uri) {
     assert(uri != null || analyzeOnly);
-    scanBuiltinLibraries();
-    if (librariesToAnalyzeWhenRun != null) {
-      for (Uri libraryUri in librariesToAnalyzeWhenRun) {
-        log('analyzing $libraryUri ($buildId)');
-        libraryLoader.loadLibrary(libraryUri, null, libraryUri);
-      }
-    }
-    if (uri != null) {
-      if (analyzeOnly) {
-        log('analyzing $uri ($buildId)');
-      } else {
-        log('compiling $uri ($buildId)');
-      }
-      mainApp = libraryLoader.loadLibrary(uri, null, uri);
-    }
-    Element main = null;
-    if (mainApp != null) {
-      main = mainApp.find(MAIN);
-      if (main == null) {
-        if (!analyzeOnly) {
-          // Allow analyze only of libraries with no main.
-          reportFatalError('Could not find $MAIN', mainApp);
-        } else if (!analyzeAll) {
-          reportFatalError(
-              "Could not find $MAIN. "
-              "No source will be analyzed. "
-              "Use '--analyze-all' to analyze all code in the library.",
-              mainApp);
-        }
-      } else {
-        if (!main.isFunction()) {
-          reportFatalError('main is not a function', main);
-        }
-        FunctionElement mainMethod = main;
-        FunctionSignature parameters = mainMethod.computeSignature(this);
-        parameters.forEachParameter((Element parameter) {
-          reportFatalError('main cannot have parameters', parameter);
+    return scanBuiltinLibraries().then((_) {
+      if (librariesToAnalyzeWhenRun != null) {
+        return Future.forEach(librariesToAnalyzeWhenRun, (libraryUri) {
+          log('analyzing $libraryUri ($buildId)');
+          return libraryLoader.loadLibrary(libraryUri, null, libraryUri);
         });
       }
+    }).then((_) {
+      if (uri != null) {
+        if (analyzeOnly) {
+          log('analyzing $uri ($buildId)');
+        } else {
+          log('compiling $uri ($buildId)');
+        }
+        return libraryLoader.loadLibrary(uri, null, uri).then((library) {
+          mainApp = library;
+        });
+      }
+    }).then((_) {
+      Element main = null;
+      if (mainApp != null) {
+        main = mainApp.find(MAIN);
+        if (main == null) {
+          if (!analyzeOnly) {
+            // Allow analyze only of libraries with no main.
+            reportFatalError('Could not find $MAIN', mainApp);
+          } else if (!analyzeAll) {
+            reportFatalError(
+                "Could not find $MAIN. "
+                "No source will be analyzed. "
+                "Use '--analyze-all' to analyze all code in the library.",
+                mainApp);
+          }
+        } else {
+          if (!main.isFunction()) {
+            reportFatalError('main is not a function', main);
+          }
+          FunctionElement mainMethod = main;
+          FunctionSignature parameters = mainMethod.computeSignature(this);
+          parameters.forEachParameter((Element parameter) {
+            reportFatalError('main cannot have parameters', parameter);
+          });
+        }
 
-      // In order to see if a library is deferred, we must compute the
-      // compile-time constants that are metadata.  This means adding
-      // something to the resolution queue.  So we cannot wait with
-      // this until after the resolution queue is processed.
-      // TODO(ahe): Clean this up, for example, by not enqueueing
-      // classes only used for metadata.
-      deferredLoadTask.findDeferredLibraries(mainApp);
-    }
+        // In order to see if a library is deferred, we must compute the
+        // compile-time constants that are metadata.  This means adding
+        // something to the resolution queue.  So we cannot wait with
+        // this until after the resolution queue is processed.
+        // TODO(ahe): Clean this up, for example, by not enqueueing
+        // classes only used for metadata.
+        deferredLoadTask.findDeferredLibraries(mainApp);
+      }
 
-    log('Resolving...');
-    phase = PHASE_RESOLVING;
-    if (analyzeAll) {
-      libraries.forEach(
-          (_, lib) => fullyEnqueueLibrary(lib, enqueuer.resolution));
-    }
-    // Elements required by enqueueHelpers are global dependencies
-    // that are not pulled in by a particular element.
-    backend.enqueueHelpers(enqueuer.resolution, globalDependencies);
-    resolveReflectiveDataIfNeeded();
-    processQueue(enqueuer.resolution, main);
-    enqueuer.resolution.logSummary(log);
+      log('Resolving...');
+      phase = PHASE_RESOLVING;
+      if (analyzeAll) {
+        libraries.forEach(
+            (_, lib) => fullyEnqueueLibrary(lib, enqueuer.resolution));
+      }
+      // Elements required by enqueueHelpers are global dependencies
+      // that are not pulled in by a particular element.
+      backend.enqueueHelpers(enqueuer.resolution, globalDependencies);
+      resolveReflectiveDataIfNeeded();
+      processQueue(enqueuer.resolution, main);
+      enqueuer.resolution.logSummary(log);
 
-    if (compilationFailed) return;
-    if (analyzeOnly) return;
-    assert(main != null);
-    phase = PHASE_DONE_RESOLVING;
+      if (compilationFailed) return;
+      if (analyzeOnly) return;
+      assert(main != null);
+      phase = PHASE_DONE_RESOLVING;
 
-    // TODO(ahe): Remove this line. Eventually, enqueuer.resolution
-    // should know this.
-    world.populate();
-    // Compute whole-program-knowledge that the backend needs. (This might
-    // require the information computed in [world.populate].)
-    backend.onResolutionComplete();
+      // TODO(ahe): Remove this line. Eventually, enqueuer.resolution
+      // should know this.
+      world.populate();
+      // Compute whole-program-knowledge that the backend needs. (This might
+      // require the information computed in [world.populate].)
+      backend.onResolutionComplete();
 
-    deferredLoadTask.onResolutionComplete(main);
+      deferredLoadTask.onResolutionComplete(main);
 
-    log('Inferring types...');
-    typesTask.onResolutionComplete(main);
+      log('Inferring types...');
+      typesTask.onResolutionComplete(main);
 
-    log('Compiling...');
-    phase = PHASE_COMPILING;
-    // TODO(johnniwinther): Move these to [CodegenEnqueuer].
-    if (hasIsolateSupport()) {
-      enqueuer.codegen.addToWorkList(
-          isolateHelperLibrary.find(Compiler.START_ROOT_ISOLATE));
-      enqueuer.codegen.registerGetOfStaticFunction(mainApp.find(MAIN));
-    }
-    if (enabledNoSuchMethod) {
-      enqueuer.codegen.registerInvocation(NO_SUCH_METHOD, noSuchMethodSelector);
-      enqueuer.codegen.addToWorkList(createInvocationMirrorElement);
-    }
-    if (compileAll) {
-      libraries.forEach((_, lib) => fullyEnqueueLibrary(lib, enqueuer.codegen));
-    }
-    processQueue(enqueuer.codegen, main);
-    enqueuer.codegen.logSummary(log);
+      log('Compiling...');
+      phase = PHASE_COMPILING;
+      // TODO(johnniwinther): Move these to [CodegenEnqueuer].
+      if (hasIsolateSupport()) {
+        enqueuer.codegen.addToWorkList(
+            isolateHelperLibrary.find(Compiler.START_ROOT_ISOLATE));
+        enqueuer.codegen.registerGetOfStaticFunction(mainApp.find(MAIN));
+      }
+      if (enabledNoSuchMethod) {
+        enqueuer.codegen.registerInvocation(NO_SUCH_METHOD,
+            noSuchMethodSelector);
+        enqueuer.codegen.addToWorkList(createInvocationMirrorElement);
+      }
+      if (compileAll) {
+        libraries.forEach((_, lib) => fullyEnqueueLibrary(lib,
+            enqueuer.codegen));
+      }
+      processQueue(enqueuer.codegen, main);
+      enqueuer.codegen.logSummary(log);
 
-    if (compilationFailed) return;
+      if (compilationFailed) return;
 
-    backend.assembleProgram();
+      backend.assembleProgram();
 
-    checkQueues();
+      checkQueues();
+    });
   }
 
   void resolveReflectiveDataIfNeeded() {
     // Only need reflective data when dart:mirrors is loaded.
     if (!mirrorsEnabled) return;
 
     for (LibraryElement library in libraries.values) {
       for (Link link = library.metadata; !link.isEmpty; link = link.tail) {
         link.head.ensureResolved(this);
       }
@@ skipping 316 matching lines @@
   Uri translateResolvedUri(LibraryElement importingLibrary,
                            Uri resolvedUri, Node node) {
     unimplemented('Compiler.translateResolvedUri');
   }
 
   /**
    * Reads the script specified by the [readableUri].
    *
    * See [LibraryLoader] for terminology on URIs.
    */
-  Script readScript(Uri readableUri, [Node node]) {
+  Future<Script> readScript(Uri readableUri, [Node node]) {
     unimplemented('Compiler.readScript');
   }
 
   String get legDirectory {
     unimplemented('Compiler.legDirectory');
   }
 
   // TODO(karlklose): split into findHelperFunction and findHelperClass and
   // add a check that the element has the expected kind.
   Element findHelper(SourceString name)
@@ skipping 57 matching lines @@
     } finally {
       watch.stop();
       if (previous != null) previous.watch.start();
       compiler.measuredTask = previous;
     }
   }
 
   measureElement(Element element, action()) {
     compiler.withCurrentElement(element, () => measure(action));
   }
+
+  Future measureAsync(Future action()) {
+    if (watch == null) return action();
+    CompilerTask previous = compiler.measuredTask;
+    if (identical(this, previous)) return action();
+    compiler.measuredTask = this;
+    if (previous != null) previous.watch.stop();
+    watch.start();
+    print("start measuring async $this");
+    return action().whenComplete(() {

[ahe, 2013/06/26 07:02:00]

I'm not sure measureAsync actually works.

The problem I see is that when you return from this method, previousTask is
still "running", but whatever work it does before yielding to the main event
loop will be counted as belonging to this task.

As far as I can tell, we can only count how much time is being spent reading
files by having a global counter that covers the time from the beginning of the
compiler until it reaches a point when all libraries are scanned, and subtract
the time spent in other tasks.

I think you should drop measureAsync. I think we can measure the time spent
reading files in dart2js.dart (you can just add a TODO).

[Bob Nystrom, 2013/06/27 00:38:18]

That sounds good to me.
Done.
I put this in compiler.dart above ReadingFilesTask since that seemed related.
Does that work?

+      print("done measuring async $this");

[Bob Nystrom, 2013/06/26 01:03:24]

Oops, I'll delete these debug prints.

+      watch.stop();
+      if (previous != null) previous.watch.start();
+      compiler.measuredTask = previous;
+    });
+  }
 }
 
 class CompilerCancelledException implements Exception {
   final String reason;
   CompilerCancelledException(this.reason);
 
   String toString() {
     String banner = 'compiler cancelled';
     return (reason != null) ? '$banner: $reason' : '$banner';
   }
@@ skipping 77 matching lines @@
 
   void close() {}
 
   toString() => name;
 
   /// Convenience method for getting an [api.CompilerOutputProvider].
   static NullSink outputProvider(String name, String extension) {
     return new NullSink('$name.$extension');
   }
 }