Split loader from the rest of the compiler. This adds several abstractions to

pkg/compiler/lib/src/library_loader.dart

 // Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library dart2js.library_loader;
 
 import 'dart:async';
 
 import 'common.dart';
 import 'common/names.dart' show
@@ skipping 12 matching lines @@
 import 'elements/modelx.dart' show
     CompilationUnitElementX,
     DeferredLoaderGetterElementX,
     ErroneousElementX,
     ExportElementX,
     ImportElementX,
     LibraryElementX,
     LibraryDependencyElementX,
     PrefixElementX,
     SyntheticImportElement;
-
+import 'environment.dart';
 import 'script.dart';
+import 'serialization/serialization.dart' show LibraryDeserializer;
 import 'tree/tree.dart';
 import 'util/util.dart' show
     Link,
     LinkBuilder;
 
 /**
  * [CompilerTask] for loading libraries and setting up the import/export scopes.
  *
  * The library loader uses four different kinds of URIs in different parts of
  * the loading process.
@@ skipping 53 matching lines @@
  * as when loaded through the dart URI.
  *
  * ## Readable URI ##
  *
  * A 'readable URI' is an absolute URI whose scheme is either 'package' or
  * something supported by the input provider, normally 'file'. Dart URIs such as
  * 'dart:core' and 'dart:_js_helper' are not readable themselves but are instead
  * resolved into a readable URI using the library root URI provided from the
  * command line and the list of platform libraries found in
  * 'sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart'. This is done
- * through the [Compiler.translateResolvedUri] method which checks whether a
- * library by that name exists and in case of internal libraries whether access
- * is granted.
+ * through a [ResolvedUriTranslator] provided from the compiler. The translator
+ * checks whether a library by that name exists and in case of internal
+ * libraries whether access is granted.
  *
  * ## Resource URI ##
  *
  * A 'resource URI' is an absolute URI with a scheme supported by the input
  * provider. For the standard implementation this means a URI with the 'file'
  * scheme. Readable URIs are converted into resource URIs as part of the
  * [Compiler.readScript] method. In the standard implementation the package URIs
  * are converted to file URIs using the package root URI provided on the
  * command line as base. If the package root URI is
  * 'file:///current/working/dir/' then the package URI 'package:foo/bar.dart'
  * will be resolved to the resource URI
  * 'file:///current/working/dir/foo/bar.dart'.
  *
  * The distinction between readable URI and resource URI is necessary to ensure
  * that these imports
  *
  *     import 'package:foo.dart' as a;
  *     import 'packages/foo.dart' as b;
  *
  * do _not_ resolve to the same library when the package root URI happens to
  * point to the 'packages' folder.
  *
  */
 abstract class LibraryLoaderTask implements CompilerTask {
-  factory LibraryLoaderTask(Compiler compiler) = _LibraryLoaderTask;
+  factory LibraryLoaderTask(Compiler compiler,
+      ResolvedUriTranslator uriTranslator,
+      ScriptLoader scriptLoader,
+      ElementScanner scriptScanner,
+      LibraryDeserializer deserializer,
+      LibraryLoaderListener listener,
+      Environment environment) = _LibraryLoaderTask;
 
   /// Returns all libraries that have been loaded.
   Iterable<LibraryElement> get libraries;
 
   /// Looks up the library with the [canonicalUri].
   LibraryElement lookupLibrary(Uri canonicalUri);
 
   /// Loads the library specified by the [resolvedUri] and returns its
   /// [LibraryElement].
   ///
@@ skipping 106 matching lines @@
  * A list of combinators represented as a list of element names to exclude.
  */
 class HideFilter extends CombinatorFilter {
   final Set<String> excludedNames;
 
   HideFilter(this.excludedNames);
 
   bool exclude(Element element) => excludedNames.contains(element.name);
 }
 
-/**
- * Implementation class for [LibraryLoader]. The distinction between
- * [LibraryLoader] and [LibraryLoaderTask] is made to hide internal members from
- * the [LibraryLoader] interface.
- */
+/// Implementation class for [LibraryLoaderTask]. The distinction between
+/// [LibraryLoaderTask] and [_LibraryLoaderTask] is made to hide internal
+/// members from the [LibraryLoaderTask] interface.
 class _LibraryLoaderTask extends CompilerTask implements LibraryLoaderTask {
-  _LibraryLoaderTask(Compiler compiler) : super(compiler);
+  /// Translates interal uris (like dart:core) to a disk location.

[Harry Terkelsen, 2016/03/22 21:49:28]

s/interal/internal/

[Siggi Cherem (dart-lang), 2016/03/23 22:51:05]

Done.

+  final ResolvedUriTranslator uriTranslator;
+
+  /// Loads the contents of a a script file (a .dart file). Used when loading

[Harry Terkelsen, 2016/03/22 21:49:28]

s/a a/a/

[Siggi Cherem (dart-lang), 2016/03/23 22:51:05]

Done.

+  /// libraries from source.
+  final ScriptLoader scriptLoader;
+
+  /// Provides a diet element model from a script file containing information
+  /// about imports and exports. Used when loading libraries from source.
+  final ElementScanner scanner;
+
+  /// Provides a diet element model for a library. Used when loading libraries
+  /// from a serialized form.
+  final LibraryDeserializer deserializer;
+
+  /// Hooks to inform others about progress done by this loader.
+  // TODO(sigmund): move away from this.
+  final LibraryLoaderListener listener;
+
+  /// Definitions provided via the `-D` command line flags. Used to resolve
+  /// conditional imports.
+  final Environment environment;
+
+  _LibraryLoaderTask(
+      Compiler compiler, this.uriTranslator, this.scriptLoader,
+      this.scanner, this.deserializer, this.listener, this.environment)
+      // TODO(sigmund): make measurements separate from compiler
+      : super(compiler);
 
   String get name => 'LibraryLoader';
 
   final Map<Uri, LibraryElement> libraryCanonicalUriMap =
       new Map<Uri, LibraryElement>();
   final Map<Uri, LibraryElement> libraryResourceUriMap =
       new Map<Uri, LibraryElement>();
   final Map<String, LibraryElement> libraryNames =
       new Map<String, LibraryElement>();
 
   LibraryDependencyHandler currentHandler;
 
   Iterable<LibraryElement> get libraries => libraryCanonicalUriMap.values;
 
   LibraryElement lookupLibrary(Uri canonicalUri) {
     return libraryCanonicalUriMap[canonicalUri];
   }
 
   void reset({bool reuseLibrary(LibraryElement library)}) {
     measure(() {
       assert(currentHandler == null);
 
       Iterable<LibraryElement> reusedLibraries = null;
       if (reuseLibrary != null) {
+        // TODO(sigmund): make measurements separate from compiler
         reusedLibraries = compiler.reuseLibraryTask.measure(() {
           // Call [toList] to force eager calls to [reuseLibrary].
           return libraryCanonicalUriMap.values.where(reuseLibrary).toList();
         });
       }
 
       resetImplementation(reusedLibraries);
     });
   }
 
   void resetImplementation(Iterable<LibraryElement> reusedLibraries) {
     measure(() {
       libraryCanonicalUriMap.clear();
       libraryResourceUriMap.clear();
       libraryNames.clear();
 
       if (reusedLibraries != null) {
         reusedLibraries.forEach(mapLibrary);
       }
     });
   }
 
   Future resetAsync(Future<bool> reuseLibrary(LibraryElement library)) {
     return measure(() {
       assert(currentHandler == null);
 
-      Future<LibraryElement> wrapper(LibraryElement library) {
-        try {
-          return reuseLibrary(library).then(
-              (bool reuse) => reuse ? library : null);
-        } catch (exception, trace) {
-          compiler.diagnoseCrashInUserCode(
-              'Uncaught exception in reuseLibrary', exception, trace);
-          rethrow;
-        }
-      }
-
+      wrapper(lib) => reuseLibrary(lib).then((reuse) => reuse ? lib : null);
       List<Future<LibraryElement>> reusedLibrariesFuture =
+          // TODO(sigmund): make measurements separate from compiler
           compiler.reuseLibraryTask.measure(
               () => libraryCanonicalUriMap.values.map(wrapper).toList());
 
       return Future.wait(reusedLibrariesFuture).then(
           (List<LibraryElement> reusedLibraries) {
             resetImplementation(reusedLibraries.where((e) => e != null));
           });
     });
   }
 
   /// Insert [library] in the internal maps. Used for compiler reuse.
   void mapLibrary(LibraryElement library) {
     libraryCanonicalUriMap[library.canonicalUri] = library;
 
     Uri resourceUri = library.entryCompilationUnit.script.resourceUri;
     libraryResourceUriMap[resourceUri] = library;
 
     if (library.hasLibraryName) {
       String name = library.libraryName;
       libraryNames[name] = library;
     }
   }
 
-  Future<LibraryElement> loadLibrary(
-      Uri resolvedUri,
+  Future<LibraryElement> loadLibrary(Uri resolvedUri,
       {bool skipFileWithPartOfTag: false}) {
     return measure(() {
       assert(currentHandler == null);
       // TODO(johnniwinther): Ensure that currentHandler correctly encloses the
       // loading of a library cluster.
       currentHandler = new LibraryDependencyHandler(this);
       return createLibrary(currentHandler, null, resolvedUri,
           skipFileWithPartOfTag: skipFileWithPartOfTag)
           .then((LibraryElement library) {
         if (library == null) {
           currentHandler = null;
           return null;
         }
         return reporter.withCurrentElement(library, () {
           return measure(() {
             currentHandler.computeExports();
             LoadedLibraries loadedLibraries = new _LoadedLibraries(
                 library,
                 currentHandler.newLibraries,
                 currentHandler.nodeMap,
                 this);
             currentHandler = null;
-            return compiler.onLibrariesLoaded(loadedLibraries)
+            return listener.onLibrariesLoaded(loadedLibraries)
                 .then((_) => library);
           });
         });
       });
     });
   }
 
   /**
    * Processes the library tags in [library].
    *
@@ skipping 13 matching lines @@
       return reporter.withCurrentElement(library, () {
 
         Uri computeUri(LibraryDependency node) {
           StringNode uriNode = node.uri;
           if (node.conditionalUris != null) {
             for (ConditionalUri conditionalUri in node.conditionalUris) {
               String key = conditionalUri.key.slowNameString;
               String value = conditionalUri.value == null
                   ? "true"
                   : conditionalUri.value.dartString.slowToString();
-              String actual = compiler.fromEnvironment(key);
+              String actual = environment.valueOf(key);
               if (value == actual) {
                 uriNode = conditionalUri.uri;
                 break;
               }
             }
           }
           String tagUriString = uriNode.dartString.slowToString();
           try {
             return Uri.parse(tagUriString);
           } on FormatException {
@@ skipping 42 matching lines @@
           Part part = tag;
           StringNode uri = part.uri;
           Uri resolvedUri = base.resolve(uri.dartString.slowToString());
           tagState.checkTag(TagState.PART, part, reporter);
           return scanPart(part, resolvedUri, library);
         } else {
           reporter.internalError(tag, "Unhandled library tag.");
         }
       });
     }).then((_) {
-      return compiler.onLibraryScanned(library, handler);
+      return listener.onLibraryScanned(library, handler);
     }).then((_) {
       return reporter.withCurrentElement(library, () {
         checkDuplicatedLibraryName(library);
 
         // Import dart:core if not already imported.
         if (!importsDartCore && library.canonicalUri != Uris.dart_core) {
           return createLibrary(handler, null, Uris.dart_core)
               .then((LibraryElement coreLibrary) {
             handler.registerDependency(library,
                 new SyntheticImportElement(
@@ skipping 56 matching lines @@
       }
     }
   }
 
   /**
    * Handle a part tag in the scope of [library]. The [resolvedUri] given is
    * used as is, any URI resolution should be done beforehand.
    */
   Future scanPart(Part part, Uri resolvedUri, LibraryElement library) {
     if (!resolvedUri.isAbsolute) throw new ArgumentError(resolvedUri);
-    Uri readableUri = compiler.translateResolvedUri(library, resolvedUri, part);
+    Uri readableUri = uriTranslator.translate(library, resolvedUri, part);
     if (readableUri == null) return new Future.value();
     return reporter.withCurrentElement(library, () {
-      return compiler.readScript(part, readableUri).
-          then((Script sourceScript) {
-            if (sourceScript == null) return;
-
-            CompilationUnitElementX unit =
-                new CompilationUnitElementX(sourceScript, library);
-            reporter.withCurrentElement(unit, () {
-              compiler.scanner.scan(unit);
-              if (unit.partTag == null && !sourceScript.isSynthesized) {
-                reporter.reportErrorMessage(
-                    unit, MessageKind.MISSING_PART_OF_TAG);
-              }
-            });
-          });
+      return scriptLoader.readScript(readableUri, part).then((Script script) {
+        if (script == null) return;
+        return createUnitSync(script, library);
+      });
     });
   }
 
   /**
    * Handle an import/export tag by loading the referenced library and
    * registering its dependency in [handler] for the computation of the import/
    * export scope. If the tag does not contain a valid URI, then its dependency
    * is not registered in [handler].
    */
   Future<Null> registerLibraryFromImportExport(
@@ skipping 15 matching lines @@
 
   /// Loads the deserialized [library] with the [handler].
   ///
   /// All libraries imported or exported transitively from [library] will be
   /// loaded as well.
   Future<LibraryElement> loadDeserializedLibrary(
       LibraryDependencyHandler handler,
       LibraryElement library) {
     libraryCanonicalUriMap[library.canonicalUri] = library;
     handler.registerNewLibrary(library);
-    return compiler.onLibraryScanned(library, handler).then((_) {
+    return listener.onLibraryScanned(library, handler).then((_) {
       return Future.forEach(library.imports, (ImportElement import) {
         return createLibrary(handler, library, import.uri);
       }).then((_) {
         return Future.forEach(library.exports, (ExportElement export) {
           return createLibrary(handler, library, export.uri);
         }).then((_) => library);
       });
     });
   }
 
+  Future<Script> _readScript(Spannable spannable,
+      Uri readableUri, Uri resolvedUri) {
+    if (readableUri == null) {
+      return new Future.value(new Script.synthetic(resolvedUri));
+    } else {
+      return scriptLoader.readScript(readableUri, spannable);
+    }
+  }
+
   /**
    * Create (or reuse) a library element for the library specified by the
    * [resolvedUri].
    *
    * If a new library is created, the [handler] is notified.
    */
   Future<LibraryElement> createLibrary(
       LibraryDependencyHandler handler,
       LibraryElement importingLibrary,
       Uri resolvedUri,
       {Spannable node,
        bool skipFileWithPartOfTag: false}) {
     Uri readableUri =
-        compiler.translateResolvedUri(importingLibrary, resolvedUri, node);
+        uriTranslator.translate(importingLibrary, resolvedUri, node);
     LibraryElement library = libraryCanonicalUriMap[resolvedUri];
     if (library != null) {
       return new Future.value(library);
     }
-    library = compiler.serialization.readLibrary(resolvedUri);
+    library = deserializer.readLibrary(resolvedUri);
     if (library != null) {
       return loadDeserializedLibrary(handler, library);
     }
-    var readScript = compiler.readScript;
-    if (readableUri == null) {
-      readableUri = resolvedUri;
-      readScript = compiler.synthesizeScript;
-    }
     return reporter.withCurrentElement(importingLibrary, () {
-      return readScript(node, readableUri).then((Script script) {
+      return _readScript(node, readableUri, resolvedUri).then((Script script) {
         if (script == null) return null;
         LibraryElement element =
             createLibrarySync(handler, script, resolvedUri);
         CompilationUnitElementX compilationUnit = element.entryCompilationUnit;
         if (compilationUnit.partTag != null) {
           if (skipFileWithPartOfTag) {
-            // TODO(johnniwinther): Avoid calling [Compiler.onLibraryCreated]
+            // TODO(johnniwinther): Avoid calling [listener.onLibraryCreated]
             // for this library.
             libraryCanonicalUriMap.remove(resolvedUri);
             return null;
           }
           if (importingLibrary == null) {
             DiagnosticMessage error = reporter.withCurrentElement(
                 compilationUnit,
                 () => reporter.createMessage(
                     compilationUnit.partTag, MessageKind.MAIN_HAS_PART_OF));
             reporter.reportError(error);
@@ skipping 13 matching lines @@
         return processLibraryTags(handler, element).then((_) {
           reporter.withCurrentElement(element, () {
             handler.registerLibraryExports(element);
           });
           return element;
         });
       });
     });
   }
 
-  LibraryElement createLibrarySync(
-      LibraryDependencyHandler handler,
-      Script script,
-      Uri resolvedUri) {
+  LibraryElement createLibrarySync(LibraryDependencyHandler handler,
+      Script script, Uri resolvedUri) {
     LibraryElement element = new LibraryElementX(script, resolvedUri);
     return reporter.withCurrentElement(element, () {
       if (handler != null) {
         handler.registerNewLibrary(element);
         libraryCanonicalUriMap[resolvedUri] = element;
       }
-      compiler.scanner.scanLibrary(element);
+      scanner.scanLibrary(element);
       return element;
     });
   }
+
+  CompilationUnitElement createUnitSync(Script script, LibraryElement library) {
+    CompilationUnitElementX unit = new CompilationUnitElementX(script, library);
+    reporter.withCurrentElement(unit, () {
+      scanner.scanUnit(unit);
+      if (unit.partTag == null && !script.isSynthesized) {
+        reporter.reportErrorMessage(unit, MessageKind.MISSING_PART_OF_TAG);
+      }
+    });
+    return unit;
+  }
 }
 
 
 /// A state machine for checking script tags come in the correct order.
 class TagState {
   /// Initial state.
   static const int NO_TAG_SEEN = 0;
 
   /// Passed to [checkTag] when a library declaration (the syntax "library
   /// name;") has been seen. Not an actual state.
@@ skipping 459 matching lines @@
    * Newly loaded libraries and their corresponding node in the library
    * dependency graph. Libraries that have already been fully loaded are not
    * part of the dependency graph of this handler since their export scopes have
    * already been computed.
    */
   Map<LibraryElement, LibraryDependencyNode> nodeMap =
       new Map<LibraryElement, LibraryDependencyNode>();
 
   LibraryDependencyHandler(this.task);
 
-  Compiler get compiler => task.compiler;
-
   DiagnosticReporter get reporter => task.reporter;
 
   /// The libraries created with this handler.
   Iterable<LibraryElement> get newLibraries => _newLibraries;
 
   /**
    * Performs a fixed-point computation on the export scopes of all registered
    * libraries and creates the import/export of the libraries based on the
    * fixed-point.
    */
@@ skipping 72 matching lines @@
       assert(invariant(library, importingNode != null,
           message: "$library has not been registered"));
       importingNode.registerImportDependency(libraryDependency, loadedLibrary);
     }
   }
 
   /**
    * Registers [library] for the processing of its import/export scope.
    */
   void registerNewLibrary(LibraryElement library) {
-    compiler.onLibraryCreated(library);
+    task.listener.onLibraryCreated(library);
     _newLibraries.add(library);
     if (!library.exportsHandled) {
       nodeMap[library] = new LibraryDependencyNode(library);
     }
   }
 
   /**
    * Registers all top-level entities of [library] as starting point for the
    * fixed-point computation of the import/export scopes.
    */
@@ skipping 131 matching lines @@
       }
       suffixChainMap[library] = suffixes;
       return;
     }
 
     computeSuffixes(rootLibrary, const Link<Uri>());
   }
 
   String toString() => 'root=$rootLibrary,libraries=${loadedLibraries.keys}';
 }
+
+/// API used by the library loader to translate internal SDK uri's into file

[Harry Terkelsen, 2016/03/22 21:49:28]

Be consistent with "uri's" or "URIs"

[Siggi Cherem (dart-lang), 2016/03/23 22:51:05]

Done.

+/// system readable URIs.
+abstract class ResolvedUriTranslator {
+  // TODO(sigmund): move here the comments from library loader.
+  /// Translate the resolved [uri] in the context of [importingLibrary].
+  ///
+  /// Use [spannable] for error reporting.
+  Uri translate(
+      LibraryElement importingLibrary, Uri uri, [Spannable spannable]);
+}
+
+
+// TODO(sigmund): remove ScriptLoader & ElementScanner. Such abstraction seems
+// rather low-level. It might be more practical to split the library-loading
+// task itself.  The task would continue to do the work of recursively loading
+// dependencies, but it can delegate to a set of subloaders how to do the actual
+// loading. We would then have a list of subloaders that use different
+// implementations: in-memory cache, deserialization, scanning from files.
+//
+// For example, the API might look like this:
+//
+// /// APIs to create [LibraryElement] and [CompilationUnitElements] given it's
+// /// URI.
+// abstract class SubLoader {
+//   /// Return the library corresponding to the script at [uri].
+//   ///
+//   /// Use [spannable] for error reporting.
+//   Future<LibraryElement> createLibrary(Uri uri, [Spannable spannable]);
+//
+//   /// Return the compilation unit at [uri] that is a part of [library].
+//   Future<CompilationUnitElement> createUnit(Uri uri, LibraryElement library,
+//       [Spannable spannable]);
+// }
+//
+// /// A [SubLoader] that parses a serialized form of the element model to
+// /// produce the results.
+// class DeserializingUnitElementCreator implements SubLoader {
+// ...
+// }
+//
+// /// A [SubLoader] that finds the script sources and does a diet parse
+// /// on them to produces the results.
+// class ScanningUnitElementCreator implements SubLoader {
+// ...
+// }
+//
+// Each subloader would internally create what they need (a scanner, a
+// deserializer), and we wouldn't need to create abstractions to pass in
+// something that is only used by the loader.
+
+/// API used by the library loader to request scripts from the compiler system.
+abstract class ScriptLoader {
+  /// Load script from a readable [uri], report any errors using the location of
+  /// the given [spannable].
+  Future<Script> readScript(Uri uri, [Spannable spannable]);
+}
+
+/// API used byt he library loader to sychronously scan a library or compilation

[Harry Terkelsen, 2016/03/22 21:49:28]

byt he
sychronously

[Siggi Cherem (dart-lang), 2016/03/23 22:51:05]

Done.

+/// unit and ensure that their library tags are computed.
+abstract class ElementScanner {
+  void scanLibrary(LibraryElement library);
+  void scanUnit(CompilationUnitElement unit);
+}
+
+
+/// TODO(sigmund): remove this abstraction. Ideally the loader can produce the
+/// LoadedLibraries results once, and the compiler and choose what to do with
+/// it instead.
+abstract class LibraryLoaderListener {

[Harry Terkelsen, 2016/03/22 21:49:28]

Maybe the loader can just publish a stream of libraries that it loads?

[Siggi Cherem (dart-lang), 2016/03/23 22:51:05]

Might be worth looking into a similar idea in the future. Johnni pointed out
that a stream might not be enough because we don't know when we are done that
easily (some callbacks enqueue more libraries to load). Maybe a FutureGroup is
more appropriate?

I want to see though if we can move away from the callbacks entirely, so there
would be no listener and just a full list of loaded libaries. Maybe a list of
libraries + extra info needed to figure out what else to load (e.g. if there is
something specific to the backend)

+  /// Called after a request to load a library. The [results] will include all
+  /// transitive libraries loaded as a result of the initial request.
+  Future onLibrariesLoaded(LoadedLibraries results);
+
+  /// Called whenever a library element is created.
+  void onLibraryCreated(LibraryElement library);
+
+  /// Called whenever a library is scanned from a script file.
+  Future onLibraryScanned(LibraryElement library, LibraryLoader loader);
+}