Initial implementation of pub summary manager.

pkg/analyzer/lib/src/summary/pub_summary.dart

+// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:collection';
+import 'dart:core' hide Resource;
+
+import 'package:analyzer/dart/ast/ast.dart';
+import 'package:analyzer/file_system/file_system.dart';
+import 'package:analyzer/src/generated/engine.dart';
+import 'package:analyzer/src/generated/sdk.dart';
+import 'package:analyzer/src/generated/source.dart';
+import 'package:analyzer/src/summary/format.dart';
+import 'package:analyzer/src/summary/idl.dart';
+import 'package:analyzer/src/summary/package_bundle_reader.dart'
+    show ResynthesizerResultProvider;
+import 'package:analyzer/src/summary/summarize_ast.dart'
+    show serializeAstUnlinked;
+import 'package:analyzer/src/summary/summarize_elements.dart'
+    show PackageBundleAssembler;
+import 'package:analyzer/src/util/fast_uri.dart';
+import 'package:analyzer/task/dart.dart';
+import 'package:path/path.dart' as pathos;
+
+/**
+ * A package in the pub cache.
+ */
+class PubPackage {
+  final String name;
+  final Folder libFolder;
+
+  PubPackage(this.name, this.libFolder);
+
+  Folder get folder => libFolder.parent;
+
+  @override
+  int get hashCode => libFolder.hashCode;
+
+  @override
+  bool operator ==(other) {
+    return other is PubPackage && other.libFolder == libFolder;
+  }
+
+  @override
+  String toString() => '($name in $folder)';
+}
+
+/**
+ * Class that manages summaries for pub packages.
+ *
+ * The client should call [getLinkedBundles] after creating a new
+ * [AnalysisContext] and configuring its source factory, but before computing
+ * any analysis results.  The returned linked bundles can be used to create and
+ * configure [ResynthesizerResultProvider] for the context.
+ */
+class PubSummaryManager {
+  static const UNLINKED_BUNDLE_FILE_NAME = 'unlinked.ds';
+
+  final ResourceProvider resourceProvider;
+
+  /**
+   * A [DartSdk]. While we create only unlinked summaries, we just need it to
+   * be able to resolve `dart:core` (parse task requires this).
+   */
+  final DartSdk defaultDartSdk;
+
+  /**
+   * The map from [PubPackage]s to their unlinked [PackageBundle]s in the pub
+   * cache.
+   */
+  final Map<PubPackage, PackageBundle> unlinkedBundleMap =
+      new HashMap<PubPackage, PackageBundle>();
+
+  /**
+   * The set of packages to compute unlinked summaries for.
+   */
+  final Set<PubPackage> packagesToComputeUnlinked = new Set<PubPackage>();
+
+  /**
+   * The set of already processed packages, which we have already checked
+   * for their unlinked bundle existence, or scheduled its computing.
+   */
+  final Set<PubPackage> seenPackages = new Set<PubPackage>();
+
+  /**
+   * The [Completer] that completes when computing of all scheduled unlinked
+   * bundles is complete.
+   */
+  Completer _onUnlinkedCompleteCompleter;
+
+  PubSummaryManager(this.resourceProvider, this.defaultDartSdk);
+
+  /**
+   * The [Future] that completes when computing of all scheduled unlinked
+   * bundles is complete.
+   */
+  Future get onUnlinkedComplete {
+    if (packagesToComputeUnlinked.isEmpty) {
+      return new Future.value();
+    }
+    _onUnlinkedCompleteCompleter ??= new Completer();
+    return _onUnlinkedCompleteCompleter.future;
+  }
+
+  /**
+   * Return the [pathos.Context] corresponding to the [resourceProvider].
+   */
+  pathos.Context get pathContext => resourceProvider.pathContext;
+
+  /**
+   * Return the list of linked [PackageBundle]s that can be provided at this
+   * time for a subset of the packages used by the given [context].  If
+   * information about some of the used packages is not available yet, schedule
+   * its computation, so that it might be available later for other contexts
+   * referencing the same packages.
+   */
+  List<PackageBundle> getLinkedBundles(AnalysisContext context) {

[Paul Berry, 2016/08/08 12:24:59]

Sorry for not picking up on this during the last review, but a concern I have
about this API is that it forces linked bundle computation to be done eagerly. 
I believe that in the typical use case the number of linked bundles that will
actually be needed for a given instantiation of analysis server is going to be
small, so it might make more sense to compute them on demand at the time they
are needed.  (Note: we will probably want to try both alternatives and make some
profiling measurements, though).

Consider shifting to an API that would allow for the possibility of changing the
implementation in the future to computing the linked bundles on demand instead
of eagerly.

Note also that I am in the process of adding information to summaries to make it
easier for the analysis server to locate the summaries they depend on (and to
figure out the mapping between those summaries and the URIs they contain
information for); we should talk sometime today about this and see if there's a
way this information can save you some I/O in the case where not all the
summaries are needed.

(I don't want to hold up progress, though.  If you'd prefer to address this at a
later date, that's ok with me).

[scheglov, 2016/08/09 03:41:07]

Yes, I plan to perform profiling on some dependencies-heavy applications to see
whether loading summaries contributes much to the total analysis time.

+    Map<String, PackageBundle> unlinkedBundles = getUnlinkedBundles(context);
+    // TODO(scheglov) actually compute available linked bundles
+    return <PackageBundle>[];
+  }
+
+  /**
+   * Return all available unlinked [PackageBundle]s for the given [context],
+   * maybe an empty list, but not `null`.
+   */
+  Map<String, PackageBundle> getUnlinkedBundles(AnalysisContext context) {

[Paul Berry, 2016/08/08 12:24:59]

Similar concern here; also ok if you want to defer until later.

+    Map<String, PackageBundle> unlinkedBundles =
+        new HashMap<String, PackageBundle>();
+    Map<String, List<Folder>> packageMap = context.sourceFactory.packageMap;
+    if (packageMap != null) {
+      packageMap.forEach((String packageName, List<Folder> libFolders) {
+        if (libFolders.length == 1) {
+          Folder libFolder = libFolders.first;
+          if (isPathInPubCache(pathContext, libFolder.path)) {
+            PubPackage package = new PubPackage(packageName, libFolder);
+            PackageBundle unlinkedBundle = _getUnlinkedOrSchedule(package);
+            if (unlinkedBundle != null) {
+              unlinkedBundles[packageName] = unlinkedBundle;
+            }
+          }
+        }
+      });
+    }
+    return unlinkedBundles;
+  }
+
+  /**
+   * Compute unlinked bundle for a package from [packagesToComputeUnlinked],
+   * and schedule delayed computation for the next package, if any.
+   */
+  void _computeNextUnlinked() {
+    if (packagesToComputeUnlinked.isNotEmpty) {
+      PubPackage package = packagesToComputeUnlinked.first;
+      _computeUnlinked(package);
+      packagesToComputeUnlinked.remove(package);
+      _scheduleNextUnlinked();
+    } else {
+      if (_onUnlinkedCompleteCompleter != null) {
+        _onUnlinkedCompleteCompleter.complete(true);
+        _onUnlinkedCompleteCompleter = null;
+      }
+    }
+  }
+
+  /**
+   * Compute the unlinked bundle for the package with the given path, put
+   * it in the [unlinkedBundleMap] and store into the [resourceProvider].
+   *
+   * TODO(scheglov) Consider moving into separate isolate(s).
+   */
+  void _computeUnlinked(PubPackage package) {
+    AnalysisContext context = AnalysisEngine.instance.createAnalysisContext();
+    context.sourceFactory =
+        new SourceFactory(<UriResolver>[new DartUriResolver(defaultDartSdk)]);
+
+    Folder libFolder = package.libFolder;
+    String libPath = libFolder.path + pathContext.separator;
+    PackageBundleAssembler assembler = new PackageBundleAssembler();
+
+    /**
+     * If the given [file] is a Dart file, add its unlinked unit.
+     */
+    void addDartFile(File file) {
+      String path = file.path;
+      if (AnalysisEngine.isDartFileName(path)) {
+        String pathInLib = path.substring(libPath.length);
+        String uriPath = pathos.posix.joinAll(pathContext.split(pathInLib));
+        String uriStr = 'package:${package.name}/$uriPath';
+        Uri uri = FastUri.parse(uriStr);
+        Source source = file.createSource(uri);
+        CompilationUnit unit = context.computeResult(source, PARSED_UNIT);

[Brian Wilkerson, 2016/08/08 14:32:55]

Do we want to discard all of the data computed while parsing, or do we want to
capture the context in which the work is to be done? Even if all of the data is
recoverable from the summary (is LineInfo recoverable?) it seems a shame to
compute it, discard it, and then deserialize it. But maybe it's fast enough to
not be worth the extra effort.

+        UnlinkedUnitBuilder unlinkedUnit = serializeAstUnlinked(unit);
+        assembler.addUnlinkedUnit(source, unlinkedUnit);
+      }
+    }
+
+    /**
+     * Visit the [folder] recursively.
+     */
+    void addDartFiles(Folder folder) {
+      List<Resource> children = folder.getChildren();
+      for (Resource child in children) {
+        if (child is File) {
+          addDartFile(child);
+        }
+      }
+      for (Resource child in children) {
+        if (child is Folder) {
+          addDartFiles(child);
+        }
+      }
+    }
+
+    try {
+      addDartFiles(libFolder);
+      List<int> bytes = assembler.assemble().toBuffer();
+      package.folder
+          .getChildAssumingFile(UNLINKED_BUNDLE_FILE_NAME)
+          .writeAsBytesSync(bytes);
+    } on FileSystemException {
+      // Ignore file system exceptions.
+    }
+  }
+
+  /**
+   * Return the unlinked [PackageBundle] for the given [package]. If the bundle
+   * has not been compute yet, return `null` and schedule its computation.
+   */
+  PackageBundle _getUnlinkedOrSchedule(PubPackage package) {
+    // Try to find in the cache.
+    PackageBundle bundle = unlinkedBundleMap[package];
+    if (bundle != null) {
+      return bundle;
+    }
+    // Try to read from the file system.
+    File unlinkedFile =
+        package.folder.getChildAssumingFile(UNLINKED_BUNDLE_FILE_NAME);
+    if (unlinkedFile.exists) {
+      try {
+        List<int> bytes = unlinkedFile.readAsBytesSync();
+        bundle = new PackageBundle.fromBuffer(bytes);
+        unlinkedBundleMap[package] = bundle;
+        return bundle;
+      } on FileSystemException {
+        // Ignore file system exceptions.
+      }
+    }
+    // Schedule computation in the background.
+    if (package != null && seenPackages.add(package)) {
+      packagesToComputeUnlinked.add(package);
+      if (packagesToComputeUnlinked.length == 1) {

[Brian Wilkerson, 2016/08/08 14:32:56]

Given that `packagesToComputeUnlinked` is a set, I think this allows us to
schedule multiple times if the same package is requested multiple times. Not
sure whether it matters (although we might get multiple "completed"
notifications), but I think we need to test whether the set was empty before
adding the package.

[scheglov, 2016/08/09 03:41:07]

Fixed.

+        _scheduleNextUnlinked();
+      }
+    }
+    // The bundle is for available.
+    return null;
+  }
+
+  /**
+   * Schedule delayed computation of the next package unlinked bundle from the
+   * set of [packagesToComputeUnlinked].  We delay each computation because we
+   * want operations in analysis server to proceed, and computing bundles of
+   * packages is a background task.
+   */
+  void _scheduleNextUnlinked() {
+    new Future.delayed(new Duration(milliseconds: 10), _computeNextUnlinked);

[Brian Wilkerson, 2016/08/08 14:32:55]

It seems odd to me that we're scheduling work here. Seems like the constraints
on whether and when to perform work need to be under the client's control. For
example, in the command-line analyzer (assuming we want to generate summaries
for the next run) there isn't any reason to delay the computation.

+  }
+
+  /**
+   * Return `true` if the given absolute [path] is in the pub cache.
+   */
+  static bool isPathInPubCache(pathos.Context pathContext, String path) {
+    List<String> parts = pathContext.split(path);
+    for (int i = 0; i < parts.length - 1; i++) {
+      if (parts[i] == '.pub-cache') {
+        return true;
+      }
+      if (parts[i] == 'Pub' && parts[i + 1] == 'Cache') {
+        return true;
+      }
+    }
+    return false;
+  }
+}