Initial LibraryBundleCache implementation.

pkg/analyzer/lib/src/summary/incremental_cache.dart

Index: pkg/analyzer/lib/src/summary/incremental_cache.dart
diff --git a/pkg/analyzer/lib/src/summary/incremental_cache.dart b/pkg/analyzer/lib/src/summary/incremental_cache.dart
new file mode 100644
index 0000000000000000000000000000000000000000..18f783cdaf288bc9e6c124a1fd862c3019c9a308
--- /dev/null
+++ b/pkg/analyzer/lib/src/summary/incremental_cache.dart
@@ -0,0 +1,221 @@
+// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert' show UTF8;
+import 'dart:core' hide Resource;
+
+import 'package:analyzer/dart/element/element.dart';
+import 'package:analyzer/file_system/file_system.dart';
+import 'package:analyzer/src/generated/engine.dart';
+import 'package:analyzer/src/generated/source.dart';
+import 'package:analyzer/src/summary/format.dart';
+import 'package:analyzer/src/summary/idl.dart';
+import 'package:analyzer/src/summary/summarize_elements.dart';
+import 'package:crypto/crypto.dart';
+
+/**
+ * The cache of per-library [PackageBundle]s.
+ *
+ * Note that currently this class is not intended for interactive use.
+ */
+class LibraryBundleCache {
+  final Folder cacheFolder;
+  final AnalysisContext context;
+  final List<int> configSalt;
+
+  final Map<Source, List<Source>> _libraryClosureMap = <Source, List<Source>>{};
+  final Map<Source, List<int>> _sourceContentHashMap = <Source, List<int>>{};
+
+  LibraryBundleCache(this.cacheFolder, this.context, this.configSalt);
+
+  /**
+   * Clear internal caches so that we read from file system again.
+   */
+  void clearInternalCaches() {
+    _libraryClosureMap.clear();
+    _sourceContentHashMap.clear();
+  }
+
+  /**
+   * Write information about the [library] into the cache.
+   */
+  void putLibrary(LibraryElement library) {
+    _writeUris(library);
+    List<int> hash = _getLibraryClosureHash(library.source);

[Paul Berry, 2016/03/17 18:11:32]

This seems wasteful.  _getLibraryClosureHash() is going to immediately read all
the files that _writeUris() just wrote into the cache.  Could we avoid the extra
disk I/O by having _writeUris return a map containing the data it just wrote?

Also, I assume that in the long term you are planning to add some logic which
removes files from the cache if it gets too big.  Once that logic is added,
there will be a race condition: if another process remove a file from the cache
after it is written by _writeUris() and before it is read by
_getLibraryClosureHash(), _getLibraryClosureHash will fail.  This race condition
would also be solved by having _writeUris return a map, since
_getLibraryClosureHash would just use the data in the map rather than reading it
from disk. 

[scheglov, 2016/03/17 21:08:14]

Done.
We now cache CacheLibraryUris objects.

+    if (hash != null) {
+      String hashStr = CryptoUtils.bytesToHex(hash);
+      PackageBundleAssembler assembler = new PackageBundleAssembler();
+      assembler.serializeLibraryElement(library);
+      List<int> bytes = assembler.assemble().toBuffer();
+      (cacheFolder.getChild('$hashStr.sum') as File).writeAsBytesSync(bytes);

[Paul Berry, 2016/03/17 18:11:32]

There is a small but nonzero danger that this could lead to corrupted data, if
(a) the analyzer is killed while the file operation is in progress, or (b) there
are two analyzer processes running, and one of them tries to read a file that is
still in the process of being written by the other.

The usual way of dealing with this is to write the file to a temporary filename
that is unique to this process (e.g. "$pid.tmp") and then once it has been fully
written, rename it to the final filename.  That way readers only ever see a
complete file.

(Note that if the analyzer is killed while writing to "$pid.tmp", that fill will
remain in the filesystem, so before writing "$pid.tmp", you should check whether
such a file already exists, and remove it if it does).

Also make sure to handle the case where two processes simultaneously try to
generate the same output file.  They will write their two files to separate
"$pid.tmp" files, but then they will both try to rename them to the same final
filename.  On some OSes (Windows I think), the second rename will fail.  That's
ok, though, because as long as one of the renames succeeded, the correct file
has been written to its final location.  So you should be able to just ignore
the error if the rename fails.

[scheglov, 2016/03/17 21:08:14]

Done.

+    }
+  }
+
+  /**
+   * Read the [PackageBundle] for the library with the given [source] from
+   * the cache. The returned bundle will correspond to the state when the set
+   * of direct and indirect dependencies is resolved in the [context]. Return
+   * `null` if such bundle does not exist.
+   */
+  PackageBundle readBundle(Source source) {
+    try {
+      List<int> hash = _getLibraryClosureHash(source);
+      String hashStr = CryptoUtils.bytesToHex(hash);
+      List<int> bytes = _safeReadBytes('$hashStr.sum');
+      if (bytes == null) {
+        return null;
+      }
+      return new PackageBundle.fromBuffer(bytes);
+    } on StateError {
+      return null;
+    }
+  }
+
+  /**
+   * Fill the whole source closure of the library with the given
+   * [librarySource]. It includes defining units and parts of the library and
+   * all its directly or indirectly imported or exported libraries.
+   */
+  void _appendLibraryClosure(Set<Source> closure, Source librarySource) {
+    if (closure.add(librarySource)) {
+      CacheLibraryUris libraryUris = _getUris(librarySource);
+      if (libraryUris == null) {
+        throw new StateError('No URIs for $librarySource');
+      }
+      // Append parts.
+      for (String partUri in libraryUris.partUris) {
+        Source partSource =
+            context.sourceFactory.resolveUri(librarySource, partUri);
+        if (partSource == null) {
+          throw new StateError('Unable to resolve $partUri in $librarySource');

[Paul Berry, 2016/03/17 18:11:32]

As a general rule the analyzer should be able to do all of its tasks without
throwing exceptions even when presented with illegal code from the user.  I
believe this breaks that, because illegal code from the user could cause an
unresolvable URI.

[scheglov, 2016/03/17 21:08:14]

readBundle() already ignores exception, as if there were no corresponding
bundle. I've added the corresponding try/catch to putLibrary() so that we don't
fail on write. I doubt that many packages have illegal code, so we can check and
decide how to handle it better later.

+        }
+        closure.add(partSource);
+      }
+      // Append imports and exports.
+      void appendLibrarySources(String refUri) {
+        Source refSource =
+            context.sourceFactory.resolveUri(librarySource, refUri);
+        if (refSource == null) {
+          throw new StateError('Unable to resolve $refUri in $librarySource');

[Paul Berry, 2016/03/17 18:11:32]

Similar issue here.

+        }
+        _appendLibraryClosure(closure, refSource);
+      }
+      libraryUris.importedUris.forEach(appendLibrarySources);
+      libraryUris.exportedUris.forEach(appendLibrarySources);
+    }
+  }
+
+  /**
+   * Return the whole source closure of the library with the given
+   * [librarySource]. It includes defining units and parts of the library and
+   * all its directly or indirectly imported or exported libraries.
+   */
+  List<Source> _getLibraryClosure(Source librarySource) {
+    return _libraryClosureMap.putIfAbsent(librarySource, () {
+      Set<Source> closure = new Set<Source>();
+      _appendLibraryClosure(closure, librarySource);
+      return closure.toList();
+    });
+  }
+
+  /**
+   * Return the [context]-specific hash of the closure of the library with
+   * the given [librarySource], or `null` if the cache does not have enough
+   * information.
+   */
+  List<int> _getLibraryClosureHash(Source librarySource) {
+    List<Source> closure = _getLibraryClosure(librarySource);
+    MD5 md5 = new MD5();
+    for (Source source in closure) {
+      List<int> sourceHash = _getSourceContentHash(source);
+      md5.add(sourceHash);
+    }
+    md5.add(configSalt);
+    return md5.close();
+  }
+
+  /**
+   * Compute a hash of the given [source] contents.
+   */
+  List<int> _getSourceContentHash(Source source) {
+    return _sourceContentHashMap.putIfAbsent(source, () {
+      String sourceText = source.contents.data;

[Paul Berry, 2016/03/17 18:11:32]

What happens if source refers to a non-existent file?  (I believe this could
happen if the code being analyzed contains an import of a non-existent file)

[scheglov, 2016/03/17 21:08:14]

It will throw an exception, which we will catch and treat is a not available
summary.

+      List<int> sourceBytes = UTF8.encode(sourceText);
+      return (new MD5()..add(sourceBytes)).close();
+    });
+  }
+
+  /**
+   * Get the URIs information of the library with the given [source], maybe
+   * `null` if the information is not in the cache.
+   */
+  CacheLibraryUris _getUris(Source source) {
+    String fileName = _getUrisFileName(source);
+    List<int> bytes = _safeReadBytes(fileName);
+    if (bytes != null) {
+      return new CacheLibraryUris.fromBuffer(bytes);
+    }
+    return null;
+  }
+
+  /**
+   * Return the name of the file with the [source] library URIs information.
+   */
+  String _getUrisFileName(Source source) {
+    List<int> hash = _getSourceContentHash(source);
+    String hashStr = CryptoUtils.bytesToHex(hash);
+    return '$hashStr.uris';
+  }
+
+  /**
+   * Return bytes of the file with the given [relPath] in the cache, or `null`
+   * if the file does not exist.
+   */
+  List<int> _safeReadBytes(String relPath) {
+    Resource urisFile = cacheFolder.getChild(relPath);
+    if (urisFile is File) {
+      try {
+        return urisFile.readAsBytesSync();
+      } on FileSystemException {}
+    }
+    return null;
+  }
+
+  /**
+   * Write URIs information for the given [library] and its direct and
+   * indirect imports/exports.
+   */
+  void _writeUris(LibraryElement library,
+      [Set<LibraryElement> writtenLibraries]) {
+    writtenLibraries ??= new Set<LibraryElement>();
+    if (!writtenLibraries.add(library)) {
+      return;
+    }
+    // Prepare import/export URIs.
+    List<String> importUris = <String>[];
+    List<String> exportUris = <String>[];
+    for (ImportElement element in library.imports) {
+      String uri = element.uri;
+      if (uri != null) {
+        importUris.add(uri);
+        _writeUris(element.importedLibrary, writtenLibraries);
+      }
+    }
+    for (ExportElement element in library.exports) {
+      String uri = element.uri;
+      if (uri != null) {
+        exportUris.add(uri);
+        _writeUris(element.exportedLibrary, writtenLibraries);
+      }
+    }
+    // Write the URIs.
+    CacheLibraryUrisBuilder b = new CacheLibraryUrisBuilder(
+        importedUris: importUris,
+        exportedUris: exportUris,
+        partUris: library.parts.map((e) => e.uri).toList());
+    String fileName = _getUrisFileName(library.source);
+    File file = cacheFolder.getChild(fileName);
+    file.writeAsBytesSync(b.toBuffer());
+  }
+}