Cache linked bundles for Bazel packages in files..

pkg/analyzer/lib/src/summary/bazel_summary.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:convert';
 
 import 'package:analyzer/file_system/file_system.dart';
 import 'package:analyzer/src/generated/engine.dart';
 import 'package:analyzer/src/generated/source.dart';
 import 'package:analyzer/src/generated/utilities_collection.dart';
+import 'package:analyzer/src/summary/api_signature.dart';
 import 'package:analyzer/src/summary/format.dart';
 import 'package:analyzer/src/summary/idl.dart';
 import 'package:analyzer/src/summary/link.dart';
 import 'package:analyzer/src/summary/package_bundle_reader.dart';
 import 'package:analyzer/src/summary/summarize_elements.dart';
 import 'package:analyzer/src/util/fast_uri.dart';
 import 'package:convert/convert.dart';
 import 'package:crypto/crypto.dart';
 import 'package:meta/meta.dart';
 
@@ skipping 70 matching lines @@
 
 /**
  * Class that reads summaries of Bazel packages.
  *
  * When the client needs to produce a resolution result for a new [Source], it
  * should call [getLinkedPackages] to check whether there is the set of
  * packages to resynthesize resolution results.
  */
 class SummaryProvider {
   final ResourceProvider provider;
+  final String tempFileName;
   final GetOutputFolder getOutputFolder;
+  final Folder linkedCacheFolder;
   final AnalysisContext context;
   final PackageBundle sdkBundle;
 
   /**
+   * If `true` (by default), then linking new bundles is allowed.
+   * Otherwise only using existing cached bundles can be used.
+   */
+  final bool allowLinking;
+
+  /**
+   * See [PackageBundleAssembler.currentMajorVersion].
+   */
+  final int majorVersion;
+
+  /**
    * Mapping from bundle paths to corresponding [Package]s.  The packages in
    * the map were consistent with their constituent sources at the moment when
    * they were put into the map.
    */
   final Map<Folder, List<Package>> folderToPackagesMap = {};
 
   /**
    * Mapping from [Uri]s to corresponding [_LinkNode]s.
    */
   final Map<Uri, _LinkNode> uriToNodeMap = {};
 
-  SummaryProvider(this.provider, this.getOutputFolder, AnalysisContext context)
+  SummaryProvider(
+      this.provider,
+      this.tempFileName,
+      this.getOutputFolder,
+      this.linkedCacheFolder,
+      AnalysisContext context,
+      {@visibleForTesting
+          this.allowLinking: true,
+      @visibleForTesting
+          this.majorVersion: PackageBundleAssembler.currentMajorVersion})
       : context = context,
         sdkBundle = context.sourceFactory.dartSdk?.getLinkedBundle();
 
   /**
    * Return the complete list of [Package]s that are required to provide all
    * resolution results for the given [source].
    *
    * The same list of packages is returned for the same [Source], i.e. always
    * the full list, not a difference with a previous request.  It is up to the
    * client to decide whether some of the returned packages should be excluded
@@ skipping 12 matching lines @@
     // Compute all transitive dependencies.
     node.computeTransitiveDependencies();
     List<_LinkNode> nodes = node.transitiveDependencies.toList();
     nodes.forEach((dependency) => dependency.computeTransitiveDependencies());
 
     // Fail if any dependency cannot be resolved.
     if (node.failed) {
       return null;
     }
 
-    _link(nodes);
+    // Read existing cached linked bundles.
+    for (_LinkNode node in nodes) {
+      _readLinked(node);
+    }
+
+    // Link new packages, if allowed.
+    if (allowLinking) {
+      _link(nodes);
+    }
 
     // Create successfully linked packages.
     return nodes
         .map((node) => node.package)
         .where((package) => package.linked != null)
         .toList();
   }
 
   /**
    * Return the [Package] that contains information about the source with
@@ skipping 19 matching lines @@
    * given [source] in [context].
    */
   String _computeSourceHashHex(Source source) {
     String text = context.getContents(source).data;
     List<int> bytes = UTF8.encode(text);
     List<int> hashBytes = md5.convert(bytes).bytes;
     return hex.encode(hashBytes);
   }
 
   /**
+   * Return the name of the file for a linked bundle, in strong or spec mode.
+   */
+  String _getLinkedName(String hash) {
+    if (context.analysisOptions.strongMode) {
+      return 'linked_$hash.ds';
+    } else {
+      return 'linked_spec_$hash.ds';
+    }
+  }
+
+  /**
    * Return the node for the given [uri], or `null` if there is no unlinked
    * bundle that contains [uri].
    */
   _LinkNode _getLinkNodeForUri(Uri uri) {
     return uriToNodeMap.putIfAbsent(uri, () {
       Package package = getUnlinkedForUri(uri);
       if (package == null) {
         return null;
       }
       return new _LinkNode(this, package);
@@ skipping 90 matching lines @@
     for (_LinkNode node in nodes) {
       if (!node.isReady) {
         PackageBundleAssembler assembler = new PackageBundleAssembler();
         linkedLibraries.forEach((uri, linkedLibrary) {
           if (identical(uriToNode[uri], node)) {
             assembler.addLinkedLibrary(uri, linkedLibrary);
           }
         });
         List<int> bytes = assembler.assemble().toBuffer();
         node.package._linked = new PackageBundle.fromBuffer(bytes);
+        _writeLinked(node, bytes);
       }
     }
   }
+
+  /**
+   * Attempt to read the linked bundle that corresponds to the given [node]
+   * with all its transitive dependencies.
+   */
+  void _readLinked(_LinkNode node) {
+    if (!node.isReady && node.linkedHash != null) {
+      String fileName = _getLinkedName(node.linkedHash);
+      File file = linkedCacheFolder.getChildAssumingFile(fileName);
+      // Try to read from the file system.
+      if (file.exists) {
+        try {
+          List<int> bytes = file.readAsBytesSync();
+          node.package._linked = new PackageBundle.fromBuffer(bytes);
+        } on FileSystemException {
+          // Ignore file system exceptions.
+        }
+      }
+    }
+  }
 
   /**
    * Read the unlinked [Package] from the given [file], or return `null` if the
    * file does not exist, or it cannot be read, or is not consistent with the
    * constituent sources on the file system.
    */
   Package _readUnlinkedPackage(File file) {
     try {
       List<int> bytes = file.readAsBytesSync();
       PackageBundle bundle = new PackageBundle.fromBuffer(bytes);
+      // Check the major version.
+      if (bundle.majorVersion != majorVersion) {
+        return null;
+      }
       // Check for consistency, and fail if it's not.
       if (!_isUnlinkedBundleConsistent(bundle)) {
         return null;
       }
       // OK, use the bundle.
       return new Package(file, bundle);
     } on FileSystemException {}
     return null;
   }
+
+  /**
+   * Atomically write the given [bytes] into the file in the [folder].
+   */
+  void _writeAtomic(Folder folder, String fileName, List<int> bytes) {
+    String filePath = folder.getChildAssumingFile(fileName).path;
+    File tempFile = folder.getChildAssumingFile(tempFileName);
+    tempFile.writeAsBytesSync(bytes);
+    tempFile.renameSync(filePath);
+  }
+
+  /**
+   * If a new linked bundle was linked for the given [node], write the bundle
+   * into the memory cache and the file system.
+   */
+  void _writeLinked(_LinkNode node, List<int> bytes) {
+    String hash = node.linkedHash;
+    if (hash != null) {
+      String fileName = _getLinkedName(hash);
+//      File file = linkedCacheFolder.getChildAssumingFile(fileName);

[Paul Berry, 2016/09/30 20:30:47]

Remove commented out lines

[scheglov, 2016/09/30 20:38:28]

Done.

+//      manager.linkedBundleMap[file.path] = node.linked;
+      _writeAtomic(linkedCacheFolder, fileName, bytes);
+    }
+  }
 }
 
 /**
  * Information about a single [Package].
  */
 class _LinkNode {
   final SummaryProvider linker;
   final Package package;
 
   bool failed = false;
   Set<_LinkNode> transitiveDependencies;
 
   List<_LinkNode> _dependencies;
+  String _linkedHash;
 
   _LinkNode(this.linker, this.package);
 
   /**
    * Retrieve the dependencies of this node.
    */
   List<_LinkNode> get dependencies {
     if (_dependencies == null) {
       Set<_LinkNode> dependencies = new Set<_LinkNode>();
 
@@ skipping 33 matching lines @@
     return _dependencies;
   }
 
   /**
    * Return `true` is the node is ready - has the linked bundle or failed (does
    * not have all required dependencies).
    */
   bool get isReady => package.linked != null || failed;
 
   /**
+   * Return the hash string that corresponds to this linked bundle in the
+   * context of its SDK bundle and transitive dependencies.  Return `null` if
+   * the hash computation fails, because for example the full transitive
+   * dependencies cannot computed.
+   */
+  String get linkedHash {

[Paul Berry, 2016/09/30 20:30:47]

This method and the one below are copied almost exactly from pub_summary.dart. 
Can we share code?

[scheglov, 2016/09/30 20:38:28]

Yes, these two methods are similar to the ones we have in pub_summary.dart. 
I will look into merging them a bit later.
I want to get some performance results first.

+    if (_linkedHash == null && transitiveDependencies != null && !failed) {
+      ApiSignature signature = new ApiSignature();
+      // Add all unlinked API signatures.
+      List<String> signatures = <String>[];
+      signatures.add(linker.sdkBundle.apiSignature);
+      transitiveDependencies
+          .map((node) => node.package.unlinked.apiSignature)
+          .forEach(signatures.add);
+      signatures.sort();
+      signatures.forEach(signature.addString);
+      // Combine into a single hash.
+      appendDeclaredVariables(signature);
+      _linkedHash = signature.toHex();
+    }
+    return _linkedHash;
+  }
+
+  /**
+   * Append names and values of all referenced declared variables (even the
+   * ones without actually declared values) to the given [signature].
+   */
+  void appendDeclaredVariables(ApiSignature signature) {
+    Set<String> nameSet = new Set<String>();
+    for (_LinkNode node in transitiveDependencies) {
+      for (UnlinkedUnit unit in node.package.unlinked.unlinkedUnits) {
+        for (UnlinkedImport import in unit.imports) {
+          for (UnlinkedConfiguration configuration in import.configurations) {
+            nameSet.add(configuration.name);
+          }
+        }
+        for (UnlinkedExportPublic export in unit.publicNamespace.exports) {
+          for (UnlinkedConfiguration configuration in export.configurations) {
+            nameSet.add(configuration.name);
+          }
+        }
+      }
+    }
+    List<String> sortedNameList = nameSet.toList()..sort();
+    signature.addInt(sortedNameList.length);
+    for (String name in sortedNameList) {
+      signature.addString(name);
+      signature.addString(linker.context.declaredVariables.get(name) ?? '');
+    }
+  }
+
+  /**
    * Compute the set of existing transitive dependencies for this node.
    * If any dependency cannot be resolved, then set [failed] to `true`.
    * Only unlinked bundle is used, so this method can be called before linking.
    */
   void computeTransitiveDependencies() {
     if (transitiveDependencies == null) {
       transitiveDependencies = new Set<_LinkNode>();
 
       void appendDependencies(_LinkNode node) {
         if (transitiveDependencies.add(node)) {
           node.dependencies.forEach(appendDependencies);
         }
       }
 
       appendDependencies(this);
       if (transitiveDependencies.any((node) => node.failed)) {
         failed = true;
       }
     }
   }
 
   @override
   String toString() => package.toString();
 }