Compute API signatures of files.

pkg/front_end/lib/src/incremental/file_state.dart

 // Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:async';
 import 'dart:typed_data';
 
 import 'package:crypto/crypto.dart';
 import 'package:front_end/file_system.dart';
+import 'package:front_end/src/base/api_signature.dart';
 import 'package:front_end/src/base/resolve_relative_uri.dart';
 import 'package:front_end/src/dependency_walker.dart' as graph;
 import 'package:front_end/src/fasta/parser/dart_vm_native.dart';
+import 'package:front_end/src/fasta/parser/listener.dart' show Listener;
+import 'package:front_end/src/fasta/parser/parser.dart' show Parser, optional;
 import 'package:front_end/src/fasta/parser/top_level_parser.dart';
 import 'package:front_end/src/fasta/scanner.dart';
+import 'package:front_end/src/fasta/scanner/token_constants.dart'
+    show STRING_TOKEN;
 import 'package:front_end/src/fasta/source/directive_listener.dart';
 import 'package:front_end/src/fasta/translate_uri.dart';
 import 'package:kernel/target/vm.dart';
 
 /// Information about a file being compiled, explicitly or implicitly.
 ///
 /// It provides a consistent view on its properties.
 ///
 /// The properties are not guaranteed to represent the most recent state
 /// of the file system. To update the file to the most recent state, [refresh]
 /// should be called.
 class FileState {
   final FileSystemState _fsState;
 
   /// The absolute URI of the file.
   final Uri uri;
 
   /// The resolved URI of the file in the file system.
   final Uri fileUri;
 
   bool _exists;
   List<int> _content;
   List<int> _contentHash;
+  List<int> _apiSignature;
 
   List<NamespaceExport> _exports;
   List<FileState> _importedLibraries;
   List<FileState> _exportedLibraries;
   List<FileState> _partFiles;
 
   Set<FileState> _directReferencedFiles = new Set<FileState>();
   List<FileState> _directReferencedLibraries = <FileState>[];
 
   FileState._(this._fsState, this.uri, this.fileUri);
 
+  /// The MD5 signature of the file API.
+  /// It depends on all non-comment tokens outside the block bodies.

[ahe, 2017/06/08 12:33:12]

How is the MD5 sum represented? As a byte array? Could you elaborate on this in
the comment?

[scheglov, 2017/06/09 18:40:59]

Done.

+  List<int> get apiSignature => _apiSignature;
+
   /// The content of the file.
   List<int> get content => _content;
 
   /// The MD5 hash of the [content].
   List<int> get contentHash => _contentHash;
 
   /// Libraries that this library file directly imports or exports.
   List<FileState> get directReferencedLibraries => _directReferencedLibraries;
 
   /// Whether the file exists.
@@ skipping 51 matching lines @@
       _content = await entry.readAsBytes();
       _exists = true;
     } catch (_) {
       _content = new Uint8List(0);
       _exists = false;
     }
 
     // Compute the content hash.
     _contentHash = md5.convert(_content).bytes;
 
+    // Scan the content.
+    ScannerResult scanResult = _scan();

[ahe, 2017/06/08 12:33:12]

Thinking out loud here, not a review comment:

Since you need the tokens to compute the MD5 sum, I don't think I there's
anything we can tweak in the scanner API.

[scheglov, 2017/06/09 18:40:59]

I don't understand what this means.

+
+    // Compute the API signature.
+    _apiSignature = _computeApiSignature(scanResult.tokens);
+
     // Parse directives.
-    ScannerResult scannerResults = _scan();
     var listener = new _DirectiveListenerWithNative();
-    new TopLevelParser(listener).parseUnit(scannerResults.tokens);
+    new TopLevelParser(listener).parseUnit(scanResult.tokens);
 
     // Build the graph.
     _importedLibraries = <FileState>[];
     _exportedLibraries = <FileState>[];
     _partFiles = <FileState>[];
     _exports = <NamespaceExport>[];
     {
       FileState coreFile = await _getFileForRelativeUri('dart:core');
       // TODO(scheglov) add error handling
       if (coreFile != null) {
@@ skipping 46 matching lines @@
     if (uri.toString() != 'dart:core') return;
     for (String uri in new VmTarget(null).extraRequiredLibraries) {
       FileState file = await _getFileForRelativeUri(uri);
       // TODO(scheglov) add error handling
       if (file != null) {
         _importedLibraries.add(file);
       }
     }
   }
 
+  /// Compute and return the API signature of the file.
+  ///
+  /// The signature is based on non-comment tokens of the file outside
+  /// of function bodies.
+  List<int> _computeApiSignature(Token token) {
+    var parser = new _BodySkippingParser();
+    parser.parseUnit(token);
+
+    ApiSignature apiSignature = new ApiSignature();
+    apiSignature.addBytes(_fsState._salt);
+
+    // Iterate over tokens and skip bodies.
+    Iterator<_BodyRange> bodyIterator = parser.bodyRanges.iterator;
+    bodyIterator.moveNext();
+    for (; token.kind != EOF_TOKEN; token = token.next) {
+      // Move to the body range that ends after the token.
+      while (bodyIterator.current != null &&
+          bodyIterator.current.last < token.charOffset) {
+        bodyIterator.moveNext();
+      }
+      // If the current body range starts before or at the token, skip it.
+      if (bodyIterator.current != null &&
+          bodyIterator.current.first <= token.charOffset) {
+        continue;
+      }
+      // The token is outside of a function body, add it.
+      apiSignature.addString(token.lexeme);
+    }
+
+    return apiSignature.toByteList();
+  }
+
+  /// Exclude all `native 'xyz';` token sequences.
+  void _excludeNativeClauses(Token token) {
+    for (; token.kind != EOF_TOKEN; token = token.next) {
+      if (optional('native', token) &&
+          token.next.kind == STRING_TOKEN &&
+          optional(';', token.next.next)) {
+        token.previous.next = token.next.next;
+      }
+    }
+  }
+
   /// Return the [FileState] for the given [relativeUri] or `null` if the URI
   /// cannot be parsed, cannot correspond any file, etc.
   Future<FileState> _getFileForRelativeUri(String relativeUri) async {
     if (relativeUri.isEmpty) return null;
 
     // Resolve the relative URI into absolute.
     // The result is either:
     //   1) The absolute file URI.
     //   2) The absolute non-file URI, e.g. `package:foo/foo.dart`.
     Uri absoluteUri;
     try {
       absoluteUri = resolveRelativeUri(uri, Uri.parse(relativeUri));
     } on FormatException {
       return null;
     }
 
     return await _fsState.getFile(absoluteUri);
   }
 
   /// Scan the content of the file.
   ScannerResult _scan() {
     var zeroTerminatedBytes = new Uint8List(_content.length + 1);
     zeroTerminatedBytes.setRange(0, _content.length, _content);
-    return scan(zeroTerminatedBytes);
+    ScannerResult result = scan(zeroTerminatedBytes);
+    _excludeNativeClauses(result.tokens);
+    return result;
   }
 }
 
 /// Information about known file system state.
 class FileSystemState {
   final FileSystem fileSystem;
   final TranslateUri uriTranslator;
+  final List<int> _salt;
 
   _FileSystemView _fileSystemView;
 
   /// Mapping from file URIs to corresponding [FileState]s.
   final Map<Uri, FileState> _uriToFile = {};
 
   /// Mapping from file URIs to corresponding [FileState]s.
   final Map<Uri, FileState> _fileUriToFile = {};
 
-  FileSystemState(this.fileSystem, this.uriTranslator);
+  FileSystemState(this.fileSystem, this.uriTranslator, this._salt);
 
   /// Return the [FileSystem] that is backed by this [FileSystemState].  The
   /// files in this [FileSystem] always have the same content as the
   /// corresponding [FileState]s, thus avoiding race conditions when a file
   /// is updated on the actual file system.
   FileSystem get fileSystemView {
     return _fileSystemView ??= new _FileSystemView(this);
   }
 
   /// Return the [FileState] for the given [absoluteUri], or `null` if the
@@ skipping 61 matching lines @@
       } else {
         if (combinator.names.contains(name)) {
           return false;
         }
       }
     }
     return true;
   }
 }
 
+/// The char range of a function body.
+class _BodyRange {
+  /// The char offset of the first token in the range.
+  final int first;
+
+  /// The char offset of the last token in the range.
+  final int last;
+
+  _BodyRange(this.first, this.last);
+
+  @override
+  String toString() => '[$first, $last]';
+}
+
+/// The [Parser] that skips function bodies and remembers their token ranges.
+class _BodySkippingParser extends Parser {
+  final List<_BodyRange> bodyRanges = [];
+
+  _BodySkippingParser() : super(new Listener());
+
+  @override
+  Token parseFunctionBody(Token token, bool isExpression, bool allowAbstract) {
+    if (identical('{', token.lexeme)) {
+      Token close = skipBlock(token);
+      bodyRanges.add(new _BodyRange(token.charOffset, close.charOffset));
+      return close;
+    }
+    return super.parseFunctionBody(token, isExpression, allowAbstract);
+  }
+}
+
 /// [DirectiveListener] that skips native clauses.
 class _DirectiveListenerWithNative extends DirectiveListener {
   @override
   Token handleNativeClause(Token token) => skipNativeClause(token);
 }
 
 /// [FileSystemState] based implementation of [FileSystem].
 /// It provides a consistent view on the known file system state.
 class _FileSystemView implements FileSystem {
   final FileSystemState fsState;
@@ skipping 76 matching lines @@
       node.isEvaluated = true;
       cycle.libraries.add(node.file);
     }
     topologicallySortedCycles.add(cycle);
   }
 
   _LibraryNode getNode(FileState file) {
     return nodesOfFiles.putIfAbsent(file, () => new _LibraryNode(this, file));
   }
 }