Compute API signatures of files.

pkg/front_end/lib/src/incremental/file_state.dart

Index: pkg/front_end/lib/src/incremental/file_state.dart
diff --git a/pkg/front_end/lib/src/incremental/file_state.dart b/pkg/front_end/lib/src/incremental/file_state.dart
index ac7e14ad1e310d7bbfb5e5f542f8d82bc971c059..eba7359d38d0d4c0db23fa389f499d7c02e0bf0b 100644
--- a/pkg/front_end/lib/src/incremental/file_state.dart
+++ b/pkg/front_end/lib/src/incremental/file_state.dart
@@ -7,11 +7,16 @@ import 'dart:typed_data';
 
 import 'package:crypto/crypto.dart';
 import 'package:front_end/file_system.dart';
+import 'package:front_end/src/base/api_signature.dart';
 import 'package:front_end/src/base/resolve_relative_uri.dart';
 import 'package:front_end/src/dependency_walker.dart' as graph;
 import 'package:front_end/src/fasta/parser/dart_vm_native.dart';
+import 'package:front_end/src/fasta/parser/listener.dart' show Listener;
+import 'package:front_end/src/fasta/parser/parser.dart' show Parser, optional;
 import 'package:front_end/src/fasta/parser/top_level_parser.dart';
 import 'package:front_end/src/fasta/scanner.dart';
+import 'package:front_end/src/fasta/scanner/token_constants.dart'
+    show STRING_TOKEN;
 import 'package:front_end/src/fasta/source/directive_listener.dart';
 import 'package:front_end/src/fasta/translate_uri.dart';
 import 'package:kernel/target/vm.dart';
@@ -35,6 +40,7 @@ class FileState {
   bool _exists;
   List<int> _content;
   List<int> _contentHash;
+  List<int> _apiSignature;
 
   List<NamespaceExport> _exports;
   List<FileState> _importedLibraries;
@@ -46,6 +52,10 @@ class FileState {
 
   FileState._(this._fsState, this.uri, this.fileUri);
 
+  /// The MD5 signature of the file API.
+  /// It depends on all non-comment tokens outside the block bodies.

[ahe, 2017/06/08 12:33:12]

How is the MD5 sum represented? As a byte array? Could you elaborate on this in
the comment?

[scheglov, 2017/06/09 18:40:59]

Done.

+  List<int> get apiSignature => _apiSignature;
+
   /// The content of the file.
   List<int> get content => _content;
 
@@ -117,10 +127,15 @@ class FileState {
     // Compute the content hash.
     _contentHash = md5.convert(_content).bytes;
 
+    // Scan the content.
+    ScannerResult scanResult = _scan();

[ahe, 2017/06/08 12:33:12]

Thinking out loud here, not a review comment:

Since you need the tokens to compute the MD5 sum, I don't think I there's
anything we can tweak in the scanner API.

[scheglov, 2017/06/09 18:40:59]

I don't understand what this means.

+
+    // Compute the API signature.
+    _apiSignature = _computeApiSignature(scanResult.tokens);
+
     // Parse directives.
-    ScannerResult scannerResults = _scan();
     var listener = new _DirectiveListenerWithNative();
-    new TopLevelParser(listener).parseUnit(scannerResults.tokens);
+    new TopLevelParser(listener).parseUnit(scanResult.tokens);
 
     // Build the graph.
     _importedLibraries = <FileState>[];
@@ -187,6 +202,49 @@ class FileState {
     }
   }
 
+  /// Compute and return the API signature of the file.
+  ///
+  /// The signature is based on non-comment tokens of the file outside
+  /// of function bodies.
+  List<int> _computeApiSignature(Token token) {
+    var parser = new _BodySkippingParser();
+    parser.parseUnit(token);
+
+    ApiSignature apiSignature = new ApiSignature();
+    apiSignature.addBytes(_fsState._salt);
+
+    // Iterate over tokens and skip bodies.
+    Iterator<_BodyRange> bodyIterator = parser.bodyRanges.iterator;
+    bodyIterator.moveNext();
+    for (; token.kind != EOF_TOKEN; token = token.next) {
+      // Move to the body range that ends after the token.
+      while (bodyIterator.current != null &&
+          bodyIterator.current.last < token.charOffset) {
+        bodyIterator.moveNext();
+      }
+      // If the current body range starts before or at the token, skip it.
+      if (bodyIterator.current != null &&
+          bodyIterator.current.first <= token.charOffset) {
+        continue;
+      }
+      // The token is outside of a function body, add it.
+      apiSignature.addString(token.lexeme);
+    }
+
+    return apiSignature.toByteList();
+  }
+
+  /// Exclude all `native 'xyz';` token sequences.
+  void _excludeNativeClauses(Token token) {
+    for (; token.kind != EOF_TOKEN; token = token.next) {
+      if (optional('native', token) &&
+          token.next.kind == STRING_TOKEN &&
+          optional(';', token.next.next)) {
+        token.previous.next = token.next.next;
+      }
+    }
+  }
+
   /// Return the [FileState] for the given [relativeUri] or `null` if the URI
   /// cannot be parsed, cannot correspond any file, etc.
   Future<FileState> _getFileForRelativeUri(String relativeUri) async {
@@ -210,7 +268,9 @@ class FileState {
   ScannerResult _scan() {
     var zeroTerminatedBytes = new Uint8List(_content.length + 1);
     zeroTerminatedBytes.setRange(0, _content.length, _content);
-    return scan(zeroTerminatedBytes);
+    ScannerResult result = scan(zeroTerminatedBytes);
+    _excludeNativeClauses(result.tokens);
+    return result;
   }
 }
 
@@ -218,6 +278,7 @@ class FileState {
 class FileSystemState {
   final FileSystem fileSystem;
   final TranslateUri uriTranslator;
+  final List<int> _salt;
 
   _FileSystemView _fileSystemView;
 
@@ -227,7 +288,7 @@ class FileSystemState {
   /// Mapping from file URIs to corresponding [FileState]s.
   final Map<Uri, FileState> _fileUriToFile = {};
 
-  FileSystemState(this.fileSystem, this.uriTranslator);
+  FileSystemState(this.fileSystem, this.uriTranslator, this._salt);
 
   /// Return the [FileSystem] that is backed by this [FileSystemState].  The
   /// files in this [FileSystem] always have the same content as the
@@ -309,6 +370,37 @@ class NamespaceExport {
   }
 }
 
+/// The char range of a function body.
+class _BodyRange {
+  /// The char offset of the first token in the range.
+  final int first;
+
+  /// The char offset of the last token in the range.
+  final int last;
+
+  _BodyRange(this.first, this.last);
+
+  @override
+  String toString() => '[$first, $last]';
+}
+
+/// The [Parser] that skips function bodies and remembers their token ranges.
+class _BodySkippingParser extends Parser {
+  final List<_BodyRange> bodyRanges = [];
+
+  _BodySkippingParser() : super(new Listener());
+
+  @override
+  Token parseFunctionBody(Token token, bool isExpression, bool allowAbstract) {
+    if (identical('{', token.lexeme)) {
+      Token close = skipBlock(token);
+      bodyRanges.add(new _BodyRange(token.charOffset, close.charOffset));
+      return close;
+    }
+    return super.parseFunctionBody(token, isExpression, allowAbstract);
+  }
+}
+
 /// [DirectiveListener] that skips native clauses.
 class _DirectiveListenerWithNative extends DirectiveListener {
   @override