more work aligning fasta.Token with analyzer.Token

pkg/front_end/lib/src/fasta/scanner/token.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.scanner.token;
 
 import '../../scanner/token.dart' as analyzer;
 import '../../scanner/token.dart' show TokenType;
 
 import 'token_constants.dart' show IDENTIFIER_TOKEN;
 
 import 'string_canonicalizer.dart';
 
 /**
  * A token that doubles as a linked list.
  */
 abstract class Token implements analyzer.TokenWithComment {
   @override
   int charOffset;
 
   Token(this.charOffset);
 
-  /**
-   * The next token in the token stream.
-   */
-  Token next;
-
-  /**
-   * The previous token in the token stream.
-   *
-   * Deprecated :: This exists for compatibility with the Analyzer token stream
-   * and will be removed at some future date.
-   */
-  @deprecated
-  Token previousToken;
-
-  /**
-   * Return the first comment in the list of comments that precede this token,
-   * or `null` if there are no comments preceding this token. Additional
-   * comments can be reached by following the token stream using [next] until
-   * `null` is returned.
-   */
-  CommentToken precedingCommentTokens;
+  @override
+  analyzer.Token next;
 
   @override
-  analyzer.CommentToken get precedingComments => precedingCommentTokens;
+  analyzer.Token previous;
 
   @override
-  void set precedingComments(analyzer.CommentToken token) {
-    precedingCommentTokens = token;
-  }
-
-  /**
-   * The string represented by this token, a substring of the source code.
-   *
-   * For [StringToken]s the [lexeme] includes the quotes, explicit escapes, etc.
-   */
-  String get lexeme;
+  analyzer.CommentToken precedingComments;
 
   @override
   String get stringValue => type.stringValue;
 
-  /**
-   * The kind enum of this token as determined by its [type].
-   */
+  @override
   int get kind => type.kind;
 
   /**
-   * The precedence level for this token.
-   */
-  int get precedence => type.precedence;
-
-  /**
-   * True if this token is an identifier. Some keywords allowed as identifiers,
-   * see implementation in [KeywordToken].
-   */
-  bool get isIdentifier;
-
-  bool get isPseudo => false;
-
-  /**
    * Returns a textual representation of this token to be used for debugging
    * purposes. The resulting string might contain information about the
    * structure of the token, for example 'StringToken(foo)' for the identifier
    * token 'foo'.
    *
    * Use [lexeme] for the text actually parsed by the token.
    */
   String toString();
 
   @override
-  int get charCount {
-    if (type == analyzer.TokenType.BAD_INPUT) {
-      // This is a token that wraps around an error message. Return 1
-      // instead of the size of the length of the error message.
-      return 1;
-    } else {
-      return lexeme.length;
-    }
-  }
+  int get charCount => lexeme.length;
 
   @override
   int get charEnd => charOffset + charCount;
 
   @override
   bool get isEof => type == analyzer.TokenType.EOF;
 
   bool get isBuiltInIdentifier => false;
 
   @override
@@ skipping 10 matching lines @@
     charOffset = newOffset;
   }
 
   @override
   int get length => charCount;
 
   @override
   int get end => charEnd;
 
   @override
-  analyzer.Token get previous => previousToken;
-
-  @override
-  set previous(analyzer.Token newToken) {
-    previousToken = newToken as Token;
-  }
-
-  @override
   void applyDelta(int delta) {
     charOffset += delta;
     CommentToken token = precedingComments;
     while (token != null) {
       token.applyDelta(delta);
       token = token.next;
     }
   }
 
   @override
@@ skipping 32 matching lines @@
       if (this.type == type) {
         return true;
       }
     }
     return false;
   }
 
   @override
   analyzer.Token setNext(analyzer.Token token) {
     next = token as Token;
-    next.previousToken = this;
+    next.previous = this;
     return token;
   }
 
   @override
   analyzer.Token setNextWithoutSettingPrevious(analyzer.Token token) {
     next = token as Token;
     return token;
   }
 
   @override
   Object value() => lexeme;
 }
 
 /**
  * A [SymbolToken] represents the symbol in its precedence info.
  * Also used for end of file with EOF_INFO.
  */
 class SymbolToken extends Token {
   final TokenType type;
 
   SymbolToken(this.type, int charOffset) : super(charOffset);
 
   factory SymbolToken.eof(int charOffset) {
     var eof = new SyntheticSymbolToken(analyzer.TokenType.EOF, charOffset);
     // EOF points to itself so there's always infinite look-ahead.
-    eof.previousToken = eof;
+    eof.previous = eof;
     eof.next = eof;
     return eof;
   }
 
   @override
   String get lexeme => type.value;
 
   @override
   bool get isIdentifier => false;
 
@@ skipping 53 matching lines @@
 }
 
 /**
  * A keyword token.
  */
 class KeywordToken extends Token implements analyzer.KeywordTokenWithComment {
   final analyzer.Keyword keyword;
 
   KeywordToken(this.keyword, int charOffset) : super(charOffset);
 
-  TokenType get info => keyword;
-
+  @override
   String get lexeme => keyword.lexeme;
 
+  @override
   bool get isIdentifier => keyword.isPseudo || keyword.isBuiltIn;
 
-  bool get isPseudo => keyword.isPseudo;
-
+  @override
   bool get isBuiltInIdentifier => keyword.isBuiltIn;
 
+  @override
   String toString() => "KeywordToken($lexeme)";
 
   @override
   Token copyWithoutComments() => new KeywordToken(keyword, charOffset);
 
   @override
   analyzer.Keyword value() => keyword;
 
   @override
   analyzer.TokenType get type => keyword;
@@ skipping 25 matching lines @@
  * A String-valued token. Represents identifiers, string literals,
  * number literals, comments, and error tokens, using the corresponding
  * precedence info.
  */
 class StringToken extends Token implements analyzer.StringTokenWithComment {
   /**
    * The length threshold above which substring tokens are computed lazily.
    *
    * For string tokens that are substrings of the program source, the actual
    * substring extraction is performed lazily. This is beneficial because
-   * not all scanned code is actually used. For unused parts, the substrings
+   * not all scanned code are actually used. For unused parts, the substrings
    * are never computed and allocated.
    */
   static const int LAZY_THRESHOLD = 4;
 
   var /* String | LazySubtring */ valueOrLazySubstring;
 
+  @override
   final TokenType type;
 
   /**
    * Creates a non-lazy string token. If [canonicalize] is true, the string
    * is canonicalized before the token is created.
    */
   StringToken.fromString(this.type, String value, int charOffset,
       {bool canonicalize: false})
       : valueOrLazySubstring =
             canonicalizedString(value, 0, value.length, canonicalize),
         super(charOffset);
 
   /**
    * Creates a lazy string token. If [canonicalize] is true, the string
    * is canonicalized before the token is created.
    */
   StringToken.fromSubstring(
       this.type, String data, int start, int end, int charOffset,
       {bool canonicalize: false})
       : super(charOffset) {
     int length = end - start;
     if (length <= LAZY_THRESHOLD) {
       valueOrLazySubstring =
           canonicalizedString(data, start, end, canonicalize);
     } else {
       valueOrLazySubstring =
-          new LazySubstring(data, start, length, canonicalize);
+          new _LazySubstring(data, start, length, canonicalize);
     }
   }
 
   /**
    * Creates a lazy string token. If [asciiOnly] is false, the byte array
    * is passed through a UTF-8 decoder.
    */
   StringToken.fromUtf8Bytes(this.type, List<int> data, int start, int end,
       bool asciiOnly, int charOffset)
       : super(charOffset) {
     int length = end - start;
     if (length <= LAZY_THRESHOLD) {
       valueOrLazySubstring = decodeUtf8(data, start, end, asciiOnly);
     } else {
-      valueOrLazySubstring = new LazySubstring(data, start, length, asciiOnly);
+      valueOrLazySubstring = new _LazySubstring(data, start, length, asciiOnly);
     }
   }
 
   StringToken._(this.type, this.valueOrLazySubstring, int charOffset)
       : super(charOffset);
 
+  @override
   String get lexeme {
     if (valueOrLazySubstring is String) {
       return valueOrLazySubstring;
     } else {
-      assert(valueOrLazySubstring is LazySubstring);
+      assert(valueOrLazySubstring is _LazySubstring);
       var data = valueOrLazySubstring.data;
       int start = valueOrLazySubstring.start;
       int end = start + valueOrLazySubstring.length;
       if (data is String) {
         valueOrLazySubstring = canonicalizedString(
             data, start, end, valueOrLazySubstring.boolValue);
       } else {
         valueOrLazySubstring =
             decodeUtf8(data, start, end, valueOrLazySubstring.boolValue);
       }
       return valueOrLazySubstring;
     }
   }
 
+  @override
   bool get isIdentifier => identical(kind, IDENTIFIER_TOKEN);
 
+  @override
   String toString() => "StringToken($lexeme)";
 
   static final StringCanonicalizer canonicalizer = new StringCanonicalizer();
 
   static String canonicalizedString(
       String s, int start, int end, bool canonicalize) {
     if (!canonicalize) return s;
     return canonicalizer.canonicalize(s, start, end, false);
   }
 
@@ skipping 70 matching lines @@
       next?.previous = previous;
     } else {
       assert(parent.precedingComments == this);
       parent.precedingComments = next as CommentToken;
     }
   }
 }
 
 class DartDocToken extends CommentToken
     implements analyzer.DocumentationCommentToken {
-  /**
-   * The references embedded within the documentation comment.
-   * This list will be empty unless this is a documentation comment that has
-   * references embedded within it.
-   */
+  @override
   final List<Token> references = <Token>[];
 
   /**
    * Creates a lazy comment token. If [canonicalize] is true, the string
    * is canonicalized before the token is created.
    */
   DartDocToken.fromSubstring(
       TokenType type, String data, int start, int end, int charOffset,
       {bool canonicalize: false})
       : super.fromSubstring(type, data, start, end, charOffset,
@@ skipping 17 matching lines @@
     references.forEach((ref) => copy.references.add(ref.copy()));
     return copy;
   }
 }
 
 /**
  * This class represents the necessary information to compute a substring
  * lazily. The substring can either originate from a string or from
  * a [:List<int>:] of UTF-8 bytes.
  */
-abstract class LazySubstring {
+abstract class _LazySubstring {
   /** The original data, either a string or a List<int> */
   get data;
 
   int get start;
   int get length;
 
   /**
    * If this substring is based on a String, the [boolValue] indicates wheter
    * the resulting substring should be canonicalized.
    *
    * For substrings based on a byte array, the [boolValue] is true if the
    * array only holds ASCII characters. The resulting substring will be
    * canonicalized after decoding.
    */
   bool get boolValue;
 
-  LazySubstring.internal();
+  _LazySubstring.internal();
 
-  factory LazySubstring(data, int start, int length, bool b) {
+  factory _LazySubstring(data, int start, int length, bool b) {
     // See comment on [CompactLazySubstring].
     if (start < 0x100000 && length < 0x200) {
       int fields = (start << 9);
       fields = fields | length;
       fields = fields << 1;
       if (b) fields |= 1;
-      return new CompactLazySubstring(data, fields);
+      return new _CompactLazySubstring(data, fields);
     } else {
-      return new FullLazySubstring(data, start, length, b);
+      return new _FullLazySubstring(data, start, length, b);
     }
   }
 }
 
 /**
  * This class encodes [start], [length] and [boolValue] in a single
  * 30 bit integer. It uses 20 bits for [start], which covers source files
  * of 1MB. [length] has 9 bits, which covers 512 characters.
  *
  * The file html_dart2js.dart is currently around 1MB.
  */
-class CompactLazySubstring extends LazySubstring {
+class _CompactLazySubstring extends _LazySubstring {
   final data;
   final int fields;
 
-  CompactLazySubstring(this.data, this.fields) : super.internal();
+  _CompactLazySubstring(this.data, this.fields) : super.internal();
 
   int get start => fields >> 10;
   int get length => (fields >> 1) & 0x1ff;
   bool get boolValue => (fields & 1) == 1;
 }
 
-class FullLazySubstring extends LazySubstring {
+class _FullLazySubstring extends _LazySubstring {
   final data;
   final int start;
   final int length;
   final bool boolValue;
-  FullLazySubstring(this.data, this.start, this.length, this.boolValue)
+  _FullLazySubstring(this.data, this.start, this.length, this.boolValue)
       : super.internal();
 }
 
 bool isUserDefinableOperator(String value) {
   return isBinaryOperator(value) ||
       isMinusOperator(value) ||
       isTernaryOperator(value) ||
       isUnaryOperator(value);
 }
 
@@ skipping 14 matching lines @@
       identical(value, "<=") ||
       identical(value, "<") ||
       identical(value, "&") ||
       identical(value, "^") ||
       identical(value, "|");
 }
 
 bool isTernaryOperator(String value) => identical(value, "[]=");
 
 bool isMinusOperator(String value) => identical(value, "-");