more work aligning fasta.Token with analyzer.Token

pkg/front_end/lib/src/fasta/scanner/token.dart

Index: pkg/front_end/lib/src/fasta/scanner/token.dart
diff --git a/pkg/front_end/lib/src/fasta/scanner/token.dart b/pkg/front_end/lib/src/fasta/scanner/token.dart
index 92d4a538f5a6c274236b84fa3596fd40a9badc27..e32d2c0149236c320951f757103ff98645195d4e 100644
--- a/pkg/front_end/lib/src/fasta/scanner/token.dart
+++ b/pkg/front_end/lib/src/fasta/scanner/token.dart
@@ -20,64 +20,21 @@ abstract class Token implements analyzer.TokenWithComment {
 
   Token(this.charOffset);
 
-  /**
-   * The next token in the token stream.
-   */
-  Token next;
-
-  /**
-   * The previous token in the token stream.
-   *
-   * Deprecated :: This exists for compatibility with the Analyzer token stream
-   * and will be removed at some future date.
-   */
-  @deprecated
-  Token previousToken;
-
-  /**
-   * Return the first comment in the list of comments that precede this token,
-   * or `null` if there are no comments preceding this token. Additional
-   * comments can be reached by following the token stream using [next] until
-   * `null` is returned.
-   */
-  CommentToken precedingCommentTokens;
-
   @override
-  analyzer.CommentToken get precedingComments => precedingCommentTokens;
+  analyzer.Token next;
 
   @override
-  void set precedingComments(analyzer.CommentToken token) {
-    precedingCommentTokens = token;
-  }
+  analyzer.Token previous;
 
-  /**
-   * The string represented by this token, a substring of the source code.
-   *
-   * For [StringToken]s the [lexeme] includes the quotes, explicit escapes, etc.
-   */
-  String get lexeme;
+  @override
+  analyzer.CommentToken precedingComments;
 
   @override
   String get stringValue => type.stringValue;
 
-  /**
-   * The kind enum of this token as determined by its [type].
-   */
+  @override
   int get kind => type.kind;
 
-  /**
-   * The precedence level for this token.
-   */
-  int get precedence => type.precedence;
-
-  /**
-   * True if this token is an identifier. Some keywords allowed as identifiers,
-   * see implementation in [KeywordToken].
-   */
-  bool get isIdentifier;
-
-  bool get isPseudo => false;
-
   /**
    * Returns a textual representation of this token to be used for debugging
    * purposes. The resulting string might contain information about the
@@ -89,15 +46,7 @@ abstract class Token implements analyzer.TokenWithComment {
   String toString();
 
   @override
-  int get charCount {
-    if (type == analyzer.TokenType.BAD_INPUT) {
-      // This is a token that wraps around an error message. Return 1
-      // instead of the size of the length of the error message.
-      return 1;
-    } else {
-      return lexeme.length;
-    }
-  }
+  int get charCount => lexeme.length;
 
   @override
   int get charEnd => charOffset + charCount;
@@ -127,14 +76,6 @@ abstract class Token implements analyzer.TokenWithComment {
   @override
   int get end => charEnd;
 
-  @override
-  analyzer.Token get previous => previousToken;
-
-  @override
-  set previous(analyzer.Token newToken) {
-    previousToken = newToken as Token;
-  }
-
   @override
   void applyDelta(int delta) {
     charOffset += delta;
@@ -188,7 +129,7 @@ abstract class Token implements analyzer.TokenWithComment {
   @override
   analyzer.Token setNext(analyzer.Token token) {
     next = token as Token;
-    next.previousToken = this;
+    next.previous = this;
     return token;
   }
 
@@ -214,7 +155,7 @@ class SymbolToken extends Token {
   factory SymbolToken.eof(int charOffset) {
     var eof = new SyntheticSymbolToken(analyzer.TokenType.EOF, charOffset);
     // EOF points to itself so there's always infinite look-ahead.
-    eof.previousToken = eof;
+    eof.previous = eof;
     eof.next = eof;
     return eof;
   }
@@ -288,16 +229,16 @@ class KeywordToken extends Token implements analyzer.KeywordTokenWithComment {
 
   KeywordToken(this.keyword, int charOffset) : super(charOffset);
 
-  TokenType get info => keyword;
-
+  @override
   String get lexeme => keyword.lexeme;
 
+  @override
   bool get isIdentifier => keyword.isPseudo || keyword.isBuiltIn;
 
-  bool get isPseudo => keyword.isPseudo;
-
+  @override
   bool get isBuiltInIdentifier => keyword.isBuiltIn;
 
+  @override
   String toString() => "KeywordToken($lexeme)";
 
   @override
@@ -343,13 +284,14 @@ class StringToken extends Token implements analyzer.StringTokenWithComment {
    *
    * For string tokens that are substrings of the program source, the actual
    * substring extraction is performed lazily. This is beneficial because
-   * not all scanned code is actually used. For unused parts, the substrings
+   * not all scanned code are actually used. For unused parts, the substrings
    * are never computed and allocated.
    */
   static const int LAZY_THRESHOLD = 4;
 
   var /* String | LazySubtring */ valueOrLazySubstring;
 
+  @override
   final TokenType type;
 
   /**
@@ -376,7 +318,7 @@ class StringToken extends Token implements analyzer.StringTokenWithComment {
           canonicalizedString(data, start, end, canonicalize);
     } else {
       valueOrLazySubstring =
-          new LazySubstring(data, start, length, canonicalize);
+          new _LazySubstring(data, start, length, canonicalize);
     }
   }
 
@@ -391,18 +333,19 @@ class StringToken extends Token implements analyzer.StringTokenWithComment {
     if (length <= LAZY_THRESHOLD) {
       valueOrLazySubstring = decodeUtf8(data, start, end, asciiOnly);
     } else {
-      valueOrLazySubstring = new LazySubstring(data, start, length, asciiOnly);
+      valueOrLazySubstring = new _LazySubstring(data, start, length, asciiOnly);
     }
   }
 
   StringToken._(this.type, this.valueOrLazySubstring, int charOffset)
       : super(charOffset);
 
+  @override
   String get lexeme {
     if (valueOrLazySubstring is String) {
       return valueOrLazySubstring;
     } else {
-      assert(valueOrLazySubstring is LazySubstring);
+      assert(valueOrLazySubstring is _LazySubstring);
       var data = valueOrLazySubstring.data;
       int start = valueOrLazySubstring.start;
       int end = start + valueOrLazySubstring.length;
@@ -417,8 +360,10 @@ class StringToken extends Token implements analyzer.StringTokenWithComment {
     }
   }
 
+  @override
   bool get isIdentifier => identical(kind, IDENTIFIER_TOKEN);
 
+  @override
   String toString() => "StringToken($lexeme)";
 
   static final StringCanonicalizer canonicalizer = new StringCanonicalizer();
@@ -509,11 +454,7 @@ class CommentToken extends StringToken implements analyzer.CommentToken {
 
 class DartDocToken extends CommentToken
     implements analyzer.DocumentationCommentToken {
-  /**
-   * The references embedded within the documentation comment.
-   * This list will be empty unless this is a documentation comment that has
-   * references embedded within it.
-   */
+  @override
   final List<Token> references = <Token>[];
 
   /**
@@ -551,7 +492,7 @@ class DartDocToken extends CommentToken
  * lazily. The substring can either originate from a string or from
  * a [:List<int>:] of UTF-8 bytes.
  */
-abstract class LazySubstring {
+abstract class _LazySubstring {
   /** The original data, either a string or a List<int> */
   get data;
 
@@ -568,18 +509,18 @@ abstract class LazySubstring {
    */
   bool get boolValue;
 
-  LazySubstring.internal();
+  _LazySubstring.internal();
 
-  factory LazySubstring(data, int start, int length, bool b) {
+  factory _LazySubstring(data, int start, int length, bool b) {
     // See comment on [CompactLazySubstring].
     if (start < 0x100000 && length < 0x200) {
       int fields = (start << 9);
       fields = fields | length;
       fields = fields << 1;
       if (b) fields |= 1;
-      return new CompactLazySubstring(data, fields);
+      return new _CompactLazySubstring(data, fields);
     } else {
-      return new FullLazySubstring(data, start, length, b);
+      return new _FullLazySubstring(data, start, length, b);
     }
   }
 }
@@ -591,23 +532,23 @@ abstract class LazySubstring {
  *
  * The file html_dart2js.dart is currently around 1MB.
  */
-class CompactLazySubstring extends LazySubstring {
+class _CompactLazySubstring extends _LazySubstring {
   final data;
   final int fields;
 
-  CompactLazySubstring(this.data, this.fields) : super.internal();
+  _CompactLazySubstring(this.data, this.fields) : super.internal();
 
   int get start => fields >> 10;
   int get length => (fields >> 1) & 0x1ff;
   bool get boolValue => (fields & 1) == 1;
 }
 
-class FullLazySubstring extends LazySubstring {
+class _FullLazySubstring extends _LazySubstring {
   final data;
   final int start;
   final int length;
   final bool boolValue;
-  FullLazySubstring(this.data, this.start, this.length, this.boolValue)
+  _FullLazySubstring(this.data, this.start, this.length, this.boolValue)
       : super.internal();
 }