align fasta.Token and analyzer.Token

pkg/front_end/lib/src/fasta/scanner/token.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.scanner.token;
 
 import '../../scanner/token.dart' as analyzer;
 import '../../scanner/token.dart' show TokenType;
 
 import 'token_constants.dart' show IDENTIFIER_TOKEN;
 
 import 'string_canonicalizer.dart';
 
 /**
  * A token that doubles as a linked list.
  */
 abstract class Token implements analyzer.TokenWithComment {
-  /**
-   * The character offset of the start of this token within the source text.
-   */
+  @override
   int charOffset;
 
   Token(this.charOffset);
 
   /**
    * The next token in the token stream.
    */
   Token next;
 
   /**
@@ skipping 21 matching lines @@
     precedingCommentTokens = token;
   }
 
   /**
    * The string represented by this token, a substring of the source code.
    *
    * For [StringToken]s the [lexeme] includes the quotes, explicit escapes, etc.
    */
   String get lexeme;
 
-  /**
-   * For symbol and keyword tokens, returns the string value represented by this
-   * token. For [StringToken]s this method returns [:null:].
-   *
-   * For [SymbolToken]s and [KeywordToken]s, the string value is a compile-time
-   * constant originating in the [TokenType] or in the [Keyword] instance.
-   * This allows testing for keywords and symbols using [:identical:], e.g.,
-   * [:identical('class', token.value):].
-   *
-   * Note that returning [:null:] for string tokens is important to identify
-   * symbols and keywords, we cannot use [lexeme] instead. The string literal
-   *   "$a($b"
-   * produces ..., SymbolToken($), StringToken(a), StringToken((), ...
-   *
-   * After parsing the identifier 'a', the parser tests for a function
-   * declaration using [:identical(next.stringValue, '('):], which (rightfully)
-   * returns false because stringValue returns [:null:].
-   */
-  String get stringValue;
+  @override
+  String get stringValue => type.stringValue;
 
   /**
    * The kind enum of this token as determined by its [type].
    */
   int get kind => type.kind;
 
   /**
    * The precedence level for this token.
    */
   int get precedence => type.precedence;
 
   /**
    * True if this token is an identifier. Some keywords allowed as identifiers,
    * see implementation in [KeywordToken].
    */
   bool get isIdentifier;
 
   bool get isPseudo => false;
 
   /**
    * Returns a textual representation of this token to be used for debugging
    * purposes. The resulting string might contain information about the
    * structure of the token, for example 'StringToken(foo)' for the identifier
    * token 'foo'.
    *
    * Use [lexeme] for the text actually parsed by the token.
    */
   String toString();
 
-  /**
-   * The number of characters parsed by this token.
-   */
+  @override
   int get charCount {
     if (type == analyzer.TokenType.BAD_INPUT) {
       // This is a token that wraps around an error message. Return 1
       // instead of the size of the length of the error message.
       return 1;
     } else {
       return lexeme.length;
     }
   }
 
-  /// The character offset of the end of this token within the source text.
+  @override
   int get charEnd => charOffset + charCount;
 
-  bool get isEof => false;
+  @override
+  bool get isEof => type == analyzer.TokenType.EOF;
 
   bool get isBuiltInIdentifier => false;
 
   @override
   bool get isOperator => type.isOperator;
 
   @override
   bool get isUserDefinableOperator => type.isUserDefinableOperator;
 
   @override
@@ skipping 99 matching lines @@
     // EOF points to itself so there's always infinite look-ahead.
     eof.previousToken = eof;
     eof.next = eof;
     return eof;
   }
 
   @override
   String get lexeme => type.value;
 
   @override
-  String get stringValue => type.value;
-
-  @override
   bool get isIdentifier => false;
 
   @override
   String toString() => "SymbolToken(${isEof ? '-eof-' : lexeme})";
 
   @override
-  bool get isEof => type == analyzer.TokenType.EOF;
-
-  @override
   Token copyWithoutComments() => isEof
       ? new SymbolToken.eof(charOffset)
       : new SymbolToken(type, charOffset);
 }
 
 /**
  * A [SyntheticSymbolToken] represents the symbol in its precedence info
  * which does not exist in the original source.
  * For example, if the scanner finds '(' missing a ')'
  * then it will insert an synthetic ')'.
@@ skipping 43 matching lines @@
  */
 class KeywordToken extends Token implements analyzer.KeywordTokenWithComment {
   final analyzer.Keyword keyword;
 
   KeywordToken(this.keyword, int charOffset) : super(charOffset);
 
   TokenType get info => keyword;
 
   String get lexeme => keyword.lexeme;
 
-  String get stringValue => keyword.lexeme;
-
   bool get isIdentifier => keyword.isPseudo || keyword.isBuiltIn;
 
   bool get isPseudo => keyword.isPseudo;
 
   bool get isBuiltInIdentifier => keyword.isBuiltIn;
 
   String toString() => "KeywordToken($lexeme)";
 
   @override
   Token copyWithoutComments() => new KeywordToken(keyword, charOffset);
@@ skipping 105 matching lines @@
         valueOrLazySubstring = canonicalizedString(
             data, start, end, valueOrLazySubstring.boolValue);
       } else {
         valueOrLazySubstring =
             decodeUtf8(data, start, end, valueOrLazySubstring.boolValue);
       }
       return valueOrLazySubstring;
     }
   }
 
-  /// See [Token.stringValue] for an explanation.
-  String get stringValue => null;
-
   bool get isIdentifier => identical(kind, IDENTIFIER_TOKEN);
 
   String toString() => "StringToken($lexeme)";
 
   static final StringCanonicalizer canonicalizer = new StringCanonicalizer();
 
   static String canonicalizedString(
       String s, int start, int end, bool canonicalize) {
     if (!canonicalize) return s;
     return canonicalizer.canonicalize(s, start, end, false);
@@ skipping 205 matching lines @@
       identical(value, "<=") ||
       identical(value, "<") ||
       identical(value, "&") ||
       identical(value, "^") ||
       identical(value, "|");
 }
 
 bool isTernaryOperator(String value) => identical(value, "[]=");
 
 bool isMinusOperator(String value) => identical(value, "-");