align fasta.Token and analyzer.Token

pkg/front_end/lib/src/scanner/token.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Defines the tokens that are produced by the scanner, used by the parser, and
  * referenced from the [AST structure](ast.dart).
  */
 import 'dart:collection';
 
@@ skipping 521 matching lines @@
    * The next token in the token stream.
    */
   Token _next;
 
   /**
    * Initialize a newly created token to have the given [type] and [offset].
    */
   SimpleToken(this.type, this.offset);
 
   @override
+  int get charCount => length;
+
+  @override
+  int get charOffset => offset;
+
+  @override
+  int get charEnd => end;
+
+  @override
   int get end => offset + length;
 
   @override
+  bool get isEof => type == TokenType.EOF;
+
+  @override
   bool get isOperator => type.isOperator;
 
   @override
   bool get isSynthetic => length == 0;
 
   @override
   bool get isUserDefinableOperator => type.isUserDefinableOperator;
 
   @override
   Keyword get keyword => null;
 
   @override
+  int get kind => type.kind;
+
+  @override
   int get length => lexeme.length;
 
   @override
   String get lexeme => type.lexeme;
 
   @override
   Token get next => _next;
 
   @override
   CommentToken get precedingComments => null;
 
   @override
+  String get stringValue => type.stringValue;
+
+  @override
   void applyDelta(int delta) {
     offset += delta;
   }
 
   @override
   Token copy() => new Token(type, offset);
 
   @override
   Token copyComments(Token token) {
     if (token == null) {
@@ skipping 157 matching lines @@
  * precede and follow it, acting as a link in a doubly linked list of tokens.
  *
  * Clients may not extend, implement or mix-in this class.
  */
 abstract class Token implements SyntacticEntity {
   /**
    * Initialize a newly created token to have the given [type] and [offset].
    */
   factory Token(TokenType type, int offset) = SimpleToken;
 
+  /**
+   * The number of characters parsed by this token.
+   */
+  int get charCount;
+
+  /**
+   * The character offset of the start of this token within the source text.
+   */
+  int get charOffset;
+
+  /**
+   * The character offset of the end of this token within the source text.
+   */
+  int get charEnd;
+
   @override
   int get end;
 
   /**
+   * Return `true` if this token represents an end of file.
+   */
+  bool get isEof;
+
+  /**
    * Return `true` if this token represents an operator.
    */
   bool get isOperator;
 
   /**
    * Return `true` if this token is a synthetic token. A synthetic token is a
    * token that was introduced by the parser in order to recover from an error
    * in the code.
    */
   bool get isSynthetic;
 
   /**
    * Return `true` if this token represents an operator that can be defined by
    * users.
    */
   bool get isUserDefinableOperator;
 
   /**
    * Return the keyword, if a keyword token, or `null` otherwise.
    */
   Keyword get keyword;
 
+  /**
+   * The kind enum of this token as determined by its [type].
+   */
+  int get kind;
+
   @override
   int get length;
 
   /**
    * Return the lexeme that represents this token.
    */
   String get lexeme;
 
   /**
    * Return the next token in the token stream.
@@ skipping 25 matching lines @@
    * Return the previous token in the token stream.
    */
   Token get previous;
 
   /**
    * Set the previous token in the token stream to the given [token].
    */
   void set previous(Token token);
 
   /**
+   * For symbol and keyword tokens, returns the string value represented by this
+   * token. For [StringToken]s this method returns [:null:].
+   *
+   * For [SymbolToken]s and [KeywordToken]s, the string value is a compile-time
+   * constant originating in the [TokenType] or in the [Keyword] instance.
+   * This allows testing for keywords and symbols using [:identical:], e.g.,
+   * [:identical('class', token.value):].
+   *
+   * Note that returning [:null:] for string tokens is important to identify
+   * symbols and keywords, we cannot use [lexeme] instead. The string literal
+   *   "$a($b"
+   * produces ..., SymbolToken($), StringToken(a), StringToken((), ...
+   *
+   * After parsing the identifier 'a', the parser tests for a function
+   * declaration using [:identical(next.stringValue, '('):], which (rightfully)
+   * returns false because stringValue returns [:null:].
+   */
+  String get stringValue;
+
+  /**
    * Return the type of the token.
    */
   TokenType get type;
 
   /**
    * Apply (add) the given [delta] to this token's offset.
    */
   void applyDelta(int delta);
 
   /**
@@ skipping 188 matching lines @@
  *
  * Clients may not extend, implement or mix-in this class.
  */
 class TokenType {
   /**
    * The type of the token that marks the start or end of the input.
    */
   static const TokenType EOF =
       const TokenType('', 'EOF', NO_PRECEDENCE, EOF_TOKEN);
 
-  static const TokenType DOUBLE =
-      const TokenType('double', 'DOUBLE', NO_PRECEDENCE, DOUBLE_TOKEN);
+  static const TokenType DOUBLE = const TokenType(
+      'double', 'DOUBLE', NO_PRECEDENCE, DOUBLE_TOKEN,
+      stringValue: null);
 
   static const TokenType HEXADECIMAL = const TokenType(
-      'hexadecimal', 'HEXADECIMAL', NO_PRECEDENCE, HEXADECIMAL_TOKEN);
+      'hexadecimal', 'HEXADECIMAL', NO_PRECEDENCE, HEXADECIMAL_TOKEN,
+      stringValue: null);
 
   static const TokenType IDENTIFIER = const TokenType(
-      'identifier', 'STRING_INT', NO_PRECEDENCE, IDENTIFIER_TOKEN);
+      'identifier', 'STRING_INT', NO_PRECEDENCE, IDENTIFIER_TOKEN,
+      stringValue: null);
 
-  static const TokenType INT =
-      const TokenType('int', 'INT', NO_PRECEDENCE, INT_TOKEN);
+  static const TokenType INT = const TokenType(
+      'int', 'INT', NO_PRECEDENCE, INT_TOKEN,
+      stringValue: null);
 
   static const TokenType MULTI_LINE_COMMENT = const TokenType(
-      'comment', 'MULTI_LINE_COMMENT', NO_PRECEDENCE, COMMENT_TOKEN);
+      'comment', 'MULTI_LINE_COMMENT', NO_PRECEDENCE, COMMENT_TOKEN,
+      stringValue: null);
 
   static const TokenType SCRIPT_TAG =
       const TokenType('script', 'SCRIPT_TAG', NO_PRECEDENCE, SCRIPT_TOKEN);
 
   static const TokenType SINGLE_LINE_COMMENT = const TokenType(
-      'comment', 'SINGLE_LINE_COMMENT', NO_PRECEDENCE, COMMENT_TOKEN);
+      'comment', 'SINGLE_LINE_COMMENT', NO_PRECEDENCE, COMMENT_TOKEN,
+      stringValue: null);
 
-  static const TokenType STRING =
-      const TokenType('string', 'STRING', NO_PRECEDENCE, STRING_TOKEN);
+  static const TokenType STRING = const TokenType(
+      'string', 'STRING', NO_PRECEDENCE, STRING_TOKEN,
+      stringValue: null);
 
   static const TokenType AMPERSAND = const TokenType(
       '&', 'AMPERSAND', BITWISE_AND_PRECEDENCE, AMPERSAND_TOKEN,
       isOperator: true, isUserDefinableOperator: true);
 
   static const TokenType AMPERSAND_AMPERSAND = const TokenType('&&',
       'AMPERSAND_AMPERSAND', LOGICAL_AND_PRECEDENCE, AMPERSAND_AMPERSAND_TOKEN,
       isOperator: true);
 
   // This is not yet part of the language and not supported by fasta
@@ skipping 227 matching lines @@
   static const TokenType BACKSLASH =
       const TokenType('\\', 'BACKSLASH', NO_PRECEDENCE, BACKSLASH_TOKEN);
 
   static const TokenType PERIOD_PERIOD_PERIOD = const TokenType(
       '...', 'PERIOD_PERIOD_PERIOD', NO_PRECEDENCE, PERIOD_PERIOD_PERIOD_TOKEN);
 
   static const TokenType GENERIC_METHOD_TYPE_LIST = const TokenType(
       'generic_comment_list',
       'GENERIC_METHOD_TYPE_LIST',
       NO_PRECEDENCE,
-      GENERIC_METHOD_TYPE_LIST_TOKEN);
+      GENERIC_METHOD_TYPE_LIST_TOKEN,
+      stringValue: null);
 
   static const TokenType GENERIC_METHOD_TYPE_ASSIGN = const TokenType(
       'generic_comment_assign',
       'GENERIC_METHOD_TYPE_ASSIGN',
       NO_PRECEDENCE,
-      GENERIC_METHOD_TYPE_ASSIGN_TOKEN);
+      GENERIC_METHOD_TYPE_ASSIGN_TOKEN,
+      stringValue: null);
 
   static const TokenType AS = Keyword.AS;
 
   static const TokenType IS = Keyword.IS;
 
   /**
    * Token type used by error tokens.
    */
   static const TokenType BAD_INPUT = const TokenType(
-      'malformed input', 'BAD_INPUT', NO_PRECEDENCE, BAD_INPUT_TOKEN);
+      'malformed input', 'BAD_INPUT', NO_PRECEDENCE, BAD_INPUT_TOKEN,
+      stringValue: null);
 
   /**
    * Token type used by synthetic tokens that are created during parser
    * recovery (non-analyzer use case).
    */
-  static const TokenType RECOVERY =
-      const TokenType('recovery', 'RECOVERY', NO_PRECEDENCE, RECOVERY_TOKEN);
+  static const TokenType RECOVERY = const TokenType(
+      'recovery', 'RECOVERY', NO_PRECEDENCE, RECOVERY_TOKEN,
+      stringValue: null);
 
   // TODO(danrubel): "all" is misleading
   // because this list does not include all TokenType instances.
   static const List<TokenType> all = const <TokenType>[
     TokenType.EOF,
     TokenType.DOUBLE,
     TokenType.HEXADECIMAL,
     TokenType.IDENTIFIER,
     TokenType.INT,
     TokenType.MULTI_LINE_COMMENT,
@@ skipping 103 matching lines @@
    * The name of the token type.
    */
   final String name;
 
   /**
    * The precedence of this type of token,
    * or `0` if the token does not represent an operator.
    */
   final int precedence;
 
+  /**
+   * See [Token.stringValue] for an explanation.
+   */
+  final String stringValue;
+
   const TokenType(this.lexeme, this.name, this.precedence, this.kind,
-      {this.isOperator: false, this.isUserDefinableOperator: false});
+      {this.isOperator: false,
+      this.isUserDefinableOperator: false,
+      String stringValue: 'unspecified'})
+      : this.stringValue = stringValue == 'unspecified' ? lexeme : stringValue;
 
   /**
    * Return `true` if this type of token represents an additive operator.
    */
   bool get isAdditiveOperator => precedence == ADDITIVE_PRECEDENCE;
 
   /**
    * Return `true` if this type of token represents an assignment operator.
    */
   bool get isAssignmentOperator => precedence == ASSIGNMENT_PRECEDENCE;
@@ skipping 102 matching lines @@
 
   void set precedingComments(CommentToken comment) {
     _precedingComment = comment;
     _setCommentParent(_precedingComment);
   }
 
   @override
   Token copy() =>
       new TokenWithComment(type, offset, copyComments(precedingComments));
 }