more work aligning fasta.Token with analyzer.Token

pkg/front_end/lib/src/scanner/token.dart

 // Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 /**
  * Defines the tokens that are produced by the scanner, used by the parser, and
  * referenced from the [AST structure](ast.dart).
  */
 import 'dart:collection';
 
@@ skipping 366 matching lines @@
   /**
    * A table mapping the lexemes of keywords to the corresponding keyword.
    */
   static final Map<String, Keyword> keywords = _createKeywordMap();
 
   /**
    * A flag indicating whether the keyword is "built-in" identifier.
    */
   final bool isBuiltIn;
 
-  /**
-   * A flag indicating whether the keyword can be used as an identifier
-   * in some situations.
-   */
+  @override
   final bool isPseudo;
 
   /**
    * Initialize a newly created keyword.
    */
   const Keyword(String lexeme, String name,
       {this.isBuiltIn: false,
       this.isPseudo: false,
       int precedence: NO_PRECEDENCE})
       : super(lexeme, name, precedence, KEYWORD_TOKEN);
@@ skipping 48 matching lines @@
   /**
    * Initialize a newly created token to represent the given [keyword] at the
    * given [offset].
    */
   KeywordToken(this.keyword, int offset) : super(keyword, offset);
 
   @override
   Token copy() => new KeywordToken(keyword, offset);
 
   @override
+  bool get isIdentifier => keyword.isPseudo || keyword.isBuiltIn;
+
+  @override
   // Changed return type from Keyword to Object because
   // fasta considers pseudo-keywords to be keywords rather than identifiers
   Object value() => keyword;
 }
 
 /**
  * A keyword token that is preceded by comments.
  */
 class KeywordTokenWithComment extends KeywordToken implements TokenWithComment {
   /**
@@ skipping 52 matching lines @@
    */
   @override
   int offset = 0;
 
   /**
    * The previous token in the token stream.
    */
   @override
   Token previous;
 
-  /**
-   * The next token in the token stream.
-   */
-  Token _next;
+  @override
+  Token next;
 
   /**
    * Initialize a newly created token to have the given [type] and [offset].
    */
   SimpleToken(this.type, this.offset);
 
   @override
   int get charCount => length;
 
   @override
   int get charOffset => offset;
 
   @override
   int get charEnd => end;
 
   @override
   int get end => offset + length;
 
   @override
   bool get isEof => type == TokenType.EOF;
 
   @override
+  bool get isIdentifier => false;
+
+  @override
   bool get isOperator => type.isOperator;
 
   @override
   bool get isSynthetic => length == 0;
 
   @override
   bool get isUserDefinableOperator => type.isUserDefinableOperator;
 
   @override
   Keyword get keyword => null;
 
   @override
   int get kind => type.kind;
 
   @override
   int get length => lexeme.length;
 
   @override
   String get lexeme => type.lexeme;
 
   @override
-  Token get next => _next;
-
-  @override
   CommentToken get precedingComments => null;
 
   @override
   String get stringValue => type.stringValue;
 
   @override
   void applyDelta(int delta) {
     offset += delta;
   }
 
@@ skipping 20 matching lines @@
     for (TokenType type in types) {
       if (this.type == type) {
         return true;
       }
     }
     return false;
   }
 
   @override
   Token setNext(Token token) {
-    _next = token;
+    next = token;
     token.previous = this;
     return token;
   }
 
   @override
   Token setNextWithoutSettingPrevious(Token token) {
-    _next = token;
+    next = token;
     return token;
   }
 
   @override
   String toString() => lexeme;
 
   @override
   Object value() => type.lexeme;
 
   /**
@@ skipping 19 matching lines @@
 
   /**
    * Initialize a newly created token to represent a token of the given [type]
    * with the given [value] at the given [offset].
    */
   StringToken(TokenType type, String value, int offset) : super(type, offset) {
     this._value = StringUtilities.intern(value);
   }
 
   @override
+  bool get isIdentifier => identical(kind, IDENTIFIER_TOKEN);
+
+  @override
   String get lexeme => _value;
 
   @override
   Token copy() => new StringToken(type, _value, offset);
 
   @override
   String value() => _value;
 }
 
 /**
@@ skipping 100 matching lines @@
 
   @override
   int get end;
 
   /**
    * Return `true` if this token represents an end of file.
    */
   bool get isEof;
 
   /**
+   * True if this token is an identifier. Some keywords allowed as identifiers,
+   * see implementation in [KeywordToken].
+   */
+  bool get isIdentifier;
+
+  /**
    * Return `true` if this token represents an operator.
    */
   bool get isOperator;
 
   /**
    * Return `true` if this token is a synthetic token. A synthetic token is a
    * token that was introduced by the parser in order to recover from an error
    * in the code.
    */
   bool get isSynthetic;
@@ skipping 12 matching lines @@
   /**
    * The kind enum of this token as determined by its [type].
    */
   int get kind;
 
   @override
   int get length;
 
   /**
    * Return the lexeme that represents this token.
+   *
+   * For [StringToken]s the [lexeme] includes the quotes, explicit escapes, etc.
    */
   String get lexeme;
 
   /**
    * Return the next token in the token stream.
    */
   Token get next;
 
+  /**
+   * Return the next token in the token stream.
+   */
+  void set next(Token next);
+
   @override
   int get offset;
 
   /**
    * Set the offset from the beginning of the file to the first character in
    * the token to the given [offset].
    */
   void set offset(int offset);
 
   /**
@@ skipping 728 matching lines @@
    */
   bool get isIncrementOperator =>
       this == TokenType.PLUS_PLUS || this == TokenType.MINUS_MINUS;
 
   /**
    * Return `true` if this type of token is a keyword.
    */
   bool get isKeyword => kind == KEYWORD_TOKEN;
 
   /**
+   * A flag indicating whether the keyword can be used as an identifier
+   * in some situations.
+   */
+  bool get isPseudo => false;
+
+  /**
    * Return `true` if this type of token represents a multiplicative operator.
    */
   bool get isMultiplicativeOperator => precedence == MULTIPLICATIVE_PRECEDENCE;
 
   /**
    * Return `true` if this type of token represents a relational operator.
    */
   bool get isRelationalOperator =>
       this == TokenType.LT ||
       this == TokenType.LT_EQ ||
@@ skipping 52 matching lines @@
 
   void set precedingComments(CommentToken comment) {
     _precedingComment = comment;
     _setCommentParent(_precedingComment);
   }
 
   @override
   Token copy() =>
       new TokenWithComment(type, offset, copyComments(precedingComments));
 }