fasta tokens implement analyzer token API

pkg/front_end/lib/src/fasta/scanner/token.dart

 // Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
 library fasta.scanner.token;
 
 import '../../scanner/token.dart' as analyzer;
 
 import 'keyword.dart' show Keyword;
 
@@ skipping 246 matching lines @@
   SymbolToken(this.info, int charOffset) : super(charOffset);
 
   factory SymbolToken.eof(int charOffset) {
     var eof = new SyntheticSymbolToken(EOF_INFO, charOffset);
     // EOF points to itself so there's always infinite look-ahead.
     eof.previousToken = eof;
     eof.next = eof;
     return eof;
   }
 
+  @override
   String get lexeme => info.value;
 
+  @override
   String get stringValue => info.value;
 
+  @override
   bool isIdentifier() => false;
 
-  String toString() => "SymbolToken(${info == EOF_INFO ? '-eof-' : lexeme})";
+  @override
+  String toString() => "SymbolToken(${isEof ? '-eof-' : lexeme})";
 
+  @override
   bool get isEof => info == EOF_INFO;
 
   @override
-  Token copyWithoutComments() => new SymbolToken(info, charOffset);
+  Token copyWithoutComments() => isEof
+      ? new SymbolToken.eof(charOffset)
+      : new SymbolToken(info, charOffset);
 }
 
 /**
  * A [SyntheticSymbolToken] represents the symbol in its precedence info
  * which does not exist in the original source.
  * For example, if the scanner finds '(' missing a ')'
  * then it will insert an synthetic ')'.
  */
 class SyntheticSymbolToken extends SymbolToken {
   SyntheticSymbolToken(PrecedenceInfo info, int charOffset)
       : super(info, charOffset);
 
   @override
   int get charCount => 0;
 
   @override
   bool get isSynthetic => true;
+
+  @override
+  Token copyWithoutComments() => isEof
+      ? new SymbolToken.eof(charOffset)
+      : new SyntheticSymbolToken(info, charOffset);
 }
 
 /**
  * A [BeginGroupToken] represents a symbol that may be the beginning of
  * a pair of brackets, i.e., ( { [ < or ${
  * The [endGroup] token points to the matching closing bracked in case
  * it can be identified during scanning.
  */
-class BeginGroupToken extends SymbolToken implements analyzer.BeginToken {
+class BeginGroupToken extends SymbolToken
+    implements analyzer.BeginTokenWithComment {
   Token endGroup;
 
   BeginGroupToken(PrecedenceInfo info, int charOffset)
       : super(info, charOffset);
 
   @override
   analyzer.Token get endToken => endGroup;
 
   @override
   void set endToken(analyzer.Token token) {
     endGroup = token;
   }
+
+  @override
+  Token copyWithoutComments() => new BeginGroupToken(info, charOffset);
 }
 
 /**
  * A keyword token.
  */
-class KeywordToken extends Token {
+class KeywordToken extends Token implements analyzer.KeywordTokenWithComment {
   final Keyword keyword;
 
   KeywordToken(this.keyword, int charOffset) : super(charOffset);
 
   PrecedenceInfo get info => keyword.info;
 
   String get lexeme => keyword.syntax;
 
   String get stringValue => keyword.syntax;
 
   bool isIdentifier() => keyword.isPseudo || keyword.isBuiltIn;
 
   bool get isPseudo => keyword.isPseudo;
 
   bool get isBuiltInIdentifier => keyword.isBuiltIn;
 
   String toString() => "KeywordToken($lexeme)";
 
   @override
   Token copyWithoutComments() => new KeywordToken(keyword, charOffset);
 
   @override
   // Analyzer considers pseudo-keywords to have a different value
-  Object value() => isPseudo ? lexeme : keyword;
+  Keyword value() => isPseudo ? lexeme : keyword;
 
   @override
   // Analyzer considers pseudo-keywords to be identifiers
   analyzer.TokenType get type => isPseudo ? IDENTIFIER_INFO : KEYWORD_INFO;
 }
 
 /**
+ * A synthetic keyword token.
+ */
+class SyntheticKeywordToken extends KeywordToken
+    implements analyzer.SyntheticKeywordToken {
+  /**
+   * Initialize a newly created token to represent the given [keyword] at the
+   * given [offset].
+   */
+  SyntheticKeywordToken(Keyword keyword, int offset) : super(keyword, offset);
+
+  @override
+  bool get isSynthetic => true;
+
+  @override
+  int get length => 0;
+
+  @override
+  Token copyWithoutComments() => new SyntheticKeywordToken(keyword, offset);
+}
+
+/**
  * A String-valued token. Represents identifiers, string literals,
  * number literals, comments, and error tokens, using the corresponding
  * precedence info.
  */
-class StringToken extends Token implements analyzer.StringToken {
+class StringToken extends Token implements analyzer.StringTokenWithComment {
   /**
    * The length threshold above which substring tokens are computed lazily.
    *
    * For string tokens that are substrings of the program source, the actual
    * substring extraction is performed lazily. This is beneficial because
    * not all scanned code is actually used. For unused parts, the substrings
    * are never computed and allocated.
    */
   static const int LAZY_THRESHOLD = 4;
 
@@ skipping 86 matching lines @@
   }
 
   @override
   Token copyWithoutComments() =>
       new StringToken._(info, valueOrLazySubstring, charOffset);
 
   @override
   String value() => lexeme;
 }
 
+/**
+ * A String-valued token that does not exist in the original source.
+ */
+class SyntheticStringToken extends StringToken
+    implements analyzer.SyntheticStringToken {
+  SyntheticStringToken(PrecedenceInfo info, String value, int offset)
+      : super._(info, value, offset);
+
+  @override
+  bool get isSynthetic => true;
+
+  @override
+  int get length => 0;
+
+  @override
+  Token copyWithoutComments() =>
+      new SyntheticStringToken(info, valueOrLazySubstring, offset);
+}
+
 class CommentToken extends StringToken implements analyzer.CommentToken {
   /**
    * Creates a lazy comment token. If [canonicalize] is true, the string
    * is canonicalized before the token is created.
    */
   CommentToken.fromSubstring(
       PrecedenceInfo info, String data, int start, int end, int charOffset,
       {bool canonicalize: false})
       : super.fromSubstring(info, data, start, end, charOffset,
             canonicalize: canonicalize);
@@ skipping 162 matching lines @@
       identical(value, "<=") ||
       identical(value, "<") ||
       identical(value, "&") ||
       identical(value, "^") ||
       identical(value, "|");
 }
 
 bool isTernaryOperator(String value) => identical(value, "[]=");
 
 bool isMinusOperator(String value) => identical(value, "-");