Move scanner into pkg/front_end/lib/src/scanner.

pkg/front_end/lib/src/scanner/token.dart

-// Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
+// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-/**
- * Defines the tokens that are produced by the scanner, used by the parser, and
- * referenced from the [AST structure](ast.dart).
- */
-library analyzer.dart.ast.token;
+library front_end.src.scanner.token;

[Brian Wilkerson, 2016/11/08 20:53:24]

Not sure why we dropped the doc comment.

[Paul Berry, 2016/11/08 21:47:06]

Re-added.

 
 import 'dart:collection';
 
-import 'package:analyzer/dart/ast/syntactic_entity.dart';
-import 'package:analyzer/src/dart/ast/token.dart' show SimpleToken, TokenClass;
+import 'package:front_end/src/scanner/string_utilities.dart';
+import 'package:front_end/src/scanner/syntactic_entity.dart';
+
+/**
+ * The opening half of a grouping pair of tokens. This is used for curly
+ * brackets ('{'), parentheses ('('), and square brackets ('[').
+ */
+class BeginToken extends SimpleToken {

[Brian Wilkerson, 2016/11/08 20:53:24]

As discussed off-line, I liked the previous separation between the Token
interface, which was public, and the specific implementations of tokens, which
were not public. Your call, though.

[Paul Berry, 2016/11/08 21:47:06]

Acknowledged.

+  /**
+   * The token that corresponds to this token.
+   */
+  Token endToken;
+
+  /**
+   * Initialize a newly created token to have the given [type] at the given
+   * [offset].
+   */
+  BeginToken(TokenType type, int offset) : super(type, offset) {
+    assert(type == TokenType.OPEN_CURLY_BRACKET ||
+        type == TokenType.OPEN_PAREN ||
+        type == TokenType.OPEN_SQUARE_BRACKET ||
+        type == TokenType.STRING_INTERPOLATION_EXPRESSION);
+  }
+
+  @override
+  Token copy() => new BeginToken(type, offset);
+}
+
+/**
+ * A begin token that is preceded by comments.
+ */
+class BeginTokenWithComment extends BeginToken implements TokenWithComment {
+  /**
+   * The first comment in the list of comments that precede this token.
+   */
+  @override
+  CommentToken _precedingComment;
+
+  /**
+   * Initialize a newly created token to have the given [type] at the given
+   * [offset] and to be preceded by the comments reachable from the given
+   * [_precedingComment].
+   */
+  BeginTokenWithComment(TokenType type, int offset, this._precedingComment)
+      : super(type, offset) {
+    _setCommentParent(_precedingComment);
+  }
+
+  @override
+  CommentToken get precedingComments => _precedingComment;
+
+  @override
+  void set precedingComments(CommentToken comment) {
+    _precedingComment = comment;
+    _setCommentParent(_precedingComment);
+  }
+
+  @override
+  void applyDelta(int delta) {
+    super.applyDelta(delta);
+    Token token = precedingComments;
+    while (token != null) {
+      token.applyDelta(delta);
+      token = token.next;
+    }
+  }
+
+  @override
+  Token copy() =>
+      new BeginTokenWithComment(type, offset, copyComments(precedingComments));
+}
+
+/**
+ * A token representing a comment.
+ */
+class CommentToken extends StringToken {
+  /**
+   * The token that contains this comment.
+   */
+  TokenWithComment parent;
+
+  /**
+   * Initialize a newly created token to represent a token of the given [type]
+   * with the given [value] at the given [offset].
+   */
+  CommentToken(TokenType type, String value, int offset)
+      : super(type, value, offset);
+
+  @override
+  CommentToken copy() => new CommentToken(type, _value, offset);
+
+  /**
+   * Remove this comment token from the list.
+   *
+   * This is used when we decide to interpret the comment as syntax.
+   */
+  void remove() {
+    if (previous != null) {
+      previous.setNextWithoutSettingPrevious(next);
+      next?.previous = previous;
+    } else {
+      assert(parent.precedingComments == this);
+      parent.precedingComments = next;
+    }
+  }
+}
+
+/**
+ * A documentation comment token.
+ */
+class DocumentationCommentToken extends CommentToken {
+  /**
+   * The references embedded within the documentation comment.
+   * This list will be empty unless this is a documentation comment that has
+   * references embedded within it.
+   */
+  final List<Token> references = <Token>[];
+
+  /**
+   * Initialize a newly created token to represent a token of the given [type]
+   * with the given [value] at the given [offset].
+   */
+  DocumentationCommentToken(TokenType type, String value, int offset)
+      : super(type, value, offset);
+
+  @override
+  CommentToken copy() {
+    DocumentationCommentToken copy =
+        new DocumentationCommentToken(type, _value, offset);
+    references.forEach((ref) => copy.references.add(ref.copy()));
+    return copy;
+  }
+}
 
 /**
  * The keywords in the Dart programming language.
  *
  * Clients may not extend, implement or mix-in this class.
  */
 class Keyword {
   static const Keyword ABSTRACT = const Keyword._('ABSTRACT', "abstract", true);
 
   static const Keyword AS = const Keyword._('AS', "as", true);
@@ skipping 183 matching lines @@
     LinkedHashMap<String, Keyword> result =
         new LinkedHashMap<String, Keyword>();
     for (Keyword keyword in values) {
       result[keyword.syntax] = keyword;
     }
     return result;
   }
 }
 
 /**
+ * A token representing a keyword in the language.
+ */
+class KeywordToken extends SimpleToken {
+  @override
+  final Keyword keyword;
+
+  /**
+   * Initialize a newly created token to represent the given [keyword] at the
+   * given [offset].
+   */
+  KeywordToken(this.keyword, int offset) : super(TokenType.KEYWORD, offset);
+
+  @override
+  String get lexeme => keyword.syntax;
+
+  @override
+  Token copy() => new KeywordToken(keyword, offset);
+
+  @override
+  Keyword value() => keyword;
+}
+
+/**
+ * A keyword token that is preceded by comments.
+ */
+class KeywordTokenWithComment extends KeywordToken implements TokenWithComment {
+  /**
+   * The first comment in the list of comments that precede this token.
+   */
+  @override
+  CommentToken _precedingComment;
+
+  /**
+   * Initialize a newly created token to to represent the given [keyword] at the
+   * given [offset] and to be preceded by the comments reachable from the given
+   * [_precedingComment].
+   */
+  KeywordTokenWithComment(Keyword keyword, int offset, this._precedingComment)
+      : super(keyword, offset) {
+    _setCommentParent(_precedingComment);
+  }
+
+  @override
+  CommentToken get precedingComments => _precedingComment;
+
+  void set precedingComments(CommentToken comment) {
+    _precedingComment = comment;
+    _setCommentParent(_precedingComment);
+  }
+
+  @override
+  void applyDelta(int delta) {
+    super.applyDelta(delta);
+    Token token = precedingComments;
+    while (token != null) {
+      token.applyDelta(delta);
+      token = token.next;
+    }
+  }
+
+  @override
+  Token copy() => new KeywordTokenWithComment(
+      keyword, offset, copyComments(precedingComments));
+}
+
+/**
+ * A token that was scanned from the input. Each token knows which tokens
+ * precede and follow it, acting as a link in a doubly linked list of tokens.
+ */
+class SimpleToken implements Token {
+  /**
+   * The type of the token.
+   */
+  @override
+  final TokenType type;
+
+  /**
+   * The offset from the beginning of the file to the first character in the
+   * token.
+   */
+  @override
+  int offset = 0;
+
+  /**
+   * The previous token in the token stream.
+   */
+  @override
+  Token previous;
+
+  /**
+   * The next token in the token stream.
+   */
+  Token _next;
+
+  /**
+   * Initialize a newly created token to have the given [type] and [offset].
+   */
+  SimpleToken(this.type, this.offset);
+
+  @override
+  int get end => offset + length;
+
+  @override
+  bool get isOperator => type.isOperator;
+
+  @override
+  bool get isSynthetic => length == 0;
+
+  @override
+  bool get isUserDefinableOperator => type.isUserDefinableOperator;
+
+  @override
+  Keyword get keyword => null;
+
+  @override
+  int get length => lexeme.length;
+
+  @override
+  String get lexeme => type.lexeme;
+
+  @override
+  Token get next => _next;
+
+  @override
+  CommentToken get precedingComments => null;
+
+  @override
+  void applyDelta(int delta) {
+    offset += delta;
+  }
+
+  @override
+  Token copy() => new Token(type, offset);
+
+  @override
+  Token copyComments(Token token) {
+    if (token == null) {
+      return null;
+    }
+    Token head = token.copy();
+    Token tail = head;
+    token = token.next;
+    while (token != null) {
+      tail = tail.setNext(token.copy());
+      token = token.next;
+    }
+    return head;
+  }
+
+  @override
+  bool matchesAny(List<TokenType> types) {
+    for (TokenType type in types) {
+      if (this.type == type) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  @override
+  Token setNext(Token token) {
+    _next = token;
+    token.previous = this;
+    return token;
+  }
+
+  @override
+  Token setNextWithoutSettingPrevious(Token token) {
+    _next = token;
+    return token;
+  }
+
+  @override
+  String toString() => lexeme;
+
+  @override
+  Object value() => type.lexeme;
+
+  /**
+   * Sets the `parent` property to `this` for the given [comment] and all the
+   * next tokens.
+   */
+  void _setCommentParent(CommentToken comment) {
+    while (comment != null) {
+      comment.parent = this;
+      comment = comment.next;
+    }
+  }
+}
+
+/**
+ * A token whose value is independent of it's type.
+ */
+class StringToken extends SimpleToken {
+  /**
+   * The lexeme represented by this token.
+   */
+  String _value;
+
+  /**
+   * Initialize a newly created token to represent a token of the given [type]
+   * with the given [value] at the given [offset].
+   */
+  StringToken(TokenType type, String value, int offset) : super(type, offset) {
+    this._value = StringUtilities.intern(value);
+  }
+
+  @override
+  String get lexeme => _value;
+
+  @override
+  Token copy() => new StringToken(type, _value, offset);
+
+  @override
+  String value() => _value;
+}
+
+/**
+ * A string token that is preceded by comments.
+ */
+class StringTokenWithComment extends StringToken implements TokenWithComment {
+  /**
+   * The first comment in the list of comments that precede this token.
+   */
+  CommentToken _precedingComment;
+
+  /**
+   * Initialize a newly created token to have the given [type] at the given
+   * [offset] and to be preceded by the comments reachable from the given
+   * [comment].
+   */
+  StringTokenWithComment(
+      TokenType type, String value, int offset, this._precedingComment)
+      : super(type, value, offset) {
+    _setCommentParent(_precedingComment);
+  }
+
+  @override
+  CommentToken get precedingComments => _precedingComment;
+
+  void set precedingComments(CommentToken comment) {
+    _precedingComment = comment;
+    _setCommentParent(_precedingComment);
+  }
+
+  @override
+  void applyDelta(int delta) {
+    super.applyDelta(delta);
+    Token token = precedingComments;
+    while (token != null) {
+      token.applyDelta(delta);
+      token = token.next;
+    }
+  }
+
+  @override
+  Token copy() => new StringTokenWithComment(
+      type, lexeme, offset, copyComments(precedingComments));
+}
+
+/**
  * A token that was scanned from the input. Each token knows which tokens
  * precede and follow it, acting as a link in a doubly linked list of tokens.
  *
  * Clients may not extend, implement or mix-in this class.
  */
 abstract class Token implements SyntacticEntity {
   /**
    * Initialize a newly created token to have the given [type] and [offset].
    */
   factory Token(TokenType type, int offset) = SimpleToken;
@@ skipping 131 matching lines @@
       if (token != null && (offset < 0 || token.offset < offset)) {
         first = token;
         offset = token.offset;
       }
     }
     return first;
   }
 }
 
 /**
+ * The classes (or groups) of tokens with a similar use.
+ */
+class TokenClass {
+  /**
+   * A value used to indicate that the token type is not part of any specific
+   * class of token.
+   */
+  static const TokenClass NO_CLASS = const TokenClass('NO_CLASS');
+
+  /**
+   * A value used to indicate that the token type is an additive operator.
+   */
+  static const TokenClass ADDITIVE_OPERATOR =
+      const TokenClass('ADDITIVE_OPERATOR', 13);
+
+  /**
+   * A value used to indicate that the token type is an assignment operator.
+   */
+  static const TokenClass ASSIGNMENT_OPERATOR =
+      const TokenClass('ASSIGNMENT_OPERATOR', 1);
+
+  /**
+   * A value used to indicate that the token type is a bitwise-and operator.
+   */
+  static const TokenClass BITWISE_AND_OPERATOR =
+      const TokenClass('BITWISE_AND_OPERATOR', 11);
+
+  /**
+   * A value used to indicate that the token type is a bitwise-or operator.
+   */
+  static const TokenClass BITWISE_OR_OPERATOR =
+      const TokenClass('BITWISE_OR_OPERATOR', 9);
+
+  /**
+   * A value used to indicate that the token type is a bitwise-xor operator.
+   */
+  static const TokenClass BITWISE_XOR_OPERATOR =
+      const TokenClass('BITWISE_XOR_OPERATOR', 10);
+
+  /**
+   * A value used to indicate that the token type is a cascade operator.
+   */
+  static const TokenClass CASCADE_OPERATOR =
+      const TokenClass('CASCADE_OPERATOR', 2);
+
+  /**
+   * A value used to indicate that the token type is a conditional operator.
+   */
+  static const TokenClass CONDITIONAL_OPERATOR =
+      const TokenClass('CONDITIONAL_OPERATOR', 3);
+
+  /**
+   * A value used to indicate that the token type is an equality operator.
+   */
+  static const TokenClass EQUALITY_OPERATOR =
+      const TokenClass('EQUALITY_OPERATOR', 7);
+
+  /**
+   * A value used to indicate that the token type is an if-null operator.
+   */
+  static const TokenClass IF_NULL_OPERATOR =
+      const TokenClass('IF_NULL_OPERATOR', 4);
+
+  /**
+   * A value used to indicate that the token type is a logical-and operator.
+   */
+  static const TokenClass LOGICAL_AND_OPERATOR =
+      const TokenClass('LOGICAL_AND_OPERATOR', 6);
+
+  /**
+   * A value used to indicate that the token type is a logical-or operator.
+   */
+  static const TokenClass LOGICAL_OR_OPERATOR =
+      const TokenClass('LOGICAL_OR_OPERATOR', 5);
+
+  /**
+   * A value used to indicate that the token type is a multiplicative operator.
+   */
+  static const TokenClass MULTIPLICATIVE_OPERATOR =
+      const TokenClass('MULTIPLICATIVE_OPERATOR', 14);
+
+  /**
+   * A value used to indicate that the token type is a relational operator.
+   */
+  static const TokenClass RELATIONAL_OPERATOR =
+      const TokenClass('RELATIONAL_OPERATOR', 8);
+
+  /**
+   * A value used to indicate that the token type is a shift operator.
+   */
+  static const TokenClass SHIFT_OPERATOR =
+      const TokenClass('SHIFT_OPERATOR', 12);
+
+  /**
+   * A value used to indicate that the token type is a unary operator.
+   */
+  static const TokenClass UNARY_POSTFIX_OPERATOR =
+      const TokenClass('UNARY_POSTFIX_OPERATOR', 16);
+
+  /**
+   * A value used to indicate that the token type is a unary operator.
+   */
+  static const TokenClass UNARY_PREFIX_OPERATOR =
+      const TokenClass('UNARY_PREFIX_OPERATOR', 15);
+
+  /**
+   * The name of the token class.
+   */
+  final String name;
+
+  /**
+   * The precedence of tokens of this class, or `0` if the such tokens do not
+   * represent an operator.
+   */
+  final int precedence;
+
+  /**
+   * Initialize a newly created class of tokens to have the given [name] and
+   * [precedence].
+   */
+  const TokenClass(this.name, [this.precedence = 0]);
+
+  @override
+  String toString() => name;
+}
+
+/**
  * The types of tokens that can be returned by the scanner.
  *
  * Clients may not extend, implement or mix-in this class.
  */
 class TokenType {
   /**
    * The type of the token that marks the start or end of the input.
    */
   static const TokenType EOF = const _EndOfFileTokenType();
 
@@ skipping 344 matching lines @@
    * Return the precedence of the token, or `0` if the token does not represent
    * an operator.
    */
   int get precedence => _tokenClass.precedence;
 
   @override
   String toString() => name;
 }
 
 /**
+ * A normal token that is preceded by comments.
+ */
+class TokenWithComment extends SimpleToken {
+  /**
+   * The first comment in the list of comments that precede this token.
+   */
+  CommentToken _precedingComment;
+
+  /**
+   * Initialize a newly created token to have the given [type] at the given
+   * [offset] and to be preceded by the comments reachable from the given
+   * [comment].
+   */
+  TokenWithComment(TokenType type, int offset, this._precedingComment)
+      : super(type, offset) {
+    _setCommentParent(_precedingComment);
+  }
+
+  @override
+  CommentToken get precedingComments => _precedingComment;
+
+  void set precedingComments(CommentToken comment) {
+    _precedingComment = comment;
+    _setCommentParent(_precedingComment);
+  }
+
+  @override
+  Token copy() =>
+      new TokenWithComment(type, offset, copyComments(precedingComments));
+}
+
+/**
  * A token representing the end (either the head or the tail) of a stream of
  * tokens.
  */
 class _EndOfFileTokenType extends TokenType {
   /**
    * Initialize a newly created token.
    */
   const _EndOfFileTokenType() : super._('EOF', TokenClass.NO_CLASS, '');
 
   @override
   String toString() => '-eof-';
 }