Index: pkg/front_end/lib/src/scanner/token.dart |
diff --git a/pkg/analyzer/lib/dart/ast/token.dart b/pkg/front_end/lib/src/scanner/token.dart |
similarity index 61% |
copy from pkg/analyzer/lib/dart/ast/token.dart |
copy to pkg/front_end/lib/src/scanner/token.dart |
index fce7f249ef6d8b07d99e32b5280442cf3c154093..6e08ddd73a78afe8112609eb0ea7e70f64afb973 100644 |
--- a/pkg/analyzer/lib/dart/ast/token.dart |
+++ b/pkg/front_end/lib/src/scanner/token.dart |
@@ -1,17 +1,144 @@ |
-// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file |
+// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file |
// for details. All rights reserved. Use of this source code is governed by a |
// BSD-style license that can be found in the LICENSE file. |
+library front_end.src.scanner.token; |
Brian Wilkerson
2016/11/08 20:53:24
Not sure why we dropped the doc comment.
Paul Berry
2016/11/08 21:47:06
Re-added.
|
+ |
+import 'dart:collection'; |
+ |
+import 'package:front_end/src/scanner/string_utilities.dart'; |
+import 'package:front_end/src/scanner/syntactic_entity.dart'; |
+ |
/** |
- * Defines the tokens that are produced by the scanner, used by the parser, and |
- * referenced from the [AST structure](ast.dart). |
+ * The opening half of a grouping pair of tokens. This is used for curly |
+ * brackets ('{'), parentheses ('('), and square brackets ('['). |
*/ |
-library analyzer.dart.ast.token; |
+class BeginToken extends SimpleToken { |
Brian Wilkerson
2016/11/08 20:53:24
As discussed off-line, I liked the previous separa
Paul Berry
2016/11/08 21:47:06
Acknowledged.
|
+ /** |
+ * The token that corresponds to this token. |
+ */ |
+ Token endToken; |
-import 'dart:collection'; |
+ /** |
+ * Initialize a newly created token to have the given [type] at the given |
+ * [offset]. |
+ */ |
+ BeginToken(TokenType type, int offset) : super(type, offset) { |
+ assert(type == TokenType.OPEN_CURLY_BRACKET || |
+ type == TokenType.OPEN_PAREN || |
+ type == TokenType.OPEN_SQUARE_BRACKET || |
+ type == TokenType.STRING_INTERPOLATION_EXPRESSION); |
+ } |
-import 'package:analyzer/dart/ast/syntactic_entity.dart'; |
-import 'package:analyzer/src/dart/ast/token.dart' show SimpleToken, TokenClass; |
+ @override |
+ Token copy() => new BeginToken(type, offset); |
+} |
+ |
+/** |
+ * A begin token that is preceded by comments. |
+ */ |
+class BeginTokenWithComment extends BeginToken implements TokenWithComment { |
+ /** |
+ * The first comment in the list of comments that precede this token. |
+ */ |
+ @override |
+ CommentToken _precedingComment; |
+ |
+ /** |
+ * Initialize a newly created token to have the given [type] at the given |
+ * [offset] and to be preceded by the comments reachable from the given |
+ * [_precedingComment]. |
+ */ |
+ BeginTokenWithComment(TokenType type, int offset, this._precedingComment) |
+ : super(type, offset) { |
+ _setCommentParent(_precedingComment); |
+ } |
+ |
+ @override |
+ CommentToken get precedingComments => _precedingComment; |
+ |
+ @override |
+ void set precedingComments(CommentToken comment) { |
+ _precedingComment = comment; |
+ _setCommentParent(_precedingComment); |
+ } |
+ |
+ @override |
+ void applyDelta(int delta) { |
+ super.applyDelta(delta); |
+ Token token = precedingComments; |
+ while (token != null) { |
+ token.applyDelta(delta); |
+ token = token.next; |
+ } |
+ } |
+ |
+ @override |
+ Token copy() => |
+ new BeginTokenWithComment(type, offset, copyComments(precedingComments)); |
+} |
+ |
+/** |
+ * A token representing a comment. |
+ */ |
+class CommentToken extends StringToken { |
+ /** |
+ * The token that contains this comment. |
+ */ |
+ TokenWithComment parent; |
+ |
+ /** |
+ * Initialize a newly created token to represent a token of the given [type] |
+ * with the given [value] at the given [offset]. |
+ */ |
+ CommentToken(TokenType type, String value, int offset) |
+ : super(type, value, offset); |
+ |
+ @override |
+ CommentToken copy() => new CommentToken(type, _value, offset); |
+ |
+ /** |
+ * Remove this comment token from the list. |
+ * |
+ * This is used when we decide to interpret the comment as syntax. |
+ */ |
+ void remove() { |
+ if (previous != null) { |
+ previous.setNextWithoutSettingPrevious(next); |
+ next?.previous = previous; |
+ } else { |
+ assert(parent.precedingComments == this); |
+ parent.precedingComments = next; |
+ } |
+ } |
+} |
+ |
+/** |
+ * A documentation comment token. |
+ */ |
+class DocumentationCommentToken extends CommentToken { |
+ /** |
+ * The references embedded within the documentation comment. |
+ * This list will be empty unless this is a documentation comment that has |
+ * references embedded within it. |
+ */ |
+ final List<Token> references = <Token>[]; |
+ |
+ /** |
+ * Initialize a newly created token to represent a token of the given [type] |
+ * with the given [value] at the given [offset]. |
+ */ |
+ DocumentationCommentToken(TokenType type, String value, int offset) |
+ : super(type, value, offset); |
+ |
+ @override |
+ CommentToken copy() { |
+ DocumentationCommentToken copy = |
+ new DocumentationCommentToken(type, _value, offset); |
+ references.forEach((ref) => copy.references.add(ref.copy())); |
+ return copy; |
+ } |
+} |
/** |
* The keywords in the Dart programming language. |
@@ -215,6 +342,267 @@ class Keyword { |
} |
/** |
+ * A token representing a keyword in the language. |
+ */ |
+class KeywordToken extends SimpleToken { |
+ @override |
+ final Keyword keyword; |
+ |
+ /** |
+ * Initialize a newly created token to represent the given [keyword] at the |
+ * given [offset]. |
+ */ |
+ KeywordToken(this.keyword, int offset) : super(TokenType.KEYWORD, offset); |
+ |
+ @override |
+ String get lexeme => keyword.syntax; |
+ |
+ @override |
+ Token copy() => new KeywordToken(keyword, offset); |
+ |
+ @override |
+ Keyword value() => keyword; |
+} |
+ |
+/** |
+ * A keyword token that is preceded by comments. |
+ */ |
+class KeywordTokenWithComment extends KeywordToken implements TokenWithComment { |
+ /** |
+ * The first comment in the list of comments that precede this token. |
+ */ |
+ @override |
+ CommentToken _precedingComment; |
+ |
+ /** |
+ * Initialize a newly created token to to represent the given [keyword] at the |
+ * given [offset] and to be preceded by the comments reachable from the given |
+ * [_precedingComment]. |
+ */ |
+ KeywordTokenWithComment(Keyword keyword, int offset, this._precedingComment) |
+ : super(keyword, offset) { |
+ _setCommentParent(_precedingComment); |
+ } |
+ |
+ @override |
+ CommentToken get precedingComments => _precedingComment; |
+ |
+ void set precedingComments(CommentToken comment) { |
+ _precedingComment = comment; |
+ _setCommentParent(_precedingComment); |
+ } |
+ |
+ @override |
+ void applyDelta(int delta) { |
+ super.applyDelta(delta); |
+ Token token = precedingComments; |
+ while (token != null) { |
+ token.applyDelta(delta); |
+ token = token.next; |
+ } |
+ } |
+ |
+ @override |
+ Token copy() => new KeywordTokenWithComment( |
+ keyword, offset, copyComments(precedingComments)); |
+} |
+ |
+/** |
+ * A token that was scanned from the input. Each token knows which tokens |
+ * precede and follow it, acting as a link in a doubly linked list of tokens. |
+ */ |
+class SimpleToken implements Token { |
+ /** |
+ * The type of the token. |
+ */ |
+ @override |
+ final TokenType type; |
+ |
+ /** |
+ * The offset from the beginning of the file to the first character in the |
+ * token. |
+ */ |
+ @override |
+ int offset = 0; |
+ |
+ /** |
+ * The previous token in the token stream. |
+ */ |
+ @override |
+ Token previous; |
+ |
+ /** |
+ * The next token in the token stream. |
+ */ |
+ Token _next; |
+ |
+ /** |
+ * Initialize a newly created token to have the given [type] and [offset]. |
+ */ |
+ SimpleToken(this.type, this.offset); |
+ |
+ @override |
+ int get end => offset + length; |
+ |
+ @override |
+ bool get isOperator => type.isOperator; |
+ |
+ @override |
+ bool get isSynthetic => length == 0; |
+ |
+ @override |
+ bool get isUserDefinableOperator => type.isUserDefinableOperator; |
+ |
+ @override |
+ Keyword get keyword => null; |
+ |
+ @override |
+ int get length => lexeme.length; |
+ |
+ @override |
+ String get lexeme => type.lexeme; |
+ |
+ @override |
+ Token get next => _next; |
+ |
+ @override |
+ CommentToken get precedingComments => null; |
+ |
+ @override |
+ void applyDelta(int delta) { |
+ offset += delta; |
+ } |
+ |
+ @override |
+ Token copy() => new Token(type, offset); |
+ |
+ @override |
+ Token copyComments(Token token) { |
+ if (token == null) { |
+ return null; |
+ } |
+ Token head = token.copy(); |
+ Token tail = head; |
+ token = token.next; |
+ while (token != null) { |
+ tail = tail.setNext(token.copy()); |
+ token = token.next; |
+ } |
+ return head; |
+ } |
+ |
+ @override |
+ bool matchesAny(List<TokenType> types) { |
+ for (TokenType type in types) { |
+ if (this.type == type) { |
+ return true; |
+ } |
+ } |
+ return false; |
+ } |
+ |
+ @override |
+ Token setNext(Token token) { |
+ _next = token; |
+ token.previous = this; |
+ return token; |
+ } |
+ |
+ @override |
+ Token setNextWithoutSettingPrevious(Token token) { |
+ _next = token; |
+ return token; |
+ } |
+ |
+ @override |
+ String toString() => lexeme; |
+ |
+ @override |
+ Object value() => type.lexeme; |
+ |
+ /** |
+ * Sets the `parent` property to `this` for the given [comment] and all the |
+ * next tokens. |
+ */ |
+ void _setCommentParent(CommentToken comment) { |
+ while (comment != null) { |
+ comment.parent = this; |
+ comment = comment.next; |
+ } |
+ } |
+} |
+ |
+/** |
+ * A token whose value is independent of it's type. |
+ */ |
+class StringToken extends SimpleToken { |
+ /** |
+ * The lexeme represented by this token. |
+ */ |
+ String _value; |
+ |
+ /** |
+ * Initialize a newly created token to represent a token of the given [type] |
+ * with the given [value] at the given [offset]. |
+ */ |
+ StringToken(TokenType type, String value, int offset) : super(type, offset) { |
+ this._value = StringUtilities.intern(value); |
+ } |
+ |
+ @override |
+ String get lexeme => _value; |
+ |
+ @override |
+ Token copy() => new StringToken(type, _value, offset); |
+ |
+ @override |
+ String value() => _value; |
+} |
+ |
+/** |
+ * A string token that is preceded by comments. |
+ */ |
+class StringTokenWithComment extends StringToken implements TokenWithComment { |
+ /** |
+ * The first comment in the list of comments that precede this token. |
+ */ |
+ CommentToken _precedingComment; |
+ |
+ /** |
+ * Initialize a newly created token to have the given [type] at the given |
+ * [offset] and to be preceded by the comments reachable from the given |
+ * [comment]. |
+ */ |
+ StringTokenWithComment( |
+ TokenType type, String value, int offset, this._precedingComment) |
+ : super(type, value, offset) { |
+ _setCommentParent(_precedingComment); |
+ } |
+ |
+ @override |
+ CommentToken get precedingComments => _precedingComment; |
+ |
+ void set precedingComments(CommentToken comment) { |
+ _precedingComment = comment; |
+ _setCommentParent(_precedingComment); |
+ } |
+ |
+ @override |
+ void applyDelta(int delta) { |
+ super.applyDelta(delta); |
+ Token token = precedingComments; |
+ while (token != null) { |
+ token.applyDelta(delta); |
+ token = token.next; |
+ } |
+ } |
+ |
+ @override |
+ Token copy() => new StringTokenWithComment( |
+ type, lexeme, offset, copyComments(precedingComments)); |
+} |
+ |
+/** |
* A token that was scanned from the input. Each token knows which tokens |
* precede and follow it, acting as a link in a doubly linked list of tokens. |
* |
@@ -366,6 +754,133 @@ abstract class Token implements SyntacticEntity { |
} |
/** |
+ * The classes (or groups) of tokens with a similar use. |
+ */ |
+class TokenClass { |
+ /** |
+ * A value used to indicate that the token type is not part of any specific |
+ * class of token. |
+ */ |
+ static const TokenClass NO_CLASS = const TokenClass('NO_CLASS'); |
+ |
+ /** |
+ * A value used to indicate that the token type is an additive operator. |
+ */ |
+ static const TokenClass ADDITIVE_OPERATOR = |
+ const TokenClass('ADDITIVE_OPERATOR', 13); |
+ |
+ /** |
+ * A value used to indicate that the token type is an assignment operator. |
+ */ |
+ static const TokenClass ASSIGNMENT_OPERATOR = |
+ const TokenClass('ASSIGNMENT_OPERATOR', 1); |
+ |
+ /** |
+ * A value used to indicate that the token type is a bitwise-and operator. |
+ */ |
+ static const TokenClass BITWISE_AND_OPERATOR = |
+ const TokenClass('BITWISE_AND_OPERATOR', 11); |
+ |
+ /** |
+ * A value used to indicate that the token type is a bitwise-or operator. |
+ */ |
+ static const TokenClass BITWISE_OR_OPERATOR = |
+ const TokenClass('BITWISE_OR_OPERATOR', 9); |
+ |
+ /** |
+ * A value used to indicate that the token type is a bitwise-xor operator. |
+ */ |
+ static const TokenClass BITWISE_XOR_OPERATOR = |
+ const TokenClass('BITWISE_XOR_OPERATOR', 10); |
+ |
+ /** |
+ * A value used to indicate that the token type is a cascade operator. |
+ */ |
+ static const TokenClass CASCADE_OPERATOR = |
+ const TokenClass('CASCADE_OPERATOR', 2); |
+ |
+ /** |
+ * A value used to indicate that the token type is a conditional operator. |
+ */ |
+ static const TokenClass CONDITIONAL_OPERATOR = |
+ const TokenClass('CONDITIONAL_OPERATOR', 3); |
+ |
+ /** |
+ * A value used to indicate that the token type is an equality operator. |
+ */ |
+ static const TokenClass EQUALITY_OPERATOR = |
+ const TokenClass('EQUALITY_OPERATOR', 7); |
+ |
+ /** |
+ * A value used to indicate that the token type is an if-null operator. |
+ */ |
+ static const TokenClass IF_NULL_OPERATOR = |
+ const TokenClass('IF_NULL_OPERATOR', 4); |
+ |
+ /** |
+ * A value used to indicate that the token type is a logical-and operator. |
+ */ |
+ static const TokenClass LOGICAL_AND_OPERATOR = |
+ const TokenClass('LOGICAL_AND_OPERATOR', 6); |
+ |
+ /** |
+ * A value used to indicate that the token type is a logical-or operator. |
+ */ |
+ static const TokenClass LOGICAL_OR_OPERATOR = |
+ const TokenClass('LOGICAL_OR_OPERATOR', 5); |
+ |
+ /** |
+ * A value used to indicate that the token type is a multiplicative operator. |
+ */ |
+ static const TokenClass MULTIPLICATIVE_OPERATOR = |
+ const TokenClass('MULTIPLICATIVE_OPERATOR', 14); |
+ |
+ /** |
+ * A value used to indicate that the token type is a relational operator. |
+ */ |
+ static const TokenClass RELATIONAL_OPERATOR = |
+ const TokenClass('RELATIONAL_OPERATOR', 8); |
+ |
+ /** |
+ * A value used to indicate that the token type is a shift operator. |
+ */ |
+ static const TokenClass SHIFT_OPERATOR = |
+ const TokenClass('SHIFT_OPERATOR', 12); |
+ |
+ /** |
+ * A value used to indicate that the token type is a unary operator. |
+ */ |
+ static const TokenClass UNARY_POSTFIX_OPERATOR = |
+ const TokenClass('UNARY_POSTFIX_OPERATOR', 16); |
+ |
+ /** |
+ * A value used to indicate that the token type is a unary operator. |
+ */ |
+ static const TokenClass UNARY_PREFIX_OPERATOR = |
+ const TokenClass('UNARY_PREFIX_OPERATOR', 15); |
+ |
+ /** |
+ * The name of the token class. |
+ */ |
+ final String name; |
+ |
+ /** |
+ * The precedence of tokens of this class, or `0` if the such tokens do not |
+ * represent an operator. |
+ */ |
+ final int precedence; |
+ |
+ /** |
+ * Initialize a newly created class of tokens to have the given [name] and |
+ * [precedence]. |
+ */ |
+ const TokenClass(this.name, [this.precedence = 0]); |
+ |
+ @override |
+ String toString() => name; |
+} |
+ |
+/** |
* The types of tokens that can be returned by the scanner. |
* |
* Clients may not extend, implement or mix-in this class. |
@@ -730,6 +1245,38 @@ class TokenType { |
} |
/** |
+ * A normal token that is preceded by comments. |
+ */ |
+class TokenWithComment extends SimpleToken { |
+ /** |
+ * The first comment in the list of comments that precede this token. |
+ */ |
+ CommentToken _precedingComment; |
+ |
+ /** |
+ * Initialize a newly created token to have the given [type] at the given |
+ * [offset] and to be preceded by the comments reachable from the given |
+ * [comment]. |
+ */ |
+ TokenWithComment(TokenType type, int offset, this._precedingComment) |
+ : super(type, offset) { |
+ _setCommentParent(_precedingComment); |
+ } |
+ |
+ @override |
+ CommentToken get precedingComments => _precedingComment; |
+ |
+ void set precedingComments(CommentToken comment) { |
+ _precedingComment = comment; |
+ _setCommentParent(_precedingComment); |
+ } |
+ |
+ @override |
+ Token copy() => |
+ new TokenWithComment(type, offset, copyComments(precedingComments)); |
+} |
+ |
+/** |
* A token representing the end (either the head or the tail) of a stream of |
* tokens. |
*/ |