Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(475)

Side by Side Diff: pkg/analyzer/lib/src/generated/parser.dart

Issue 2842643004: flatten Keyword into TokenType (Closed)
Patch Set: Created 3 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library analyzer.src.generated.parser; 5 library analyzer.src.generated.parser;
6 6
7 import 'dart:collection'; 7 import 'dart:collection';
8 import "dart:math" as math; 8 import "dart:math" as math;
9 9
10 import 'package:analyzer/dart/ast/ast.dart'; 10 import 'package:analyzer/dart/ast/ast.dart';
(...skipping 140 matching lines...) Expand 10 before | Expand all | Expand 10 after
151 return true; 151 return true;
152 } 152 }
153 return needsSpace; 153 return needsSpace;
154 } 154 }
155 } 155 }
156 156
157 /** 157 /**
158 * A parser used to parse tokens into an AST structure. 158 * A parser used to parse tokens into an AST structure.
159 */ 159 */
160 class Parser { 160 class Parser {
161 static String ASYNC = Keyword.ASYNC.syntax; 161 static String ASYNC = Keyword.ASYNC.lexeme;
162 162
163 static String _AWAIT = Keyword.AWAIT.syntax; 163 static String _AWAIT = Keyword.AWAIT.lexeme;
164 164
165 static String _HIDE = Keyword.HIDE.syntax; 165 static String _HIDE = Keyword.HIDE.lexeme;
166 166
167 static String _SHOW = Keyword.SHOW.syntax; 167 static String _SHOW = Keyword.SHOW.lexeme;
168 168
169 static String SYNC = Keyword.SYNC.syntax; 169 static String SYNC = Keyword.SYNC.lexeme;
170 170
171 static String _YIELD = Keyword.YIELD.syntax; 171 static String _YIELD = Keyword.YIELD.lexeme;
172 172
173 static const int _MAX_TREE_DEPTH = 300; 173 static const int _MAX_TREE_DEPTH = 300;
174 174
175 /** 175 /**
176 * The source being parsed. 176 * The source being parsed.
177 */ 177 */
178 final Source _source; 178 final Source _source;
179 179
180 /** 180 /**
181 * The error listener that will be informed of any errors that are found 181 * The error listener that will be informed of any errors that are found
(...skipping 202 matching lines...) Expand 10 before | Expand all | Expand 10 after
384 index = _translateCharacter(buffer, lexeme, index); 384 index = _translateCharacter(buffer, lexeme, index);
385 } 385 }
386 return buffer.toString(); 386 return buffer.toString();
387 } 387 }
388 388
389 /** 389 /**
390 * Return a synthetic identifier. 390 * Return a synthetic identifier.
391 */ 391 */
392 SimpleIdentifier createSyntheticIdentifier({bool isDeclaration: false}) { 392 SimpleIdentifier createSyntheticIdentifier({bool isDeclaration: false}) {
393 Token syntheticToken; 393 Token syntheticToken;
394 if (_currentToken.type == TokenType.KEYWORD) { 394 if (_currentToken.type.isKeyword) {
395 // Consider current keyword token as an identifier. 395 // Consider current keyword token as an identifier.
396 // It is not always true, e.g. "^is T" where "^" is place the place for 396 // It is not always true, e.g. "^is T" where "^" is place the place for
397 // synthetic identifier. By creating SyntheticStringToken we can 397 // synthetic identifier. By creating SyntheticStringToken we can
398 // distinguish a real identifier from synthetic. In the code completion 398 // distinguish a real identifier from synthetic. In the code completion
399 // behavior will depend on a cursor position - before or on "is". 399 // behavior will depend on a cursor position - before or on "is".
400 syntheticToken = _injectToken(new SyntheticStringToken( 400 syntheticToken = _injectToken(new SyntheticStringToken(
401 TokenType.IDENTIFIER, _currentToken.lexeme, _currentToken.offset)); 401 TokenType.IDENTIFIER, _currentToken.lexeme, _currentToken.offset));
402 } else { 402 } else {
403 syntheticToken = _createSyntheticToken(TokenType.IDENTIFIER); 403 syntheticToken = _createSyntheticToken(TokenType.IDENTIFIER);
404 } 404 }
(...skipping 173 matching lines...) Expand 10 before | Expand all | Expand 10 after
578 } 578 }
579 // It is OK to parse as a variable declaration in these cases: 579 // It is OK to parse as a variable declaration in these cases:
580 // String v } 580 // String v }
581 // String v if (true) print('OK'); 581 // String v if (true) print('OK');
582 // String v { print(42); } 582 // String v { print(42); }
583 // ...but not in these cases: 583 // ...but not in these cases:
584 // get getterName { 584 // get getterName {
585 // String get getterName 585 // String get getterName
586 if (allowAdditionalTokens) { 586 if (allowAdditionalTokens) {
587 if (type == TokenType.CLOSE_CURLY_BRACKET || 587 if (type == TokenType.CLOSE_CURLY_BRACKET ||
588 type == TokenType.KEYWORD || 588 type.isKeyword ||
589 type == TokenType.IDENTIFIER || 589 type == TokenType.IDENTIFIER ||
590 type == TokenType.OPEN_CURLY_BRACKET) { 590 type == TokenType.OPEN_CURLY_BRACKET) {
591 return true; 591 return true;
592 } 592 }
593 } 593 }
594 return false; 594 return false;
595 } 595 }
596 596
597 /** 597 /**
598 * Return `true` if the current token appears to be the beginning of a switch 598 * Return `true` if the current token appears to be the beginning of a switch
(...skipping 2561 matching lines...) Expand 10 before | Expand all | Expand 10 after
3160 if (type == TokenType.SEMICOLON) { 3160 if (type == TokenType.SEMICOLON) {
3161 if (!mayBeEmpty) { 3161 if (!mayBeEmpty) {
3162 _reportErrorForCurrentToken(emptyErrorCode); 3162 _reportErrorForCurrentToken(emptyErrorCode);
3163 } 3163 }
3164 return astFactory.emptyFunctionBody(getAndAdvance()); 3164 return astFactory.emptyFunctionBody(getAndAdvance());
3165 } 3165 }
3166 Token keyword = null; 3166 Token keyword = null;
3167 Token star = null; 3167 Token star = null;
3168 bool foundAsync = false; 3168 bool foundAsync = false;
3169 bool foundSync = false; 3169 bool foundSync = false;
3170 if (type == TokenType.KEYWORD) { 3170 if (type.isKeyword) {
3171 String lexeme = _currentToken.lexeme; 3171 String lexeme = _currentToken.lexeme;
3172 if (lexeme == ASYNC) { 3172 if (lexeme == ASYNC) {
3173 foundAsync = true; 3173 foundAsync = true;
3174 keyword = getAndAdvance(); 3174 keyword = getAndAdvance();
3175 if (_matches(TokenType.STAR)) { 3175 if (_matches(TokenType.STAR)) {
3176 star = getAndAdvance(); 3176 star = getAndAdvance();
3177 _inGenerator = true; 3177 _inGenerator = true;
3178 } 3178 }
3179 type = _currentToken.type; 3179 type = _currentToken.type;
3180 _inAsync = true; 3180 _inAsync = true;
(...skipping 845 matching lines...) Expand 10 before | Expand all | Expand 10 after
4026 TokenType type = _currentToken.type; 4026 TokenType type = _currentToken.type;
4027 if (type == TokenType.OPEN_CURLY_BRACKET) { 4027 if (type == TokenType.OPEN_CURLY_BRACKET) {
4028 if (_tokenMatches(_peek(), TokenType.STRING)) { 4028 if (_tokenMatches(_peek(), TokenType.STRING)) {
4029 Token afterString = skipStringLiteral(_currentToken.next); 4029 Token afterString = skipStringLiteral(_currentToken.next);
4030 if (afterString != null && afterString.type == TokenType.COLON) { 4030 if (afterString != null && afterString.type == TokenType.COLON) {
4031 return astFactory.expressionStatement( 4031 return astFactory.expressionStatement(
4032 parseExpression2(), _expect(TokenType.SEMICOLON)); 4032 parseExpression2(), _expect(TokenType.SEMICOLON));
4033 } 4033 }
4034 } 4034 }
4035 return parseBlock(); 4035 return parseBlock();
4036 } else if (type == TokenType.KEYWORD && 4036 } else if (type.isKeyword && !_currentToken.keyword.isBuiltInOrPseudo) {
4037 !_currentToken.keyword.isBuiltInOrPseudo) {
4038 Keyword keyword = _currentToken.keyword; 4037 Keyword keyword = _currentToken.keyword;
4039 // TODO(jwren) compute some metrics to figure out a better order for this 4038 // TODO(jwren) compute some metrics to figure out a better order for this
4040 // if-then sequence to optimize performance 4039 // if-then sequence to optimize performance
4041 if (keyword == Keyword.ASSERT) { 4040 if (keyword == Keyword.ASSERT) {
4042 return parseAssertStatement(); 4041 return parseAssertStatement();
4043 } else if (keyword == Keyword.BREAK) { 4042 } else if (keyword == Keyword.BREAK) {
4044 return parseBreakStatement(); 4043 return parseBreakStatement();
4045 } else if (keyword == Keyword.CONTINUE) { 4044 } else if (keyword == Keyword.CONTINUE) {
4046 return parseContinueStatement(); 4045 return parseContinueStatement();
4047 } else if (keyword == Keyword.DO) { 4046 } else if (keyword == Keyword.DO) {
(...skipping 1956 matching lines...) Expand 10 before | Expand all | Expand 10 after
6004 * after advancing to the next token. Otherwise report an error and return the 6003 * after advancing to the next token. Otherwise report an error and return the
6005 * current token without advancing. 6004 * current token without advancing.
6006 */ 6005 */
6007 Token _expectKeyword(Keyword keyword) { 6006 Token _expectKeyword(Keyword keyword) {
6008 if (_matchesKeyword(keyword)) { 6007 if (_matchesKeyword(keyword)) {
6009 return getAndAdvance(); 6008 return getAndAdvance();
6010 } 6009 }
6011 // Remove uses of this method in favor of matches? 6010 // Remove uses of this method in favor of matches?
6012 // Pass in the error code to use to report the error? 6011 // Pass in the error code to use to report the error?
6013 _reportErrorForCurrentToken( 6012 _reportErrorForCurrentToken(
6014 ParserErrorCode.EXPECTED_TOKEN, [keyword.syntax]); 6013 ParserErrorCode.EXPECTED_TOKEN, [keyword.lexeme]);
6015 return _currentToken; 6014 return _currentToken;
6016 } 6015 }
6017 6016
6018 /** 6017 /**
6019 * Search the given list of [ranges] for a range that contains the given 6018 * Search the given list of [ranges] for a range that contains the given
6020 * [index]. Return the range that was found, or `null` if none of the ranges 6019 * [index]. Return the range that was found, or `null` if none of the ranges
6021 * contain the index. 6020 * contain the index.
6022 */ 6021 */
6023 List<int> _findRange(List<List<int>> ranges, int index) { 6022 List<int> _findRange(List<List<int>> ranges, int index) {
6024 int rangeCount = ranges.length; 6023 int rangeCount = ranges.length;
(...skipping 537 matching lines...) Expand 10 before | Expand all | Expand 10 after
6562 Token classKeyword, 6561 Token classKeyword,
6563 SimpleIdentifier className, 6562 SimpleIdentifier className,
6564 TypeParameterList typeParameters) { 6563 TypeParameterList typeParameters) {
6565 Token equals = _expect(TokenType.EQ); 6564 Token equals = _expect(TokenType.EQ);
6566 TypeName superclass = parseTypeName(false); 6565 TypeName superclass = parseTypeName(false);
6567 WithClause withClause = null; 6566 WithClause withClause = null;
6568 if (_matchesKeyword(Keyword.WITH)) { 6567 if (_matchesKeyword(Keyword.WITH)) {
6569 withClause = parseWithClause(); 6568 withClause = parseWithClause();
6570 } else { 6569 } else {
6571 _reportErrorForCurrentToken( 6570 _reportErrorForCurrentToken(
6572 ParserErrorCode.EXPECTED_TOKEN, [Keyword.WITH.syntax]); 6571 ParserErrorCode.EXPECTED_TOKEN, [Keyword.WITH.lexeme]);
6573 } 6572 }
6574 ImplementsClause implementsClause = null; 6573 ImplementsClause implementsClause = null;
6575 if (_matchesKeyword(Keyword.IMPLEMENTS)) { 6574 if (_matchesKeyword(Keyword.IMPLEMENTS)) {
6576 implementsClause = parseImplementsClause(); 6575 implementsClause = parseImplementsClause();
6577 } 6576 }
6578 Token semicolon; 6577 Token semicolon;
6579 if (_matches(TokenType.SEMICOLON)) { 6578 if (_matches(TokenType.SEMICOLON)) {
6580 semicolon = getAndAdvance(); 6579 semicolon = getAndAdvance();
6581 } else { 6580 } else {
6582 if (_matches(TokenType.OPEN_CURLY_BRACKET)) { 6581 if (_matches(TokenType.OPEN_CURLY_BRACKET)) {
(...skipping 977 matching lines...) Expand 10 before | Expand all | Expand 10 after
7560 } 7559 }
7561 7560
7562 /** 7561 /**
7563 * Parse a string literal representing a URI. Return the string literal that 7562 * Parse a string literal representing a URI. Return the string literal that
7564 * was parsed. 7563 * was parsed.
7565 */ 7564 */
7566 StringLiteral _parseUri() { 7565 StringLiteral _parseUri() {
7567 // TODO(brianwilkerson) Should this function also return true for valid 7566 // TODO(brianwilkerson) Should this function also return true for valid
7568 // top-level keywords? 7567 // top-level keywords?
7569 bool isKeywordAfterUri(Token token) => 7568 bool isKeywordAfterUri(Token token) =>
7570 token.lexeme == Keyword.AS.syntax || 7569 token.lexeme == Keyword.AS.lexeme ||
7571 token.lexeme == _HIDE || 7570 token.lexeme == _HIDE ||
7572 token.lexeme == _SHOW; 7571 token.lexeme == _SHOW;
7573 TokenType type = _currentToken.type; 7572 TokenType type = _currentToken.type;
7574 if (type != TokenType.STRING && 7573 if (type != TokenType.STRING &&
7575 type != TokenType.SEMICOLON && 7574 type != TokenType.SEMICOLON &&
7576 !isKeywordAfterUri(_currentToken)) { 7575 !isKeywordAfterUri(_currentToken)) {
7577 // Attempt to recover in the case where the URI was not enclosed in 7576 // Attempt to recover in the case where the URI was not enclosed in
7578 // quotes. 7577 // quotes.
7579 Token token = _currentToken; 7578 Token token = _currentToken;
7580 bool isValidInUri(Token token) { 7579 bool isValidInUri(Token token) {
(...skipping 494 matching lines...) Expand 10 before | Expand all | Expand 10 after
8075 * include built-in identifiers (pseudo-keywords). 8074 * include built-in identifiers (pseudo-keywords).
8076 */ 8075 */
8077 bool _tokenMatchesIdentifier(Token token) => 8076 bool _tokenMatchesIdentifier(Token token) =>
8078 _tokenMatches(token, TokenType.IDENTIFIER) || 8077 _tokenMatches(token, TokenType.IDENTIFIER) ||
8079 _tokenMatchesPseudoKeyword(token); 8078 _tokenMatchesPseudoKeyword(token);
8080 8079
8081 /** 8080 /**
8082 * Return `true` if the given [token] is either an identifier or a keyword. 8081 * Return `true` if the given [token] is either an identifier or a keyword.
8083 */ 8082 */
8084 bool _tokenMatchesIdentifierOrKeyword(Token token) => 8083 bool _tokenMatchesIdentifierOrKeyword(Token token) =>
8085 _tokenMatches(token, TokenType.IDENTIFIER) || 8084 _tokenMatches(token, TokenType.IDENTIFIER) || token.type.isKeyword;
8086 _tokenMatches(token, TokenType.KEYWORD);
8087 8085
8088 /** 8086 /**
8089 * Return `true` if the given [token] matches the given [keyword]. 8087 * Return `true` if the given [token] matches the given [keyword].
8090 */ 8088 */
8091 bool _tokenMatchesKeyword(Token token, Keyword keyword) => 8089 bool _tokenMatchesKeyword(Token token, Keyword keyword) =>
8092 token.keyword == keyword; 8090 token.keyword == keyword;
8093 8091
8094 /** 8092 /**
8095 * Return `true` if the given [token] matches a pseudo keyword. 8093 * Return `true` if the given [token] matches a pseudo keyword.
8096 */ 8094 */
(...skipping 503 matching lines...) Expand 10 before | Expand all | Expand 10 after
8600 } 8598 }
8601 } 8599 }
8602 } 8600 }
8603 8601
8604 /** 8602 /**
8605 * Instances of this class are thrown when the parser detects that AST has 8603 * Instances of this class are thrown when the parser detects that AST has
8606 * too many nested expressions to be parsed safely and avoid possibility of 8604 * too many nested expressions to be parsed safely and avoid possibility of
8607 * [StackOverflowError] in the parser or during later analysis. 8605 * [StackOverflowError] in the parser or during later analysis.
8608 */ 8606 */
8609 class _TooDeepTreeError {} 8607 class _TooDeepTreeError {}
OLDNEW
« no previous file with comments | « pkg/analyzer/lib/src/generated/error_verifier.dart ('k') | pkg/analyzer/lib/src/generated/resolver.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698