Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(60)

Side by Side Diff: pkg/analyzer/lib/src/generated/parser.dart

Issue 2842213003: new TokenType and Keyword API (Closed)
Patch Set: rebase Created 3 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 4
5 library analyzer.src.generated.parser; 5 library analyzer.src.generated.parser;
6 6
7 import 'dart:collection'; 7 import 'dart:collection';
8 import "dart:math" as math; 8 import "dart:math" as math;
9 9
10 import 'package:analyzer/dart/ast/ast.dart'; 10 import 'package:analyzer/dart/ast/ast.dart';
(...skipping 140 matching lines...) Expand 10 before | Expand all | Expand 10 after
151 return true; 151 return true;
152 } 152 }
153 return needsSpace; 153 return needsSpace;
154 } 154 }
155 } 155 }
156 156
157 /** 157 /**
158 * A parser used to parse tokens into an AST structure. 158 * A parser used to parse tokens into an AST structure.
159 */ 159 */
160 class Parser { 160 class Parser {
161 static String ASYNC = Keyword.ASYNC.syntax; 161 static String ASYNC = Keyword.ASYNC.lexeme;
162 162
163 static String _AWAIT = Keyword.AWAIT.syntax; 163 static String _AWAIT = Keyword.AWAIT.lexeme;
164 164
165 static String _HIDE = Keyword.HIDE.syntax; 165 static String _HIDE = Keyword.HIDE.lexeme;
166 166
167 static String _SHOW = Keyword.SHOW.syntax; 167 static String _SHOW = Keyword.SHOW.lexeme;
168 168
169 static String SYNC = Keyword.SYNC.syntax; 169 static String SYNC = Keyword.SYNC.lexeme;
170 170
171 static String _YIELD = Keyword.YIELD.syntax; 171 static String _YIELD = Keyword.YIELD.lexeme;
172 172
173 static const int _MAX_TREE_DEPTH = 300; 173 static const int _MAX_TREE_DEPTH = 300;
174 174
175 /** 175 /**
176 * The source being parsed. 176 * The source being parsed.
177 */ 177 */
178 final Source _source; 178 final Source _source;
179 179
180 /** 180 /**
181 * The error listener that will be informed of any errors that are found 181 * The error listener that will be informed of any errors that are found
(...skipping 202 matching lines...) Expand 10 before | Expand all | Expand 10 after
384 index = _translateCharacter(buffer, lexeme, index); 384 index = _translateCharacter(buffer, lexeme, index);
385 } 385 }
386 return buffer.toString(); 386 return buffer.toString();
387 } 387 }
388 388
389 /** 389 /**
390 * Return a synthetic identifier. 390 * Return a synthetic identifier.
391 */ 391 */
392 SimpleIdentifier createSyntheticIdentifier({bool isDeclaration: false}) { 392 SimpleIdentifier createSyntheticIdentifier({bool isDeclaration: false}) {
393 Token syntheticToken; 393 Token syntheticToken;
394 if (_currentToken.type == TokenType.KEYWORD) { 394 if (_currentToken.type.isKeyword) {
395 // Consider current keyword token as an identifier. 395 // Consider current keyword token as an identifier.
396 // It is not always true, e.g. "^is T" where "^" is place the place for 396 // It is not always true, e.g. "^is T" where "^" is place the place for
397 // synthetic identifier. By creating SyntheticStringToken we can 397 // synthetic identifier. By creating SyntheticStringToken we can
398 // distinguish a real identifier from synthetic. In the code completion 398 // distinguish a real identifier from synthetic. In the code completion
399 // behavior will depend on a cursor position - before or on "is". 399 // behavior will depend on a cursor position - before or on "is".
400 syntheticToken = _injectToken(new SyntheticStringToken( 400 syntheticToken = _injectToken(new SyntheticStringToken(
401 TokenType.IDENTIFIER, _currentToken.lexeme, _currentToken.offset)); 401 TokenType.IDENTIFIER, _currentToken.lexeme, _currentToken.offset));
402 } else { 402 } else {
403 syntheticToken = _createSyntheticToken(TokenType.IDENTIFIER); 403 syntheticToken = _createSyntheticToken(TokenType.IDENTIFIER);
404 } 404 }
(...skipping 173 matching lines...) Expand 10 before | Expand all | Expand 10 after
578 } 578 }
579 // It is OK to parse as a variable declaration in these cases: 579 // It is OK to parse as a variable declaration in these cases:
580 // String v } 580 // String v }
581 // String v if (true) print('OK'); 581 // String v if (true) print('OK');
582 // String v { print(42); } 582 // String v { print(42); }
583 // ...but not in these cases: 583 // ...but not in these cases:
584 // get getterName { 584 // get getterName {
585 // String get getterName 585 // String get getterName
586 if (allowAdditionalTokens) { 586 if (allowAdditionalTokens) {
587 if (type == TokenType.CLOSE_CURLY_BRACKET || 587 if (type == TokenType.CLOSE_CURLY_BRACKET ||
588 type == TokenType.KEYWORD || 588 type.isKeyword ||
589 type == TokenType.IDENTIFIER || 589 type == TokenType.IDENTIFIER ||
590 type == TokenType.OPEN_CURLY_BRACKET) { 590 type == TokenType.OPEN_CURLY_BRACKET) {
591 return true; 591 return true;
592 } 592 }
593 } 593 }
594 return false; 594 return false;
595 } 595 }
596 596
597 /** 597 /**
598 * Return `true` if the current token appears to be the beginning of a switch 598 * Return `true` if the current token appears to be the beginning of a switch
(...skipping 2561 matching lines...) Expand 10 before | Expand all | Expand 10 after
3160 if (type == TokenType.SEMICOLON) { 3160 if (type == TokenType.SEMICOLON) {
3161 if (!mayBeEmpty) { 3161 if (!mayBeEmpty) {
3162 _reportErrorForCurrentToken(emptyErrorCode); 3162 _reportErrorForCurrentToken(emptyErrorCode);
3163 } 3163 }
3164 return astFactory.emptyFunctionBody(getAndAdvance()); 3164 return astFactory.emptyFunctionBody(getAndAdvance());
3165 } 3165 }
3166 Token keyword = null; 3166 Token keyword = null;
3167 Token star = null; 3167 Token star = null;
3168 bool foundAsync = false; 3168 bool foundAsync = false;
3169 bool foundSync = false; 3169 bool foundSync = false;
3170 if (type == TokenType.KEYWORD) { 3170 if (type.isKeyword) {
3171 String lexeme = _currentToken.lexeme; 3171 String lexeme = _currentToken.lexeme;
3172 if (lexeme == ASYNC) { 3172 if (lexeme == ASYNC) {
3173 foundAsync = true; 3173 foundAsync = true;
3174 keyword = getAndAdvance(); 3174 keyword = getAndAdvance();
3175 if (_matches(TokenType.STAR)) { 3175 if (_matches(TokenType.STAR)) {
3176 star = getAndAdvance(); 3176 star = getAndAdvance();
3177 _inGenerator = true; 3177 _inGenerator = true;
3178 } 3178 }
3179 type = _currentToken.type; 3179 type = _currentToken.type;
3180 _inAsync = true; 3180 _inAsync = true;
(...skipping 845 matching lines...) Expand 10 before | Expand all | Expand 10 after
4026 TokenType type = _currentToken.type; 4026 TokenType type = _currentToken.type;
4027 if (type == TokenType.OPEN_CURLY_BRACKET) { 4027 if (type == TokenType.OPEN_CURLY_BRACKET) {
4028 if (_tokenMatches(_peek(), TokenType.STRING)) { 4028 if (_tokenMatches(_peek(), TokenType.STRING)) {
4029 Token afterString = skipStringLiteral(_currentToken.next); 4029 Token afterString = skipStringLiteral(_currentToken.next);
4030 if (afterString != null && afterString.type == TokenType.COLON) { 4030 if (afterString != null && afterString.type == TokenType.COLON) {
4031 return astFactory.expressionStatement( 4031 return astFactory.expressionStatement(
4032 parseExpression2(), _expect(TokenType.SEMICOLON)); 4032 parseExpression2(), _expect(TokenType.SEMICOLON));
4033 } 4033 }
4034 } 4034 }
4035 return parseBlock(); 4035 return parseBlock();
4036 } else if (type == TokenType.KEYWORD && 4036 } else if (type.isKeyword && !_currentToken.keyword.isBuiltInOrPseudo) {
4037 !_currentToken.keyword.isBuiltInOrPseudo) {
4038 Keyword keyword = _currentToken.keyword; 4037 Keyword keyword = _currentToken.keyword;
4039 // TODO(jwren) compute some metrics to figure out a better order for this 4038 // TODO(jwren) compute some metrics to figure out a better order for this
4040 // if-then sequence to optimize performance 4039 // if-then sequence to optimize performance
4041 if (keyword == Keyword.ASSERT) { 4040 if (keyword == Keyword.ASSERT) {
4042 return parseAssertStatement(); 4041 return parseAssertStatement();
4043 } else if (keyword == Keyword.BREAK) { 4042 } else if (keyword == Keyword.BREAK) {
4044 return parseBreakStatement(); 4043 return parseBreakStatement();
4045 } else if (keyword == Keyword.CONTINUE) { 4044 } else if (keyword == Keyword.CONTINUE) {
4046 return parseContinueStatement(); 4045 return parseContinueStatement();
4047 } else if (keyword == Keyword.DO) { 4046 } else if (keyword == Keyword.DO) {
(...skipping 1959 matching lines...) Expand 10 before | Expand all | Expand 10 after
6007 * after advancing to the next token. Otherwise report an error and return the 6006 * after advancing to the next token. Otherwise report an error and return the
6008 * current token without advancing. 6007 * current token without advancing.
6009 */ 6008 */
6010 Token _expectKeyword(Keyword keyword) { 6009 Token _expectKeyword(Keyword keyword) {
6011 if (_matchesKeyword(keyword)) { 6010 if (_matchesKeyword(keyword)) {
6012 return getAndAdvance(); 6011 return getAndAdvance();
6013 } 6012 }
6014 // Remove uses of this method in favor of matches? 6013 // Remove uses of this method in favor of matches?
6015 // Pass in the error code to use to report the error? 6014 // Pass in the error code to use to report the error?
6016 _reportErrorForCurrentToken( 6015 _reportErrorForCurrentToken(
6017 ParserErrorCode.EXPECTED_TOKEN, [keyword.syntax]); 6016 ParserErrorCode.EXPECTED_TOKEN, [keyword.lexeme]);
6018 return _currentToken; 6017 return _currentToken;
6019 } 6018 }
6020 6019
6021 /** 6020 /**
6022 * Search the given list of [ranges] for a range that contains the given 6021 * Search the given list of [ranges] for a range that contains the given
6023 * [index]. Return the range that was found, or `null` if none of the ranges 6022 * [index]. Return the range that was found, or `null` if none of the ranges
6024 * contain the index. 6023 * contain the index.
6025 */ 6024 */
6026 List<int> _findRange(List<List<int>> ranges, int index) { 6025 List<int> _findRange(List<List<int>> ranges, int index) {
6027 int rangeCount = ranges.length; 6026 int rangeCount = ranges.length;
(...skipping 537 matching lines...) Expand 10 before | Expand all | Expand 10 after
6565 Token classKeyword, 6564 Token classKeyword,
6566 SimpleIdentifier className, 6565 SimpleIdentifier className,
6567 TypeParameterList typeParameters) { 6566 TypeParameterList typeParameters) {
6568 Token equals = _expect(TokenType.EQ); 6567 Token equals = _expect(TokenType.EQ);
6569 TypeName superclass = parseTypeName(false); 6568 TypeName superclass = parseTypeName(false);
6570 WithClause withClause = null; 6569 WithClause withClause = null;
6571 if (_matchesKeyword(Keyword.WITH)) { 6570 if (_matchesKeyword(Keyword.WITH)) {
6572 withClause = parseWithClause(); 6571 withClause = parseWithClause();
6573 } else { 6572 } else {
6574 _reportErrorForCurrentToken( 6573 _reportErrorForCurrentToken(
6575 ParserErrorCode.EXPECTED_TOKEN, [Keyword.WITH.syntax]); 6574 ParserErrorCode.EXPECTED_TOKEN, [Keyword.WITH.lexeme]);
6576 } 6575 }
6577 ImplementsClause implementsClause = null; 6576 ImplementsClause implementsClause = null;
6578 if (_matchesKeyword(Keyword.IMPLEMENTS)) { 6577 if (_matchesKeyword(Keyword.IMPLEMENTS)) {
6579 implementsClause = parseImplementsClause(); 6578 implementsClause = parseImplementsClause();
6580 } 6579 }
6581 Token semicolon; 6580 Token semicolon;
6582 if (_matches(TokenType.SEMICOLON)) { 6581 if (_matches(TokenType.SEMICOLON)) {
6583 semicolon = getAndAdvance(); 6582 semicolon = getAndAdvance();
6584 } else { 6583 } else {
6585 if (_matches(TokenType.OPEN_CURLY_BRACKET)) { 6584 if (_matches(TokenType.OPEN_CURLY_BRACKET)) {
(...skipping 977 matching lines...) Expand 10 before | Expand all | Expand 10 after
7563 } 7562 }
7564 7563
7565 /** 7564 /**
7566 * Parse a string literal representing a URI. Return the string literal that 7565 * Parse a string literal representing a URI. Return the string literal that
7567 * was parsed. 7566 * was parsed.
7568 */ 7567 */
7569 StringLiteral _parseUri() { 7568 StringLiteral _parseUri() {
7570 // TODO(brianwilkerson) Should this function also return true for valid 7569 // TODO(brianwilkerson) Should this function also return true for valid
7571 // top-level keywords? 7570 // top-level keywords?
7572 bool isKeywordAfterUri(Token token) => 7571 bool isKeywordAfterUri(Token token) =>
7573 token.lexeme == Keyword.AS.syntax || 7572 token.lexeme == Keyword.AS.lexeme ||
7574 token.lexeme == _HIDE || 7573 token.lexeme == _HIDE ||
7575 token.lexeme == _SHOW; 7574 token.lexeme == _SHOW;
7576 TokenType type = _currentToken.type; 7575 TokenType type = _currentToken.type;
7577 if (type != TokenType.STRING && 7576 if (type != TokenType.STRING &&
7578 type != TokenType.SEMICOLON && 7577 type != TokenType.SEMICOLON &&
7579 !isKeywordAfterUri(_currentToken)) { 7578 !isKeywordAfterUri(_currentToken)) {
7580 // Attempt to recover in the case where the URI was not enclosed in 7579 // Attempt to recover in the case where the URI was not enclosed in
7581 // quotes. 7580 // quotes.
7582 Token token = _currentToken; 7581 Token token = _currentToken;
7583 bool isValidInUri(Token token) { 7582 bool isValidInUri(Token token) {
(...skipping 494 matching lines...) Expand 10 before | Expand all | Expand 10 after
8078 * include built-in identifiers (pseudo-keywords). 8077 * include built-in identifiers (pseudo-keywords).
8079 */ 8078 */
8080 bool _tokenMatchesIdentifier(Token token) => 8079 bool _tokenMatchesIdentifier(Token token) =>
8081 _tokenMatches(token, TokenType.IDENTIFIER) || 8080 _tokenMatches(token, TokenType.IDENTIFIER) ||
8082 _tokenMatchesPseudoKeyword(token); 8081 _tokenMatchesPseudoKeyword(token);
8083 8082
8084 /** 8083 /**
8085 * Return `true` if the given [token] is either an identifier or a keyword. 8084 * Return `true` if the given [token] is either an identifier or a keyword.
8086 */ 8085 */
8087 bool _tokenMatchesIdentifierOrKeyword(Token token) => 8086 bool _tokenMatchesIdentifierOrKeyword(Token token) =>
8088 _tokenMatches(token, TokenType.IDENTIFIER) || 8087 _tokenMatches(token, TokenType.IDENTIFIER) || token.type.isKeyword;
8089 _tokenMatches(token, TokenType.KEYWORD);
8090 8088
8091 /** 8089 /**
8092 * Return `true` if the given [token] matches the given [keyword]. 8090 * Return `true` if the given [token] matches the given [keyword].
8093 */ 8091 */
8094 bool _tokenMatchesKeyword(Token token, Keyword keyword) => 8092 bool _tokenMatchesKeyword(Token token, Keyword keyword) =>
8095 token.keyword == keyword; 8093 token.keyword == keyword;
8096 8094
8097 /** 8095 /**
8098 * Return `true` if the given [token] matches a pseudo keyword. 8096 * Return `true` if the given [token] matches a pseudo keyword.
8099 */ 8097 */
(...skipping 503 matching lines...) Expand 10 before | Expand all | Expand 10 after
8603 } 8601 }
8604 } 8602 }
8605 } 8603 }
8606 8604
8607 /** 8605 /**
8608 * Instances of this class are thrown when the parser detects that AST has 8606 * Instances of this class are thrown when the parser detects that AST has
8609 * too many nested expressions to be parsed safely and avoid possibility of 8607 * too many nested expressions to be parsed safely and avoid possibility of
8610 * [StackOverflowError] in the parser or during later analysis. 8608 * [StackOverflowError] in the parser or during later analysis.
8611 */ 8609 */
8612 class _TooDeepTreeError {} 8610 class _TooDeepTreeError {}
OLDNEW
« no previous file with comments | « pkg/analyzer/lib/src/generated/error_verifier.dart ('k') | pkg/analyzer/lib/src/generated/resolver.dart » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698