| OLD | NEW |
| 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.analyzer.token_utils; | 5 library fasta.analyzer.token_utils; |
| 6 | 6 |
| 7 import 'package:front_end/src/fasta/scanner/error_token.dart' show ErrorToken; | 7 import 'package:front_end/src/fasta/scanner/error_token.dart' show ErrorToken; |
| 8 | 8 |
| 9 import 'package:front_end/src/scanner/token.dart' show Keyword; | 9 import 'package:front_end/src/scanner/token.dart' show Keyword; |
| 10 | 10 |
| (...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 81 /// Converts a stream of Fasta tokens (starting with [token] and continuing to | 81 /// Converts a stream of Fasta tokens (starting with [token] and continuing to |
| 82 /// EOF) to a stream of analyzer tokens. | 82 /// EOF) to a stream of analyzer tokens. |
| 83 analyzer.Token convertTokens(Token token) { | 83 analyzer.Token convertTokens(Token token) { |
| 84 _analyzerTokenHead = new analyzer.Token(TokenType.EOF, -1); | 84 _analyzerTokenHead = new analyzer.Token(TokenType.EOF, -1); |
| 85 _analyzerTokenHead.previous = _analyzerTokenHead; | 85 _analyzerTokenHead.previous = _analyzerTokenHead; |
| 86 _analyzerTokenTail = _analyzerTokenHead; | 86 _analyzerTokenTail = _analyzerTokenHead; |
| 87 _beginTokenStack = [null]; | 87 _beginTokenStack = [null]; |
| 88 _endTokenStack = <Token>[null]; | 88 _endTokenStack = <Token>[null]; |
| 89 | 89 |
| 90 while (true) { | 90 while (true) { |
| 91 if (token.info.kind == BAD_INPUT_TOKEN) { | 91 if (token.type.kind == BAD_INPUT_TOKEN) { |
| 92 ErrorToken errorToken = token; | 92 ErrorToken errorToken = token; |
| 93 translateErrorToken(errorToken, reportError); | 93 translateErrorToken(errorToken, reportError); |
| 94 } else { | 94 } else { |
| 95 var translatedToken = translateToken( | 95 var translatedToken = translateToken( |
| 96 token, translateCommentTokens(token.precedingCommentTokens)); | 96 token, translateCommentTokens(token.precedingCommentTokens)); |
| 97 _matchGroups(token, translatedToken); | 97 _matchGroups(token, translatedToken); |
| 98 translatedToken.setNext(translatedToken); | 98 translatedToken.setNext(translatedToken); |
| 99 _analyzerTokenTail.setNext(translatedToken); | 99 _analyzerTokenTail.setNext(translatedToken); |
| 100 translatedToken.previous = _analyzerTokenTail; | 100 translatedToken.previous = _analyzerTokenTail; |
| 101 _analyzerTokenTail = translatedToken; | 101 _analyzerTokenTail = translatedToken; |
| (...skipping 99 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 201 // so that we don't have to check if they're empty. | 201 // so that we don't have to check if they're empty. |
| 202 var beginTokenStack = <BeginGroupToken>[null]; | 202 var beginTokenStack = <BeginGroupToken>[null]; |
| 203 var endTokenStack = <analyzer.Token>[null]; | 203 var endTokenStack = <analyzer.Token>[null]; |
| 204 var angleBracketStack = <BeginGroupToken>[]; | 204 var angleBracketStack = <BeginGroupToken>[]; |
| 205 void matchGroups(analyzer.Token analyzerToken, Token translatedToken) { | 205 void matchGroups(analyzer.Token analyzerToken, Token translatedToken) { |
| 206 if (identical(endTokenStack.last, analyzerToken)) { | 206 if (identical(endTokenStack.last, analyzerToken)) { |
| 207 angleBracketStack.clear(); | 207 angleBracketStack.clear(); |
| 208 beginTokenStack.last.endGroup = translatedToken; | 208 beginTokenStack.last.endGroup = translatedToken; |
| 209 beginTokenStack.removeLast(); | 209 beginTokenStack.removeLast(); |
| 210 endTokenStack.removeLast(); | 210 endTokenStack.removeLast(); |
| 211 } else if (translatedToken.info.kind == LT_TOKEN) { | 211 } else if (translatedToken.type.kind == LT_TOKEN) { |
| 212 BeginGroupToken beginGroupToken = translatedToken; | 212 BeginGroupToken beginGroupToken = translatedToken; |
| 213 angleBracketStack.add(beginGroupToken); | 213 angleBracketStack.add(beginGroupToken); |
| 214 } else if (translatedToken.info.kind == GT_TOKEN && | 214 } else if (translatedToken.type.kind == GT_TOKEN && |
| 215 angleBracketStack.isNotEmpty) { | 215 angleBracketStack.isNotEmpty) { |
| 216 angleBracketStack.removeLast().endGroup = translatedToken; | 216 angleBracketStack.removeLast().endGroup = translatedToken; |
| 217 } else if (translatedToken.info.kind == GT_GT_TOKEN && | 217 } else if (translatedToken.type.kind == GT_GT_TOKEN && |
| 218 angleBracketStack.isNotEmpty) { | 218 angleBracketStack.isNotEmpty) { |
| 219 angleBracketStack.removeLast(); | 219 angleBracketStack.removeLast(); |
| 220 if (angleBracketStack.isNotEmpty) { | 220 if (angleBracketStack.isNotEmpty) { |
| 221 angleBracketStack.removeLast().endGroup = translatedToken; | 221 angleBracketStack.removeLast().endGroup = translatedToken; |
| 222 } | 222 } |
| 223 } | 223 } |
| 224 // TODO(paulberry): generate synthetic closer tokens and "UnmatchedToken" | 224 // TODO(paulberry): generate synthetic closer tokens and "UnmatchedToken" |
| 225 // tokens as appropriate. | 225 // tokens as appropriate. |
| 226 if (translatedToken is BeginGroupToken && | 226 if (translatedToken is BeginGroupToken && |
| 227 analyzerToken is analyzer.BeginToken && | 227 analyzerToken is analyzer.BeginToken && |
| (...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 268 translateComments(analyzerToken.precedingComments); | 268 translateComments(analyzerToken.precedingComments); |
| 269 tokenTail.next.next = tokenTail.next; | 269 tokenTail.next.next = tokenTail.next; |
| 270 return tokenHead.next; | 270 return tokenHead.next; |
| 271 } | 271 } |
| 272 analyzerToken = translateAndAppend(analyzerToken); | 272 analyzerToken = translateAndAppend(analyzerToken); |
| 273 } | 273 } |
| 274 } | 274 } |
| 275 | 275 |
| 276 /// Converts a single analyzer token into a Fasta token. | 276 /// Converts a single analyzer token into a Fasta token. |
| 277 Token fromAnalyzerToken(analyzer.Token token) { | 277 Token fromAnalyzerToken(analyzer.Token token) { |
| 278 Token beginGroup(TokenType info) => new BeginGroupToken(info, token.offset); | 278 Token beginGroup(TokenType type) => new BeginGroupToken(type, token.offset); |
| 279 Token string(TokenType info) => | 279 Token string(TokenType type) => |
| 280 new StringToken.fromString(info, token.lexeme, token.offset); | 280 new StringToken.fromString(type, token.lexeme, token.offset); |
| 281 Token symbol(TokenType info) => new SymbolToken(info, token.offset); | 281 Token symbol(TokenType type) => new SymbolToken(type, token.offset); |
| 282 if (token.type.isKeyword) { | 282 if (token.type.isKeyword) { |
| 283 var keyword = Keyword.keywords[token.lexeme]; | 283 var keyword = Keyword.keywords[token.lexeme]; |
| 284 if (keyword != null) { | 284 if (keyword != null) { |
| 285 return new KeywordToken(keyword, token.offset); | 285 return new KeywordToken(keyword, token.offset); |
| 286 } else { | 286 } else { |
| 287 return internalError("Unrecognized keyword: '${token.lexeme}'."); | 287 return internalError("Unrecognized keyword: '${token.lexeme}'."); |
| 288 } | 288 } |
| 289 } | 289 } |
| 290 switch (token.type) { | 290 switch (token.type) { |
| 291 case TokenType.DOUBLE: | 291 case TokenType.DOUBLE: |
| (...skipping 298 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 590 "hide": analyzer.Keyword.HIDE, | 590 "hide": analyzer.Keyword.HIDE, |
| 591 "native": analyzer.Keyword.NATIVE, | 591 "native": analyzer.Keyword.NATIVE, |
| 592 "of": analyzer.Keyword.OF, | 592 "of": analyzer.Keyword.OF, |
| 593 "on": analyzer.Keyword.ON, | 593 "on": analyzer.Keyword.ON, |
| 594 "patch": analyzer.Keyword.PATCH, | 594 "patch": analyzer.Keyword.PATCH, |
| 595 "show": analyzer.Keyword.SHOW, | 595 "show": analyzer.Keyword.SHOW, |
| 596 "source": analyzer.Keyword.SOURCE, | 596 "source": analyzer.Keyword.SOURCE, |
| 597 "sync": analyzer.Keyword.SYNC, | 597 "sync": analyzer.Keyword.SYNC, |
| 598 "yield": analyzer.Keyword.YIELD, | 598 "yield": analyzer.Keyword.YIELD, |
| 599 }; | 599 }; |
| OLD | NEW |