| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.scanner.array_based_scanner; | 5 library fasta.scanner.array_based_scanner; |
| 6 | 6 |
| 7 import 'error_token.dart' show ErrorToken, UnmatchedToken; | 7 import 'error_token.dart' show ErrorToken, UnmatchedToken; |
| 8 | 8 |
| 9 import 'keyword.dart' show Keyword; | 9 import 'keyword.dart' show Keyword; |
| 10 | 10 |
| 11 import 'precedence.dart' show PrecedenceInfo; | 11 import 'precedence.dart' show PrecedenceInfo; |
| 12 | 12 |
| 13 import 'token.dart' | 13 import 'token.dart' |
| 14 show | 14 show |
| 15 BeginGroupToken, | 15 BeginGroupToken, |
| 16 CommentToken, | |
| 17 DartDocToken, | |
| 18 KeywordToken, | 16 KeywordToken, |
| 19 StringToken, | 17 StringToken, |
| 20 SymbolToken, | 18 SymbolToken, |
| 21 SyntheticSymbolToken, | 19 SyntheticSymbolToken, |
| 22 Token; | 20 Token; |
| 23 | 21 |
| 24 import 'token_constants.dart' | 22 import 'token_constants.dart' |
| 25 show | 23 show |
| 26 LT_TOKEN, | 24 LT_TOKEN, |
| 27 OPEN_CURLY_BRACKET_TOKEN, | 25 OPEN_CURLY_BRACKET_TOKEN, |
| (...skipping 13 matching lines...) Expand all Loading... |
| 41 : super(includeComments, numberOfBytesHint: numberOfBytesHint); | 39 : super(includeComments, numberOfBytesHint: numberOfBytesHint); |
| 42 | 40 |
| 43 /** | 41 /** |
| 44 * The stack of open groups, e.g [: { ... ( .. :] | 42 * The stack of open groups, e.g [: { ... ( .. :] |
| 45 * Each BeginGroupToken has a pointer to the token where the group | 43 * Each BeginGroupToken has a pointer to the token where the group |
| 46 * ends. This field is set when scanning the end group token. | 44 * ends. This field is set when scanning the end group token. |
| 47 */ | 45 */ |
| 48 Link<BeginGroupToken> groupingStack = const Link<BeginGroupToken>(); | 46 Link<BeginGroupToken> groupingStack = const Link<BeginGroupToken>(); |
| 49 | 47 |
| 50 /** | 48 /** |
| 51 * Append the given token to the [tail] of the current stream of tokens. | |
| 52 */ | |
| 53 void appendToken(Token token) { | |
| 54 tail.next = token; | |
| 55 tail.next.previousToken = tail; | |
| 56 tail = tail.next; | |
| 57 if (comments != null) { | |
| 58 tail.precedingCommentTokens = comments; | |
| 59 comments = null; | |
| 60 commentsTail = null; | |
| 61 } | |
| 62 } | |
| 63 | |
| 64 /** | |
| 65 * Appends a fixed token whose kind and content is determined by [info]. | 49 * Appends a fixed token whose kind and content is determined by [info]. |
| 66 * Appends an *operator* token from [info]. | 50 * Appends an *operator* token from [info]. |
| 67 * | 51 * |
| 68 * An operator token represent operators like ':', '.', ';', '&&', '==', '--', | 52 * An operator token represent operators like ':', '.', ';', '&&', '==', '--', |
| 69 * '=>', etc. | 53 * '=>', etc. |
| 70 */ | 54 */ |
| 71 void appendPrecedenceToken(PrecedenceInfo info) { | 55 void appendPrecedenceToken(PrecedenceInfo info) { |
| 72 appendToken(new SymbolToken(info, tokenStart)); | 56 appendToken(new SymbolToken(info, tokenStart)); |
| 73 } | 57 } |
| 74 | 58 |
| (...skipping 149 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 224 // '<', the inner '<' is left without endGroup. | 208 // '<', the inner '<' is left without endGroup. |
| 225 groupingStack = groupingStack.tail; | 209 groupingStack = groupingStack.tail; |
| 226 } | 210 } |
| 227 if (groupingStack.isEmpty) return; | 211 if (groupingStack.isEmpty) return; |
| 228 if (identical(groupingStack.head.kind, LT_TOKEN)) { | 212 if (identical(groupingStack.head.kind, LT_TOKEN)) { |
| 229 groupingStack.head.endGroup = tail; | 213 groupingStack.head.endGroup = tail; |
| 230 groupingStack = groupingStack.tail; | 214 groupingStack = groupingStack.tail; |
| 231 } | 215 } |
| 232 } | 216 } |
| 233 | 217 |
| 234 void appendComment(start, PrecedenceInfo info, bool asciiOnly) { | |
| 235 if (!includeComments) return; | |
| 236 Token newComment = createCommentToken(info, start, asciiOnly); | |
| 237 _appendToCommentStream(newComment); | |
| 238 } | |
| 239 | |
| 240 void appendDartDoc(start, PrecedenceInfo info, bool asciiOnly) { | |
| 241 if (!includeComments) return; | |
| 242 Token newComment = createDartDocToken(info, start, asciiOnly); | |
| 243 _appendToCommentStream(newComment); | |
| 244 } | |
| 245 | |
| 246 void _appendToCommentStream(Token newComment) { | |
| 247 if (comments == null) { | |
| 248 comments = newComment; | |
| 249 commentsTail = comments; | |
| 250 } else { | |
| 251 commentsTail.next = newComment; | |
| 252 commentsTail.next.previousToken = commentsTail; | |
| 253 commentsTail = commentsTail.next; | |
| 254 } | |
| 255 } | |
| 256 | |
| 257 void appendErrorToken(ErrorToken token) { | 218 void appendErrorToken(ErrorToken token) { |
| 258 hasErrors = true; | 219 hasErrors = true; |
| 259 appendToken(token); | 220 appendToken(token); |
| 260 } | 221 } |
| 261 | 222 |
| 262 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, | 223 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, |
| 263 [int extraOffset = 0]) { | 224 [int extraOffset = 0]) { |
| 264 appendToken(createSubstringToken(info, start, asciiOnly, extraOffset)); | 225 appendToken(createSubstringToken(info, start, asciiOnly, extraOffset)); |
| 265 } | 226 } |
| 266 | 227 |
| 267 /** | 228 /** |
| 268 * Returns a new substring from the scan offset [start] to the current | 229 * Returns a new substring from the scan offset [start] to the current |
| 269 * [scanOffset] plus the [extraOffset]. For example, if the current | 230 * [scanOffset] plus the [extraOffset]. For example, if the current |
| 270 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the | 231 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the |
| 271 * substring string [5,9). | 232 * substring string [5,9). |
| 272 * | 233 * |
| 273 * Note that [extraOffset] can only be used if the covered character(s) are | 234 * Note that [extraOffset] can only be used if the covered character(s) are |
| 274 * known to be ASCII. | 235 * known to be ASCII. |
| 275 */ | 236 */ |
| 276 StringToken createSubstringToken( | 237 StringToken createSubstringToken( |
| 277 PrecedenceInfo info, int start, bool asciiOnly, | 238 PrecedenceInfo info, int start, bool asciiOnly, |
| 278 [int extraOffset = 0]); | 239 [int extraOffset = 0]); |
| 279 | 240 |
| 280 /** | 241 /** |
| 281 * Returns a new comment from the scan offset [start] to the current | |
| 282 * [scanOffset] plus the [extraOffset]. For example, if the current | |
| 283 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the | |
| 284 * substring string [5,9). | |
| 285 * | |
| 286 * Note that [extraOffset] can only be used if the covered character(s) are | |
| 287 * known to be ASCII. | |
| 288 */ | |
| 289 CommentToken createCommentToken( | |
| 290 PrecedenceInfo info, int start, bool asciiOnly, | |
| 291 [int extraOffset = 0]); | |
| 292 | |
| 293 /** | |
| 294 * Returns a new dartdoc from the scan offset [start] to the current | |
| 295 * [scanOffset] plus the [extraOffset]. For example, if the current | |
| 296 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the | |
| 297 * substring string [5,9). | |
| 298 * | |
| 299 * Note that [extraOffset] can only be used if the covered character(s) are | |
| 300 * known to be ASCII. | |
| 301 */ | |
| 302 DartDocToken createDartDocToken( | |
| 303 PrecedenceInfo info, int start, bool asciiOnly, | |
| 304 [int extraOffset = 0]); | |
| 305 | |
| 306 /** | |
| 307 * This method is called to discard '<' from the "grouping" stack. | 242 * This method is called to discard '<' from the "grouping" stack. |
| 308 * | 243 * |
| 309 * [PartialParser.skipExpression] relies on the fact that we do not | 244 * [PartialParser.skipExpression] relies on the fact that we do not |
| 310 * create groups for stuff like: | 245 * create groups for stuff like: |
| 311 * [:a = b < c, d = e > f:]. | 246 * [:a = b < c, d = e > f:]. |
| 312 * | 247 * |
| 313 * In other words, this method is called when the scanner recognizes | 248 * In other words, this method is called when the scanner recognizes |
| 314 * something which cannot possibly be part of a type parameter/argument | 249 * something which cannot possibly be part of a type parameter/argument |
| 315 * list, like the '=' in the above example. | 250 * list, like the '=' in the above example. |
| 316 */ | 251 */ |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 365 // | | 300 // | |
| 366 // next | 301 // next |
| 367 // v | 302 // v |
| 368 // EOF | 303 // EOF |
| 369 PrecedenceInfo info = closeBraceInfoFor(begin); | 304 PrecedenceInfo info = closeBraceInfoFor(begin); |
| 370 appendToken(new SyntheticSymbolToken(info, tokenStart)); | 305 appendToken(new SyntheticSymbolToken(info, tokenStart)); |
| 371 begin.endGroup = tail; | 306 begin.endGroup = tail; |
| 372 appendErrorToken(new UnmatchedToken(begin)); | 307 appendErrorToken(new UnmatchedToken(begin)); |
| 373 } | 308 } |
| 374 } | 309 } |
| OLD | NEW |