| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.scanner.array_based_scanner; | 5 library fasta.scanner.array_based_scanner; |
| 6 | 6 |
| 7 import 'error_token.dart' show ErrorToken, UnmatchedToken; | 7 import 'error_token.dart' show ErrorToken, UnmatchedToken; |
| 8 | 8 |
| 9 import '../../scanner/token.dart' show Keyword, TokenType; | 9 import '../../scanner/token.dart' show Keyword, TokenType; |
| 10 | 10 |
| (...skipping 28 matching lines...) Expand all Loading... |
| 39 numberOfBytesHint: numberOfBytesHint); | 39 numberOfBytesHint: numberOfBytesHint); |
| 40 | 40 |
| 41 /** | 41 /** |
| 42 * The stack of open groups, e.g [: { ... ( .. :] | 42 * The stack of open groups, e.g [: { ... ( .. :] |
| 43 * Each BeginGroupToken has a pointer to the token where the group | 43 * Each BeginGroupToken has a pointer to the token where the group |
| 44 * ends. This field is set when scanning the end group token. | 44 * ends. This field is set when scanning the end group token. |
| 45 */ | 45 */ |
| 46 Link<BeginGroupToken> groupingStack = const Link<BeginGroupToken>(); | 46 Link<BeginGroupToken> groupingStack = const Link<BeginGroupToken>(); |
| 47 | 47 |
| 48 /** | 48 /** |
| 49 * Appends a fixed token whose kind and content is determined by [info]. | 49 * Appends a fixed token whose kind and content is determined by [type]. |
| 50 * Appends an *operator* token from [info]. | 50 * Appends an *operator* token from [type]. |
| 51 * | 51 * |
| 52 * An operator token represent operators like ':', '.', ';', '&&', '==', '--', | 52 * An operator token represent operators like ':', '.', ';', '&&', '==', '--', |
| 53 * '=>', etc. | 53 * '=>', etc. |
| 54 */ | 54 */ |
| 55 void appendPrecedenceToken(TokenType info) { | 55 void appendPrecedenceToken(TokenType type) { |
| 56 appendToken(new SymbolToken(info, tokenStart)); | 56 appendToken(new SymbolToken(type, tokenStart)); |
| 57 } | 57 } |
| 58 | 58 |
| 59 /** | 59 /** |
| 60 * Appends a fixed token based on whether the current char is [choice] or not. | 60 * Appends a fixed token based on whether the current char is [choice] or not. |
| 61 * If the current char is [choice] a fixed token whose kind and content | 61 * If the current char is [choice] a fixed token whose kind and content |
| 62 * is determined by [yes] is appended, otherwise a fixed token whose kind | 62 * is determined by [yes] is appended, otherwise a fixed token whose kind |
| 63 * and content is determined by [no] is appended. | 63 * and content is determined by [no] is appended. |
| 64 */ | 64 */ |
| 65 int select(int choice, TokenType yes, TokenType no) { | 65 int select(int choice, TokenType yes, TokenType no) { |
| 66 int next = advance(); | 66 int next = advance(); |
| (...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 112 * Notifies on [$LF] characters in multi-line comments or strings. | 112 * Notifies on [$LF] characters in multi-line comments or strings. |
| 113 * | 113 * |
| 114 * This method is used by the scanners to track line breaks and create the | 114 * This method is used by the scanners to track line breaks and create the |
| 115 * [lineStarts] map. | 115 * [lineStarts] map. |
| 116 */ | 116 */ |
| 117 void lineFeedInMultiline() { | 117 void lineFeedInMultiline() { |
| 118 lineStarts.add(stringOffset + 1); | 118 lineStarts.add(stringOffset + 1); |
| 119 } | 119 } |
| 120 | 120 |
| 121 /** | 121 /** |
| 122 * Appends a token that begins a new group, represented by [info]. | 122 * Appends a token that begins a new group, represented by [type]. |
| 123 * Group begin tokens are '{', '(', '[' and '${'. | 123 * Group begin tokens are '{', '(', '[' and '${'. |
| 124 */ | 124 */ |
| 125 void appendBeginGroup(TokenType info) { | 125 void appendBeginGroup(TokenType type) { |
| 126 Token token = new BeginGroupToken(info, tokenStart); | 126 Token token = new BeginGroupToken(type, tokenStart); |
| 127 appendToken(token); | 127 appendToken(token); |
| 128 | 128 |
| 129 // { [ ${ cannot appear inside a type parameters / arguments. | 129 // { [ ${ cannot appear inside a type parameters / arguments. |
| 130 if (!identical(info.kind, LT_TOKEN) && | 130 if (!identical(type.kind, LT_TOKEN) && |
| 131 !identical(info.kind, OPEN_PAREN_TOKEN)) { | 131 !identical(type.kind, OPEN_PAREN_TOKEN)) { |
| 132 discardOpenLt(); | 132 discardOpenLt(); |
| 133 } | 133 } |
| 134 groupingStack = groupingStack.prepend(token); | 134 groupingStack = groupingStack.prepend(token); |
| 135 } | 135 } |
| 136 | 136 |
| 137 /** | 137 /** |
| 138 * Appends a token that begins an end group, represented by [info]. | 138 * Appends a token that begins an end group, represented by [type]. |
| 139 * It handles the group end tokens '}', ')' and ']'. The tokens '>' and | 139 * It handles the group end tokens '}', ')' and ']'. The tokens '>' and |
| 140 * '>>' are handled separately bo [appendGt] and [appendGtGt]. | 140 * '>>' are handled separately bo [appendGt] and [appendGtGt]. |
| 141 */ | 141 */ |
| 142 int appendEndGroup(TokenType info, int openKind) { | 142 int appendEndGroup(TokenType type, int openKind) { |
| 143 assert(!identical(openKind, LT_TOKEN)); // openKind is < for > and >> | 143 assert(!identical(openKind, LT_TOKEN)); // openKind is < for > and >> |
| 144 discardBeginGroupUntil(openKind); | 144 discardBeginGroupUntil(openKind); |
| 145 appendPrecedenceToken(info); | 145 appendPrecedenceToken(type); |
| 146 Token close = tail; | 146 Token close = tail; |
| 147 if (groupingStack.isEmpty) { | 147 if (groupingStack.isEmpty) { |
| 148 return advance(); | 148 return advance(); |
| 149 } | 149 } |
| 150 BeginGroupToken begin = groupingStack.head; | 150 BeginGroupToken begin = groupingStack.head; |
| 151 if (!identical(begin.kind, openKind)) { | 151 if (!identical(begin.kind, openKind)) { |
| 152 assert(begin.kind == STRING_INTERPOLATION_TOKEN && | 152 assert(begin.kind == STRING_INTERPOLATION_TOKEN && |
| 153 openKind == OPEN_CURLY_BRACKET_TOKEN); | 153 openKind == OPEN_CURLY_BRACKET_TOKEN); |
| 154 // We're ending an interpolated expression. | 154 // We're ending an interpolated expression. |
| 155 begin.endGroup = close; | 155 begin.endGroup = close; |
| (...skipping 23 matching lines...) Expand all Loading... |
| 179 unmatchedBeginGroup(begin); | 179 unmatchedBeginGroup(begin); |
| 180 groupingStack = groupingStack.tail; | 180 groupingStack = groupingStack.tail; |
| 181 } | 181 } |
| 182 } | 182 } |
| 183 | 183 |
| 184 /** | 184 /** |
| 185 * Appends a token for '>'. | 185 * Appends a token for '>'. |
| 186 * This method does not issue unmatched errors, because > is also the | 186 * This method does not issue unmatched errors, because > is also the |
| 187 * greater-than operator. It does not necessarily have to close a group. | 187 * greater-than operator. It does not necessarily have to close a group. |
| 188 */ | 188 */ |
| 189 void appendGt(TokenType info) { | 189 void appendGt(TokenType type) { |
| 190 appendPrecedenceToken(info); | 190 appendPrecedenceToken(type); |
| 191 if (groupingStack.isEmpty) return; | 191 if (groupingStack.isEmpty) return; |
| 192 if (identical(groupingStack.head.kind, LT_TOKEN)) { | 192 if (identical(groupingStack.head.kind, LT_TOKEN)) { |
| 193 groupingStack.head.endGroup = tail; | 193 groupingStack.head.endGroup = tail; |
| 194 groupingStack = groupingStack.tail; | 194 groupingStack = groupingStack.tail; |
| 195 } | 195 } |
| 196 } | 196 } |
| 197 | 197 |
| 198 /** | 198 /** |
| 199 * Appends a token for '>>'. | 199 * Appends a token for '>>'. |
| 200 * This method does not issue unmatched errors, because >> is also the | 200 * This method does not issue unmatched errors, because >> is also the |
| 201 * shift operator. It does not necessarily have to close a group. | 201 * shift operator. It does not necessarily have to close a group. |
| 202 */ | 202 */ |
| 203 void appendGtGt(TokenType info) { | 203 void appendGtGt(TokenType type) { |
| 204 appendPrecedenceToken(info); | 204 appendPrecedenceToken(type); |
| 205 if (groupingStack.isEmpty) return; | 205 if (groupingStack.isEmpty) return; |
| 206 if (identical(groupingStack.head.kind, LT_TOKEN)) { | 206 if (identical(groupingStack.head.kind, LT_TOKEN)) { |
| 207 // Don't assign endGroup: in "T<U<V>>", the '>>' token closes the outer | 207 // Don't assign endGroup: in "T<U<V>>", the '>>' token closes the outer |
| 208 // '<', the inner '<' is left without endGroup. | 208 // '<', the inner '<' is left without endGroup. |
| 209 groupingStack = groupingStack.tail; | 209 groupingStack = groupingStack.tail; |
| 210 } | 210 } |
| 211 if (groupingStack.isEmpty) return; | 211 if (groupingStack.isEmpty) return; |
| 212 if (identical(groupingStack.head.kind, LT_TOKEN)) { | 212 if (identical(groupingStack.head.kind, LT_TOKEN)) { |
| 213 groupingStack.head.endGroup = tail; | 213 groupingStack.head.endGroup = tail; |
| 214 groupingStack = groupingStack.tail; | 214 groupingStack = groupingStack.tail; |
| 215 } | 215 } |
| 216 } | 216 } |
| 217 | 217 |
| 218 void appendErrorToken(ErrorToken token) { | 218 void appendErrorToken(ErrorToken token) { |
| 219 hasErrors = true; | 219 hasErrors = true; |
| 220 appendToken(token); | 220 appendToken(token); |
| 221 } | 221 } |
| 222 | 222 |
| 223 void appendSubstringToken(TokenType info, int start, bool asciiOnly, | 223 void appendSubstringToken(TokenType type, int start, bool asciiOnly, |
| 224 [int extraOffset = 0]) { | 224 [int extraOffset = 0]) { |
| 225 appendToken(createSubstringToken(info, start, asciiOnly, extraOffset)); | 225 appendToken(createSubstringToken(type, start, asciiOnly, extraOffset)); |
| 226 } | 226 } |
| 227 | 227 |
| 228 /** | 228 /** |
| 229 * Returns a new substring from the scan offset [start] to the current | 229 * Returns a new substring from the scan offset [start] to the current |
| 230 * [scanOffset] plus the [extraOffset]. For example, if the current | 230 * [scanOffset] plus the [extraOffset]. For example, if the current |
| 231 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the | 231 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the |
| 232 * substring string [5,9). | 232 * substring string [5,9). |
| 233 * | 233 * |
| 234 * Note that [extraOffset] can only be used if the covered character(s) are | 234 * Note that [extraOffset] can only be used if the covered character(s) are |
| 235 * known to be ASCII. | 235 * known to be ASCII. |
| 236 */ | 236 */ |
| 237 StringToken createSubstringToken(TokenType info, int start, bool asciiOnly, | 237 StringToken createSubstringToken(TokenType type, int start, bool asciiOnly, |
| 238 [int extraOffset = 0]); | 238 [int extraOffset = 0]); |
| 239 | 239 |
| 240 /** | 240 /** |
| 241 * This method is called to discard '<' from the "grouping" stack. | 241 * This method is called to discard '<' from the "grouping" stack. |
| 242 * | 242 * |
| 243 * [PartialParser.skipExpression] relies on the fact that we do not | 243 * [PartialParser.skipExpression] relies on the fact that we do not |
| 244 * create groups for stuff like: | 244 * create groups for stuff like: |
| 245 * [:a = b < c, d = e > f:]. | 245 * [:a = b < c, d = e > f:]. |
| 246 * | 246 * |
| 247 * In other words, this method is called when the scanner recognizes | 247 * In other words, this method is called when the scanner recognizes |
| (...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 293 // v | | 293 // v | |
| 294 // SymbolToken(})<----------------------------------+ | 294 // SymbolToken(})<----------------------------------+ |
| 295 // | | 295 // | |
| 296 // next | 296 // next |
| 297 // v | 297 // v |
| 298 // SymbolToken(;) | 298 // SymbolToken(;) |
| 299 // | | 299 // | |
| 300 // next | 300 // next |
| 301 // v | 301 // v |
| 302 // EOF | 302 // EOF |
| 303 TokenType info = closeBraceInfoFor(begin); | 303 TokenType type = closeBraceInfoFor(begin); |
| 304 appendToken(new SyntheticSymbolToken(info, tokenStart)); | 304 appendToken(new SyntheticSymbolToken(type, tokenStart)); |
| 305 begin.endGroup = tail; | 305 begin.endGroup = tail; |
| 306 appendErrorToken(new UnmatchedToken(begin)); | 306 appendErrorToken(new UnmatchedToken(begin)); |
| 307 } | 307 } |
| 308 } | 308 } |
| OLD | NEW |