| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 library fasta.scanner.array_based_scanner; | 5 library fasta.scanner.array_based_scanner; |
| 6 | 6 |
| 7 import 'error_token.dart' show ErrorToken, UnmatchedToken; | 7 import 'error_token.dart' show ErrorToken, UnmatchedToken; |
| 8 | 8 |
| 9 import '../../scanner/token.dart' show Keyword; | 9 import '../../scanner/token.dart' show Keyword, TokenType; |
| 10 | |
| 11 import 'precedence.dart' show PrecedenceInfo; | |
| 12 | 10 |
| 13 import 'token.dart' | 11 import 'token.dart' |
| 14 show | 12 show |
| 15 BeginGroupToken, | 13 BeginGroupToken, |
| 16 KeywordToken, | 14 KeywordToken, |
| 17 StringToken, | 15 StringToken, |
| 18 SymbolToken, | 16 SymbolToken, |
| 19 SyntheticSymbolToken, | 17 SyntheticSymbolToken, |
| 20 Token; | 18 Token; |
| 21 | 19 |
| (...skipping 23 matching lines...) Expand all Loading... |
| 45 */ | 43 */ |
| 46 Link<BeginGroupToken> groupingStack = const Link<BeginGroupToken>(); | 44 Link<BeginGroupToken> groupingStack = const Link<BeginGroupToken>(); |
| 47 | 45 |
| 48 /** | 46 /** |
| 49 * Appends a fixed token whose kind and content is determined by [info]. | 47 * Appends a fixed token whose kind and content is determined by [info]. |
| 50 * Appends an *operator* token from [info]. | 48 * Appends an *operator* token from [info]. |
| 51 * | 49 * |
| 52 * An operator token represent operators like ':', '.', ';', '&&', '==', '--', | 50 * An operator token represent operators like ':', '.', ';', '&&', '==', '--', |
| 53 * '=>', etc. | 51 * '=>', etc. |
| 54 */ | 52 */ |
| 55 void appendPrecedenceToken(PrecedenceInfo info) { | 53 void appendPrecedenceToken(TokenType info) { |
| 56 appendToken(new SymbolToken(info, tokenStart)); | 54 appendToken(new SymbolToken(info, tokenStart)); |
| 57 } | 55 } |
| 58 | 56 |
| 59 /** | 57 /** |
| 60 * Appends a fixed token based on whether the current char is [choice] or not. | 58 * Appends a fixed token based on whether the current char is [choice] or not. |
| 61 * If the current char is [choice] a fixed token whose kind and content | 59 * If the current char is [choice] a fixed token whose kind and content |
| 62 * is determined by [yes] is appended, otherwise a fixed token whose kind | 60 * is determined by [yes] is appended, otherwise a fixed token whose kind |
| 63 * and content is determined by [no] is appended. | 61 * and content is determined by [no] is appended. |
| 64 */ | 62 */ |
| 65 int select(int choice, PrecedenceInfo yes, PrecedenceInfo no) { | 63 int select(int choice, TokenType yes, TokenType no) { |
| 66 int next = advance(); | 64 int next = advance(); |
| 67 if (identical(next, choice)) { | 65 if (identical(next, choice)) { |
| 68 appendPrecedenceToken(yes); | 66 appendPrecedenceToken(yes); |
| 69 return advance(); | 67 return advance(); |
| 70 } else { | 68 } else { |
| 71 appendPrecedenceToken(no); | 69 appendPrecedenceToken(no); |
| 72 return next; | 70 return next; |
| 73 } | 71 } |
| 74 } | 72 } |
| 75 | 73 |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 115 * [lineStarts] map. | 113 * [lineStarts] map. |
| 116 */ | 114 */ |
| 117 void lineFeedInMultiline() { | 115 void lineFeedInMultiline() { |
| 118 lineStarts.add(stringOffset + 1); | 116 lineStarts.add(stringOffset + 1); |
| 119 } | 117 } |
| 120 | 118 |
| 121 /** | 119 /** |
| 122 * Appends a token that begins a new group, represented by [info]. | 120 * Appends a token that begins a new group, represented by [info]. |
| 123 * Group begin tokens are '{', '(', '[' and '${'. | 121 * Group begin tokens are '{', '(', '[' and '${'. |
| 124 */ | 122 */ |
| 125 void appendBeginGroup(PrecedenceInfo info) { | 123 void appendBeginGroup(TokenType info) { |
| 126 Token token = new BeginGroupToken(info, tokenStart); | 124 Token token = new BeginGroupToken(info, tokenStart); |
| 127 appendToken(token); | 125 appendToken(token); |
| 128 | 126 |
| 129 // { [ ${ cannot appear inside a type parameters / arguments. | 127 // { [ ${ cannot appear inside a type parameters / arguments. |
| 130 if (!identical(info.kind, LT_TOKEN) && | 128 if (!identical(info.kind, LT_TOKEN) && |
| 131 !identical(info.kind, OPEN_PAREN_TOKEN)) { | 129 !identical(info.kind, OPEN_PAREN_TOKEN)) { |
| 132 discardOpenLt(); | 130 discardOpenLt(); |
| 133 } | 131 } |
| 134 groupingStack = groupingStack.prepend(token); | 132 groupingStack = groupingStack.prepend(token); |
| 135 } | 133 } |
| 136 | 134 |
| 137 /** | 135 /** |
| 138 * Appends a token that begins an end group, represented by [info]. | 136 * Appends a token that begins an end group, represented by [info]. |
| 139 * It handles the group end tokens '}', ')' and ']'. The tokens '>' and | 137 * It handles the group end tokens '}', ')' and ']'. The tokens '>' and |
| 140 * '>>' are handled separately bo [appendGt] and [appendGtGt]. | 138 * '>>' are handled separately bo [appendGt] and [appendGtGt]. |
| 141 */ | 139 */ |
| 142 int appendEndGroup(PrecedenceInfo info, int openKind) { | 140 int appendEndGroup(TokenType info, int openKind) { |
| 143 assert(!identical(openKind, LT_TOKEN)); // openKind is < for > and >> | 141 assert(!identical(openKind, LT_TOKEN)); // openKind is < for > and >> |
| 144 discardBeginGroupUntil(openKind); | 142 discardBeginGroupUntil(openKind); |
| 145 appendPrecedenceToken(info); | 143 appendPrecedenceToken(info); |
| 146 Token close = tail; | 144 Token close = tail; |
| 147 if (groupingStack.isEmpty) { | 145 if (groupingStack.isEmpty) { |
| 148 return advance(); | 146 return advance(); |
| 149 } | 147 } |
| 150 BeginGroupToken begin = groupingStack.head; | 148 BeginGroupToken begin = groupingStack.head; |
| 151 if (!identical(begin.kind, openKind)) { | 149 if (!identical(begin.kind, openKind)) { |
| 152 assert(begin.kind == STRING_INTERPOLATION_TOKEN && | 150 assert(begin.kind == STRING_INTERPOLATION_TOKEN && |
| (...skipping 26 matching lines...) Expand all Loading... |
| 179 unmatchedBeginGroup(begin); | 177 unmatchedBeginGroup(begin); |
| 180 groupingStack = groupingStack.tail; | 178 groupingStack = groupingStack.tail; |
| 181 } | 179 } |
| 182 } | 180 } |
| 183 | 181 |
| 184 /** | 182 /** |
| 185 * Appends a token for '>'. | 183 * Appends a token for '>'. |
| 186 * This method does not issue unmatched errors, because > is also the | 184 * This method does not issue unmatched errors, because > is also the |
| 187 * greater-than operator. It does not necessarily have to close a group. | 185 * greater-than operator. It does not necessarily have to close a group. |
| 188 */ | 186 */ |
| 189 void appendGt(PrecedenceInfo info) { | 187 void appendGt(TokenType info) { |
| 190 appendPrecedenceToken(info); | 188 appendPrecedenceToken(info); |
| 191 if (groupingStack.isEmpty) return; | 189 if (groupingStack.isEmpty) return; |
| 192 if (identical(groupingStack.head.kind, LT_TOKEN)) { | 190 if (identical(groupingStack.head.kind, LT_TOKEN)) { |
| 193 groupingStack.head.endGroup = tail; | 191 groupingStack.head.endGroup = tail; |
| 194 groupingStack = groupingStack.tail; | 192 groupingStack = groupingStack.tail; |
| 195 } | 193 } |
| 196 } | 194 } |
| 197 | 195 |
| 198 /** | 196 /** |
| 199 * Appends a token for '>>'. | 197 * Appends a token for '>>'. |
| 200 * This method does not issue unmatched errors, because >> is also the | 198 * This method does not issue unmatched errors, because >> is also the |
| 201 * shift operator. It does not necessarily have to close a group. | 199 * shift operator. It does not necessarily have to close a group. |
| 202 */ | 200 */ |
| 203 void appendGtGt(PrecedenceInfo info) { | 201 void appendGtGt(TokenType info) { |
| 204 appendPrecedenceToken(info); | 202 appendPrecedenceToken(info); |
| 205 if (groupingStack.isEmpty) return; | 203 if (groupingStack.isEmpty) return; |
| 206 if (identical(groupingStack.head.kind, LT_TOKEN)) { | 204 if (identical(groupingStack.head.kind, LT_TOKEN)) { |
| 207 // Don't assign endGroup: in "T<U<V>>", the '>>' token closes the outer | 205 // Don't assign endGroup: in "T<U<V>>", the '>>' token closes the outer |
| 208 // '<', the inner '<' is left without endGroup. | 206 // '<', the inner '<' is left without endGroup. |
| 209 groupingStack = groupingStack.tail; | 207 groupingStack = groupingStack.tail; |
| 210 } | 208 } |
| 211 if (groupingStack.isEmpty) return; | 209 if (groupingStack.isEmpty) return; |
| 212 if (identical(groupingStack.head.kind, LT_TOKEN)) { | 210 if (identical(groupingStack.head.kind, LT_TOKEN)) { |
| 213 groupingStack.head.endGroup = tail; | 211 groupingStack.head.endGroup = tail; |
| 214 groupingStack = groupingStack.tail; | 212 groupingStack = groupingStack.tail; |
| 215 } | 213 } |
| 216 } | 214 } |
| 217 | 215 |
| 218 void appendErrorToken(ErrorToken token) { | 216 void appendErrorToken(ErrorToken token) { |
| 219 hasErrors = true; | 217 hasErrors = true; |
| 220 appendToken(token); | 218 appendToken(token); |
| 221 } | 219 } |
| 222 | 220 |
| 223 void appendSubstringToken(PrecedenceInfo info, int start, bool asciiOnly, | 221 void appendSubstringToken(TokenType info, int start, bool asciiOnly, |
| 224 [int extraOffset = 0]) { | 222 [int extraOffset = 0]) { |
| 225 appendToken(createSubstringToken(info, start, asciiOnly, extraOffset)); | 223 appendToken(createSubstringToken(info, start, asciiOnly, extraOffset)); |
| 226 } | 224 } |
| 227 | 225 |
| 228 /** | 226 /** |
| 229 * Returns a new substring from the scan offset [start] to the current | 227 * Returns a new substring from the scan offset [start] to the current |
| 230 * [scanOffset] plus the [extraOffset]. For example, if the current | 228 * [scanOffset] plus the [extraOffset]. For example, if the current |
| 231 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the | 229 * scanOffset is 10, then [appendSubstringToken(5, -1)] will append the |
| 232 * substring string [5,9). | 230 * substring string [5,9). |
| 233 * | 231 * |
| 234 * Note that [extraOffset] can only be used if the covered character(s) are | 232 * Note that [extraOffset] can only be used if the covered character(s) are |
| 235 * known to be ASCII. | 233 * known to be ASCII. |
| 236 */ | 234 */ |
| 237 StringToken createSubstringToken( | 235 StringToken createSubstringToken(TokenType info, int start, bool asciiOnly, |
| 238 PrecedenceInfo info, int start, bool asciiOnly, | |
| 239 [int extraOffset = 0]); | 236 [int extraOffset = 0]); |
| 240 | 237 |
| 241 /** | 238 /** |
| 242 * This method is called to discard '<' from the "grouping" stack. | 239 * This method is called to discard '<' from the "grouping" stack. |
| 243 * | 240 * |
| 244 * [PartialParser.skipExpression] relies on the fact that we do not | 241 * [PartialParser.skipExpression] relies on the fact that we do not |
| 245 * create groups for stuff like: | 242 * create groups for stuff like: |
| 246 * [:a = b < c, d = e > f:]. | 243 * [:a = b < c, d = e > f:]. |
| 247 * | 244 * |
| 248 * In other words, this method is called when the scanner recognizes | 245 * In other words, this method is called when the scanner recognizes |
| (...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 294 // v | | 291 // v | |
| 295 // SymbolToken(})<----------------------------------+ | 292 // SymbolToken(})<----------------------------------+ |
| 296 // | | 293 // | |
| 297 // next | 294 // next |
| 298 // v | 295 // v |
| 299 // SymbolToken(;) | 296 // SymbolToken(;) |
| 300 // | | 297 // | |
| 301 // next | 298 // next |
| 302 // v | 299 // v |
| 303 // EOF | 300 // EOF |
| 304 PrecedenceInfo info = closeBraceInfoFor(begin); | 301 TokenType info = closeBraceInfoFor(begin); |
| 305 appendToken(new SyntheticSymbolToken(info, tokenStart)); | 302 appendToken(new SyntheticSymbolToken(info, tokenStart)); |
| 306 begin.endGroup = tail; | 303 begin.endGroup = tail; |
| 307 appendErrorToken(new UnmatchedToken(begin)); | 304 appendErrorToken(new UnmatchedToken(begin)); |
| 308 } | 305 } |
| 309 } | 306 } |
| OLD | NEW |