| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 part of scanner; | 5 part of scanner; |
| 6 | 6 |
| 7 abstract class Scanner { | 7 abstract class Scanner { |
| 8 Token tokenize(); | 8 Token tokenize(); |
| 9 | 9 |
| 10 factory Scanner(SourceFile file, {bool includeComments: false}) { | 10 factory Scanner(SourceFile file, {bool includeComments: false}) { |
| (...skipping 1078 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 1089 int advanceAfterError(bool shouldAdvance) { | 1089 int advanceAfterError(bool shouldAdvance) { |
| 1090 if (atEndOfFile()) return $EOF; | 1090 if (atEndOfFile()) return $EOF; |
| 1091 if (shouldAdvance) { | 1091 if (shouldAdvance) { |
| 1092 return advance(); // Ensure progress. | 1092 return advance(); // Ensure progress. |
| 1093 } else { | 1093 } else { |
| 1094 return -1; | 1094 return -1; |
| 1095 } | 1095 } |
| 1096 } | 1096 } |
| 1097 | 1097 |
| 1098 void unmatchedBeginGroup(BeginGroupToken begin) { | 1098 void unmatchedBeginGroup(BeginGroupToken begin) { |
| 1099 String error = 'unmatched "${begin.stringValue}"'; | 1099 // We want to ensure that unmatched BeginGroupTokens are reported as |
| 1100 Token close = | 1100 // errors. However, the diet parser assumes that groups are well-balanced |
| 1101 new StringToken.fromString( | 1101 // and will never look at the endGroup token. This is a nice property that |
| 1102 BAD_INPUT_INFO, error, begin.charOffset, canonicalize: true); | 1102 // allows us to skip quickly over correct code. By inserting an additional |
| 1103 | 1103 // synthetic token in the stream, we can keep ignoring endGroup tokens. |
| 1104 // We want to ensure that unmatched BeginGroupTokens are reported | |
| 1105 // as errors. However, the rest of the parser assumes the groups | |
| 1106 // are well-balanced and will never look at the endGroup | |
| 1107 // token. This is a nice property that allows us to skip quickly | |
| 1108 // over correct code. By inserting an additional error token in | |
| 1109 // the stream, we can keep ignoring endGroup tokens. | |
| 1110 // | 1104 // |
| 1111 // [begin] --next--> [tail] | 1105 // [begin] --next--> [tail] |
| 1112 // [begin] --endG--> [close] --next--> [next] --next--> [tail] | 1106 // [begin] --endG--> [synthetic] --next--> [next] --next--> [tail] |
| 1113 // | 1107 // |
| 1114 // This allows the parser to skip from [begin] via endGroup to [close] and | 1108 // This allows the diet parser to skip from [begin] via endGroup to |
| 1115 // ignore the [close] token (assuming it's correct), then the error will be | 1109 // [synthetic] and ignore the [synthetic] token (assuming it's correct), |
| 1116 // reported when parsing the [next] token. | 1110 // then the error will be reported when parsing the [next] token. |
| 1117 | 1111 // |
| 1118 Token next = new StringToken.fromString( | 1112 // For example, tokenize("{[1};") produces: |
| 1119 BAD_INPUT_INFO, error, begin.charOffset, canonicalize: true); | 1113 // |
| 1120 begin.endGroup = close; | 1114 // SymbolToken({) --endGroup-----+ |
| 1121 close.next = next; | 1115 // | | |
| 1122 next.next = begin.next; | 1116 // next | |
| 1117 // v | |
| 1118 // SymbolToken([) --endGroup--+ | |
| 1119 // | | | |
| 1120 // next | | |
| 1121 // v | | |
| 1122 // StringToken(1) | | |
| 1123 // | v | |
| 1124 // next SymbolToken(]) | <- Synthetic token. |
| 1125 // | | | |
| 1126 // | next | |
| 1127 // v | | |
| 1128 // UnmatchedToken([)<---------+ | |
| 1129 // | | |
| 1130 // next | |
| 1131 // v | |
| 1132 // SymbolToken(})<---------------+ |
| 1133 // | |
| 1134 // next |
| 1135 // v |
| 1136 // SymbolToken(;) |
| 1137 // | |
| 1138 // next |
| 1139 // v |
| 1140 // EOF |
| 1141 Token synthetic = |
| 1142 new SymbolToken(closeBraceInfoFor(begin), begin.charOffset); |
| 1143 UnmatchedToken next = new UnmatchedToken(begin); |
| 1144 begin.endGroup = synthetic; |
| 1145 synthetic.next = next; |
| 1146 appendErrorToken(next); |
| 1123 } | 1147 } |
| 1124 } | 1148 } |
| 1149 |
| 1150 PrecedenceInfo closeBraceInfoFor(BeginGroupToken begin) { |
| 1151 return const { |
| 1152 '(': CLOSE_PAREN_INFO, |
| 1153 '[': CLOSE_SQUARE_BRACKET_INFO, |
| 1154 '{': CLOSE_CURLY_BRACKET_INFO, |
| 1155 '<': GT_INFO, |
| 1156 r'${': CLOSE_CURLY_BRACKET_INFO, |
| 1157 }[begin.value]; |
| 1158 } |
| OLD | NEW |