OLD | NEW |
| (Empty) |
1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | |
2 // for details. All rights reserved. Use of this source code is governed by a | |
3 // BSD-style license that can be found in the LICENSE file. | |
4 | |
5 part of scanner; | |
6 | |
7 abstract class ArrayBasedScanner extends AbstractScanner { | |
8 ArrayBasedScanner(SourceFile file, bool includeComments) | |
9 : super(file, includeComments); | |
10 | |
11 /** | |
12 * The stack of open groups, e.g [: { ... ( .. :] | |
13 * Each BeginGroupToken has a pointer to the token where the group | |
14 * ends. This field is set when scanning the end group token. | |
15 */ | |
16 Link<BeginGroupToken> groupingStack = const Link<BeginGroupToken>(); | |
17 | |
18 /** | |
19 * Appends a fixed token whose kind and content is determined by [info]. | |
20 * Appends an *operator* token from [info]. | |
21 * | |
22 * An operator token represent operators like ':', '.', ';', '&&', '==', '--', | |
23 * '=>', etc. | |
24 */ | |
25 void appendPrecedenceToken(PrecedenceInfo info) { | |
26 tail.next = new SymbolToken(info, tokenStart); | |
27 tail = tail.next; | |
28 } | |
29 | |
30 /** | |
31 * Appends a fixed token based on whether the current char is [choice] or not. | |
32 * If the current char is [choice] a fixed token whose kind and content | |
33 * is determined by [yes] is appended, otherwise a fixed token whose kind | |
34 * and content is determined by [no] is appended. | |
35 */ | |
36 int select(int choice, PrecedenceInfo yes, PrecedenceInfo no) { | |
37 int next = advance(); | |
38 if (identical(next, choice)) { | |
39 appendPrecedenceToken(yes); | |
40 return advance(); | |
41 } else { | |
42 appendPrecedenceToken(no); | |
43 return next; | |
44 } | |
45 } | |
46 | |
47 /** | |
48 * Appends a keyword token whose kind is determined by [keyword]. | |
49 */ | |
50 void appendKeywordToken(Keyword keyword) { | |
51 String syntax = keyword.syntax; | |
52 // Type parameters and arguments cannot contain 'this' or 'super'. | |
53 if (identical(syntax, 'this') || identical(syntax, 'super')) { | |
54 discardOpenLt(); | |
55 } | |
56 tail.next = new KeywordToken(keyword, tokenStart); | |
57 tail = tail.next; | |
58 } | |
59 | |
60 void appendEofToken() { | |
61 beginToken(); | |
62 discardOpenLt(); | |
63 while (!groupingStack.isEmpty) { | |
64 unmatchedBeginGroup(groupingStack.head); | |
65 groupingStack = groupingStack.tail; | |
66 } | |
67 tail.next = new SymbolToken(EOF_INFO, tokenStart); | |
68 tail = tail.next; | |
69 // EOF points to itself so there's always infinite look-ahead. | |
70 tail.next = tail; | |
71 } | |
72 | |
73 /** | |
74 * Notifies scanning a whitespace character. Note that [appendWhiteSpace] is | |
75 * not always invoked for [$SPACE] characters. | |
76 * | |
77 * This method is used by the scanners to track line breaks and create the | |
78 * [lineStarts] map. | |
79 */ | |
80 void appendWhiteSpace(int next) { | |
81 if (next == $LF && file != null) { | |
82 lineStarts.add(stringOffset + 1); // +1, the line starts after the $LF. | |
83 } | |
84 } | |
85 | |
86 /** | |
87 * Notifies on [$LF] characters in multi-line comments or strings. | |
88 * | |
89 * This method is used by the scanners to track line breaks and create the | |
90 * [lineStarts] map. | |
91 */ | |
92 void lineFeedInMultiline() { | |
93 if (file != null) { | |
94 lineStarts.add(stringOffset + 1); | |
95 } | |
96 } | |
97 | |
98 /** | |
99 * Appends a token that begins a new group, represented by [value]. | |
100 * Group begin tokens are '{', '(', '[' and '${'. | |
101 */ | |
102 void appendBeginGroup(PrecedenceInfo info) { | |
103 Token token = new BeginGroupToken(info, tokenStart); | |
104 tail.next = token; | |
105 tail = tail.next; | |
106 | |
107 // { ( [ ${ cannot appear inside a type parameters / arguments. | |
108 if (!identical(info.kind, LT_TOKEN)) discardOpenLt(); | |
109 groupingStack = groupingStack.prepend(token); | |
110 } | |
111 | |
112 /** | |
113 * Appends a token that begins an end group, represented by [value]. | |
114 * It handles the group end tokens '}', ')' and ']'. The tokens '>' and | |
115 * '>>' are handled separately bo [appendGt] and [appendGtGt]. | |
116 */ | |
117 int appendEndGroup(PrecedenceInfo info, int openKind) { | |
118 assert(!identical(openKind, LT_TOKEN)); // openKind is < for > and >> | |
119 discardBeginGroupUntil(openKind); | |
120 appendPrecedenceToken(info); | |
121 Token close = tail; | |
122 if (groupingStack.isEmpty) { | |
123 return advance(); | |
124 } | |
125 BeginGroupToken begin = groupingStack.head; | |
126 if (!identical(begin.kind, openKind)) { | |
127 assert(begin.kind == STRING_INTERPOLATION_TOKEN && | |
128 openKind == OPEN_CURLY_BRACKET_TOKEN); | |
129 // We're ending an interpolated expression. | |
130 begin.endGroup = close; | |
131 groupingStack = groupingStack.tail; | |
132 // Using "start-of-text" to signal that we're back in string | |
133 // scanning mode. | |
134 return $STX; | |
135 } | |
136 begin.endGroup = close; | |
137 groupingStack = groupingStack.tail; | |
138 return advance(); | |
139 } | |
140 | |
141 /** | |
142 * Discards begin group tokens until a match with [openKind] is found. | |
143 * This recovers nicely from from a situation like "{[}". | |
144 */ | |
145 void discardBeginGroupUntil(int openKind) { | |
146 while (!groupingStack.isEmpty) { | |
147 // Don't report unmatched errors for <; it is also the less-than operator. | |
148 discardOpenLt(); | |
149 if (groupingStack.isEmpty) return; | |
150 BeginGroupToken begin = groupingStack.head; | |
151 if (openKind == begin.kind) return; | |
152 if (openKind == OPEN_CURLY_BRACKET_TOKEN && | |
153 begin.kind == STRING_INTERPOLATION_TOKEN) return; | |
154 unmatchedBeginGroup(begin); | |
155 groupingStack = groupingStack.tail; | |
156 } | |
157 } | |
158 | |
159 /** | |
160 * Appends a token for '>'. | |
161 * This method does not issue unmatched errors, because > is also the | |
162 * greater-than operator. It does not necessarily have to close a group. | |
163 */ | |
164 void appendGt(PrecedenceInfo info) { | |
165 appendPrecedenceToken(info); | |
166 if (groupingStack.isEmpty) return; | |
167 if (identical(groupingStack.head.kind, LT_TOKEN)) { | |
168 groupingStack.head.endGroup = tail; | |
169 groupingStack = groupingStack.tail; | |
170 } | |
171 } | |
172 | |
173 /** | |
174 * Appends a token for '>>'. | |
175 * This method does not issue unmatched errors, because >> is also the | |
176 * shift operator. It does not necessarily have to close a group. | |
177 */ | |
178 void appendGtGt(PrecedenceInfo info) { | |
179 appendPrecedenceToken(info); | |
180 if (groupingStack.isEmpty) return; | |
181 if (identical(groupingStack.head.kind, LT_TOKEN)) { | |
182 // Don't assign endGroup: in "T<U<V>>", the '>>' token closes the outer | |
183 // '<', the inner '<' is left without endGroup. | |
184 groupingStack = groupingStack.tail; | |
185 } | |
186 if (groupingStack.isEmpty) return; | |
187 if (identical(groupingStack.head.kind, LT_TOKEN)) { | |
188 groupingStack.head.endGroup = tail; | |
189 groupingStack = groupingStack.tail; | |
190 } | |
191 } | |
192 | |
193 void appendComment(start, bool asciiOnly) { | |
194 if (!includeComments) return; | |
195 appendSubstringToken(COMMENT_INFO, start, asciiOnly); | |
196 } | |
197 | |
198 void appendErrorToken(ErrorToken token) { | |
199 tail.next = token; | |
200 tail = token; | |
201 } | |
202 | |
203 /** | |
204 * This method is called to discard '<' from the "grouping" stack. | |
205 * | |
206 * [PartialParser.skipExpression] relies on the fact that we do not | |
207 * create groups for stuff like: | |
208 * [:a = b < c, d = e > f:]. | |
209 * | |
210 * In other words, this method is called when the scanner recognizes | |
211 * something which cannot possibly be part of a type parameter/argument | |
212 * list, like the '=' in the above example. | |
213 */ | |
214 void discardOpenLt() { | |
215 while (!groupingStack.isEmpty | |
216 && identical(groupingStack.head.kind, LT_TOKEN)) { | |
217 groupingStack = groupingStack.tail; | |
218 } | |
219 } | |
220 } | |
OLD | NEW |