OLD | NEW |
1 library angular.core.parser.lexer; | 1 library angular.core.parser.lexer; |
2 | 2 |
3 import 'package:angular/core/module.dart' show NgInjectableService; | 3 import 'package:angular/core/annotation_src.dart'; |
4 import 'package:angular/core/parser/characters.dart'; | 4 import 'package:angular/core/parser/characters.dart'; |
5 | 5 |
6 class Token { | 6 part 'tokens.dart'; |
7 final int index; | |
8 final String text; | |
9 | 7 |
10 var value; | 8 @Injectable() |
11 // Tokens should have one of these set. | |
12 String opKey; | |
13 String key; | |
14 | |
15 Token(this.index, this.text); | |
16 | |
17 withOp(op) { | |
18 this.opKey = op; | |
19 } | |
20 | |
21 withGetterSetter(key) { | |
22 this.key = key; | |
23 } | |
24 | |
25 withValue(value) { this.value = value; } | |
26 | |
27 toString() => "Token($text)"; | |
28 } | |
29 | |
30 | |
31 @NgInjectableService() | |
32 class Lexer { | 9 class Lexer { |
33 List<Token> call(String text) { | 10 List<Token> call(String text) { |
34 Scanner scanner = new Scanner(text); | 11 Scanner scanner = new Scanner(text); |
35 List<Token> tokens = []; | 12 List<Token> tokens = []; |
36 Token token = scanner.scanToken(); | 13 Token token = scanner.scanToken(); |
37 while (token != null) { | 14 while (token != null) { |
38 tokens.add(token); | 15 tokens.add(token); |
39 token = scanner.scanToken(); | 16 token = scanner.scanToken(); |
40 } | 17 } |
41 return tokens; | 18 return tokens; |
(...skipping 23 matching lines...) Expand all Loading... |
65 } | 42 } |
66 | 43 |
67 // Handle identifiers and numbers. | 44 // Handle identifiers and numbers. |
68 if (isIdentifierStart(peek)) return scanIdentifier(); | 45 if (isIdentifierStart(peek)) return scanIdentifier(); |
69 if (isDigit(peek)) return scanNumber(index); | 46 if (isDigit(peek)) return scanNumber(index); |
70 | 47 |
71 int start = index; | 48 int start = index; |
72 switch (peek) { | 49 switch (peek) { |
73 case $PERIOD: | 50 case $PERIOD: |
74 advance(); | 51 advance(); |
75 return isDigit(peek) ? scanNumber(start) : new Token(start, '.'); | 52 return isDigit(peek) ? scanNumber(start) : new CharacterToken(start, $PE
RIOD); |
76 case $LPAREN: | 53 case $LPAREN: |
77 case $RPAREN: | 54 case $RPAREN: |
78 case $LBRACE: | 55 case $LBRACE: |
79 case $RBRACE: | 56 case $RBRACE: |
80 case $LBRACKET: | 57 case $LBRACKET: |
81 case $RBRACKET: | 58 case $RBRACKET: |
82 case $COMMA: | 59 case $COMMA: |
83 case $COLON: | 60 case $COLON: |
84 case $SEMICOLON: | 61 case $SEMICOLON: |
85 return scanCharacter(start, new String.fromCharCode(peek)); | 62 return scanCharacter(start, peek); |
86 case $SQ: | 63 case $SQ: |
87 case $DQ: | 64 case $DQ: |
88 return scanString(); | 65 return scanString(); |
89 case $PLUS: | 66 case $PLUS: |
90 case $MINUS: | 67 case $MINUS: |
91 case $STAR: | 68 case $STAR: |
92 case $SLASH: | 69 case $SLASH: |
93 case $PERCENT: | 70 case $PERCENT: |
94 case $CARET: | 71 case $CARET: |
95 case $QUESTION: | 72 case $QUESTION: |
(...skipping 12 matching lines...) Expand all Loading... |
108 case $NBSP: | 85 case $NBSP: |
109 while (isWhitespace(peek)) advance(); | 86 while (isWhitespace(peek)) advance(); |
110 return scanToken(); | 87 return scanToken(); |
111 } | 88 } |
112 | 89 |
113 String character = new String.fromCharCode(peek); | 90 String character = new String.fromCharCode(peek); |
114 error('Unexpected character [$character]'); | 91 error('Unexpected character [$character]'); |
115 return null; | 92 return null; |
116 } | 93 } |
117 | 94 |
118 Token scanCharacter(int start, String string) { | 95 Token scanCharacter(int start, int code) { |
119 assert(peek == string.codeUnitAt(0)); | 96 assert(peek == code); |
120 advance(); | 97 advance(); |
121 return new Token(start, string); | 98 return new CharacterToken(start, code); |
122 } | 99 } |
123 | 100 |
124 Token scanOperator(int start, String string) { | 101 Token scanOperator(int start, String string) { |
125 assert(peek == string.codeUnitAt(0)); | 102 assert(peek == string.codeUnitAt(0)); |
126 assert(OPERATORS.contains(string)); | 103 assert(OPERATORS.contains(string)); |
127 advance(); | 104 advance(); |
128 return new Token(start, string)..withOp(string); | 105 return new OperatorToken(start, string); |
129 } | 106 } |
130 | 107 |
131 Token scanComplexOperator(int start, int code, String one, String two) { | 108 Token scanComplexOperator(int start, int code, String one, String two) { |
132 assert(peek == one.codeUnitAt(0)); | 109 assert(peek == one.codeUnitAt(0)); |
133 advance(); | 110 advance(); |
134 String string = one; | 111 String string = one; |
135 if (peek == code) { | 112 if (peek == code) { |
136 advance(); | 113 advance(); |
137 string += two; | 114 string += two; |
138 } | 115 } |
139 assert(OPERATORS.contains(string)); | 116 assert(OPERATORS.contains(string)); |
140 return new Token(start, string)..withOp(string); | 117 return new OperatorToken(start, string); |
141 } | 118 } |
142 | 119 |
143 Token scanIdentifier() { | 120 Token scanIdentifier() { |
144 assert(isIdentifierStart(peek)); | 121 assert(isIdentifierStart(peek)); |
145 int start = index; | 122 int start = index; |
146 advance(); | 123 advance(); |
147 while (isIdentifierPart(peek)) advance(); | 124 while (isIdentifierPart(peek)) advance(); |
148 String string = input.substring(start, index); | 125 String string = input.substring(start, index); |
149 Token result = new Token(start, string); | 126 return new IdentifierToken(start, string, KEYWORDS.contains(string)); |
150 // TODO(kasperl): Deal with null, undefined, true, and false in | |
151 // a cleaner and faster way. | |
152 if (OPERATORS.contains(string)) { | |
153 result.withOp(string); | |
154 } else { | |
155 result.withGetterSetter(string); | |
156 } | |
157 return result; | |
158 } | 127 } |
159 | 128 |
160 Token scanNumber(int start) { | 129 Token scanNumber(int start) { |
161 assert(isDigit(peek)); | 130 assert(isDigit(peek)); |
162 bool simple = (index == start); | 131 bool simple = (index == start); |
163 advance(); // Skip initial digit. | 132 advance(); // Skip initial digit. |
164 while (true) { | 133 while (true) { |
165 if (isDigit(peek)) { | 134 if (isDigit(peek)) { |
166 // Do nothing. | 135 // Do nothing. |
167 } else if (peek == $PERIOD) { | 136 } else if (peek == $PERIOD) { |
168 simple = false; | 137 simple = false; |
169 } else if (isExponentStart(peek)) { | 138 } else if (isExponentStart(peek)) { |
170 advance(); | 139 advance(); |
171 if (isExponentSign(peek)) advance(); | 140 if (isExponentSign(peek)) advance(); |
172 if (!isDigit(peek)) error('Invalid exponent', -1); | 141 if (!isDigit(peek)) error('Invalid exponent', -1); |
173 simple = false; | 142 simple = false; |
174 } else { | 143 } else { |
175 break; | 144 break; |
176 } | 145 } |
177 advance(); | 146 advance(); |
178 } | 147 } |
179 String string = input.substring(start, index); | 148 String string = input.substring(start, index); |
180 num value = simple ? int.parse(string) : double.parse(string); | 149 num value = simple ? int.parse(string) : double.parse(string); |
181 return new Token(start, string)..withValue(value); | 150 return new NumberToken(start, value); |
182 } | 151 } |
183 | 152 |
184 Token scanString() { | 153 Token scanString() { |
185 assert(peek == $SQ || peek == $DQ); | 154 assert(peek == $SQ || peek == $DQ); |
186 int start = index; | 155 int start = index; |
187 int quote = peek; | 156 int quote = peek; |
188 advance(); // Skip initial quote. | 157 advance(); // Skip initial quote. |
189 | 158 |
190 StringBuffer buffer; | 159 StringBuffer buffer; |
191 int marker = index; | 160 int marker = index; |
192 | 161 |
193 while (peek != quote) { | 162 while (peek != quote) { |
194 if (peek == $BACKSLASH) { | 163 if (peek == $BACKSLASH) { |
195 if (buffer == null) buffer = new StringBuffer(); | 164 if (buffer == null) buffer = new StringBuffer(); |
196 buffer.write(input.substring(marker, index)); | 165 buffer.write(input.substring(marker, index)); |
197 advance(); | 166 advance(); |
198 int unescaped; | 167 int unescaped; |
199 if (peek == $u) { | 168 if (peek == $u) { |
200 // TODO(kasperl): Check bounds? Make sure we have test | 169 // TODO(kasperl): Check bounds? Make sure we have test |
201 // coverage for this. | 170 // coverage for this. |
202 String hex = input.substring(index + 1, index + 5); | 171 String hex = input.substring(index + 1, index + 5); |
203 unescaped = int.parse(hex, radix: 16, onError: (ignore) { | 172 unescaped = int.parse(hex, radix: 16, onError: (ignore) { |
204 error('Invalid unicode escape [\\u$hex]'); }); | 173 error('Invalid unicode escape [\\u$hex]'); }); |
205 for (int i = 0; i < 5; i++) advance(); | 174 for (int i = 0; i < 5; i++) { |
| 175 advance(); |
| 176 } |
206 } else { | 177 } else { |
207 unescaped = unescape(peek); | 178 unescaped = unescape(peek); |
208 advance(); | 179 advance(); |
209 } | 180 } |
210 buffer.writeCharCode(unescaped); | 181 buffer.writeCharCode(unescaped); |
211 marker = index; | 182 marker = index; |
212 } else if (peek == $EOF) { | 183 } else if (peek == $EOF) { |
213 error('Unterminated quote'); | 184 error('Unterminated quote'); |
214 } else { | 185 } else { |
215 advance(); | 186 advance(); |
216 } | 187 } |
217 } | 188 } |
218 | 189 |
219 String last = input.substring(marker, index); | 190 String last = input.substring(marker, index); |
220 advance(); // Skip terminating quote. | 191 advance(); // Skip terminating quote. |
221 String string = input.substring(start, index); | 192 String string = input.substring(start, index); |
222 | 193 |
223 // Compute the unescaped string value. | 194 // Compute the unescaped string value. |
224 String unescaped = last; | 195 String unescaped = last; |
225 if (buffer != null) { | 196 if (buffer != null) { |
226 buffer.write(last); | 197 buffer.write(last); |
227 unescaped = buffer.toString(); | 198 unescaped = buffer.toString(); |
228 } | 199 } |
229 return new Token(start, string)..withValue(unescaped); | 200 return new StringToken(start, string, unescaped); |
230 } | 201 } |
231 | 202 |
232 void advance() { | 203 void advance() { |
233 if (++index >= length) peek = $EOF; | 204 peek = ++index >= length ? $EOF : input.codeUnitAt(index); |
234 else peek = input.codeUnitAt(index); | |
235 } | 205 } |
236 | 206 |
237 void error(String message, [int offset = 0]) { | 207 void error(String message, [int offset = 0]) { |
238 // TODO(kasperl): Try to get rid of the offset. It is only used to match | 208 // TODO(kasperl): Try to get rid of the offset. It is only used to match |
239 // the error expectations in the lexer tests for numbers with exponents. | 209 // the error expectations in the lexer tests for numbers with exponents. |
240 int position = index + offset; | 210 int position = index + offset; |
241 throw "Lexer Error: $message at column $position in expression [$input]"; | 211 throw "Lexer Error: $message at column $position in expression [$input]"; |
242 } | 212 } |
243 } | 213 } |
244 | 214 |
245 Set<String> OPERATORS = new Set<String>.from([ | 215 Set<String> KEYWORDS = new Set<String>.from([ |
| 216 'null', |
246 'undefined', | 217 'undefined', |
247 'null', | |
248 'true', | 218 'true', |
249 'false', | 219 'false', |
| 220 ]); |
| 221 |
| 222 Set<String> OPERATORS = new Set<String>.from([ |
250 '+', | 223 '+', |
251 '-', | 224 '-', |
252 '*', | 225 '*', |
253 '/', | 226 '/', |
254 '~/', | 227 '~/', |
255 '%', | 228 '%', |
256 '^', | 229 '^', |
257 '=', | 230 '=', |
258 '==', | 231 '==', |
259 '!=', | 232 '!=', |
260 '<', | 233 '<', |
261 '>', | 234 '>', |
262 '<=', | 235 '<=', |
263 '>=', | 236 '>=', |
264 '&&', | 237 '&&', |
265 '||', | 238 '||', |
266 '&', | 239 '&', |
267 '|', | 240 '|', |
268 '!', | 241 '!', |
269 '?', | 242 '?', |
270 ]); | 243 ]); |
271 | 244 |
OLD | NEW |