OLD | NEW |
1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library dart2js.tokens; | 5 library dart2js.tokens; |
6 | 6 |
7 import 'dart:convert' show | 7 import 'dart:convert' show UTF8; |
8 UTF8; | 8 import 'dart:collection' show HashSet; |
9 import 'dart:collection' show | |
10 HashSet; | |
11 | 9 |
12 import '../common.dart'; | 10 import '../common.dart'; |
13 import '../util/util.dart' show | 11 import '../util/util.dart' show computeHashCode; |
14 computeHashCode; | |
15 | 12 |
16 import 'keyword.dart' show | 13 import 'keyword.dart' show Keyword; |
17 Keyword; | 14 import 'precedence.dart' show PrecedenceInfo; |
18 import 'precedence.dart' show | 15 import 'precedence_constants.dart' as Precedence show BAD_INPUT_INFO; |
19 PrecedenceInfo; | 16 import 'token_constants.dart' as Tokens show IDENTIFIER_TOKEN; |
20 import 'precedence_constants.dart' as Precedence show | |
21 BAD_INPUT_INFO; | |
22 import 'token_constants.dart' as Tokens show | |
23 IDENTIFIER_TOKEN; | |
24 | 17 |
25 /** | 18 /** |
26 * A token that doubles as a linked list. | 19 * A token that doubles as a linked list. |
27 */ | 20 */ |
28 abstract class Token implements Spannable { | 21 abstract class Token implements Spannable { |
29 /** | 22 /** |
30 * The character offset of the start of this token within the source text. | 23 * The character offset of the start of this token within the source text. |
31 */ | 24 */ |
32 final int charOffset; | 25 final int charOffset; |
33 | 26 |
(...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
125 final Token end; | 118 final Token end; |
126 | 119 |
127 TokenPair(this.begin, this.end); | 120 TokenPair(this.begin, this.end); |
128 } | 121 } |
129 | 122 |
130 /** | 123 /** |
131 * A [SymbolToken] represents the symbol in its precendence info. | 124 * A [SymbolToken] represents the symbol in its precendence info. |
132 * Also used for end of file with EOF_INFO. | 125 * Also used for end of file with EOF_INFO. |
133 */ | 126 */ |
134 class SymbolToken extends Token { | 127 class SymbolToken extends Token { |
135 | |
136 final PrecedenceInfo info; | 128 final PrecedenceInfo info; |
137 | 129 |
138 SymbolToken(this.info, int charOffset) : super(charOffset); | 130 SymbolToken(this.info, int charOffset) : super(charOffset); |
139 | 131 |
140 String get value => info.value; | 132 String get value => info.value; |
141 | 133 |
142 String get stringValue => info.value; | 134 String get stringValue => info.value; |
143 | 135 |
144 bool isIdentifier() => false; | 136 bool isIdentifier() => false; |
145 | 137 |
(...skipping 26 matching lines...) Expand all Loading... |
172 String get value => keyword.syntax; | 164 String get value => keyword.syntax; |
173 | 165 |
174 String get stringValue => keyword.syntax; | 166 String get stringValue => keyword.syntax; |
175 | 167 |
176 bool isIdentifier() => keyword.isPseudo || keyword.isBuiltIn; | 168 bool isIdentifier() => keyword.isPseudo || keyword.isBuiltIn; |
177 | 169 |
178 String toString() => "KeywordToken($value)"; | 170 String toString() => "KeywordToken($value)"; |
179 } | 171 } |
180 | 172 |
181 abstract class ErrorToken extends Token { | 173 abstract class ErrorToken extends Token { |
182 ErrorToken(int charOffset) | 174 ErrorToken(int charOffset) : super(charOffset); |
183 : super(charOffset); | |
184 | 175 |
185 PrecedenceInfo get info => Precedence.BAD_INPUT_INFO; | 176 PrecedenceInfo get info => Precedence.BAD_INPUT_INFO; |
186 | 177 |
187 String get value { | 178 String get value { |
188 throw new SpannableAssertionFailure(this, assertionMessage); | 179 throw new SpannableAssertionFailure(this, assertionMessage); |
189 } | 180 } |
190 | 181 |
191 String get stringValue => null; | 182 String get stringValue => null; |
192 | 183 |
193 bool isIdentifier() => false; | 184 bool isIdentifier() => false; |
194 | 185 |
195 String get assertionMessage; | 186 String get assertionMessage; |
196 } | 187 } |
197 | 188 |
198 class BadInputToken extends ErrorToken { | 189 class BadInputToken extends ErrorToken { |
199 final int character; | 190 final int character; |
200 | 191 |
201 BadInputToken(this.character, int charOffset) | 192 BadInputToken(this.character, int charOffset) : super(charOffset); |
202 : super(charOffset); | |
203 | 193 |
204 String toString() => "BadInputToken($character)"; | 194 String toString() => "BadInputToken($character)"; |
205 | 195 |
206 String get assertionMessage { | 196 String get assertionMessage { |
207 return 'Character U+${character.toRadixString(16)} not allowed here.'; | 197 return 'Character U+${character.toRadixString(16)} not allowed here.'; |
208 } | 198 } |
209 } | 199 } |
210 | 200 |
211 class UnterminatedToken extends ErrorToken { | 201 class UnterminatedToken extends ErrorToken { |
212 final String start; | 202 final String start; |
(...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
252 | 242 |
253 var /* String | LazySubtring */ valueOrLazySubstring; | 243 var /* String | LazySubtring */ valueOrLazySubstring; |
254 | 244 |
255 final PrecedenceInfo info; | 245 final PrecedenceInfo info; |
256 | 246 |
257 /** | 247 /** |
258 * Creates a non-lazy string token. If [canonicalize] is true, the string | 248 * Creates a non-lazy string token. If [canonicalize] is true, the string |
259 * is canonicalized before the token is created. | 249 * is canonicalized before the token is created. |
260 */ | 250 */ |
261 StringToken.fromString(this.info, String value, int charOffset, | 251 StringToken.fromString(this.info, String value, int charOffset, |
262 {bool canonicalize : false}) | 252 {bool canonicalize: false}) |
263 : valueOrLazySubstring = canonicalizedString(value, canonicalize), | 253 : valueOrLazySubstring = canonicalizedString(value, canonicalize), |
264 super(charOffset); | 254 super(charOffset); |
265 | 255 |
266 /** | 256 /** |
267 * Creates a lazy string token. If [canonicalize] is true, the string | 257 * Creates a lazy string token. If [canonicalize] is true, the string |
268 * is canonicalized before the token is created. | 258 * is canonicalized before the token is created. |
269 */ | 259 */ |
270 StringToken.fromSubstring(this.info, String data, int start, int end, | 260 StringToken.fromSubstring( |
271 int charOffset, {bool canonicalize : false}) | 261 this.info, String data, int start, int end, int charOffset, |
| 262 {bool canonicalize: false}) |
272 : super(charOffset) { | 263 : super(charOffset) { |
273 int length = end - start; | 264 int length = end - start; |
274 if (length <= LAZY_THRESHOLD) { | 265 if (length <= LAZY_THRESHOLD) { |
275 valueOrLazySubstring = canonicalizedString(data.substring(start, end), | 266 valueOrLazySubstring = |
276 canonicalize); | 267 canonicalizedString(data.substring(start, end), canonicalize); |
277 } else { | 268 } else { |
278 valueOrLazySubstring = | 269 valueOrLazySubstring = |
279 new LazySubstring(data, start, length, canonicalize); | 270 new LazySubstring(data, start, length, canonicalize); |
280 } | 271 } |
281 } | 272 } |
282 | 273 |
283 /** | 274 /** |
284 * Creates a lazy string token. If [asciiOnly] is false, the byte array | 275 * Creates a lazy string token. If [asciiOnly] is false, the byte array |
285 * is passed through a UTF-8 decoder. | 276 * is passed through a UTF-8 decoder. |
286 */ | 277 */ |
287 StringToken.fromUtf8Bytes(this.info, List<int> data, int start, int end, | 278 StringToken.fromUtf8Bytes(this.info, List<int> data, int start, int end, |
288 bool asciiOnly, int charOffset) | 279 bool asciiOnly, int charOffset) |
289 : super(charOffset) { | 280 : super(charOffset) { |
290 int length = end - start; | 281 int length = end - start; |
291 if (length <= LAZY_THRESHOLD) { | 282 if (length <= LAZY_THRESHOLD) { |
292 valueOrLazySubstring = decodeUtf8(data, start, end, asciiOnly); | 283 valueOrLazySubstring = decodeUtf8(data, start, end, asciiOnly); |
293 } else { | 284 } else { |
294 valueOrLazySubstring = new LazySubstring(data, start, length, asciiOnly); | 285 valueOrLazySubstring = new LazySubstring(data, start, length, asciiOnly); |
295 } | 286 } |
296 } | 287 } |
297 | 288 |
298 String get value { | 289 String get value { |
299 if (valueOrLazySubstring is String) { | 290 if (valueOrLazySubstring is String) { |
300 return valueOrLazySubstring; | 291 return valueOrLazySubstring; |
301 } else { | 292 } else { |
302 assert(valueOrLazySubstring is LazySubstring); | 293 assert(valueOrLazySubstring is LazySubstring); |
303 var data = valueOrLazySubstring.data; | 294 var data = valueOrLazySubstring.data; |
304 int start = valueOrLazySubstring.start; | 295 int start = valueOrLazySubstring.start; |
305 int end = start + valueOrLazySubstring.length; | 296 int end = start + valueOrLazySubstring.length; |
306 if (data is String) { | 297 if (data is String) { |
307 valueOrLazySubstring = canonicalizedString( | 298 valueOrLazySubstring = canonicalizedString( |
308 data.substring(start, end), valueOrLazySubstring.boolValue); | 299 data.substring(start, end), valueOrLazySubstring.boolValue); |
309 } else { | 300 } else { |
310 valueOrLazySubstring = decodeUtf8( | 301 valueOrLazySubstring = |
311 data, start, end, valueOrLazySubstring.boolValue); | 302 decodeUtf8(data, start, end, valueOrLazySubstring.boolValue); |
312 } | 303 } |
313 return valueOrLazySubstring; | 304 return valueOrLazySubstring; |
314 } | 305 } |
315 } | 306 } |
316 | 307 |
317 /// See [Token.stringValue] for an explanation. | 308 /// See [Token.stringValue] for an explanation. |
318 String get stringValue => null; | 309 String get stringValue => null; |
319 | 310 |
320 bool isIdentifier() => identical(kind, Tokens.IDENTIFIER_TOKEN); | 311 bool isIdentifier() => identical(kind, Tokens.IDENTIFIER_TOKEN); |
321 | 312 |
322 String toString() => "StringToken($value)"; | 313 String toString() => "StringToken($value)"; |
323 | 314 |
324 static final HashSet<String> canonicalizedSubstrings = | 315 static final HashSet<String> canonicalizedSubstrings = new HashSet<String>(); |
325 new HashSet<String>(); | |
326 | 316 |
327 static String canonicalizedString(String s, bool canonicalize) { | 317 static String canonicalizedString(String s, bool canonicalize) { |
328 if (!canonicalize) return s; | 318 if (!canonicalize) return s; |
329 var result = canonicalizedSubstrings.lookup(s); | 319 var result = canonicalizedSubstrings.lookup(s); |
330 if (result != null) return result; | 320 if (result != null) return result; |
331 canonicalizedSubstrings.add(s); | 321 canonicalizedSubstrings.add(s); |
332 return s; | 322 return s; |
333 } | 323 } |
334 | 324 |
335 static String decodeUtf8(List<int> data, int start, int end, bool asciiOnly) { | 325 static String decodeUtf8(List<int> data, int start, int end, bool asciiOnly) { |
(...skipping 66 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
402 class FullLazySubstring extends LazySubstring { | 392 class FullLazySubstring extends LazySubstring { |
403 final data; | 393 final data; |
404 final int start; | 394 final int start; |
405 final int length; | 395 final int length; |
406 final bool boolValue; | 396 final bool boolValue; |
407 FullLazySubstring(this.data, this.start, this.length, this.boolValue) | 397 FullLazySubstring(this.data, this.start, this.length, this.boolValue) |
408 : super.internal(); | 398 : super.internal(); |
409 } | 399 } |
410 | 400 |
411 bool isUserDefinableOperator(String value) { | 401 bool isUserDefinableOperator(String value) { |
412 return | 402 return isBinaryOperator(value) || |
413 isBinaryOperator(value) || | |
414 isMinusOperator(value) || | 403 isMinusOperator(value) || |
415 isTernaryOperator(value) || | 404 isTernaryOperator(value) || |
416 isUnaryOperator(value); | 405 isUnaryOperator(value); |
417 } | 406 } |
418 | 407 |
419 bool isUnaryOperator(String value) => value == '~'; | 408 bool isUnaryOperator(String value) => value == '~'; |
420 | 409 |
421 bool isBinaryOperator(String value) { | 410 bool isBinaryOperator(String value) { |
422 return | 411 return value == '==' || |
423 value == '==' || | |
424 value == '[]' || | 412 value == '[]' || |
425 value == '*' || | 413 value == '*' || |
426 value == '/' || | 414 value == '/' || |
427 value == '%' || | 415 value == '%' || |
428 value == '~/' || | 416 value == '~/' || |
429 value == '+' || | 417 value == '+' || |
430 value == '<<' || | 418 value == '<<' || |
431 value == '>>' || | 419 value == '>>' || |
432 value == '>=' || | 420 value == '>=' || |
433 value == '>' || | 421 value == '>' || |
434 value == '<=' || | 422 value == '<=' || |
435 value == '<' || | 423 value == '<' || |
436 value == '&' || | 424 value == '&' || |
437 value == '^' || | 425 value == '^' || |
438 value == '|'; | 426 value == '|'; |
439 } | 427 } |
440 | 428 |
441 bool isTernaryOperator(String value) => value == '[]='; | 429 bool isTernaryOperator(String value) => value == '[]='; |
442 | 430 |
443 bool isMinusOperator(String value) => value == '-'; | 431 bool isMinusOperator(String value) => value == '-'; |
OLD | NEW |