OLD | NEW |
1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 // Generated by scripts/tokenizer_gen.py. | 4 // Generated by scripts/tokenizer_gen.py. |
5 | 5 |
6 | 6 |
7 /** A generated file that extends the hand coded methods in TokenizerBase. */ | 7 /** A generated file that extends the hand coded methods in TokenizerBase. */ |
8 class Tokenizer extends TokenizerBase { | 8 class Tokenizer extends TokenizerBase { |
9 | 9 |
10 Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0]) | 10 Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0]) |
11 : super(source, skipWhitespace, index); | 11 : super(source, skipWhitespace, index); |
(...skipping 206 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
218 if (_maybeEatChar(47/*'/'*/)) { | 218 if (_maybeEatChar(47/*'/'*/)) { |
219 if (_maybeEatChar(61/*=*/)) { | 219 if (_maybeEatChar(61/*=*/)) { |
220 return _finishToken(TokenKind.ASSIGN_TRUNCDIV); | 220 return _finishToken(TokenKind.ASSIGN_TRUNCDIV); |
221 } else { | 221 } else { |
222 return _finishToken(TokenKind.TRUNCDIV); | 222 return _finishToken(TokenKind.TRUNCDIV); |
223 } | 223 } |
224 } else { | 224 } else { |
225 return _finishToken(TokenKind.BIT_NOT); | 225 return _finishToken(TokenKind.BIT_NOT); |
226 } | 226 } |
227 default: | 227 default: |
228 if (isIdentifierStart(ch)) { | 228 if (TokenizerHelpers.isIdentifierStart(ch)) { |
229 return this.finishIdentifier(ch); | 229 return this.finishIdentifier(ch); |
230 } else if (isDigit(ch)) { | 230 } else if (TokenizerHelpers.isDigit(ch)) { |
231 return this.finishNumber(); | 231 return this.finishNumber(); |
232 } else { | 232 } else { |
233 return _errorToken(); | 233 return _errorToken(); |
234 } | 234 } |
235 } | 235 } |
236 | 236 |
237 } | 237 } |
238 | 238 |
239 int getIdentifierKind() { | 239 int getIdentifierKind() { |
240 final i0 = _startIndex; | 240 final i0 = _startIndex; |
(...skipping 149 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
390 if (_text.charCodeAt(i0) == 105/*i*/ && _text.charCodeAt(i0+1) == 109/*m
*/ && _text.charCodeAt(i0+2) == 112/*p*/ && _text.charCodeAt(i0+3) == 108/*l*/ &
& _text.charCodeAt(i0+4) == 101/*e*/ && _text.charCodeAt(i0+5) == 109/*m*/ && _t
ext.charCodeAt(i0+6) == 101/*e*/ && _text.charCodeAt(i0+7) == 110/*n*/ && _text.
charCodeAt(i0+8) == 116/*t*/ && _text.charCodeAt(i0+9) == 115/*s*/) return Token
Kind.IMPLEMENTS; | 390 if (_text.charCodeAt(i0) == 105/*i*/ && _text.charCodeAt(i0+1) == 109/*m
*/ && _text.charCodeAt(i0+2) == 112/*p*/ && _text.charCodeAt(i0+3) == 108/*l*/ &
& _text.charCodeAt(i0+4) == 101/*e*/ && _text.charCodeAt(i0+5) == 109/*m*/ && _t
ext.charCodeAt(i0+6) == 101/*e*/ && _text.charCodeAt(i0+7) == 110/*n*/ && _text.
charCodeAt(i0+8) == 116/*t*/ && _text.charCodeAt(i0+9) == 115/*s*/) return Token
Kind.IMPLEMENTS; |
391 return TokenKind.IDENTIFIER; | 391 return TokenKind.IDENTIFIER; |
392 default: return TokenKind.IDENTIFIER; | 392 default: return TokenKind.IDENTIFIER; |
393 } | 393 } |
394 } | 394 } |
395 | 395 |
396 } | 396 } |
397 | 397 |
398 /** Static helper methods. */ | 398 /** Static helper methods. */ |
399 class TokenizerHelpers { | 399 class TokenizerHelpers { |
400 | 400 |
401 static bool isIdentifierStart(int c) { | 401 static bool isIdentifierStart(int c) { |
402 return ((c >= 97/*a*/ && c <= 122/*z*/) || (c >= 65/*A*/ && c <= 90/*Z*/) ||
c == 95/*_*/); | 402 return ((c >= 97/*a*/ && c <= 122/*z*/) || (c >= 65/*A*/ && c <= 90/*Z*/) ||
c == 95/*_*/); |
403 } | 403 } |
404 | 404 |
405 static bool isDigit(int c) { | 405 static bool isDigit(int c) { |
406 return (c >= 48/*0*/ && c <= 57/*9*/); | 406 return (c >= 48/*0*/ && c <= 57/*9*/); |
407 } | 407 } |
408 | 408 |
409 static bool isHexDigit(int c) { | 409 static bool isHexDigit(int c) { |
410 return (isDigit(c) || (c >= 97/*a*/ && c <= 102/*f*/) || (c >= 65/*A*/ && c
<= 70/*F*/)); | 410 return (isDigit(c) || (c >= 97/*a*/ && c <= 102/*f*/) || (c >= 65/*A*/ && c
<= 70/*F*/)); |
411 } | 411 } |
412 | 412 |
413 static bool isWhitespace(int c) { | 413 static bool isWhitespace(int c) { |
414 return (c == 32/*' '*/ || c == 9/*'\t'*/ || c == 10/*'\n'*/ || c == 13/*'\r'
*/); | 414 return (c == 32/*' '*/ || c == 9/*'\t'*/ || c == 10/*'\n'*/ || c == 13/*'\r'
*/); |
415 } | 415 } |
416 | 416 |
417 static bool isIdentifierPart(int c) { | 417 static bool isIdentifierPart(int c) { |
418 return (isIdentifierStart(c) || isDigit(c) || c == 36/*$*/); | 418 return (isIdentifierStart(c) || isDigit(c) || c == 36/*$*/); |
419 } | 419 } |
420 | 420 |
421 static bool isInterpIdentifierPart(int c) { | 421 static bool isInterpIdentifierPart(int c) { |
422 return (isIdentifierStart(c) || isDigit(c)); | 422 return (isIdentifierStart(c) || isDigit(c)); |
423 } | 423 } |
424 | 424 |
425 | 425 |
426 } | 426 } |
OLD | NEW |