OLD | NEW |
1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 // Generated by scripts/tokenizer_gen.py. | 4 // Generated by scripts/tokenizer_gen.py. |
5 | 5 |
6 | 6 |
7 abstract class TokenSource { | 7 abstract class TokenSource { |
8 Token next(); | 8 Token next(); |
9 } | 9 } |
10 | 10 |
(...skipping 34 matching lines...) Loading... |
45 TokenizerBase(this._source, this._skipWhitespace, [index = 0]) | 45 TokenizerBase(this._source, this._skipWhitespace, [index = 0]) |
46 : this._index = index { | 46 : this._index = index { |
47 _text = _source.text; | 47 _text = _source.text; |
48 } | 48 } |
49 | 49 |
50 abstract Token next(); | 50 abstract Token next(); |
51 abstract int getIdentifierKind(); | 51 abstract int getIdentifierKind(); |
52 | 52 |
53 int _nextChar() { | 53 int _nextChar() { |
54 if (_index < _text.length) { | 54 if (_index < _text.length) { |
55 return _text.charCodeAt(_index++); | 55 return _text.codeUnitAt(_index++); |
56 } else { | 56 } else { |
57 return 0; | 57 return 0; |
58 } | 58 } |
59 } | 59 } |
60 | 60 |
61 int _peekChar() { | 61 int _peekChar() { |
62 if (_index < _text.length) { | 62 if (_index < _text.length) { |
63 return _text.charCodeAt(_index); | 63 return _text.codeUnitAt(_index); |
64 } else { | 64 } else { |
65 return 0; | 65 return 0; |
66 } | 66 } |
67 } | 67 } |
68 | 68 |
69 bool _maybeEatChar(int ch) { | 69 bool _maybeEatChar(int ch) { |
70 if (_index < _text.length) { | 70 if (_index < _text.length) { |
71 if (_text.charCodeAt(_index) == ch) { | 71 if (_text.codeUnitAt(_index) == ch) { |
72 _index++; | 72 _index++; |
73 return true; | 73 return true; |
74 } else { | 74 } else { |
75 return false; | 75 return false; |
76 } | 76 } |
77 } else { | 77 } else { |
78 return false; | 78 return false; |
79 } | 79 } |
80 } | 80 } |
81 | 81 |
(...skipping 10 matching lines...) Loading... |
92 } | 92 } |
93 | 93 |
94 Token _errorToken([String message = null]) { | 94 Token _errorToken([String message = null]) { |
95 return new ErrorToken( | 95 return new ErrorToken( |
96 TokenKind.ERROR, _source, _startIndex, _index, message); | 96 TokenKind.ERROR, _source, _startIndex, _index, message); |
97 } | 97 } |
98 | 98 |
99 Token finishWhitespace() { | 99 Token finishWhitespace() { |
100 _index--; | 100 _index--; |
101 while (_index < _text.length) { | 101 while (_index < _text.length) { |
102 final ch = _text.charCodeAt(_index++); | 102 final ch = _text.codeUnitAt(_index++); |
103 if (ch == 32/*' '*/ || ch == 9/*'\t'*/ || ch == 13/*'\r'*/) { | 103 if (ch == 32/*' '*/ || ch == 9/*'\t'*/ || ch == 13/*'\r'*/) { |
104 // do nothing | 104 // do nothing |
105 } else if (ch == 10/*'\n'*/) { | 105 } else if (ch == 10/*'\n'*/) { |
106 if (!_skipWhitespace) { | 106 if (!_skipWhitespace) { |
107 return _finishToken(TokenKind.WHITESPACE); // note the newline? | 107 return _finishToken(TokenKind.WHITESPACE); // note the newline? |
108 } | 108 } |
109 } else { | 109 } else { |
110 _index--; | 110 _index--; |
111 if (_skipWhitespace) { | 111 if (_skipWhitespace) { |
112 return next(); | 112 return next(); |
(...skipping 38 matching lines...) Loading... |
151 | 151 |
152 if (_skipWhitespace) { | 152 if (_skipWhitespace) { |
153 return next(); | 153 return next(); |
154 } else { | 154 } else { |
155 return _finishToken(TokenKind.COMMENT); | 155 return _finishToken(TokenKind.COMMENT); |
156 } | 156 } |
157 } | 157 } |
158 | 158 |
159 void eatDigits() { | 159 void eatDigits() { |
160 while (_index < _text.length) { | 160 while (_index < _text.length) { |
161 if (TokenizerHelpers.isDigit(_text.charCodeAt(_index))) { | 161 if (TokenizerHelpers.isDigit(_text.codeUnitAt(_index))) { |
162 _index++; | 162 _index++; |
163 } else { | 163 } else { |
164 return; | 164 return; |
165 } | 165 } |
166 } | 166 } |
167 } | 167 } |
168 | 168 |
169 static int _hexDigit(int c) { | 169 static int _hexDigit(int c) { |
170 if(c >= 48/*0*/ && c <= 57/*9*/) { | 170 if(c >= 48/*0*/ && c <= 57/*9*/) { |
171 return c - 48; | 171 return c - 48; |
(...skipping 10 matching lines...) Loading... |
182 int maxIndex; | 182 int maxIndex; |
183 if (hexLength == null) { | 183 if (hexLength == null) { |
184 maxIndex = _text.length - 1; | 184 maxIndex = _text.length - 1; |
185 } else { | 185 } else { |
186 // TODO(jimhug): What if this is too long? | 186 // TODO(jimhug): What if this is too long? |
187 maxIndex = _index + hexLength; | 187 maxIndex = _index + hexLength; |
188 if (maxIndex >= _text.length) return -1; | 188 if (maxIndex >= _text.length) return -1; |
189 } | 189 } |
190 var result = 0; | 190 var result = 0; |
191 while (_index < maxIndex) { | 191 while (_index < maxIndex) { |
192 final digit = _hexDigit(_text.charCodeAt(_index)); | 192 final digit = _hexDigit(_text.codeUnitAt(_index)); |
193 if (digit == -1) { | 193 if (digit == -1) { |
194 if (hexLength == null) { | 194 if (hexLength == null) { |
195 return result; | 195 return result; |
196 } else { | 196 } else { |
197 return -1; | 197 return -1; |
198 } | 198 } |
199 } | 199 } |
200 _hexDigit(_text.charCodeAt(_index)); | 200 _hexDigit(_text.codeUnitAt(_index)); |
201 // Multiply by 16 rather than shift by 4 since that will result in a | 201 // Multiply by 16 rather than shift by 4 since that will result in a |
202 // correct value for numbers that exceed the 32 bit precision of JS | 202 // correct value for numbers that exceed the 32 bit precision of JS |
203 // 'integers'. | 203 // 'integers'. |
204 // TODO: Figure out a better solution to integer truncation. Issue 638. | 204 // TODO: Figure out a better solution to integer truncation. Issue 638. |
205 result = (result * 16) + digit; | 205 result = (result * 16) + digit; |
206 _index++; | 206 _index++; |
207 } | 207 } |
208 | 208 |
209 return result; | 209 return result; |
210 } | 210 } |
(...skipping 223 matching lines...) Loading... |
434 } else { | 434 } else { |
435 return _finishToken(TokenKind.DOT); | 435 return _finishToken(TokenKind.DOT); |
436 } | 436 } |
437 } | 437 } |
438 | 438 |
439 Token finishIdentifier() { | 439 Token finishIdentifier() { |
440 if (_interpStack != null && _interpStack.depth == -1) { | 440 if (_interpStack != null && _interpStack.depth == -1) { |
441 _interpStack.depth = 0; | 441 _interpStack.depth = 0; |
442 while (_index < _text.length) { | 442 while (_index < _text.length) { |
443 if (!TokenizerHelpers.isInterpIdentifierPart( | 443 if (!TokenizerHelpers.isInterpIdentifierPart( |
444 _text.charCodeAt(_index++))) { | 444 _text.codeUnitAt(_index++))) { |
445 _index--; | 445 _index--; |
446 break; | 446 break; |
447 } | 447 } |
448 } | 448 } |
449 } else { | 449 } else { |
450 while (_index < _text.length) { | 450 while (_index < _text.length) { |
451 if (!TokenizerHelpers.isIdentifierPart(_text.charCodeAt(_index++))) { | 451 if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index++))) { |
452 _index--; | 452 _index--; |
453 break; | 453 break; |
454 } | 454 } |
455 } | 455 } |
456 } | 456 } |
457 int kind = getIdentifierKind(); | 457 int kind = getIdentifierKind(); |
458 if (kind == TokenKind.IDENTIFIER) { | 458 if (kind == TokenKind.IDENTIFIER) { |
459 return _finishToken(TokenKind.IDENTIFIER); | 459 return _finishToken(TokenKind.IDENTIFIER); |
460 } else { | 460 } else { |
461 return _finishToken(kind); | 461 return _finishToken(kind); |
462 } | 462 } |
463 } | 463 } |
464 } | 464 } |
465 | 465 |
OLD | NEW |