| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 // Generated by scripts/tokenizer_gen.py. | 4 // Generated by scripts/tokenizer_gen.py. |
| 5 | 5 |
| 6 | 6 |
| 7 abstract class TokenSource { | 7 abstract class TokenSource { |
| 8 Token next(); | 8 Token next(); |
| 9 } | 9 } |
| 10 | 10 |
| (...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 45 TokenizerBase(this._source, this._skipWhitespace, [index = 0]) | 45 TokenizerBase(this._source, this._skipWhitespace, [index = 0]) |
| 46 : this._index = index { | 46 : this._index = index { |
| 47 _text = _source.text; | 47 _text = _source.text; |
| 48 } | 48 } |
| 49 | 49 |
| 50 abstract Token next(); | 50 abstract Token next(); |
| 51 abstract int getIdentifierKind(); | 51 abstract int getIdentifierKind(); |
| 52 | 52 |
| 53 int _nextChar() { | 53 int _nextChar() { |
| 54 if (_index < _text.length) { | 54 if (_index < _text.length) { |
| 55 return _text.charCodeAt(_index++); | 55 return _text.codeUnitAt(_index++); |
| 56 } else { | 56 } else { |
| 57 return 0; | 57 return 0; |
| 58 } | 58 } |
| 59 } | 59 } |
| 60 | 60 |
| 61 int _peekChar() { | 61 int _peekChar() { |
| 62 if (_index < _text.length) { | 62 if (_index < _text.length) { |
| 63 return _text.charCodeAt(_index); | 63 return _text.codeUnitAt(_index); |
| 64 } else { | 64 } else { |
| 65 return 0; | 65 return 0; |
| 66 } | 66 } |
| 67 } | 67 } |
| 68 | 68 |
| 69 bool _maybeEatChar(int ch) { | 69 bool _maybeEatChar(int ch) { |
| 70 if (_index < _text.length) { | 70 if (_index < _text.length) { |
| 71 if (_text.charCodeAt(_index) == ch) { | 71 if (_text.codeUnitAt(_index) == ch) { |
| 72 _index++; | 72 _index++; |
| 73 return true; | 73 return true; |
| 74 } else { | 74 } else { |
| 75 return false; | 75 return false; |
| 76 } | 76 } |
| 77 } else { | 77 } else { |
| 78 return false; | 78 return false; |
| 79 } | 79 } |
| 80 } | 80 } |
| 81 | 81 |
| (...skipping 10 matching lines...) Expand all Loading... |
| 92 } | 92 } |
| 93 | 93 |
| 94 Token _errorToken([String message = null]) { | 94 Token _errorToken([String message = null]) { |
| 95 return new ErrorToken( | 95 return new ErrorToken( |
| 96 TokenKind.ERROR, _source, _startIndex, _index, message); | 96 TokenKind.ERROR, _source, _startIndex, _index, message); |
| 97 } | 97 } |
| 98 | 98 |
| 99 Token finishWhitespace() { | 99 Token finishWhitespace() { |
| 100 _index--; | 100 _index--; |
| 101 while (_index < _text.length) { | 101 while (_index < _text.length) { |
| 102 final ch = _text.charCodeAt(_index++); | 102 final ch = _text.codeUnitAt(_index++); |
| 103 if (ch == 32/*' '*/ || ch == 9/*'\t'*/ || ch == 13/*'\r'*/) { | 103 if (ch == 32/*' '*/ || ch == 9/*'\t'*/ || ch == 13/*'\r'*/) { |
| 104 // do nothing | 104 // do nothing |
| 105 } else if (ch == 10/*'\n'*/) { | 105 } else if (ch == 10/*'\n'*/) { |
| 106 if (!_skipWhitespace) { | 106 if (!_skipWhitespace) { |
| 107 return _finishToken(TokenKind.WHITESPACE); // note the newline? | 107 return _finishToken(TokenKind.WHITESPACE); // note the newline? |
| 108 } | 108 } |
| 109 } else { | 109 } else { |
| 110 _index--; | 110 _index--; |
| 111 if (_skipWhitespace) { | 111 if (_skipWhitespace) { |
| 112 return next(); | 112 return next(); |
| (...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 151 | 151 |
| 152 if (_skipWhitespace) { | 152 if (_skipWhitespace) { |
| 153 return next(); | 153 return next(); |
| 154 } else { | 154 } else { |
| 155 return _finishToken(TokenKind.COMMENT); | 155 return _finishToken(TokenKind.COMMENT); |
| 156 } | 156 } |
| 157 } | 157 } |
| 158 | 158 |
| 159 void eatDigits() { | 159 void eatDigits() { |
| 160 while (_index < _text.length) { | 160 while (_index < _text.length) { |
| 161 if (TokenizerHelpers.isDigit(_text.charCodeAt(_index))) { | 161 if (TokenizerHelpers.isDigit(_text.codeUnitAt(_index))) { |
| 162 _index++; | 162 _index++; |
| 163 } else { | 163 } else { |
| 164 return; | 164 return; |
| 165 } | 165 } |
| 166 } | 166 } |
| 167 } | 167 } |
| 168 | 168 |
| 169 static int _hexDigit(int c) { | 169 static int _hexDigit(int c) { |
| 170 if(c >= 48/*0*/ && c <= 57/*9*/) { | 170 if(c >= 48/*0*/ && c <= 57/*9*/) { |
| 171 return c - 48; | 171 return c - 48; |
| (...skipping 10 matching lines...) Expand all Loading... |
| 182 int maxIndex; | 182 int maxIndex; |
| 183 if (hexLength == null) { | 183 if (hexLength == null) { |
| 184 maxIndex = _text.length - 1; | 184 maxIndex = _text.length - 1; |
| 185 } else { | 185 } else { |
| 186 // TODO(jimhug): What if this is too long? | 186 // TODO(jimhug): What if this is too long? |
| 187 maxIndex = _index + hexLength; | 187 maxIndex = _index + hexLength; |
| 188 if (maxIndex >= _text.length) return -1; | 188 if (maxIndex >= _text.length) return -1; |
| 189 } | 189 } |
| 190 var result = 0; | 190 var result = 0; |
| 191 while (_index < maxIndex) { | 191 while (_index < maxIndex) { |
| 192 final digit = _hexDigit(_text.charCodeAt(_index)); | 192 final digit = _hexDigit(_text.codeUnitAt(_index)); |
| 193 if (digit == -1) { | 193 if (digit == -1) { |
| 194 if (hexLength == null) { | 194 if (hexLength == null) { |
| 195 return result; | 195 return result; |
| 196 } else { | 196 } else { |
| 197 return -1; | 197 return -1; |
| 198 } | 198 } |
| 199 } | 199 } |
| 200 _hexDigit(_text.charCodeAt(_index)); | 200 _hexDigit(_text.codeUnitAt(_index)); |
| 201 // Multiply by 16 rather than shift by 4 since that will result in a | 201 // Multiply by 16 rather than shift by 4 since that will result in a |
| 202 // correct value for numbers that exceed the 32 bit precision of JS | 202 // correct value for numbers that exceed the 32 bit precision of JS |
| 203 // 'integers'. | 203 // 'integers'. |
| 204 // TODO: Figure out a better solution to integer truncation. Issue 638. | 204 // TODO: Figure out a better solution to integer truncation. Issue 638. |
| 205 result = (result * 16) + digit; | 205 result = (result * 16) + digit; |
| 206 _index++; | 206 _index++; |
| 207 } | 207 } |
| 208 | 208 |
| 209 return result; | 209 return result; |
| 210 } | 210 } |
| (...skipping 223 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 434 } else { | 434 } else { |
| 435 return _finishToken(TokenKind.DOT); | 435 return _finishToken(TokenKind.DOT); |
| 436 } | 436 } |
| 437 } | 437 } |
| 438 | 438 |
| 439 Token finishIdentifier() { | 439 Token finishIdentifier() { |
| 440 if (_interpStack != null && _interpStack.depth == -1) { | 440 if (_interpStack != null && _interpStack.depth == -1) { |
| 441 _interpStack.depth = 0; | 441 _interpStack.depth = 0; |
| 442 while (_index < _text.length) { | 442 while (_index < _text.length) { |
| 443 if (!TokenizerHelpers.isInterpIdentifierPart( | 443 if (!TokenizerHelpers.isInterpIdentifierPart( |
| 444 _text.charCodeAt(_index++))) { | 444 _text.codeUnitAt(_index++))) { |
| 445 _index--; | 445 _index--; |
| 446 break; | 446 break; |
| 447 } | 447 } |
| 448 } | 448 } |
| 449 } else { | 449 } else { |
| 450 while (_index < _text.length) { | 450 while (_index < _text.length) { |
| 451 if (!TokenizerHelpers.isIdentifierPart(_text.charCodeAt(_index++))) { | 451 if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index++))) { |
| 452 _index--; | 452 _index--; |
| 453 break; | 453 break; |
| 454 } | 454 } |
| 455 } | 455 } |
| 456 } | 456 } |
| 457 int kind = getIdentifierKind(); | 457 int kind = getIdentifierKind(); |
| 458 if (kind == TokenKind.IDENTIFIER) { | 458 if (kind == TokenKind.IDENTIFIER) { |
| 459 return _finishToken(TokenKind.IDENTIFIER); | 459 return _finishToken(TokenKind.IDENTIFIER); |
| 460 } else { | 460 } else { |
| 461 return _finishToken(kind); | 461 return _finishToken(kind); |
| 462 } | 462 } |
| 463 } | 463 } |
| 464 } | 464 } |
| 465 | 465 |
| OLD | NEW |