| OLD | NEW |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 class Tokenizer extends TokenizerBase { | 5 class Tokenizer extends TokenizerBase { |
| 6 TokenKind tmplTokens; | 6 TokenKind tmplTokens; |
| 7 | 7 |
| 8 bool _selectorParsing; | 8 bool _selectorParsing; |
| 9 | 9 |
| 10 Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0]) | 10 Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0]) |
| (...skipping 115 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 126 if (tokId == -1) { | 126 if (tokId == -1) { |
| 127 tokId = TokenKind.matchKeywords(_text, _startIndex, _index - _startIndex); | 127 tokId = TokenKind.matchKeywords(_text, _startIndex, _index - _startIndex); |
| 128 } | 128 } |
| 129 | 129 |
| 130 return tokId >= 0 ? tokId : TokenKind.IDENTIFIER; | 130 return tokId >= 0 ? tokId : TokenKind.IDENTIFIER; |
| 131 } | 131 } |
| 132 | 132 |
| 133 // Need to override so CSS version of isIdentifierPart is used. | 133 // Need to override so CSS version of isIdentifierPart is used. |
| 134 Token finishIdentifier() { | 134 Token finishIdentifier() { |
| 135 while (_index < _text.length) { | 135 while (_index < _text.length) { |
| 136 // if (!TokenizerHelpers.isIdentifierPart(_text.charCodeAt(_index++))) { | 136 // if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index++))) { |
| 137 if (!TokenizerHelpers.isIdentifierPart(_text.charCodeAt(_index))) { | 137 if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index))) { |
| 138 // _index--; | 138 // _index--; |
| 139 break; | 139 break; |
| 140 } else { | 140 } else { |
| 141 _index += 1; | 141 _index += 1; |
| 142 } | 142 } |
| 143 } | 143 } |
| 144 if (_interpStack != null && _interpStack.depth == -1) { | 144 if (_interpStack != null && _interpStack.depth == -1) { |
| 145 _interpStack.depth = 0; | 145 _interpStack.depth = 0; |
| 146 } | 146 } |
| 147 int kind = getIdentifierKind(); | 147 int kind = getIdentifierKind(); |
| (...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 202 } else { | 202 } else { |
| 203 _index -= 1; | 203 _index -= 1; |
| 204 } | 204 } |
| 205 } | 205 } |
| 206 | 206 |
| 207 return _finishToken(TokenKind.INTEGER); | 207 return _finishToken(TokenKind.INTEGER); |
| 208 } | 208 } |
| 209 | 209 |
| 210 bool maybeEatDigit() { | 210 bool maybeEatDigit() { |
| 211 if (_index < _text.length && TokenizerHelpers.isDigit( | 211 if (_index < _text.length && TokenizerHelpers.isDigit( |
| 212 _text.charCodeAt(_index))) { | 212 _text.codeUnitAt(_index))) { |
| 213 _index += 1; | 213 _index += 1; |
| 214 return true; | 214 return true; |
| 215 } | 215 } |
| 216 return false; | 216 return false; |
| 217 } | 217 } |
| 218 | 218 |
| 219 void eatHexDigits() { | 219 void eatHexDigits() { |
| 220 while (_index < _text.length) { | 220 while (_index < _text.length) { |
| 221 if (TokenizerHelpers.isHexDigit(_text.charCodeAt(_index))) { | 221 if (TokenizerHelpers.isHexDigit(_text.codeUnitAt(_index))) { |
| 222 _index += 1; | 222 _index += 1; |
| 223 } else { | 223 } else { |
| 224 return; | 224 return; |
| 225 } | 225 } |
| 226 } | 226 } |
| 227 } | 227 } |
| 228 | 228 |
| 229 bool maybeEatHexDigit() { | 229 bool maybeEatHexDigit() { |
| 230 if (_index < _text.length && TokenizerHelpers.isHexDigit( | 230 if (_index < _text.length && TokenizerHelpers.isHexDigit( |
| 231 _text.charCodeAt(_index))) { | 231 _text.codeUnitAt(_index))) { |
| 232 _index += 1; | 232 _index += 1; |
| 233 return true; | 233 return true; |
| 234 } | 234 } |
| 235 return false; | 235 return false; |
| 236 } | 236 } |
| 237 | 237 |
| 238 Token finishMultiLineComment() { | 238 Token finishMultiLineComment() { |
| 239 while (true) { | 239 while (true) { |
| 240 int ch = _nextChar(); | 240 int ch = _nextChar(); |
| 241 if (ch == 0) { | 241 if (ch == 0) { |
| (...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 302 } | 302 } |
| 303 | 303 |
| 304 static bool isSlash(int c) { | 304 static bool isSlash(int c) { |
| 305 return (c == 47/* / */); | 305 return (c == 47/* / */); |
| 306 } | 306 } |
| 307 | 307 |
| 308 static bool isCloseTag(int c) { | 308 static bool isCloseTag(int c) { |
| 309 return (c == 62/* > */); | 309 return (c == 62/* > */); |
| 310 } | 310 } |
| 311 } | 311 } |
| OLD | NEW |