OLD | NEW |
1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 class Tokenizer extends TokenizerBase { | 5 class Tokenizer extends TokenizerBase { |
6 TokenKind tmplTokens; | 6 TokenKind tmplTokens; |
7 | 7 |
8 bool _selectorParsing; | 8 bool _selectorParsing; |
9 | 9 |
10 Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0]) | 10 Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0]) |
(...skipping 115 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
126 if (tokId == -1) { | 126 if (tokId == -1) { |
127 tokId = TokenKind.matchKeywords(_text, _startIndex, _index - _startIndex); | 127 tokId = TokenKind.matchKeywords(_text, _startIndex, _index - _startIndex); |
128 } | 128 } |
129 | 129 |
130 return tokId >= 0 ? tokId : TokenKind.IDENTIFIER; | 130 return tokId >= 0 ? tokId : TokenKind.IDENTIFIER; |
131 } | 131 } |
132 | 132 |
133 // Need to override so CSS version of isIdentifierPart is used. | 133 // Need to override so CSS version of isIdentifierPart is used. |
134 Token finishIdentifier() { | 134 Token finishIdentifier() { |
135 while (_index < _text.length) { | 135 while (_index < _text.length) { |
136 // if (!TokenizerHelpers.isIdentifierPart(_text.charCodeAt(_index++))) { | 136 // if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index++))) { |
137 if (!TokenizerHelpers.isIdentifierPart(_text.charCodeAt(_index))) { | 137 if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index))) { |
138 // _index--; | 138 // _index--; |
139 break; | 139 break; |
140 } else { | 140 } else { |
141 _index += 1; | 141 _index += 1; |
142 } | 142 } |
143 } | 143 } |
144 if (_interpStack != null && _interpStack.depth == -1) { | 144 if (_interpStack != null && _interpStack.depth == -1) { |
145 _interpStack.depth = 0; | 145 _interpStack.depth = 0; |
146 } | 146 } |
147 int kind = getIdentifierKind(); | 147 int kind = getIdentifierKind(); |
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
202 } else { | 202 } else { |
203 _index -= 1; | 203 _index -= 1; |
204 } | 204 } |
205 } | 205 } |
206 | 206 |
207 return _finishToken(TokenKind.INTEGER); | 207 return _finishToken(TokenKind.INTEGER); |
208 } | 208 } |
209 | 209 |
210 bool maybeEatDigit() { | 210 bool maybeEatDigit() { |
211 if (_index < _text.length && TokenizerHelpers.isDigit( | 211 if (_index < _text.length && TokenizerHelpers.isDigit( |
212 _text.charCodeAt(_index))) { | 212 _text.codeUnitAt(_index))) { |
213 _index += 1; | 213 _index += 1; |
214 return true; | 214 return true; |
215 } | 215 } |
216 return false; | 216 return false; |
217 } | 217 } |
218 | 218 |
219 void eatHexDigits() { | 219 void eatHexDigits() { |
220 while (_index < _text.length) { | 220 while (_index < _text.length) { |
221 if (TokenizerHelpers.isHexDigit(_text.charCodeAt(_index))) { | 221 if (TokenizerHelpers.isHexDigit(_text.codeUnitAt(_index))) { |
222 _index += 1; | 222 _index += 1; |
223 } else { | 223 } else { |
224 return; | 224 return; |
225 } | 225 } |
226 } | 226 } |
227 } | 227 } |
228 | 228 |
229 bool maybeEatHexDigit() { | 229 bool maybeEatHexDigit() { |
230 if (_index < _text.length && TokenizerHelpers.isHexDigit( | 230 if (_index < _text.length && TokenizerHelpers.isHexDigit( |
231 _text.charCodeAt(_index))) { | 231 _text.codeUnitAt(_index))) { |
232 _index += 1; | 232 _index += 1; |
233 return true; | 233 return true; |
234 } | 234 } |
235 return false; | 235 return false; |
236 } | 236 } |
237 | 237 |
238 Token finishMultiLineComment() { | 238 Token finishMultiLineComment() { |
239 while (true) { | 239 while (true) { |
240 int ch = _nextChar(); | 240 int ch = _nextChar(); |
241 if (ch == 0) { | 241 if (ch == 0) { |
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
302 } | 302 } |
303 | 303 |
304 static bool isSlash(int c) { | 304 static bool isSlash(int c) { |
305 return (c == 47/* / */); | 305 return (c == 47/* / */); |
306 } | 306 } |
307 | 307 |
308 static bool isCloseTag(int c) { | 308 static bool isCloseTag(int c) { |
309 return (c == 62/* > */); | 309 return (c == 62/* > */); |
310 } | 310 } |
311 } | 311 } |
OLD | NEW |