OLD | NEW |
1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 class Tokenizer extends CSSTokenizerBase { | 5 class Tokenizer extends CSSTokenizerBase { |
6 TokenKind cssTokens; | 6 TokenKind cssTokens; |
7 | 7 |
8 bool _selectorParsing; | 8 bool _selectorParsing; |
9 | 9 |
10 Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0]) | 10 Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0]) |
(...skipping 173 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
184 tokId = (_text.substring(_startIndex, _index) == '!important') ? | 184 tokId = (_text.substring(_startIndex, _index) == '!important') ? |
185 TokenKind.IMPORTANT : -1; | 185 TokenKind.IMPORTANT : -1; |
186 } | 186 } |
187 | 187 |
188 return tokId >= 0 ? tokId : TokenKind.IDENTIFIER; | 188 return tokId >= 0 ? tokId : TokenKind.IDENTIFIER; |
189 } | 189 } |
190 | 190 |
191 // Need to override so CSS version of isIdentifierPart is used. | 191 // Need to override so CSS version of isIdentifierPart is used. |
192 Token finishIdentifier(int ch) { | 192 Token finishIdentifier(int ch) { |
193 while (_index < _text.length) { | 193 while (_index < _text.length) { |
194 // if (!TokenizerHelpers.isIdentifierPart(_text.charCodeAt(_index++))) { | 194 // if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index++))) { |
195 if (!TokenizerHelpers.isIdentifierPart(_text.charCodeAt(_index))) { | 195 if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index))) { |
196 // _index--; | 196 // _index--; |
197 break; | 197 break; |
198 } else { | 198 } else { |
199 _index += 1; | 199 _index += 1; |
200 } | 200 } |
201 } | 201 } |
202 if (_interpStack != null && _interpStack.depth == -1) { | 202 if (_interpStack != null && _interpStack.depth == -1) { |
203 _interpStack.depth = 0; | 203 _interpStack.depth = 0; |
204 } | 204 } |
205 int kind = getIdentifierKind(); | 205 int kind = getIdentifierKind(); |
206 if (kind == TokenKind.IDENTIFIER) { | 206 if (kind == TokenKind.IDENTIFIER) { |
207 return _finishToken(TokenKind.IDENTIFIER); | 207 return _finishToken(TokenKind.IDENTIFIER); |
208 } else { | 208 } else { |
209 return _finishToken(kind); | 209 return _finishToken(kind); |
210 } | 210 } |
211 } | 211 } |
212 | 212 |
213 Token finishImportant() { | 213 Token finishImportant() { |
214 | 214 |
215 } | 215 } |
216 | 216 |
217 Token finishNumber() { | 217 Token finishNumber() { |
218 eatDigits(); | 218 eatDigits(); |
219 | 219 |
220 if (_peekChar() == 46/*.*/) { | 220 if (_peekChar() == 46/*.*/) { |
221 // Handle the case of 1.toString(). | 221 // Handle the case of 1.toString(). |
222 _nextChar(); | 222 _nextChar(); |
223 if (TokenizerHelpers.isDigit(_peekChar())) { | 223 if (TokenizerHelpers.isDigit(_peekChar())) { |
224 eatDigits(); | 224 eatDigits(); |
225 return _finishToken(TokenKind.DOUBLE); | 225 return _finishToken(TokenKind.DOUBLE); |
226 } else { | 226 } else { |
227 _index -= 1; | 227 _index -= 1; |
228 } | 228 } |
229 } | 229 } |
230 | 230 |
231 return _finishToken(TokenKind.INTEGER); | 231 return _finishToken(TokenKind.INTEGER); |
232 } | 232 } |
233 | 233 |
234 bool maybeEatDigit() { | 234 bool maybeEatDigit() { |
235 if (_index < _text.length | 235 if (_index < _text.length |
236 && TokenizerHelpers.isDigit(_text.charCodeAt(_index))) { | 236 && TokenizerHelpers.isDigit(_text.codeUnitAt(_index))) { |
237 _index += 1; | 237 _index += 1; |
238 return true; | 238 return true; |
239 } | 239 } |
240 return false; | 240 return false; |
241 } | 241 } |
242 | 242 |
243 void eatHexDigits() { | 243 void eatHexDigits() { |
244 while (_index < _text.length) { | 244 while (_index < _text.length) { |
245 if (TokenizerHelpers.isHexDigit(_text.charCodeAt(_index))) { | 245 if (TokenizerHelpers.isHexDigit(_text.codeUnitAt(_index))) { |
246 _index += 1; | 246 _index += 1; |
247 } else { | 247 } else { |
248 return; | 248 return; |
249 } | 249 } |
250 } | 250 } |
251 } | 251 } |
252 | 252 |
253 bool maybeEatHexDigit() { | 253 bool maybeEatHexDigit() { |
254 if (_index < _text.length | 254 if (_index < _text.length |
255 && TokenizerHelpers.isHexDigit(_text.charCodeAt(_index))) { | 255 && TokenizerHelpers.isHexDigit(_text.codeUnitAt(_index))) { |
256 _index += 1; | 256 _index += 1; |
257 return true; | 257 return true; |
258 } | 258 } |
259 return false; | 259 return false; |
260 } | 260 } |
261 | 261 |
262 Token finishMultiLineComment() { | 262 Token finishMultiLineComment() { |
263 while (true) { | 263 while (true) { |
264 int ch = _nextChar(); | 264 int ch = _nextChar(); |
265 if (ch == 0) { | 265 if (ch == 0) { |
(...skipping 20 matching lines...) Expand all Loading... |
286 } | 286 } |
287 } | 287 } |
288 return _errorToken(); | 288 return _errorToken(); |
289 } | 289 } |
290 | 290 |
291 } | 291 } |
292 | 292 |
293 /** Static helper methods. */ | 293 /** Static helper methods. */ |
294 /** Static helper methods. */ | 294 /** Static helper methods. */ |
295 class TokenizerHelpers { | 295 class TokenizerHelpers { |
296 | 296 |
297 static bool isIdentifierStart(int c) { | 297 static bool isIdentifierStart(int c) { |
298 return ((c >= 97/*a*/ && c <= 122/*z*/) || (c >= 65/*A*/ && c <= 90/*Z*/) || | 298 return ((c >= 97/*a*/ && c <= 122/*z*/) || (c >= 65/*A*/ && c <= 90/*Z*/) || |
299 c == 95/*_*/ || c == 45 /*-*/); | 299 c == 95/*_*/ || c == 45 /*-*/); |
300 } | 300 } |
301 | 301 |
302 static bool isDigit(int c) { | 302 static bool isDigit(int c) { |
303 return (c >= 48/*0*/ && c <= 57/*9*/); | 303 return (c >= 48/*0*/ && c <= 57/*9*/); |
304 } | 304 } |
305 | 305 |
306 static bool isHexDigit(int c) { | 306 static bool isHexDigit(int c) { |
307 return (isDigit(c) || (c >= 97/*a*/ && c <= 102/*f*/) | 307 return (isDigit(c) || (c >= 97/*a*/ && c <= 102/*f*/) |
308 || (c >= 65/*A*/ && c <= 70/*F*/)); | 308 || (c >= 65/*A*/ && c <= 70/*F*/)); |
309 } | 309 } |
310 | 310 |
311 static bool isIdentifierPart(int c) { | 311 static bool isIdentifierPart(int c) { |
312 return (isIdentifierStart(c) || isDigit(c) || c == 45 /*-*/); | 312 return (isIdentifierStart(c) || isDigit(c) || c == 45 /*-*/); |
313 } | 313 } |
314 } | 314 } |
315 | 315 |
OLD | NEW |