OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 part of csslib.parser; | 5 part of csslib.parser; |
6 | 6 |
7 class Tokenizer extends TokenizerBase { | 7 class Tokenizer extends TokenizerBase { |
8 /** U+ prefix for unicode characters. */ | 8 /** U+ prefix for unicode characters. */ |
9 final UNICODE_U = 'U'.codeUnitAt(0); | 9 final UNICODE_U = 'U'.codeUnitAt(0); |
10 final UNICODE_LOWER_U = 'u'.codeUnitAt(0); | 10 final UNICODE_LOWER_U = 'u'.codeUnitAt(0); |
(...skipping 22 matching lines...) Expand all Loading... |
33 case TokenChar.END_OF_FILE: | 33 case TokenChar.END_OF_FILE: |
34 return _finishToken(TokenKind.END_OF_FILE); | 34 return _finishToken(TokenKind.END_OF_FILE); |
35 case TokenChar.AT: | 35 case TokenChar.AT: |
36 int peekCh = _peekChar(); | 36 int peekCh = _peekChar(); |
37 if (TokenizerHelpers.isIdentifierStart(peekCh)) { | 37 if (TokenizerHelpers.isIdentifierStart(peekCh)) { |
38 var oldIndex = _index; | 38 var oldIndex = _index; |
39 var oldStartIndex = _startIndex; | 39 var oldStartIndex = _startIndex; |
40 | 40 |
41 _startIndex = _index; | 41 _startIndex = _index; |
42 ch = _nextChar(); | 42 ch = _nextChar(); |
43 Token ident = finishIdentifier(); | 43 finishIdentifier(); |
44 | 44 |
45 // Is it a directive? | 45 // Is it a directive? |
46 int tokId = TokenKind.matchDirectives( | 46 int tokId = TokenKind.matchDirectives( |
47 _text, _startIndex, _index - _startIndex); | 47 _text, _startIndex, _index - _startIndex); |
48 if (tokId == -1) { | 48 if (tokId == -1) { |
49 // No, is it a margin directive? | 49 // No, is it a margin directive? |
50 tokId = TokenKind.matchMarginDirectives( | 50 tokId = TokenKind.matchMarginDirectives( |
51 _text, _startIndex, _index - _startIndex); | 51 _text, _startIndex, _index - _startIndex); |
52 } | 52 } |
53 | 53 |
(...skipping 199 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
253 ? TokenKind.IMPORTANT | 253 ? TokenKind.IMPORTANT |
254 : -1; | 254 : -1; |
255 } | 255 } |
256 | 256 |
257 return tokId >= 0 ? tokId : TokenKind.IDENTIFIER; | 257 return tokId >= 0 ? tokId : TokenKind.IDENTIFIER; |
258 } | 258 } |
259 | 259 |
260 Token finishIdentifier() { | 260 Token finishIdentifier() { |
261 // If we encounter an escape sequence, remember it so we can post-process | 261 // If we encounter an escape sequence, remember it so we can post-process |
262 // to unescape. | 262 // to unescape. |
263 bool hasEscapedChars = false; | |
264 var chars = []; | 263 var chars = []; |
265 | 264 |
266 // backup so we can start with the first character | 265 // backup so we can start with the first character |
267 int validateFrom = _index; | 266 int validateFrom = _index; |
268 _index = _startIndex; | 267 _index = _startIndex; |
269 while (_index < _text.length) { | 268 while (_index < _text.length) { |
270 int ch = _text.codeUnitAt(_index); | 269 int ch = _text.codeUnitAt(_index); |
271 | 270 |
272 // If the previous character was "\" we need to escape. T | 271 // If the previous character was "\" we need to escape. T |
273 // http://www.w3.org/TR/CSS21/syndata.html#characters | 272 // http://www.w3.org/TR/CSS21/syndata.html#characters |
(...skipping 174 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
448 // http://www.w3.org/TR/CSS21/syndata.html#characters | 447 // http://www.w3.org/TR/CSS21/syndata.html#characters |
449 // Also, escaped character should be allowed. | 448 // Also, escaped character should be allowed. |
450 c == 95 /*_*/ || c >= 0xA0 || c == 92 /*\*/); | 449 c == 95 /*_*/ || c >= 0xA0 || c == 92 /*\*/); |
451 } | 450 } |
452 | 451 |
453 /** Pseudo function expressions identifiers can't have a minus sign. */ | 452 /** Pseudo function expressions identifiers can't have a minus sign. */ |
454 static bool isIdentifierPartExpr(int c) { | 453 static bool isIdentifierPartExpr(int c) { |
455 return (isIdentifierStartExpr(c) || isDigit(c)); | 454 return (isIdentifierStartExpr(c) || isDigit(c)); |
456 } | 455 } |
457 } | 456 } |
OLD | NEW |