OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 // Generated by scripts/tokenizer_gen.py. | 4 // Generated by scripts/tokenizer_gen.py. |
5 | 5 |
6 part of csslib.parser; | 6 part of csslib.parser; |
7 | 7 |
8 /** Tokenizer state to support look ahead for Less' nested selectors. */ | 8 /** Tokenizer state to support look ahead for Less' nested selectors. */ |
9 class TokenizerState { | 9 class TokenizerState { |
10 final int index; | 10 final int index; |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
48 // other units) doesn't seem valid. We probably should defer this | 48 // other units) doesn't seem valid. We probably should defer this |
49 // analysis until we reach places in the parser where units are expected. | 49 // analysis until we reach places in the parser where units are expected. |
50 // I'm not sure this is tokenizing as described in the specs: | 50 // I'm not sure this is tokenizing as described in the specs: |
51 // http://dev.w3.org/csswg/css-syntax/ | 51 // http://dev.w3.org/csswg/css-syntax/ |
52 // http://dev.w3.org/csswg/selectors4/ | 52 // http://dev.w3.org/csswg/selectors4/ |
53 bool inSelector = false; | 53 bool inSelector = false; |
54 | 54 |
55 int _index = 0; | 55 int _index = 0; |
56 int _startIndex = 0; | 56 int _startIndex = 0; |
57 | 57 |
58 static const String _CDATA_START = '<![CDATA['; | |
59 static const String _CDATA_END = ']]>'; | |
60 | |
61 TokenizerBase(this._file, this._text, this._skipWhitespace, | 58 TokenizerBase(this._file, this._text, this._skipWhitespace, |
62 [this._index = 0]); | 59 [this._index = 0]); |
63 | 60 |
64 Token next(); | 61 Token next(); |
65 int getIdentifierKind(); | 62 int getIdentifierKind(); |
66 | 63 |
67 /** Snapshot of Tokenizer scanning state. */ | 64 /** Snapshot of Tokenizer scanning state. */ |
68 TokenizerState get mark => new TokenizerState(this); | 65 TokenizerState get mark => new TokenizerState(this); |
69 | 66 |
70 /** Restore Tokenizer scanning state. */ | 67 /** Restore Tokenizer scanning state. */ |
(...skipping 26 matching lines...) Expand all Loading... |
97 _index++; | 94 _index++; |
98 return true; | 95 return true; |
99 } else { | 96 } else { |
100 return false; | 97 return false; |
101 } | 98 } |
102 } else { | 99 } else { |
103 return false; | 100 return false; |
104 } | 101 } |
105 } | 102 } |
106 | 103 |
107 String _tokenText() { | |
108 if (_index < _text.length) { | |
109 return _text.substring(_startIndex, _index); | |
110 } else { | |
111 return _text.substring(_startIndex, _text.length); | |
112 } | |
113 } | |
114 | |
115 Token _finishToken(int kind) { | 104 Token _finishToken(int kind) { |
116 return new Token(kind, _file.span(_startIndex, _index)); | 105 return new Token(kind, _file.span(_startIndex, _index)); |
117 } | 106 } |
118 | 107 |
119 Token _errorToken([String message = null]) { | 108 Token _errorToken([String message = null]) { |
120 return new ErrorToken( | 109 return new ErrorToken( |
121 TokenKind.ERROR, _file.span(_startIndex, _index), message); | 110 TokenKind.ERROR, _file.span(_startIndex, _index), message); |
122 } | 111 } |
123 | 112 |
124 Token finishWhitespace() { | 113 Token finishWhitespace() { |
(...skipping 174 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
299 return _errorToken("invalid hex escape sequence"); | 288 return _errorToken("invalid hex escape sequence"); |
300 } else { | 289 } else { |
301 buf.add(escapeVal); | 290 buf.add(escapeVal); |
302 } | 291 } |
303 } else { | 292 } else { |
304 buf.add(ch); | 293 buf.add(ch); |
305 } | 294 } |
306 } | 295 } |
307 } | 296 } |
308 | 297 |
309 Token _finishOpenBrace() { | |
310 return _finishToken(TokenKind.LBRACE); | |
311 } | |
312 | |
313 Token _finishCloseBrace() { | |
314 return _finishToken(TokenKind.RBRACE); | |
315 } | |
316 | |
317 Token finishString(int quote) { | 298 Token finishString(int quote) { |
318 if (_maybeEatChar(quote)) { | 299 if (_maybeEatChar(quote)) { |
319 if (_maybeEatChar(quote)) { | 300 if (_maybeEatChar(quote)) { |
320 // skip an initial newline | 301 // skip an initial newline |
321 _maybeEatChar(TokenChar.NEWLINE); | 302 _maybeEatChar(TokenChar.NEWLINE); |
322 return finishMultilineString(quote); | 303 return finishMultilineString(quote); |
323 } else { | 304 } else { |
324 return _makeStringToken(new List<int>(), false); | 305 return _makeStringToken(new List<int>(), false); |
325 } | 306 } |
326 } | 307 } |
(...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
428 | 409 |
429 Token finishDot() { | 410 Token finishDot() { |
430 if (TokenizerHelpers.isDigit(_peekChar())) { | 411 if (TokenizerHelpers.isDigit(_peekChar())) { |
431 eatDigits(); | 412 eatDigits(); |
432 return finishNumberExtra(TokenKind.DOUBLE); | 413 return finishNumberExtra(TokenKind.DOUBLE); |
433 } else { | 414 } else { |
434 return _finishToken(TokenKind.DOT); | 415 return _finishToken(TokenKind.DOT); |
435 } | 416 } |
436 } | 417 } |
437 } | 418 } |
OLD | NEW |