| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 // Generated by scripts/tokenizer_gen.py. | 4 // Generated by scripts/tokenizer_gen.py. |
| 5 | 5 |
| 6 part of csslib.parser; | 6 part of csslib.parser; |
| 7 | 7 |
| 8 /** Tokenizer state to support look ahead for Less' nested selectors. */ | 8 /** Tokenizer state to support look ahead for Less' nested selectors. */ |
| 9 class TokenizerState { | 9 class TokenizerState { |
| 10 final int index; | 10 final int index; |
| (...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 48 // other units) doesn't seem valid. We probably should defer this | 48 // other units) doesn't seem valid. We probably should defer this |
| 49 // analysis until we reach places in the parser where units are expected. | 49 // analysis until we reach places in the parser where units are expected. |
| 50 // I'm not sure this is tokenizing as described in the specs: | 50 // I'm not sure this is tokenizing as described in the specs: |
| 51 // http://dev.w3.org/csswg/css-syntax/ | 51 // http://dev.w3.org/csswg/css-syntax/ |
| 52 // http://dev.w3.org/csswg/selectors4/ | 52 // http://dev.w3.org/csswg/selectors4/ |
| 53 bool inSelector = false; | 53 bool inSelector = false; |
| 54 | 54 |
| 55 int _index = 0; | 55 int _index = 0; |
| 56 int _startIndex = 0; | 56 int _startIndex = 0; |
| 57 | 57 |
| 58 static const String _CDATA_START = '<![CDATA['; | |
| 59 static const String _CDATA_END = ']]>'; | |
| 60 | |
| 61 TokenizerBase(this._file, this._text, this._skipWhitespace, | 58 TokenizerBase(this._file, this._text, this._skipWhitespace, |
| 62 [this._index = 0]); | 59 [this._index = 0]); |
| 63 | 60 |
| 64 Token next(); | 61 Token next(); |
| 65 int getIdentifierKind(); | 62 int getIdentifierKind(); |
| 66 | 63 |
| 67 /** Snapshot of Tokenizer scanning state. */ | 64 /** Snapshot of Tokenizer scanning state. */ |
| 68 TokenizerState get mark => new TokenizerState(this); | 65 TokenizerState get mark => new TokenizerState(this); |
| 69 | 66 |
| 70 /** Restore Tokenizer scanning state. */ | 67 /** Restore Tokenizer scanning state. */ |
| (...skipping 26 matching lines...) Expand all Loading... |
| 97 _index++; | 94 _index++; |
| 98 return true; | 95 return true; |
| 99 } else { | 96 } else { |
| 100 return false; | 97 return false; |
| 101 } | 98 } |
| 102 } else { | 99 } else { |
| 103 return false; | 100 return false; |
| 104 } | 101 } |
| 105 } | 102 } |
| 106 | 103 |
| 107 String _tokenText() { | |
| 108 if (_index < _text.length) { | |
| 109 return _text.substring(_startIndex, _index); | |
| 110 } else { | |
| 111 return _text.substring(_startIndex, _text.length); | |
| 112 } | |
| 113 } | |
| 114 | |
| 115 Token _finishToken(int kind) { | 104 Token _finishToken(int kind) { |
| 116 return new Token(kind, _file.span(_startIndex, _index)); | 105 return new Token(kind, _file.span(_startIndex, _index)); |
| 117 } | 106 } |
| 118 | 107 |
| 119 Token _errorToken([String message = null]) { | 108 Token _errorToken([String message = null]) { |
| 120 return new ErrorToken( | 109 return new ErrorToken( |
| 121 TokenKind.ERROR, _file.span(_startIndex, _index), message); | 110 TokenKind.ERROR, _file.span(_startIndex, _index), message); |
| 122 } | 111 } |
| 123 | 112 |
| 124 Token finishWhitespace() { | 113 Token finishWhitespace() { |
| (...skipping 174 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 299 return _errorToken("invalid hex escape sequence"); | 288 return _errorToken("invalid hex escape sequence"); |
| 300 } else { | 289 } else { |
| 301 buf.add(escapeVal); | 290 buf.add(escapeVal); |
| 302 } | 291 } |
| 303 } else { | 292 } else { |
| 304 buf.add(ch); | 293 buf.add(ch); |
| 305 } | 294 } |
| 306 } | 295 } |
| 307 } | 296 } |
| 308 | 297 |
| 309 Token _finishOpenBrace() { | |
| 310 return _finishToken(TokenKind.LBRACE); | |
| 311 } | |
| 312 | |
| 313 Token _finishCloseBrace() { | |
| 314 return _finishToken(TokenKind.RBRACE); | |
| 315 } | |
| 316 | |
| 317 Token finishString(int quote) { | 298 Token finishString(int quote) { |
| 318 if (_maybeEatChar(quote)) { | 299 if (_maybeEatChar(quote)) { |
| 319 if (_maybeEatChar(quote)) { | 300 if (_maybeEatChar(quote)) { |
| 320 // skip an initial newline | 301 // skip an initial newline |
| 321 _maybeEatChar(TokenChar.NEWLINE); | 302 _maybeEatChar(TokenChar.NEWLINE); |
| 322 return finishMultilineString(quote); | 303 return finishMultilineString(quote); |
| 323 } else { | 304 } else { |
| 324 return _makeStringToken(new List<int>(), false); | 305 return _makeStringToken(new List<int>(), false); |
| 325 } | 306 } |
| 326 } | 307 } |
| (...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 428 | 409 |
| 429 Token finishDot() { | 410 Token finishDot() { |
| 430 if (TokenizerHelpers.isDigit(_peekChar())) { | 411 if (TokenizerHelpers.isDigit(_peekChar())) { |
| 431 eatDigits(); | 412 eatDigits(); |
| 432 return finishNumberExtra(TokenKind.DOUBLE); | 413 return finishNumberExtra(TokenKind.DOUBLE); |
| 433 } else { | 414 } else { |
| 434 return _finishToken(TokenKind.DOT); | 415 return _finishToken(TokenKind.DOT); |
| 435 } | 416 } |
| 436 } | 417 } |
| 437 } | 418 } |
| OLD | NEW |