| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 // Generated by scripts/tokenizer_gen.py. | 4 // Generated by scripts/tokenizer_gen.py. |
| 5 | 5 |
| 6 part of csslib.parser; | 6 part of csslib.parser; |
| 7 | 7 |
| 8 /** Tokenizer state to support look ahead for Less' nested selectors. */ | 8 /** Tokenizer state to support look ahead for Less' nested selectors. */ |
| 9 class TokenizerState { | 9 class TokenizerState { |
| 10 final int index; | 10 final int index; |
| (...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 48 // other units) doesn't seem valid. We probably should defer this | 48 // other units) doesn't seem valid. We probably should defer this |
| 49 // analysis until we reach places in the parser where units are expected. | 49 // analysis until we reach places in the parser where units are expected. |
| 50 // I'm not sure this is tokenizing as described in the specs: | 50 // I'm not sure this is tokenizing as described in the specs: |
| 51 // http://dev.w3.org/csswg/css-syntax/ | 51 // http://dev.w3.org/csswg/css-syntax/ |
| 52 // http://dev.w3.org/csswg/selectors4/ | 52 // http://dev.w3.org/csswg/selectors4/ |
| 53 bool inSelector = false; | 53 bool inSelector = false; |
| 54 | 54 |
| 55 int _index = 0; | 55 int _index = 0; |
| 56 int _startIndex = 0; | 56 int _startIndex = 0; |
| 57 | 57 |
| 58 TokenizerBase(this._file, this._text, this._inString, | 58 TokenizerBase(this._file, this._text, this._inString, [this._index = 0]); |
| 59 [this._index = 0]); | |
| 60 | 59 |
| 61 Token next(); | 60 Token next(); |
| 62 int getIdentifierKind(); | 61 int getIdentifierKind(); |
| 63 | 62 |
| 64 /** Snapshot of Tokenizer scanning state. */ | 63 /** Snapshot of Tokenizer scanning state. */ |
| 65 TokenizerState get mark => new TokenizerState(this); | 64 TokenizerState get mark => new TokenizerState(this); |
| 66 | 65 |
| 67 /** Restore Tokenizer scanning state. */ | 66 /** Restore Tokenizer scanning state. */ |
| 68 void restore(TokenizerState markedData) { | 67 void restore(TokenizerState markedData) { |
| 69 _index = markedData.index; | 68 _index = markedData.index; |
| 70 _startIndex = markedData.startIndex; | 69 _startIndex = markedData.startIndex; |
| 71 inSelectorExpression = markedData.inSelectorExpression; | 70 inSelectorExpression = markedData.inSelectorExpression; |
| 72 inSelector = markedData.inSelector; | 71 inSelector = markedData.inSelector; |
| 73 } | 72 } |
| 74 | 73 |
| 75 int _nextChar() { | 74 int _nextChar() { |
| 76 if (_index < _text.length) { | 75 if (_index < _text.length) { |
| 77 return _text.codeUnitAt(_index++); | 76 return _text.codeUnitAt(_index++); |
| 78 } else { | 77 } else { |
| 79 return 0; | 78 return 0; |
| 80 } | 79 } |
| 81 } | 80 } |
| 82 | 81 |
| 83 int _peekChar() { | 82 int _peekChar([int offset = 0]) { |
| 84 if (_index < _text.length) { | 83 if (_index + offset < _text.length) { |
| 85 return _text.codeUnitAt(_index); | 84 return _text.codeUnitAt(_index + offset); |
| 86 } else { | 85 } else { |
| 87 return 0; | 86 return 0; |
| 88 } | 87 } |
| 89 } | 88 } |
| 90 | 89 |
| 91 bool _maybeEatChar(int ch) { | 90 bool _maybeEatChar(int ch) { |
| 92 if (_index < _text.length) { | 91 if (_index < _text.length) { |
| 93 if (_text.codeUnitAt(_index) == ch) { | 92 if (_text.codeUnitAt(_index) == ch) { |
| 94 _index++; | 93 _index++; |
| 95 return true; | 94 return true; |
| 96 } else { | 95 } else { |
| 97 return false; | 96 return false; |
| 98 } | 97 } |
| 99 } else { | 98 } else { |
| 100 return false; | 99 return false; |
| 101 } | 100 } |
| 102 } | 101 } |
| 103 | 102 |
| 103 bool _nextCharsAreNumber(int first) { |
| 104 if (TokenizerHelpers.isDigit(first)) return true; |
| 105 var second = _peekChar(); |
| 106 if (first == TokenChar.DOT) return TokenizerHelpers.isDigit(second); |
| 107 if (first == TokenChar.PLUS || first == TokenChar.MINUS) { |
| 108 return TokenizerHelpers.isDigit(second) || |
| 109 (second == TokenChar.DOT && TokenizerHelpers.isDigit(_peekChar(1))); |
| 110 } |
| 111 return false; |
| 112 } |
| 113 |
| 104 Token _finishToken(int kind) { | 114 Token _finishToken(int kind) { |
| 105 return new Token(kind, _file.span(_startIndex, _index)); | 115 return new Token(kind, _file.span(_startIndex, _index)); |
| 106 } | 116 } |
| 107 | 117 |
| 108 Token _errorToken([String message = null]) { | 118 Token _errorToken([String message = null]) { |
| 109 return new ErrorToken( | 119 return new ErrorToken( |
| 110 TokenKind.ERROR, _file.span(_startIndex, _index), message); | 120 TokenKind.ERROR, _file.span(_startIndex, _index), message); |
| 111 } | 121 } |
| 112 | 122 |
| 113 Token finishWhitespace() { | 123 Token finishWhitespace() { |
| (...skipping 295 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 409 | 419 |
| 410 Token finishDot() { | 420 Token finishDot() { |
| 411 if (TokenizerHelpers.isDigit(_peekChar())) { | 421 if (TokenizerHelpers.isDigit(_peekChar())) { |
| 412 eatDigits(); | 422 eatDigits(); |
| 413 return finishNumberExtra(TokenKind.DOUBLE); | 423 return finishNumberExtra(TokenKind.DOUBLE); |
| 414 } else { | 424 } else { |
| 415 return _finishToken(TokenKind.DOT); | 425 return _finishToken(TokenKind.DOT); |
| 416 } | 426 } |
| 417 } | 427 } |
| 418 } | 428 } |
| OLD | NEW |