OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 // Generated by scripts/tokenizer_gen.py. | 4 // Generated by scripts/tokenizer_gen.py. |
5 | 5 |
6 part of csslib.parser; | 6 part of csslib.parser; |
7 | 7 |
8 /** Tokenizer state to support look ahead for Less' nested selectors. */ | 8 /** Tokenizer state to support look ahead for Less' nested selectors. */ |
9 class TokenizerState { | 9 class TokenizerState { |
10 final int index; | 10 final int index; |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
48 // other units) doesn't seem valid. We probably should defer this | 48 // other units) doesn't seem valid. We probably should defer this |
49 // analysis until we reach places in the parser where units are expected. | 49 // analysis until we reach places in the parser where units are expected. |
50 // I'm not sure this is tokenizing as described in the specs: | 50 // I'm not sure this is tokenizing as described in the specs: |
51 // http://dev.w3.org/csswg/css-syntax/ | 51 // http://dev.w3.org/csswg/css-syntax/ |
52 // http://dev.w3.org/csswg/selectors4/ | 52 // http://dev.w3.org/csswg/selectors4/ |
53 bool inSelector = false; | 53 bool inSelector = false; |
54 | 54 |
55 int _index = 0; | 55 int _index = 0; |
56 int _startIndex = 0; | 56 int _startIndex = 0; |
57 | 57 |
58 TokenizerBase(this._file, this._text, this._inString, | 58 TokenizerBase(this._file, this._text, this._inString, [this._index = 0]); |
59 [this._index = 0]); | |
60 | 59 |
61 Token next(); | 60 Token next(); |
62 int getIdentifierKind(); | 61 int getIdentifierKind(); |
63 | 62 |
64 /** Snapshot of Tokenizer scanning state. */ | 63 /** Snapshot of Tokenizer scanning state. */ |
65 TokenizerState get mark => new TokenizerState(this); | 64 TokenizerState get mark => new TokenizerState(this); |
66 | 65 |
67 /** Restore Tokenizer scanning state. */ | 66 /** Restore Tokenizer scanning state. */ |
68 void restore(TokenizerState markedData) { | 67 void restore(TokenizerState markedData) { |
69 _index = markedData.index; | 68 _index = markedData.index; |
70 _startIndex = markedData.startIndex; | 69 _startIndex = markedData.startIndex; |
71 inSelectorExpression = markedData.inSelectorExpression; | 70 inSelectorExpression = markedData.inSelectorExpression; |
72 inSelector = markedData.inSelector; | 71 inSelector = markedData.inSelector; |
73 } | 72 } |
74 | 73 |
75 int _nextChar() { | 74 int _nextChar() { |
76 if (_index < _text.length) { | 75 if (_index < _text.length) { |
77 return _text.codeUnitAt(_index++); | 76 return _text.codeUnitAt(_index++); |
78 } else { | 77 } else { |
79 return 0; | 78 return 0; |
80 } | 79 } |
81 } | 80 } |
82 | 81 |
83 int _peekChar() { | 82 int _peekChar([int offset = 0]) { |
84 if (_index < _text.length) { | 83 if (_index + offset < _text.length) { |
85 return _text.codeUnitAt(_index); | 84 return _text.codeUnitAt(_index + offset); |
86 } else { | 85 } else { |
87 return 0; | 86 return 0; |
88 } | 87 } |
89 } | 88 } |
90 | 89 |
91 bool _maybeEatChar(int ch) { | 90 bool _maybeEatChar(int ch) { |
92 if (_index < _text.length) { | 91 if (_index < _text.length) { |
93 if (_text.codeUnitAt(_index) == ch) { | 92 if (_text.codeUnitAt(_index) == ch) { |
94 _index++; | 93 _index++; |
95 return true; | 94 return true; |
96 } else { | 95 } else { |
97 return false; | 96 return false; |
98 } | 97 } |
99 } else { | 98 } else { |
100 return false; | 99 return false; |
101 } | 100 } |
102 } | 101 } |
103 | 102 |
| 103 bool _nextCharsAreNumber(int first) { |
| 104 if (TokenizerHelpers.isDigit(first)) return true; |
| 105 var second = _peekChar(); |
| 106 if (first == TokenChar.DOT) return TokenizerHelpers.isDigit(second); |
| 107 if (first == TokenChar.PLUS || first == TokenChar.MINUS) { |
| 108 return TokenizerHelpers.isDigit(second) || |
| 109 (second == TokenChar.DOT && TokenizerHelpers.isDigit(_peekChar(1))); |
| 110 } |
| 111 return false; |
| 112 } |
| 113 |
104 Token _finishToken(int kind) { | 114 Token _finishToken(int kind) { |
105 return new Token(kind, _file.span(_startIndex, _index)); | 115 return new Token(kind, _file.span(_startIndex, _index)); |
106 } | 116 } |
107 | 117 |
108 Token _errorToken([String message = null]) { | 118 Token _errorToken([String message = null]) { |
109 return new ErrorToken( | 119 return new ErrorToken( |
110 TokenKind.ERROR, _file.span(_startIndex, _index), message); | 120 TokenKind.ERROR, _file.span(_startIndex, _index), message); |
111 } | 121 } |
112 | 122 |
113 Token finishWhitespace() { | 123 Token finishWhitespace() { |
(...skipping 295 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
409 | 419 |
410 Token finishDot() { | 420 Token finishDot() { |
411 if (TokenizerHelpers.isDigit(_peekChar())) { | 421 if (TokenizerHelpers.isDigit(_peekChar())) { |
412 eatDigits(); | 422 eatDigits(); |
413 return finishNumberExtra(TokenKind.DOUBLE); | 423 return finishNumberExtra(TokenKind.DOUBLE); |
414 } else { | 424 } else { |
415 return _finishToken(TokenKind.DOT); | 425 return _finishToken(TokenKind.DOT); |
416 } | 426 } |
417 } | 427 } |
418 } | 428 } |
OLD | NEW |