OLD | NEW |
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 // Generated by scripts/tokenizer_gen.py. | 4 // Generated by scripts/tokenizer_gen.py. |
5 | 5 |
6 part of csslib.parser; | 6 part of csslib.parser; |
7 | 7 |
8 /** Tokenizer state to support look ahead for Less' nested selectors. */ | 8 /** Tokenizer state to support look ahead for Less' nested selectors. */ |
9 class TokenizerState { | 9 class TokenizerState { |
10 final int index; | 10 final int index; |
11 final int startIndex; | 11 final int startIndex; |
12 final bool inSelectorExpression; | 12 final bool inSelectorExpression; |
13 final bool inSelector; | 13 final bool inSelector; |
14 | 14 |
15 TokenizerState(TokenizerBase base) : | 15 TokenizerState(TokenizerBase base) |
16 index = base._index, | 16 : index = base._index, |
17 startIndex = base._startIndex, | 17 startIndex = base._startIndex, |
18 inSelectorExpression = base.inSelectorExpression, | 18 inSelectorExpression = base.inSelectorExpression, |
19 inSelector = base.inSelector; | 19 inSelector = base.inSelector; |
20 } | 20 } |
21 | 21 |
22 /** | 22 /** |
23 * The base class for our tokenizer. The hand coded parts are in this file, with | 23 * The base class for our tokenizer. The hand coded parts are in this file, with |
24 * the generated parts in the subclass Tokenizer. | 24 * the generated parts in the subclass Tokenizer. |
25 */ | 25 */ |
26 abstract class TokenizerBase { | 26 abstract class TokenizerBase { |
27 final SourceFile _file; | 27 final SourceFile _file; |
28 final String _text; | 28 final String _text; |
29 | 29 |
(...skipping 104 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
134 return _finishToken(TokenKind.WHITESPACE); // note the newline? | 134 return _finishToken(TokenKind.WHITESPACE); // note the newline? |
135 } | 135 } |
136 } else { | 136 } else { |
137 _index--; | 137 _index--; |
138 if (_skipWhitespace) { | 138 if (_skipWhitespace) { |
139 return next(); | 139 return next(); |
140 } else { | 140 } else { |
141 return _finishToken(TokenKind.WHITESPACE); | 141 return _finishToken(TokenKind.WHITESPACE); |
142 } | 142 } |
143 } | 143 } |
144 | |
145 } | 144 } |
146 return _finishToken(TokenKind.END_OF_FILE); | 145 return _finishToken(TokenKind.END_OF_FILE); |
147 } | 146 } |
148 | 147 |
149 Token finishMultiLineComment() { | 148 Token finishMultiLineComment() { |
150 int nesting = 1; | 149 int nesting = 1; |
151 do { | 150 do { |
152 int ch = _nextChar(); | 151 int ch = _nextChar(); |
153 if (ch == 0) { | 152 if (ch == 0) { |
154 return _errorToken(); | 153 return _errorToken(); |
(...skipping 19 matching lines...) Expand all Loading... |
174 while (_index < _text.length) { | 173 while (_index < _text.length) { |
175 if (TokenizerHelpers.isDigit(_text.codeUnitAt(_index))) { | 174 if (TokenizerHelpers.isDigit(_text.codeUnitAt(_index))) { |
176 _index++; | 175 _index++; |
177 } else { | 176 } else { |
178 return; | 177 return; |
179 } | 178 } |
180 } | 179 } |
181 } | 180 } |
182 | 181 |
183 static int _hexDigit(int c) { | 182 static int _hexDigit(int c) { |
184 if(c >= 48/*0*/ && c <= 57/*9*/) { | 183 if (c >= 48 /*0*/ && c <= 57 /*9*/) { |
185 return c - 48; | 184 return c - 48; |
186 } else if (c >= 97/*a*/ && c <= 102/*f*/) { | 185 } else if (c >= 97 /*a*/ && c <= 102 /*f*/) { |
187 return c - 87; | 186 return c - 87; |
188 } else if (c >= 65/*A*/ && c <= 70/*F*/) { | 187 } else if (c >= 65 /*A*/ && c <= 70 /*F*/) { |
189 return c - 55; | 188 return c - 55; |
190 } else { | 189 } else { |
191 return -1; | 190 return -1; |
192 } | 191 } |
193 } | 192 } |
194 | 193 |
195 int readHex([int hexLength]) { | 194 int readHex([int hexLength]) { |
196 int maxIndex; | 195 int maxIndex; |
197 if (hexLength == null) { | 196 if (hexLength == null) { |
198 maxIndex = _text.length - 1; | 197 maxIndex = _text.length - 1; |
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
234 return finishNumberExtra(TokenKind.DOUBLE); | 233 return finishNumberExtra(TokenKind.DOUBLE); |
235 } else { | 234 } else { |
236 _index--; | 235 _index--; |
237 } | 236 } |
238 } | 237 } |
239 | 238 |
240 return finishNumberExtra(TokenKind.INTEGER); | 239 return finishNumberExtra(TokenKind.INTEGER); |
241 } | 240 } |
242 | 241 |
243 Token finishNumberExtra(int kind) { | 242 Token finishNumberExtra(int kind) { |
244 if (_maybeEatChar(101/*e*/) || _maybeEatChar(69/*E*/)) { | 243 if (_maybeEatChar(101 /*e*/) || _maybeEatChar(69 /*E*/)) { |
245 kind = TokenKind.DOUBLE; | 244 kind = TokenKind.DOUBLE; |
246 _maybeEatChar(TokenKind.MINUS); | 245 _maybeEatChar(TokenKind.MINUS); |
247 _maybeEatChar(TokenKind.PLUS); | 246 _maybeEatChar(TokenKind.PLUS); |
248 eatDigits(); | 247 eatDigits(); |
249 } | 248 } |
250 if (_peekChar() != 0 && TokenizerHelpers.isIdentifierStart(_peekChar())) { | 249 if (_peekChar() != 0 && TokenizerHelpers.isIdentifierStart(_peekChar())) { |
251 _nextChar(); | 250 _nextChar(); |
252 return _errorToken("illegal character in number"); | 251 return _errorToken("illegal character in number"); |
253 } | 252 } |
254 | 253 |
(...skipping 14 matching lines...) Expand all Loading... |
269 Token _makeRawStringToken(bool isMultiline) { | 268 Token _makeRawStringToken(bool isMultiline) { |
270 var s; | 269 var s; |
271 if (isMultiline) { | 270 if (isMultiline) { |
272 // Skip initial newline in multiline strings | 271 // Skip initial newline in multiline strings |
273 int start = _startIndex + 4; | 272 int start = _startIndex + 4; |
274 if (_text[start] == '\n') start++; | 273 if (_text[start] == '\n') start++; |
275 s = _text.substring(start, _index - 3); | 274 s = _text.substring(start, _index - 3); |
276 } else { | 275 } else { |
277 s = _text.substring(_startIndex + 2, _index - 1); | 276 s = _text.substring(_startIndex + 2, _index - 1); |
278 } | 277 } |
279 return new LiteralToken(TokenKind.STRING, | 278 return new LiteralToken( |
280 _file.span(_startIndex, _index), s); | 279 TokenKind.STRING, _file.span(_startIndex, _index), s); |
281 } | 280 } |
282 | 281 |
283 Token finishMultilineString(int quote) { | 282 Token finishMultilineString(int quote) { |
284 var buf = <int>[]; | 283 var buf = <int>[]; |
285 while (true) { | 284 while (true) { |
286 int ch = _nextChar(); | 285 int ch = _nextChar(); |
287 if (ch == 0) { | 286 if (ch == 0) { |
288 return _errorToken(); | 287 return _errorToken(); |
289 } else if (ch == quote) { | 288 } else if (ch == quote) { |
290 if (_maybeEatChar(quote)) { | 289 if (_maybeEatChar(quote)) { |
(...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
375 } else { | 374 } else { |
376 buf.add(ch); | 375 buf.add(ch); |
377 } | 376 } |
378 } | 377 } |
379 } | 378 } |
380 | 379 |
381 int readEscapeSequence() { | 380 int readEscapeSequence() { |
382 final ch = _nextChar(); | 381 final ch = _nextChar(); |
383 int hexValue; | 382 int hexValue; |
384 switch (ch) { | 383 switch (ch) { |
385 case 110/*n*/: | 384 case 110 /*n*/ : |
386 return TokenChar.NEWLINE; | 385 return TokenChar.NEWLINE; |
387 case 114/*r*/: | 386 case 114 /*r*/ : |
388 return TokenChar.RETURN; | 387 return TokenChar.RETURN; |
389 case 102/*f*/: | 388 case 102 /*f*/ : |
390 return TokenChar.FF; | 389 return TokenChar.FF; |
391 case 98/*b*/: | 390 case 98 /*b*/ : |
392 return TokenChar.BACKSPACE; | 391 return TokenChar.BACKSPACE; |
393 case 116/*t*/: | 392 case 116 /*t*/ : |
394 return TokenChar.TAB; | 393 return TokenChar.TAB; |
395 case 118/*v*/: | 394 case 118 /*v*/ : |
396 return TokenChar.FF; | 395 return TokenChar.FF; |
397 case 120/*x*/: | 396 case 120 /*x*/ : |
398 hexValue = readHex(2); | 397 hexValue = readHex(2); |
399 break; | 398 break; |
400 case 117/*u*/: | 399 case 117 /*u*/ : |
401 if (_maybeEatChar(TokenChar.LBRACE)) { | 400 if (_maybeEatChar(TokenChar.LBRACE)) { |
402 hexValue = readHex(); | 401 hexValue = readHex(); |
403 if (!_maybeEatChar(TokenChar.RBRACE)) { | 402 if (!_maybeEatChar(TokenChar.RBRACE)) { |
404 return -1; | 403 return -1; |
405 } | 404 } |
406 } else { | 405 } else { |
407 hexValue = readHex(4); | 406 hexValue = readHex(4); |
408 } | 407 } |
409 break; | 408 break; |
410 default: return ch; | 409 default: |
| 410 return ch; |
411 } | 411 } |
412 | 412 |
413 if (hexValue == -1) return -1; | 413 if (hexValue == -1) return -1; |
414 | 414 |
415 // According to the Unicode standard the high and low surrogate halves | 415 // According to the Unicode standard the high and low surrogate halves |
416 // used by UTF-16 (U+D800 through U+DFFF) and values above U+10FFFF | 416 // used by UTF-16 (U+D800 through U+DFFF) and values above U+10FFFF |
417 // are not legal Unicode values. | 417 // are not legal Unicode values. |
418 if (hexValue < 0xD800 || hexValue > 0xDFFF && hexValue <= 0xFFFF) { | 418 if (hexValue < 0xD800 || hexValue > 0xDFFF && hexValue <= 0xFFFF) { |
419 return hexValue; | 419 return hexValue; |
420 } else if (hexValue <= 0x10FFFF){ | 420 } else if (hexValue <= 0x10FFFF) { |
421 messages.error('unicode values greater than 2 bytes not implemented yet', | 421 messages.error('unicode values greater than 2 bytes not implemented yet', |
422 _file.span(_startIndex, _startIndex + 1)); | 422 _file.span(_startIndex, _startIndex + 1)); |
423 return -1; | 423 return -1; |
424 } else { | 424 } else { |
425 return -1; | 425 return -1; |
426 } | 426 } |
427 } | 427 } |
428 | 428 |
429 Token finishDot() { | 429 Token finishDot() { |
430 if (TokenizerHelpers.isDigit(_peekChar())) { | 430 if (TokenizerHelpers.isDigit(_peekChar())) { |
431 eatDigits(); | 431 eatDigits(); |
432 return finishNumberExtra(TokenKind.DOUBLE); | 432 return finishNumberExtra(TokenKind.DOUBLE); |
433 } else { | 433 } else { |
434 return _finishToken(TokenKind.DOT); | 434 return _finishToken(TokenKind.DOT); |
435 } | 435 } |
436 } | 436 } |
437 } | 437 } |
OLD | NEW |