| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 // Generated by scripts/tokenizer_gen.py. | 4 // Generated by scripts/tokenizer_gen.py. |
| 5 | 5 |
| 6 part of csslib.parser; | 6 part of csslib.parser; |
| 7 | 7 |
| 8 /** Tokenizer state to support look ahead for Less' nested selectors. */ | 8 /** Tokenizer state to support look ahead for Less' nested selectors. */ |
| 9 class TokenizerState { | 9 class TokenizerState { |
| 10 final int index; | 10 final int index; |
| 11 final int startIndex; | 11 final int startIndex; |
| 12 final bool inSelectorExpression; | 12 final bool inSelectorExpression; |
| 13 final bool inSelector; | 13 final bool inSelector; |
| 14 | 14 |
| 15 TokenizerState(TokenizerBase base) : | 15 TokenizerState(TokenizerBase base) |
| 16 index = base._index, | 16 : index = base._index, |
| 17 startIndex = base._startIndex, | 17 startIndex = base._startIndex, |
| 18 inSelectorExpression = base.inSelectorExpression, | 18 inSelectorExpression = base.inSelectorExpression, |
| 19 inSelector = base.inSelector; | 19 inSelector = base.inSelector; |
| 20 } | 20 } |
| 21 | 21 |
| 22 /** | 22 /** |
| 23 * The base class for our tokenizer. The hand coded parts are in this file, with | 23 * The base class for our tokenizer. The hand coded parts are in this file, with |
| 24 * the generated parts in the subclass Tokenizer. | 24 * the generated parts in the subclass Tokenizer. |
| 25 */ | 25 */ |
| 26 abstract class TokenizerBase { | 26 abstract class TokenizerBase { |
| 27 final SourceFile _file; | 27 final SourceFile _file; |
| 28 final String _text; | 28 final String _text; |
| 29 | 29 |
| (...skipping 104 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 134 return _finishToken(TokenKind.WHITESPACE); // note the newline? | 134 return _finishToken(TokenKind.WHITESPACE); // note the newline? |
| 135 } | 135 } |
| 136 } else { | 136 } else { |
| 137 _index--; | 137 _index--; |
| 138 if (_skipWhitespace) { | 138 if (_skipWhitespace) { |
| 139 return next(); | 139 return next(); |
| 140 } else { | 140 } else { |
| 141 return _finishToken(TokenKind.WHITESPACE); | 141 return _finishToken(TokenKind.WHITESPACE); |
| 142 } | 142 } |
| 143 } | 143 } |
| 144 | |
| 145 } | 144 } |
| 146 return _finishToken(TokenKind.END_OF_FILE); | 145 return _finishToken(TokenKind.END_OF_FILE); |
| 147 } | 146 } |
| 148 | 147 |
| 149 Token finishMultiLineComment() { | 148 Token finishMultiLineComment() { |
| 150 int nesting = 1; | 149 int nesting = 1; |
| 151 do { | 150 do { |
| 152 int ch = _nextChar(); | 151 int ch = _nextChar(); |
| 153 if (ch == 0) { | 152 if (ch == 0) { |
| 154 return _errorToken(); | 153 return _errorToken(); |
| (...skipping 19 matching lines...) Expand all Loading... |
| 174 while (_index < _text.length) { | 173 while (_index < _text.length) { |
| 175 if (TokenizerHelpers.isDigit(_text.codeUnitAt(_index))) { | 174 if (TokenizerHelpers.isDigit(_text.codeUnitAt(_index))) { |
| 176 _index++; | 175 _index++; |
| 177 } else { | 176 } else { |
| 178 return; | 177 return; |
| 179 } | 178 } |
| 180 } | 179 } |
| 181 } | 180 } |
| 182 | 181 |
| 183 static int _hexDigit(int c) { | 182 static int _hexDigit(int c) { |
| 184 if(c >= 48/*0*/ && c <= 57/*9*/) { | 183 if (c >= 48 /*0*/ && c <= 57 /*9*/) { |
| 185 return c - 48; | 184 return c - 48; |
| 186 } else if (c >= 97/*a*/ && c <= 102/*f*/) { | 185 } else if (c >= 97 /*a*/ && c <= 102 /*f*/) { |
| 187 return c - 87; | 186 return c - 87; |
| 188 } else if (c >= 65/*A*/ && c <= 70/*F*/) { | 187 } else if (c >= 65 /*A*/ && c <= 70 /*F*/) { |
| 189 return c - 55; | 188 return c - 55; |
| 190 } else { | 189 } else { |
| 191 return -1; | 190 return -1; |
| 192 } | 191 } |
| 193 } | 192 } |
| 194 | 193 |
| 195 int readHex([int hexLength]) { | 194 int readHex([int hexLength]) { |
| 196 int maxIndex; | 195 int maxIndex; |
| 197 if (hexLength == null) { | 196 if (hexLength == null) { |
| 198 maxIndex = _text.length - 1; | 197 maxIndex = _text.length - 1; |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 234 return finishNumberExtra(TokenKind.DOUBLE); | 233 return finishNumberExtra(TokenKind.DOUBLE); |
| 235 } else { | 234 } else { |
| 236 _index--; | 235 _index--; |
| 237 } | 236 } |
| 238 } | 237 } |
| 239 | 238 |
| 240 return finishNumberExtra(TokenKind.INTEGER); | 239 return finishNumberExtra(TokenKind.INTEGER); |
| 241 } | 240 } |
| 242 | 241 |
| 243 Token finishNumberExtra(int kind) { | 242 Token finishNumberExtra(int kind) { |
| 244 if (_maybeEatChar(101/*e*/) || _maybeEatChar(69/*E*/)) { | 243 if (_maybeEatChar(101 /*e*/) || _maybeEatChar(69 /*E*/)) { |
| 245 kind = TokenKind.DOUBLE; | 244 kind = TokenKind.DOUBLE; |
| 246 _maybeEatChar(TokenKind.MINUS); | 245 _maybeEatChar(TokenKind.MINUS); |
| 247 _maybeEatChar(TokenKind.PLUS); | 246 _maybeEatChar(TokenKind.PLUS); |
| 248 eatDigits(); | 247 eatDigits(); |
| 249 } | 248 } |
| 250 if (_peekChar() != 0 && TokenizerHelpers.isIdentifierStart(_peekChar())) { | 249 if (_peekChar() != 0 && TokenizerHelpers.isIdentifierStart(_peekChar())) { |
| 251 _nextChar(); | 250 _nextChar(); |
| 252 return _errorToken("illegal character in number"); | 251 return _errorToken("illegal character in number"); |
| 253 } | 252 } |
| 254 | 253 |
| (...skipping 14 matching lines...) Expand all Loading... |
| 269 Token _makeRawStringToken(bool isMultiline) { | 268 Token _makeRawStringToken(bool isMultiline) { |
| 270 var s; | 269 var s; |
| 271 if (isMultiline) { | 270 if (isMultiline) { |
| 272 // Skip initial newline in multiline strings | 271 // Skip initial newline in multiline strings |
| 273 int start = _startIndex + 4; | 272 int start = _startIndex + 4; |
| 274 if (_text[start] == '\n') start++; | 273 if (_text[start] == '\n') start++; |
| 275 s = _text.substring(start, _index - 3); | 274 s = _text.substring(start, _index - 3); |
| 276 } else { | 275 } else { |
| 277 s = _text.substring(_startIndex + 2, _index - 1); | 276 s = _text.substring(_startIndex + 2, _index - 1); |
| 278 } | 277 } |
| 279 return new LiteralToken(TokenKind.STRING, | 278 return new LiteralToken( |
| 280 _file.span(_startIndex, _index), s); | 279 TokenKind.STRING, _file.span(_startIndex, _index), s); |
| 281 } | 280 } |
| 282 | 281 |
| 283 Token finishMultilineString(int quote) { | 282 Token finishMultilineString(int quote) { |
| 284 var buf = <int>[]; | 283 var buf = <int>[]; |
| 285 while (true) { | 284 while (true) { |
| 286 int ch = _nextChar(); | 285 int ch = _nextChar(); |
| 287 if (ch == 0) { | 286 if (ch == 0) { |
| 288 return _errorToken(); | 287 return _errorToken(); |
| 289 } else if (ch == quote) { | 288 } else if (ch == quote) { |
| 290 if (_maybeEatChar(quote)) { | 289 if (_maybeEatChar(quote)) { |
| (...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 375 } else { | 374 } else { |
| 376 buf.add(ch); | 375 buf.add(ch); |
| 377 } | 376 } |
| 378 } | 377 } |
| 379 } | 378 } |
| 380 | 379 |
| 381 int readEscapeSequence() { | 380 int readEscapeSequence() { |
| 382 final ch = _nextChar(); | 381 final ch = _nextChar(); |
| 383 int hexValue; | 382 int hexValue; |
| 384 switch (ch) { | 383 switch (ch) { |
| 385 case 110/*n*/: | 384 case 110 /*n*/ : |
| 386 return TokenChar.NEWLINE; | 385 return TokenChar.NEWLINE; |
| 387 case 114/*r*/: | 386 case 114 /*r*/ : |
| 388 return TokenChar.RETURN; | 387 return TokenChar.RETURN; |
| 389 case 102/*f*/: | 388 case 102 /*f*/ : |
| 390 return TokenChar.FF; | 389 return TokenChar.FF; |
| 391 case 98/*b*/: | 390 case 98 /*b*/ : |
| 392 return TokenChar.BACKSPACE; | 391 return TokenChar.BACKSPACE; |
| 393 case 116/*t*/: | 392 case 116 /*t*/ : |
| 394 return TokenChar.TAB; | 393 return TokenChar.TAB; |
| 395 case 118/*v*/: | 394 case 118 /*v*/ : |
| 396 return TokenChar.FF; | 395 return TokenChar.FF; |
| 397 case 120/*x*/: | 396 case 120 /*x*/ : |
| 398 hexValue = readHex(2); | 397 hexValue = readHex(2); |
| 399 break; | 398 break; |
| 400 case 117/*u*/: | 399 case 117 /*u*/ : |
| 401 if (_maybeEatChar(TokenChar.LBRACE)) { | 400 if (_maybeEatChar(TokenChar.LBRACE)) { |
| 402 hexValue = readHex(); | 401 hexValue = readHex(); |
| 403 if (!_maybeEatChar(TokenChar.RBRACE)) { | 402 if (!_maybeEatChar(TokenChar.RBRACE)) { |
| 404 return -1; | 403 return -1; |
| 405 } | 404 } |
| 406 } else { | 405 } else { |
| 407 hexValue = readHex(4); | 406 hexValue = readHex(4); |
| 408 } | 407 } |
| 409 break; | 408 break; |
| 410 default: return ch; | 409 default: |
| 410 return ch; |
| 411 } | 411 } |
| 412 | 412 |
| 413 if (hexValue == -1) return -1; | 413 if (hexValue == -1) return -1; |
| 414 | 414 |
| 415 // According to the Unicode standard the high and low surrogate halves | 415 // According to the Unicode standard the high and low surrogate halves |
| 416 // used by UTF-16 (U+D800 through U+DFFF) and values above U+10FFFF | 416 // used by UTF-16 (U+D800 through U+DFFF) and values above U+10FFFF |
| 417 // are not legal Unicode values. | 417 // are not legal Unicode values. |
| 418 if (hexValue < 0xD800 || hexValue > 0xDFFF && hexValue <= 0xFFFF) { | 418 if (hexValue < 0xD800 || hexValue > 0xDFFF && hexValue <= 0xFFFF) { |
| 419 return hexValue; | 419 return hexValue; |
| 420 } else if (hexValue <= 0x10FFFF){ | 420 } else if (hexValue <= 0x10FFFF) { |
| 421 messages.error('unicode values greater than 2 bytes not implemented yet', | 421 messages.error('unicode values greater than 2 bytes not implemented yet', |
| 422 _file.span(_startIndex, _startIndex + 1)); | 422 _file.span(_startIndex, _startIndex + 1)); |
| 423 return -1; | 423 return -1; |
| 424 } else { | 424 } else { |
| 425 return -1; | 425 return -1; |
| 426 } | 426 } |
| 427 } | 427 } |
| 428 | 428 |
| 429 Token finishDot() { | 429 Token finishDot() { |
| 430 if (TokenizerHelpers.isDigit(_peekChar())) { | 430 if (TokenizerHelpers.isDigit(_peekChar())) { |
| 431 eatDigits(); | 431 eatDigits(); |
| 432 return finishNumberExtra(TokenKind.DOUBLE); | 432 return finishNumberExtra(TokenKind.DOUBLE); |
| 433 } else { | 433 } else { |
| 434 return _finishToken(TokenKind.DOT); | 434 return _finishToken(TokenKind.DOT); |
| 435 } | 435 } |
| 436 } | 436 } |
| 437 } | 437 } |
| OLD | NEW |