| OLD | NEW |
| 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| 2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
| 3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
| 4 | 4 |
| 5 part of csslib.parser; | 5 part of csslib.parser; |
| 6 | 6 |
| 7 class Tokenizer extends TokenizerBase { | 7 class Tokenizer extends TokenizerBase { |
| 8 /** U+ prefix for unicode characters. */ | 8 /** U+ prefix for unicode characters. */ |
| 9 final UNICODE_U = 'U'.codeUnitAt(0); | 9 final UNICODE_U = 'U'.codeUnitAt(0); |
| 10 final UNICODE_LOWER_U = 'u'.codeUnitAt(0); | 10 final UNICODE_LOWER_U = 'u'.codeUnitAt(0); |
| (...skipping 131 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 142 return _finishToken(TokenKind.SINGLE_QUOTE); | 142 return _finishToken(TokenKind.SINGLE_QUOTE); |
| 143 case TokenChar.DOUBLE_QUOTE: | 143 case TokenChar.DOUBLE_QUOTE: |
| 144 return _finishToken(TokenKind.DOUBLE_QUOTE); | 144 return _finishToken(TokenKind.DOUBLE_QUOTE); |
| 145 case TokenChar.SLASH: | 145 case TokenChar.SLASH: |
| 146 if (_maybeEatChar(TokenChar.ASTERISK)) return finishMultiLineComment(); | 146 if (_maybeEatChar(TokenChar.ASTERISK)) return finishMultiLineComment(); |
| 147 return _finishToken(TokenKind.SLASH); | 147 return _finishToken(TokenKind.SLASH); |
| 148 case TokenChar.LESS: // <!-- | 148 case TokenChar.LESS: // <!-- |
| 149 if (_maybeEatChar(TokenChar.BANG)) { | 149 if (_maybeEatChar(TokenChar.BANG)) { |
| 150 if (_maybeEatChar(TokenChar.MINUS) && | 150 if (_maybeEatChar(TokenChar.MINUS) && |
| 151 _maybeEatChar(TokenChar.MINUS)) { | 151 _maybeEatChar(TokenChar.MINUS)) { |
| 152 return finishMultiLineComment(); | 152 return finishHtmlComment(); |
| 153 } else if (_maybeEatChar(TokenChar.LBRACK) && | 153 } else if (_maybeEatChar(TokenChar.LBRACK) && |
| 154 _maybeEatChar(CDATA_NAME[0]) && | 154 _maybeEatChar(CDATA_NAME[0]) && |
| 155 _maybeEatChar(CDATA_NAME[1]) && | 155 _maybeEatChar(CDATA_NAME[1]) && |
| 156 _maybeEatChar(CDATA_NAME[2]) && | 156 _maybeEatChar(CDATA_NAME[2]) && |
| 157 _maybeEatChar(CDATA_NAME[3]) && | 157 _maybeEatChar(CDATA_NAME[3]) && |
| 158 _maybeEatChar(CDATA_NAME[4]) && | 158 _maybeEatChar(CDATA_NAME[4]) && |
| 159 _maybeEatChar(TokenChar.LBRACK)) { | 159 _maybeEatChar(TokenChar.LBRACK)) { |
| 160 // <![CDATA[ | 160 // <![CDATA[ |
| 161 return next(); | 161 return next(); |
| 162 } | 162 } |
| (...skipping 223 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 386 return; | 386 return; |
| 387 } | 387 } |
| 388 } | 388 } |
| 389 } | 389 } |
| 390 | 390 |
| 391 Token finishUnicodeRange() { | 391 Token finishUnicodeRange() { |
| 392 eatQuestionMarks(); | 392 eatQuestionMarks(); |
| 393 return _finishToken(TokenKind.HEX_RANGE); | 393 return _finishToken(TokenKind.HEX_RANGE); |
| 394 } | 394 } |
| 395 | 395 |
| 396 Token finishMultiLineComment() { | 396 Token finishHtmlComment() { |
| 397 while (true) { | 397 while (true) { |
| 398 int ch = _nextChar(); | 398 int ch = _nextChar(); |
| 399 if (ch == 0) { | 399 if (ch == 0) { |
| 400 return _finishToken(TokenKind.INCOMPLETE_COMMENT); | 400 return _finishToken(TokenKind.INCOMPLETE_COMMENT); |
| 401 } else if (ch == 42 /*'*'*/) { | |
| 402 if (_maybeEatChar(47 /*'/'*/)) { | |
| 403 if (_inString) { | |
| 404 return next(); | |
| 405 } else { | |
| 406 return _finishToken(TokenKind.COMMENT); | |
| 407 } | |
| 408 } | |
| 409 } else if (ch == TokenChar.MINUS) { | 401 } else if (ch == TokenChar.MINUS) { |
| 410 /* Check if close part of Comment Definition --> (CDC). */ | 402 /* Check if close part of Comment Definition --> (CDC). */ |
| 411 if (_maybeEatChar(TokenChar.MINUS)) { | 403 if (_maybeEatChar(TokenChar.MINUS)) { |
| 412 if (_maybeEatChar(TokenChar.GREATER)) { | 404 if (_maybeEatChar(TokenChar.GREATER)) { |
| 413 if (_inString) { | 405 if (_inString) { |
| 414 return next(); | 406 return next(); |
| 415 } else { | 407 } else { |
| 416 return _finishToken(TokenKind.HTML_COMMENT); | 408 return _finishToken(TokenKind.HTML_COMMENT); |
| 417 } | 409 } |
| 418 } | 410 } |
| 419 } | 411 } |
| 420 } | 412 } |
| 421 } | 413 } |
| 422 } | 414 } |
| 415 |
| 416 Token finishMultiLineComment() { |
| 417 while (true) { |
| 418 int ch = _nextChar(); |
| 419 if (ch == 0) { |
| 420 return _finishToken(TokenKind.INCOMPLETE_COMMENT); |
| 421 } else if (ch == 42 /*'*'*/) { |
| 422 if (_maybeEatChar(47 /*'/'*/)) { |
| 423 if (_inString) { |
| 424 return next(); |
| 425 } else { |
| 426 return _finishToken(TokenKind.COMMENT); |
| 427 } |
| 428 } |
| 429 } |
| 430 } |
| 431 } |
| 423 } | 432 } |
| 424 | 433 |
| 425 /** Static helper methods. */ | 434 /** Static helper methods. */ |
| 426 class TokenizerHelpers { | 435 class TokenizerHelpers { |
| 427 static bool isIdentifierStart(int c) { | 436 static bool isIdentifierStart(int c) { |
| 428 return isIdentifierStartExpr(c) || c == 45 /*-*/; | 437 return isIdentifierStartExpr(c) || c == 45 /*-*/; |
| 429 } | 438 } |
| 430 | 439 |
| 431 static bool isDigit(int c) { | 440 static bool isDigit(int c) { |
| 432 return (c >= 48 /*0*/ && c <= 57 /*9*/); | 441 return (c >= 48 /*0*/ && c <= 57 /*9*/); |
| (...skipping 20 matching lines...) Expand all Loading... |
| 453 c == 95 /*_*/ || | 462 c == 95 /*_*/ || |
| 454 c >= 0xA0 || | 463 c >= 0xA0 || |
| 455 c == 92 /*\*/); | 464 c == 92 /*\*/); |
| 456 } | 465 } |
| 457 | 466 |
| 458 /** Pseudo function expressions identifiers can't have a minus sign. */ | 467 /** Pseudo function expressions identifiers can't have a minus sign. */ |
| 459 static bool isIdentifierPartExpr(int c) { | 468 static bool isIdentifierPartExpr(int c) { |
| 460 return (isIdentifierStartExpr(c) || isDigit(c)); | 469 return (isIdentifierStartExpr(c) || isDigit(c)); |
| 461 } | 470 } |
| 462 } | 471 } |
| OLD | NEW |