| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | |
| 2 // for details. All rights reserved. Use of this source code is governed by a | |
| 3 // BSD-style license that can be found in the LICENSE file. | |
| 4 | |
| 5 class Tokenizer extends CSSTokenizerBase { | |
| 6 TokenKind cssTokens; | |
| 7 | |
| 8 bool _selectorParsing; | |
| 9 | |
| 10 Tokenizer(SourceFile source, bool skipWhitespace, [int index = 0]) | |
| 11 : super(source, skipWhitespace, index), _selectorParsing = false { | |
| 12 cssTokens = new TokenKind(); | |
| 13 } | |
| 14 | |
| 15 int get startIndex => _startIndex; | |
| 16 | |
| 17 Token next() { | |
| 18 // keep track of our starting position | |
| 19 _startIndex = _index; | |
| 20 | |
| 21 if (_interpStack != null && _interpStack.depth == 0) { | |
| 22 var istack = _interpStack; | |
| 23 _interpStack = _interpStack.pop(); | |
| 24 | |
| 25 /* TODO(terry): Enable for variable and string interpolation. | |
| 26 * if (istack.isMultiline) { | |
| 27 * return finishMultilineStringBody(istack.quote); | |
| 28 * } else { | |
| 29 * return finishStringBody(istack.quote); | |
| 30 * } | |
| 31 */ | |
| 32 } | |
| 33 | |
| 34 int ch; | |
| 35 ch = _nextChar(); | |
| 36 switch(ch) { | |
| 37 case 0: | |
| 38 return _finishToken(TokenKind.END_OF_FILE); | |
| 39 case cssTokens.tokens[TokenKind.SPACE]: | |
| 40 case cssTokens.tokens[TokenKind.TAB]: | |
| 41 case cssTokens.tokens[TokenKind.NEWLINE]: | |
| 42 case cssTokens.tokens[TokenKind.RETURN]: | |
| 43 return finishWhitespace(); | |
| 44 case cssTokens.tokens[TokenKind.END_OF_FILE]: | |
| 45 return _finishToken(TokenKind.END_OF_FILE); | |
| 46 case cssTokens.tokens[TokenKind.AT]: | |
| 47 return _finishToken(TokenKind.AT); | |
| 48 case cssTokens.tokens[TokenKind.DOT]: | |
| 49 int start = _startIndex; // Start where the dot started. | |
| 50 if (maybeEatDigit()) { | |
| 51 // looks like a number dot followed by digit(s). | |
| 52 Token number = finishNumber(); | |
| 53 if (number.kind == TokenKind.INTEGER) { | |
| 54 // It's a number but it's preceeded by a dot, so make it a double. | |
| 55 _startIndex = start; | |
| 56 return _finishToken(TokenKind.DOUBLE); | |
| 57 } else { | |
| 58 // Don't allow dot followed by a double (e.g, '..1'). | |
| 59 return _errorToken(); | |
| 60 } | |
| 61 } else { | |
| 62 // It's really a dot. | |
| 63 return _finishToken(TokenKind.DOT); | |
| 64 } | |
| 65 case cssTokens.tokens[TokenKind.LPAREN]: | |
| 66 return _finishToken(TokenKind.LPAREN); | |
| 67 case cssTokens.tokens[TokenKind.RPAREN]: | |
| 68 return _finishToken(TokenKind.RPAREN); | |
| 69 case cssTokens.tokens[TokenKind.LBRACE]: | |
| 70 return _finishToken(TokenKind.LBRACE); | |
| 71 case cssTokens.tokens[TokenKind.RBRACE]: | |
| 72 return _finishToken(TokenKind.RBRACE); | |
| 73 case cssTokens.tokens[TokenKind.LBRACK]: | |
| 74 return _finishToken(TokenKind.LBRACK); | |
| 75 case cssTokens.tokens[TokenKind.RBRACK]: | |
| 76 return _finishToken(TokenKind.RBRACK); | |
| 77 case cssTokens.tokens[TokenKind.HASH]: | |
| 78 return _finishToken(TokenKind.HASH); | |
| 79 case cssTokens.tokens[TokenKind.PLUS]: | |
| 80 if (maybeEatDigit()) { | |
| 81 return finishNumber(); | |
| 82 } else { | |
| 83 return _finishToken(TokenKind.PLUS); | |
| 84 } | |
| 85 case cssTokens.tokens[TokenKind.MINUS]: | |
| 86 if (maybeEatDigit()) { | |
| 87 return finishNumber(); | |
| 88 } else if (TokenizerHelpers.isIdentifierStart(ch)) { | |
| 89 return this.finishIdentifier(ch); | |
| 90 } else { | |
| 91 return _finishToken(TokenKind.MINUS); | |
| 92 } | |
| 93 case cssTokens.tokens[TokenKind.GREATER]: | |
| 94 return _finishToken(TokenKind.GREATER); | |
| 95 case cssTokens.tokens[TokenKind.TILDE]: | |
| 96 if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) { | |
| 97 return _finishToken(TokenKind.INCLUDES); // ~= | |
| 98 } else { | |
| 99 return _finishToken(TokenKind.TILDE); | |
| 100 } | |
| 101 case cssTokens.tokens[TokenKind.ASTERISK]: | |
| 102 if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) { | |
| 103 return _finishToken(TokenKind.SUBSTRING_MATCH); // *= | |
| 104 } else { | |
| 105 return _finishToken(TokenKind.ASTERISK); | |
| 106 } | |
| 107 case cssTokens.tokens[TokenKind.NAMESPACE]: | |
| 108 return _finishToken(TokenKind.NAMESPACE); | |
| 109 case cssTokens.tokens[TokenKind.COLON]: | |
| 110 return _finishToken(TokenKind.COLON); | |
| 111 case cssTokens.tokens[TokenKind.COMMA]: | |
| 112 return _finishToken(TokenKind.COMMA); | |
| 113 case cssTokens.tokens[TokenKind.SEMICOLON]: | |
| 114 return _finishToken(TokenKind.SEMICOLON); | |
| 115 case cssTokens.tokens[TokenKind.PERCENT]: | |
| 116 return _finishToken(TokenKind.PERCENT); | |
| 117 case cssTokens.tokens[TokenKind.SINGLE_QUOTE]: | |
| 118 return _finishToken(TokenKind.SINGLE_QUOTE); | |
| 119 case cssTokens.tokens[TokenKind.DOUBLE_QUOTE]: | |
| 120 return _finishToken(TokenKind.DOUBLE_QUOTE); | |
| 121 case cssTokens.tokens[TokenKind.SLASH]: | |
| 122 if (_maybeEatChar(cssTokens.tokens[TokenKind.ASTERISK])) { | |
| 123 return finishMultiLineComment(); | |
| 124 } else { | |
| 125 return _finishToken(TokenKind.SLASH); | |
| 126 } | |
| 127 case cssTokens.tokens[TokenKind.LESS]: // <!-- | |
| 128 if (_maybeEatChar(cssTokens.tokens[TokenKind.BANG]) && | |
| 129 _maybeEatChar(cssTokens.tokens[TokenKind.MINUS]) && | |
| 130 _maybeEatChar(cssTokens.tokens[TokenKind.MINUS])) { | |
| 131 return finishMultiLineComment(); | |
| 132 } else { | |
| 133 return _finishToken(TokenKind.LESS); | |
| 134 } | |
| 135 case cssTokens.tokens[TokenKind.EQUALS]: | |
| 136 return _finishToken(TokenKind.EQUALS); | |
| 137 case cssTokens.tokens[TokenKind.OR]: | |
| 138 if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) { | |
| 139 return _finishToken(TokenKind.DASH_MATCH); // |= | |
| 140 } else { | |
| 141 return _finishToken(TokenKind.OR); | |
| 142 } | |
| 143 case cssTokens.tokens[TokenKind.CARET]: | |
| 144 if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) { | |
| 145 return _finishToken(TokenKind.PREFIX_MATCH); // ^= | |
| 146 } else { | |
| 147 return _finishToken(TokenKind.CARET); | |
| 148 } | |
| 149 case cssTokens.tokens[TokenKind.DOLLAR]: | |
| 150 if (_maybeEatChar(cssTokens.tokens[TokenKind.EQUALS])) { | |
| 151 return _finishToken(TokenKind.SUFFIX_MATCH); // $= | |
| 152 } else { | |
| 153 return _finishToken(TokenKind.DOLLAR); | |
| 154 } | |
| 155 case cssTokens.tokens[TokenKind.BANG]: | |
| 156 Token tok = finishIdentifier(ch); | |
| 157 return (tok == null) ? _finishToken(TokenKind.BANG) : tok; | |
| 158 default: | |
| 159 if (TokenizerHelpers.isIdentifierStart(ch)) { | |
| 160 return this.finishIdentifier(ch); | |
| 161 } else if (TokenizerHelpers.isDigit(ch)) { | |
| 162 return this.finishNumber(); | |
| 163 } else { | |
| 164 return _errorToken(); | |
| 165 } | |
| 166 } | |
| 167 } | |
| 168 | |
| 169 // TODO(jmesserly): we need a way to emit human readable error messages from | |
| 170 // the tokenizer. | |
| 171 Token _errorToken([String message = null]) { | |
| 172 return _finishToken(TokenKind.ERROR); | |
| 173 } | |
| 174 | |
| 175 int getIdentifierKind() { | |
| 176 // Is the identifier a unit type? | |
| 177 int tokId = TokenKind.matchUnits(_text, _startIndex, _index - _startIndex); | |
| 178 if (tokId == -1) { | |
| 179 // No, is it a directive? | |
| 180 tokId = TokenKind.matchDirectives( | |
| 181 _text, _startIndex, _index - _startIndex); | |
| 182 } | |
| 183 if (tokId == -1) { | |
| 184 tokId = (_text.substring(_startIndex, _index) == '!important') ? | |
| 185 TokenKind.IMPORTANT : -1; | |
| 186 } | |
| 187 | |
| 188 return tokId >= 0 ? tokId : TokenKind.IDENTIFIER; | |
| 189 } | |
| 190 | |
| 191 // Need to override so CSS version of isIdentifierPart is used. | |
| 192 Token finishIdentifier(int ch) { | |
| 193 while (_index < _text.length) { | |
| 194 // if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index++))) { | |
| 195 if (!TokenizerHelpers.isIdentifierPart(_text.codeUnitAt(_index))) { | |
| 196 // _index--; | |
| 197 break; | |
| 198 } else { | |
| 199 _index += 1; | |
| 200 } | |
| 201 } | |
| 202 if (_interpStack != null && _interpStack.depth == -1) { | |
| 203 _interpStack.depth = 0; | |
| 204 } | |
| 205 int kind = getIdentifierKind(); | |
| 206 if (kind == TokenKind.IDENTIFIER) { | |
| 207 return _finishToken(TokenKind.IDENTIFIER); | |
| 208 } else { | |
| 209 return _finishToken(kind); | |
| 210 } | |
| 211 } | |
| 212 | |
| 213 Token finishImportant() { | |
| 214 | |
| 215 } | |
| 216 | |
| 217 Token finishNumber() { | |
| 218 eatDigits(); | |
| 219 | |
| 220 if (_peekChar() == 46/*.*/) { | |
| 221 // Handle the case of 1.toString(). | |
| 222 _nextChar(); | |
| 223 if (TokenizerHelpers.isDigit(_peekChar())) { | |
| 224 eatDigits(); | |
| 225 return _finishToken(TokenKind.DOUBLE); | |
| 226 } else { | |
| 227 _index -= 1; | |
| 228 } | |
| 229 } | |
| 230 | |
| 231 return _finishToken(TokenKind.INTEGER); | |
| 232 } | |
| 233 | |
| 234 bool maybeEatDigit() { | |
| 235 if (_index < _text.length | |
| 236 && TokenizerHelpers.isDigit(_text.codeUnitAt(_index))) { | |
| 237 _index += 1; | |
| 238 return true; | |
| 239 } | |
| 240 return false; | |
| 241 } | |
| 242 | |
| 243 void eatHexDigits() { | |
| 244 while (_index < _text.length) { | |
| 245 if (TokenizerHelpers.isHexDigit(_text.codeUnitAt(_index))) { | |
| 246 _index += 1; | |
| 247 } else { | |
| 248 return; | |
| 249 } | |
| 250 } | |
| 251 } | |
| 252 | |
| 253 bool maybeEatHexDigit() { | |
| 254 if (_index < _text.length | |
| 255 && TokenizerHelpers.isHexDigit(_text.codeUnitAt(_index))) { | |
| 256 _index += 1; | |
| 257 return true; | |
| 258 } | |
| 259 return false; | |
| 260 } | |
| 261 | |
| 262 Token finishMultiLineComment() { | |
| 263 while (true) { | |
| 264 int ch = _nextChar(); | |
| 265 if (ch == 0) { | |
| 266 return _finishToken(TokenKind.INCOMPLETE_COMMENT); | |
| 267 } else if (ch == 42/*'*'*/) { | |
| 268 if (_maybeEatChar(47/*'/'*/)) { | |
| 269 if (_skipWhitespace) { | |
| 270 return next(); | |
| 271 } else { | |
| 272 return _finishToken(TokenKind.COMMENT); | |
| 273 } | |
| 274 } | |
| 275 } else if (ch == cssTokens.tokens[TokenKind.MINUS]) { | |
| 276 /* Check if close part of Comment Definition --> (CDC). */ | |
| 277 if (_maybeEatChar(cssTokens.tokens[TokenKind.MINUS])) { | |
| 278 if (_maybeEatChar(cssTokens.tokens[TokenKind.GREATER])) { | |
| 279 if (_skipWhitespace) { | |
| 280 return next(); | |
| 281 } else { | |
| 282 return _finishToken(TokenKind.HTML_COMMENT); | |
| 283 } | |
| 284 } | |
| 285 } | |
| 286 } | |
| 287 } | |
| 288 return _errorToken(); | |
| 289 } | |
| 290 | |
| 291 } | |
| 292 | |
| 293 /** Static helper methods. */ | |
| 294 /** Static helper methods. */ | |
| 295 class TokenizerHelpers { | |
| 296 | |
| 297 static bool isIdentifierStart(int c) { | |
| 298 return ((c >= 97/*a*/ && c <= 122/*z*/) || (c >= 65/*A*/ && c <= 90/*Z*/) || | |
| 299 c == 95/*_*/ || c == 45 /*-*/); | |
| 300 } | |
| 301 | |
| 302 static bool isDigit(int c) { | |
| 303 return (c >= 48/*0*/ && c <= 57/*9*/); | |
| 304 } | |
| 305 | |
| 306 static bool isHexDigit(int c) { | |
| 307 return (isDigit(c) || (c >= 97/*a*/ && c <= 102/*f*/) | |
| 308 || (c >= 65/*A*/ && c <= 70/*F*/)); | |
| 309 } | |
| 310 | |
| 311 static bool isIdentifierPart(int c) { | |
| 312 return (isIdentifierStart(c) || isDigit(c) || c == 45 /*-*/); | |
| 313 } | |
| 314 } | |
| 315 | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 137013002:
Removed obsolete code (Closed)
Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart