Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(170)

Side by Side Diff: lib/src/tokenizer_base.dart

Issue 998843003: pkg/csslib: formatting (Closed) Base URL: https://github.com/dart-lang/csslib@master
Patch Set: Created 5 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file 1 // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
2 // for details. All rights reserved. Use of this source code is governed by a 2 // for details. All rights reserved. Use of this source code is governed by a
3 // BSD-style license that can be found in the LICENSE file. 3 // BSD-style license that can be found in the LICENSE file.
4 // Generated by scripts/tokenizer_gen.py. 4 // Generated by scripts/tokenizer_gen.py.
5 5
6 part of csslib.parser; 6 part of csslib.parser;
7 7
8 /** Tokenizer state to support look ahead for Less' nested selectors. */ 8 /** Tokenizer state to support look ahead for Less' nested selectors. */
9 class TokenizerState { 9 class TokenizerState {
10 final int index; 10 final int index;
11 final int startIndex; 11 final int startIndex;
12 final bool inSelectorExpression; 12 final bool inSelectorExpression;
13 final bool inSelector; 13 final bool inSelector;
14 14
15 TokenizerState(TokenizerBase base) : 15 TokenizerState(TokenizerBase base)
16 index = base._index, 16 : index = base._index,
17 startIndex = base._startIndex, 17 startIndex = base._startIndex,
18 inSelectorExpression = base.inSelectorExpression, 18 inSelectorExpression = base.inSelectorExpression,
19 inSelector = base.inSelector; 19 inSelector = base.inSelector;
20 } 20 }
21 21
22 /** 22 /**
23 * The base class for our tokenizer. The hand coded parts are in this file, with 23 * The base class for our tokenizer. The hand coded parts are in this file, with
24 * the generated parts in the subclass Tokenizer. 24 * the generated parts in the subclass Tokenizer.
25 */ 25 */
26 abstract class TokenizerBase { 26 abstract class TokenizerBase {
27 final SourceFile _file; 27 final SourceFile _file;
28 final String _text; 28 final String _text;
29 29
(...skipping 104 matching lines...) Expand 10 before | Expand all | Expand 10 after
134 return _finishToken(TokenKind.WHITESPACE); // note the newline? 134 return _finishToken(TokenKind.WHITESPACE); // note the newline?
135 } 135 }
136 } else { 136 } else {
137 _index--; 137 _index--;
138 if (_skipWhitespace) { 138 if (_skipWhitespace) {
139 return next(); 139 return next();
140 } else { 140 } else {
141 return _finishToken(TokenKind.WHITESPACE); 141 return _finishToken(TokenKind.WHITESPACE);
142 } 142 }
143 } 143 }
144
145 } 144 }
146 return _finishToken(TokenKind.END_OF_FILE); 145 return _finishToken(TokenKind.END_OF_FILE);
147 } 146 }
148 147
149 Token finishMultiLineComment() { 148 Token finishMultiLineComment() {
150 int nesting = 1; 149 int nesting = 1;
151 do { 150 do {
152 int ch = _nextChar(); 151 int ch = _nextChar();
153 if (ch == 0) { 152 if (ch == 0) {
154 return _errorToken(); 153 return _errorToken();
(...skipping 19 matching lines...) Expand all
174 while (_index < _text.length) { 173 while (_index < _text.length) {
175 if (TokenizerHelpers.isDigit(_text.codeUnitAt(_index))) { 174 if (TokenizerHelpers.isDigit(_text.codeUnitAt(_index))) {
176 _index++; 175 _index++;
177 } else { 176 } else {
178 return; 177 return;
179 } 178 }
180 } 179 }
181 } 180 }
182 181
183 static int _hexDigit(int c) { 182 static int _hexDigit(int c) {
184 if(c >= 48/*0*/ && c <= 57/*9*/) { 183 if (c >= 48 /*0*/ && c <= 57 /*9*/) {
185 return c - 48; 184 return c - 48;
186 } else if (c >= 97/*a*/ && c <= 102/*f*/) { 185 } else if (c >= 97 /*a*/ && c <= 102 /*f*/) {
187 return c - 87; 186 return c - 87;
188 } else if (c >= 65/*A*/ && c <= 70/*F*/) { 187 } else if (c >= 65 /*A*/ && c <= 70 /*F*/) {
189 return c - 55; 188 return c - 55;
190 } else { 189 } else {
191 return -1; 190 return -1;
192 } 191 }
193 } 192 }
194 193
195 int readHex([int hexLength]) { 194 int readHex([int hexLength]) {
196 int maxIndex; 195 int maxIndex;
197 if (hexLength == null) { 196 if (hexLength == null) {
198 maxIndex = _text.length - 1; 197 maxIndex = _text.length - 1;
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after
234 return finishNumberExtra(TokenKind.DOUBLE); 233 return finishNumberExtra(TokenKind.DOUBLE);
235 } else { 234 } else {
236 _index--; 235 _index--;
237 } 236 }
238 } 237 }
239 238
240 return finishNumberExtra(TokenKind.INTEGER); 239 return finishNumberExtra(TokenKind.INTEGER);
241 } 240 }
242 241
243 Token finishNumberExtra(int kind) { 242 Token finishNumberExtra(int kind) {
244 if (_maybeEatChar(101/*e*/) || _maybeEatChar(69/*E*/)) { 243 if (_maybeEatChar(101 /*e*/) || _maybeEatChar(69 /*E*/)) {
245 kind = TokenKind.DOUBLE; 244 kind = TokenKind.DOUBLE;
246 _maybeEatChar(TokenKind.MINUS); 245 _maybeEatChar(TokenKind.MINUS);
247 _maybeEatChar(TokenKind.PLUS); 246 _maybeEatChar(TokenKind.PLUS);
248 eatDigits(); 247 eatDigits();
249 } 248 }
250 if (_peekChar() != 0 && TokenizerHelpers.isIdentifierStart(_peekChar())) { 249 if (_peekChar() != 0 && TokenizerHelpers.isIdentifierStart(_peekChar())) {
251 _nextChar(); 250 _nextChar();
252 return _errorToken("illegal character in number"); 251 return _errorToken("illegal character in number");
253 } 252 }
254 253
(...skipping 14 matching lines...) Expand all
269 Token _makeRawStringToken(bool isMultiline) { 268 Token _makeRawStringToken(bool isMultiline) {
270 var s; 269 var s;
271 if (isMultiline) { 270 if (isMultiline) {
272 // Skip initial newline in multiline strings 271 // Skip initial newline in multiline strings
273 int start = _startIndex + 4; 272 int start = _startIndex + 4;
274 if (_text[start] == '\n') start++; 273 if (_text[start] == '\n') start++;
275 s = _text.substring(start, _index - 3); 274 s = _text.substring(start, _index - 3);
276 } else { 275 } else {
277 s = _text.substring(_startIndex + 2, _index - 1); 276 s = _text.substring(_startIndex + 2, _index - 1);
278 } 277 }
279 return new LiteralToken(TokenKind.STRING, 278 return new LiteralToken(
280 _file.span(_startIndex, _index), s); 279 TokenKind.STRING, _file.span(_startIndex, _index), s);
281 } 280 }
282 281
283 Token finishMultilineString(int quote) { 282 Token finishMultilineString(int quote) {
284 var buf = <int>[]; 283 var buf = <int>[];
285 while (true) { 284 while (true) {
286 int ch = _nextChar(); 285 int ch = _nextChar();
287 if (ch == 0) { 286 if (ch == 0) {
288 return _errorToken(); 287 return _errorToken();
289 } else if (ch == quote) { 288 } else if (ch == quote) {
290 if (_maybeEatChar(quote)) { 289 if (_maybeEatChar(quote)) {
(...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after
375 } else { 374 } else {
376 buf.add(ch); 375 buf.add(ch);
377 } 376 }
378 } 377 }
379 } 378 }
380 379
381 int readEscapeSequence() { 380 int readEscapeSequence() {
382 final ch = _nextChar(); 381 final ch = _nextChar();
383 int hexValue; 382 int hexValue;
384 switch (ch) { 383 switch (ch) {
385 case 110/*n*/: 384 case 110 /*n*/ :
386 return TokenChar.NEWLINE; 385 return TokenChar.NEWLINE;
387 case 114/*r*/: 386 case 114 /*r*/ :
388 return TokenChar.RETURN; 387 return TokenChar.RETURN;
389 case 102/*f*/: 388 case 102 /*f*/ :
390 return TokenChar.FF; 389 return TokenChar.FF;
391 case 98/*b*/: 390 case 98 /*b*/ :
392 return TokenChar.BACKSPACE; 391 return TokenChar.BACKSPACE;
393 case 116/*t*/: 392 case 116 /*t*/ :
394 return TokenChar.TAB; 393 return TokenChar.TAB;
395 case 118/*v*/: 394 case 118 /*v*/ :
396 return TokenChar.FF; 395 return TokenChar.FF;
397 case 120/*x*/: 396 case 120 /*x*/ :
398 hexValue = readHex(2); 397 hexValue = readHex(2);
399 break; 398 break;
400 case 117/*u*/: 399 case 117 /*u*/ :
401 if (_maybeEatChar(TokenChar.LBRACE)) { 400 if (_maybeEatChar(TokenChar.LBRACE)) {
402 hexValue = readHex(); 401 hexValue = readHex();
403 if (!_maybeEatChar(TokenChar.RBRACE)) { 402 if (!_maybeEatChar(TokenChar.RBRACE)) {
404 return -1; 403 return -1;
405 } 404 }
406 } else { 405 } else {
407 hexValue = readHex(4); 406 hexValue = readHex(4);
408 } 407 }
409 break; 408 break;
410 default: return ch; 409 default:
410 return ch;
411 } 411 }
412 412
413 if (hexValue == -1) return -1; 413 if (hexValue == -1) return -1;
414 414
415 // According to the Unicode standard the high and low surrogate halves 415 // According to the Unicode standard the high and low surrogate halves
416 // used by UTF-16 (U+D800 through U+DFFF) and values above U+10FFFF 416 // used by UTF-16 (U+D800 through U+DFFF) and values above U+10FFFF
417 // are not legal Unicode values. 417 // are not legal Unicode values.
418 if (hexValue < 0xD800 || hexValue > 0xDFFF && hexValue <= 0xFFFF) { 418 if (hexValue < 0xD800 || hexValue > 0xDFFF && hexValue <= 0xFFFF) {
419 return hexValue; 419 return hexValue;
420 } else if (hexValue <= 0x10FFFF){ 420 } else if (hexValue <= 0x10FFFF) {
421 messages.error('unicode values greater than 2 bytes not implemented yet', 421 messages.error('unicode values greater than 2 bytes not implemented yet',
422 _file.span(_startIndex, _startIndex + 1)); 422 _file.span(_startIndex, _startIndex + 1));
423 return -1; 423 return -1;
424 } else { 424 } else {
425 return -1; 425 return -1;
426 } 426 }
427 } 427 }
428 428
429 Token finishDot() { 429 Token finishDot() {
430 if (TokenizerHelpers.isDigit(_peekChar())) { 430 if (TokenizerHelpers.isDigit(_peekChar())) {
431 eatDigits(); 431 eatDigits();
432 return finishNumberExtra(TokenKind.DOUBLE); 432 return finishNumberExtra(TokenKind.DOUBLE);
433 } else { 433 } else {
434 return _finishToken(TokenKind.DOT); 434 return _finishToken(TokenKind.DOT);
435 } 435 }
436 } 436 }
437 } 437 }
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698